diff options
Diffstat (limited to 'src/or')
130 files changed, 33455 insertions, 14722 deletions
diff --git a/src/or/Makefile.nmake b/src/or/Makefile.nmake index 3b627b1d06..2ac98cd372 100644 --- a/src/or/Makefile.nmake +++ b/src/or/Makefile.nmake @@ -1,6 +1,6 @@ all: tor.exe -CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common \ +CFLAGS = /O2 /MT /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common \ /I ..\ext LIBS = ..\..\..\build-alpha\lib\libevent.lib \ @@ -15,6 +15,7 @@ LIBTOR_OBJECTS = \ buffers.obj \ channel.obj \ channeltls.obj \ + circpathbias.obj \ circuitbuild.obj \ circuitlist.obj \ circuitmux.obj \ @@ -35,6 +36,7 @@ LIBTOR_OBJECTS = \ dirvote.obj \ dns.obj \ dnsserv.obj \ + ext_orport.obj \ fp_pair.obj \ entrynodes.obj \ geoip.obj \ @@ -61,6 +63,7 @@ LIBTOR_OBJECTS = \ routerlist.obj \ routerparse.obj \ routerset.obj \ + scheduler.obj \ statefile.obj \ status.obj \ transports.obj @@ -69,7 +72,7 @@ libtor.lib: $(LIBTOR_OBJECTS) lib $(LIBTOR_OBJECTS) /out:$@ tor.exe: libtor.lib tor_main.obj - $(CC) $(CFLAGS) $(LIBS) libtor.lib ..\common\*.lib tor_main.obj /Fe$@ + $(CC) $(CFLAGS) $(LIBS) libtor.lib ..\common\*.lib ..\ext\*.lib tor_main.obj /Fe$@ clean: - del $(LIBTOR_OBJECTS) *.lib tor.exe + del $(LIBTOR_OBJECTS) tor_main.obj *.lib tor.exe diff --git a/src/or/addressmap.c b/src/or/addressmap.c index 79e4b7c5e2..047a863ef5 100644 --- a/src/or/addressmap.c +++ b/src/or/addressmap.c @@ -1,9 +1,18 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * \file addressmap.c + * + * \brief The addressmap module manages the processes by which we rewrite + * addresses in client requess. It handles the MapAddress controller and + * torrc commands, and the TrackHostExits feature, and the client-side DNS + * cache (deprecated). + */ + #define ADDRESSMAP_PRIVATE #include "or.h" @@ -45,7 +54,7 @@ typedef struct { char *new_address; time_t expires; - ENUM_BF(addressmap_entry_source_t) source:3; + addressmap_entry_source_bitfield_t source:3; unsigned src_wildcard:1; unsigned dst_wildcard:1; short num_resolve_failures; @@ -94,7 +103,7 @@ addressmap_ent_free(void *_ent) tor_free(ent); } -/** Free storage held by a virtaddress_entry_t* entry in <b>ent</b>. */ +/** Free storage held by a virtaddress_entry_t* entry in <b>_ent</b>. */ static void addressmap_virtaddress_ent_free(void *_ent) { @@ -104,11 +113,13 @@ addressmap_virtaddress_ent_free(void *_ent) ent = _ent; tor_free(ent->ipv4_address); + tor_free(ent->ipv6_address); tor_free(ent->hostname_address); tor_free(ent); } -/** Free storage held by a virtaddress_entry_t* entry in <b>ent</b>. */ +/** Remove <b>address</b> (which must map to <b>ent</b>) from the + * virtual address map. */ static void addressmap_virtaddress_remove(const char *address, addressmap_entry_t *ent) { @@ -120,9 +131,11 @@ addressmap_virtaddress_remove(const char *address, addressmap_entry_t *ent) if (ve) { if (!strcmp(address, ve->ipv4_address)) tor_free(ve->ipv4_address); + if (!strcmp(address, ve->ipv6_address)) + tor_free(ve->ipv6_address); if (!strcmp(address, ve->hostname_address)) tor_free(ve->hostname_address); - if (!ve->ipv4_address && !ve->hostname_address) { + if (!ve->ipv4_address && !ve->ipv6_address && !ve->hostname_address) { tor_free(ve); strmap_remove(virtaddress_reversemap, ent->new_address); } @@ -131,7 +144,7 @@ addressmap_virtaddress_remove(const char *address, addressmap_entry_t *ent) } /** Remove <b>ent</b> (which must be mapped to by <b>address</b>) from the - * client address maps. */ + * client address maps, and then free it. */ static void addressmap_ent_remove(const char *address, addressmap_entry_t *ent) { @@ -226,6 +239,8 @@ addressmap_address_should_automap(const char *address, return 0; SMARTLIST_FOREACH_BEGIN(suffix_list, const char *, suffix) { + if (!strcmp(suffix, ".")) + return 1; if (!strcasecmpend(address, suffix)) return 1; } SMARTLIST_FOREACH_END(suffix); @@ -384,13 +399,35 @@ addressmap_rewrite(char *address, size_t maxlen, goto done; } - if (ent && ent->source == ADDRMAPSRC_DNS) { - sa_family_t f; - tor_addr_t tmp; - f = tor_addr_parse(&tmp, ent->new_address); - if (f == AF_INET && !(flags & AMR_FLAG_USE_IPV4_DNS)) - goto done; - else if (f == AF_INET6 && !(flags & AMR_FLAG_USE_IPV6_DNS)) + switch (ent->source) { + case ADDRMAPSRC_DNS: + { + sa_family_t f; + tor_addr_t tmp; + f = tor_addr_parse(&tmp, ent->new_address); + if (f == AF_INET && !(flags & AMR_FLAG_USE_IPV4_DNS)) + goto done; + else if (f == AF_INET6 && !(flags & AMR_FLAG_USE_IPV6_DNS)) + goto done; + } + break; + case ADDRMAPSRC_CONTROLLER: + case ADDRMAPSRC_TORRC: + if (!(flags & AMR_FLAG_USE_MAPADDRESS)) + goto done; + break; + case ADDRMAPSRC_AUTOMAP: + if (!(flags & AMR_FLAG_USE_AUTOMAP)) + goto done; + break; + case ADDRMAPSRC_TRACKEXIT: + if (!(flags & AMR_FLAG_USE_TRACKEXIT)) + goto done; + break; + case ADDRMAPSRC_NONE: + default: + log_warn(LD_BUG, "Unknown addrmap source value %d. Ignoring it.", + (int) ent->source); goto done; } @@ -425,7 +462,7 @@ addressmap_rewrite(char *address, size_t maxlen, if (exit_source_out) *exit_source_out = exit_source; if (expires_out) - *expires_out = TIME_MAX; + *expires_out = expires; return (rewrites > 0); } @@ -449,6 +486,8 @@ addressmap_rewrite_reverse(char *address, size_t maxlen, unsigned flags, return 0; else if (f == AF_INET6 && !(flags & AMR_FLAG_USE_IPV6_DNS)) return 0; + /* FFFF we should reverse-map virtual addresses even if we haven't + * enabled DNS cacheing. */ } tor_asprintf(&s, "REVERSE[%s]", address); @@ -496,7 +535,7 @@ addressmap_have_mapping(const char *address, int update_expiry) * equal to <b>address</b>, or any address ending with a period followed by * <b>address</b>. If <b>wildcard_addr</b> and <b>wildcard_new_addr</b> are * both true, the mapping will rewrite addresses that end with - * ".<b>address</b>" into ones that end with ".<b>new_address</b>." + * ".<b>address</b>" into ones that end with ".<b>new_address</b>". * * If <b>new_address</b> is NULL, or <b>new_address</b> is equal to * <b>address</b> and <b>wildcard_addr</b> is equal to @@ -535,9 +574,9 @@ addressmap_register(const char *address, char *new_address, time_t expires, if (expires > 1) { log_info(LD_APP,"Temporary addressmap ('%s' to '%s') not performed, " "since it's already mapped to '%s'", - safe_str_client(address), - safe_str_client(new_address), - safe_str_client(ent->new_address)); + safe_str_client(address), + safe_str_client(new_address), + safe_str_client(ent->new_address)); tor_free(new_address); return; } @@ -670,10 +709,10 @@ client_dns_set_addressmap(entry_connection_t *for_conn, return; /* If address was an IP address already, don't add a mapping. */ if (tor_addr_family(val) == AF_INET) { - if (! for_conn->cache_ipv4_answers) + if (! for_conn->entry_cfg.cache_ipv4_answers) return; } else if (tor_addr_family(val) == AF_INET6) { - if (! for_conn->cache_ipv6_answers) + if (! for_conn->entry_cfg.cache_ipv6_answers) return; } @@ -702,8 +741,8 @@ client_dns_set_reverse_addressmap(entry_connection_t *for_conn, { tor_addr_t tmp_addr; sa_family_t f = tor_addr_parse(&tmp_addr, address); - if ((f == AF_INET && ! for_conn->cache_ipv4_answers) || - (f == AF_INET6 && ! for_conn->cache_ipv6_answers)) + if ((f == AF_INET && ! for_conn->entry_cfg.cache_ipv4_answers) || + (f == AF_INET6 && ! for_conn->entry_cfg.cache_ipv6_answers)) return; } tor_asprintf(&s, "REVERSE[%s]", address); @@ -738,6 +777,12 @@ parse_virtual_addr_network(const char *val, sa_family_t family, const int max_bits = ipv6 ? 40 : 16; virtual_addr_conf_t *conf = ipv6 ? &virtaddr_conf_ipv6 : &virtaddr_conf_ipv4; + if (!val || val[0] == '\0') { + if (msg) + tor_asprintf(msg, "Value not present (%s) after VirtualAddressNetwork%s", + val?"Empty":"NULL", ipv6?"IPv6":""); + return -1; + } if (tor_addr_parse_mask_ports(val, 0, &addr, &bits, NULL, NULL) < 0) { if (msg) tor_asprintf(msg, "Error parsing VirtualAddressNetwork%s %s", @@ -798,7 +843,7 @@ address_is_in_virtual_range(const char *address) /** Return a random address conforming to the virtual address configuration * in <b>conf</b>. */ -/* private */ void +STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf, tor_addr_t *addr_out) { uint8_t tmp[4]; @@ -839,8 +884,8 @@ get_random_virtual_addr(const virtual_addr_conf_t *conf, tor_addr_t *addr_out) } /** Return a newly allocated string holding an address of <b>type</b> - * (one of RESOLVED_TYPE_{IPV4|HOSTNAME}) that has not yet been mapped, - * and that is very unlikely to be the address of any real host. + * (one of RESOLVED_TYPE_{IPV4|IPV6|HOSTNAME}) that has not yet been + * mapped, and that is very unlikely to be the address of any real host. * * May return NULL if we have run out of virtual addresses. */ @@ -888,7 +933,7 @@ addressmap_get_virtual_address(int type) /* XXXX This code is to make sure I didn't add an undecorated version * by mistake. I hope it's needless. */ char tmp[TOR_ADDR_BUF_LEN]; - tor_addr_to_str(buf, &addr, sizeof(tmp), 0); + tor_addr_to_str(tmp, &addr, sizeof(tmp), 0); if (strmap_get(addressmap, tmp)) { log_warn(LD_BUG, "%s wasn't in the addressmap, but %s was.", buf, tmp); @@ -945,7 +990,7 @@ addressmap_register_virtual_address(int type, char *new_address) !strcasecmp(new_address, ent->new_address)) { tor_free(new_address); tor_assert(!vent_needs_to_be_added); - return tor_strdup(*addrp); + return *addrp; } else { log_warn(LD_BUG, "Internal confusion: I thought that '%s' was mapped to by " @@ -969,6 +1014,8 @@ addressmap_register_virtual_address(int type, char *new_address) strmap_set(virtaddress_reversemap, new_address, vent); addressmap_register(*addrp, new_address, 2, ADDRMAPSRC_AUTOMAP, 0, 0); + /* FFFF register corresponding reverse mapping. */ + #if 0 { /* Try to catch possible bugs */ diff --git a/src/or/addressmap.h b/src/or/addressmap.h index 40210ee990..67648d0518 100644 --- a/src/or/addressmap.h +++ b/src/or/addressmap.h @@ -1,12 +1,14 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ADDRESSMAP_H #define TOR_ADDRESSMAP_H +#include "testsupport.h" + void addressmap_init(void); void addressmap_clear_excluded_trackexithosts(const or_options_t *options); void addressmap_clear_invalid_automaps(const or_options_t *options); @@ -14,8 +16,11 @@ void addressmap_clean(time_t now); void addressmap_clear_configured(void); void addressmap_clear_transient(void); void addressmap_free_all(void); -#define AMR_FLAG_USE_IPV4_DNS (1u<<0) -#define AMR_FLAG_USE_IPV6_DNS (1u<<1) +#define AMR_FLAG_USE_IPV4_DNS (1u<<0) +#define AMR_FLAG_USE_IPV6_DNS (1u<<1) +#define AMR_FLAG_USE_MAPADDRESS (1u<<2) +#define AMR_FLAG_USE_AUTOMAP (1u<<3) +#define AMR_FLAG_USE_TRACKEXIT (1u<<4) int addressmap_rewrite(char *address, size_t maxlen, unsigned flags, time_t *expires_out, addressmap_entry_source_t *exit_source_out); @@ -52,8 +57,8 @@ typedef struct virtual_addr_conf_t { maskbits_t bits; } virtual_addr_conf_t; -void get_random_virtual_addr(const virtual_addr_conf_t *conf, - tor_addr_t *addr_out); +STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf, + tor_addr_t *addr_out); #endif #endif diff --git a/src/or/buffers.c b/src/or/buffers.c index ab3346d9b7..a41af5f429 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -1,14 +1,15 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file buffers.c * \brief Implements a generic interface buffer. Buffers are * fairly opaque string holders that can read to or flush from: - * memory, file descriptors, or TLS connections. + * memory, file descriptors, or TLS connections. Buffers are implemented + * as linked lists of memory chunks. **/ #define BUFFERS_PRIVATE #include "or.h" @@ -19,8 +20,9 @@ #include "connection_or.h" #include "control.h" #include "reasons.h" -#include "../common/util.h" -#include "../common/torlog.h" +#include "ext_orport.h" +#include "util.h" +#include "torlog.h" #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -54,6 +56,9 @@ * forever. */ +static void socks_request_set_socks5_error(socks_request_t *req, + socks5_reply_status_t reason); + static int parse_socks(const char *data, size_t datalen, socks_request_t *req, int log_sockstype, int safe_socks, ssize_t *drain_out, size_t *want_length_out); @@ -63,28 +68,39 @@ static int parse_socks_client(const uint8_t *data, size_t datalen, /* Chunk manipulation functions */ -/** A single chunk on a buffer or in a freelist. */ -typedef struct chunk_t { - struct chunk_t *next; /**< The next chunk on the buffer or freelist. */ - size_t datalen; /**< The number of bytes stored in this chunk */ - size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ - char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */ - char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in - * this chunk. */ -} chunk_t; - #define CHUNK_HEADER_LEN STRUCT_OFFSET(chunk_t, mem[0]) +/* We leave this many NUL bytes at the end of the buffer. */ +#define SENTINEL_LEN 4 + +/* Header size plus NUL bytes at the end */ +#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN) + /** Return the number of bytes needed to allocate a chunk to hold * <b>memlen</b> bytes. */ -#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_HEADER_LEN + (memlen)) +#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen)) /** Return the number of usable bytes in a chunk allocated with * malloc(<b>memlen</b>). */ -#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_HEADER_LEN) +#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD) + +#define DEBUG_SENTINEL + +#ifdef DEBUG_SENTINEL +#define DBG_S(s) s +#else +#define DBG_S(s) (void)0 +#endif + +#define CHUNK_SET_SENTINEL(chunk, alloclen) do { \ + uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen]; \ + DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]); \ + DBG_S(tor_assert(a == b)); \ + memset(a,0,SENTINEL_LEN); \ + } while (0) /** Return the next character in <b>chunk</b> onto which data can be appended. * If the chunk is full, this might be off the end of chunk->mem. */ -static INLINE char * +static inline char * CHUNK_WRITE_PTR(chunk_t *chunk) { return chunk->data + chunk->datalen; @@ -92,7 +108,7 @@ CHUNK_WRITE_PTR(chunk_t *chunk) /** Return the number of bytes that can be written onto <b>chunk</b> without * running out of space. */ -static INLINE size_t +static inline size_t CHUNK_REMAINING_CAPACITY(const chunk_t *chunk) { return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen); @@ -100,7 +116,7 @@ CHUNK_REMAINING_CAPACITY(const chunk_t *chunk) /** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem, * to free up space at the end. */ -static INLINE void +static inline void chunk_repack(chunk_t *chunk) { if (chunk->datalen && chunk->data != &chunk->mem[0]) { @@ -109,133 +125,58 @@ chunk_repack(chunk_t *chunk) chunk->data = &chunk->mem[0]; } -#if defined(ENABLE_BUF_FREELISTS) || defined(RUNNING_DOXYGEN) -/** A freelist of chunks. */ -typedef struct chunk_freelist_t { - size_t alloc_size; /**< What size chunks does this freelist hold? */ - int max_length; /**< Never allow more than this number of chunks in the - * freelist. */ - int slack; /**< When trimming the freelist, leave this number of extra - * chunks beyond lowest_length.*/ - int cur_length; /**< How many chunks on the freelist now? */ - int lowest_length; /**< What's the smallest value of cur_length since the - * last time we cleaned this freelist? */ - uint64_t n_alloc; - uint64_t n_free; - uint64_t n_hit; - chunk_t *head; /**< First chunk on the freelist. */ -} chunk_freelist_t; - -/** Macro to help define freelists. */ -#define FL(a,m,s) { a, m, s, 0, 0, 0, 0, 0, NULL } - -/** Static array of freelists, sorted by alloc_len, terminated by an entry - * with alloc_size of 0. */ -static chunk_freelist_t freelists[] = { - FL(4096, 256, 8), FL(8192, 128, 4), FL(16384, 64, 4), FL(32768, 32, 2), - FL(0, 0, 0) -}; -#undef FL -/** How many times have we looked for a chunk of a size that no freelist - * could help with? */ -static uint64_t n_freelist_miss = 0; - -static void assert_freelist_ok(chunk_freelist_t *fl); - -/** Return the freelist to hold chunks of size <b>alloc</b>, or NULL if - * no freelist exists for that size. */ -static INLINE chunk_freelist_t * -get_freelist(size_t alloc) -{ - int i; - for (i=0; (freelists[i].alloc_size <= alloc && - freelists[i].alloc_size); ++i ) { - if (freelists[i].alloc_size == alloc) { - return &freelists[i]; - } - } - return NULL; -} - -/** Deallocate a chunk or put it on a freelist */ -static void -chunk_free_unchecked(chunk_t *chunk) -{ - size_t alloc; - chunk_freelist_t *freelist; - - alloc = CHUNK_ALLOC_SIZE(chunk->memlen); - freelist = get_freelist(alloc); - if (freelist && freelist->cur_length < freelist->max_length) { - chunk->next = freelist->head; - freelist->head = chunk; - ++freelist->cur_length; - } else { - if (freelist) - ++freelist->n_free; - tor_free(chunk); - } -} - -/** Allocate a new chunk with a given allocation size, or get one from the - * freelist. Note that a chunk with allocation size A can actually hold only - * CHUNK_SIZE_WITH_ALLOC(A) bytes in its mem field. */ -static INLINE chunk_t * -chunk_new_with_alloc_size(size_t alloc) -{ - chunk_t *ch; - chunk_freelist_t *freelist; - tor_assert(alloc >= sizeof(chunk_t)); - freelist = get_freelist(alloc); - if (freelist && freelist->head) { - ch = freelist->head; - freelist->head = ch->next; - if (--freelist->cur_length < freelist->lowest_length) - freelist->lowest_length = freelist->cur_length; - ++freelist->n_hit; - } else { - if (freelist) - ++freelist->n_alloc; - else - ++n_freelist_miss; - ch = tor_malloc(alloc); - } - ch->next = NULL; - ch->datalen = 0; - ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); - ch->data = &ch->mem[0]; - return ch; -} -#else +/** Keep track of total size of allocated chunks for consistency asserts */ +static size_t total_bytes_allocated_in_chunks = 0; static void chunk_free_unchecked(chunk_t *chunk) { + if (!chunk) + return; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); } -static INLINE chunk_t * +static inline chunk_t * chunk_new_with_alloc_size(size_t alloc) { chunk_t *ch; ch = tor_malloc(alloc); ch->next = NULL; ch->datalen = 0; +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); + total_bytes_allocated_in_chunks += alloc; ch->data = &ch->mem[0]; + CHUNK_SET_SENTINEL(ch, alloc); return ch; } -#endif /** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a * new pointer to <b>chunk</b>. Old pointers are no longer valid. */ -static INLINE chunk_t * +static inline chunk_t * chunk_grow(chunk_t *chunk, size_t sz) { off_t offset; + const size_t memlen_orig = chunk->memlen; + const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig); + const size_t new_alloc = CHUNK_ALLOC_SIZE(sz); tor_assert(sz > chunk->memlen); offset = chunk->data - chunk->mem; - chunk = tor_realloc(chunk, CHUNK_ALLOC_SIZE(sz)); + chunk = tor_realloc(chunk, new_alloc); chunk->memlen = sz; chunk->data = chunk->mem + offset; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == orig_alloc); + chunk->DBG_alloc = new_alloc; +#endif + total_bytes_allocated_in_chunks += new_alloc - orig_alloc; + CHUNK_SET_SENTINEL(chunk, new_alloc); return chunk; } @@ -249,7 +190,7 @@ chunk_grow(chunk_t *chunk, size_t sz) /** Return the allocation size we'd like to use to hold <b>target</b> * bytes. */ -static INLINE size_t +static inline size_t preferred_chunk_size(size_t target) { size_t sz = MIN_CHUNK_ALLOC; @@ -259,124 +200,12 @@ preferred_chunk_size(size_t target) return sz; } -/** Remove from the freelists most chunks that have not been used since the - * last call to buf_shrink_freelists(). */ -void -buf_shrink_freelists(int free_all) -{ -#ifdef ENABLE_BUF_FREELISTS - int i; - disable_control_logging(); - for (i = 0; freelists[i].alloc_size; ++i) { - int slack = freelists[i].slack; - assert_freelist_ok(&freelists[i]); - if (free_all || freelists[i].lowest_length > slack) { - int n_to_free = free_all ? freelists[i].cur_length : - (freelists[i].lowest_length - slack); - int n_to_skip = freelists[i].cur_length - n_to_free; - int orig_length = freelists[i].cur_length; - int orig_n_to_free = n_to_free, n_freed=0; - int orig_n_to_skip = n_to_skip; - int new_length = n_to_skip; - chunk_t **chp = &freelists[i].head; - chunk_t *chunk; - while (n_to_skip) { - if (! (*chp)->next) { - log_warn(LD_BUG, "I wanted to skip %d chunks in the freelist for " - "%d-byte chunks, but only found %d. (Length %d)", - orig_n_to_skip, (int)freelists[i].alloc_size, - orig_n_to_skip-n_to_skip, freelists[i].cur_length); - assert_freelist_ok(&freelists[i]); - goto done; - } - // tor_assert((*chp)->next); - chp = &(*chp)->next; - --n_to_skip; - } - chunk = *chp; - *chp = NULL; - while (chunk) { - chunk_t *next = chunk->next; - tor_free(chunk); - chunk = next; - --n_to_free; - ++n_freed; - ++freelists[i].n_free; - } - if (n_to_free) { - log_warn(LD_BUG, "Freelist length for %d-byte chunks may have been " - "messed up somehow.", (int)freelists[i].alloc_size); - log_warn(LD_BUG, "There were %d chunks at the start. I decided to " - "keep %d. I wanted to free %d. I freed %d. I somehow think " - "I have %d left to free.", - freelists[i].cur_length, n_to_skip, orig_n_to_free, - n_freed, n_to_free); - } - // tor_assert(!n_to_free); - freelists[i].cur_length = new_length; - log_info(LD_MM, "Cleaned freelist for %d-byte chunks: original " - "length %d, kept %d, dropped %d.", - (int)freelists[i].alloc_size, orig_length, - orig_n_to_skip, orig_n_to_free); - } - freelists[i].lowest_length = freelists[i].cur_length; - assert_freelist_ok(&freelists[i]); - } - done: - enable_control_logging(); -#else - (void) free_all; -#endif -} - -/** Describe the current status of the freelists at log level <b>severity</b>. - */ -void -buf_dump_freelist_sizes(int severity) -{ -#ifdef ENABLE_BUF_FREELISTS - int i; - tor_log(severity, LD_MM, "====== Buffer freelists:"); - for (i = 0; freelists[i].alloc_size; ++i) { - uint64_t total = ((uint64_t)freelists[i].cur_length) * - freelists[i].alloc_size; - tor_log(severity, LD_MM, - U64_FORMAT" bytes in %d %d-byte chunks ["U64_FORMAT - " misses; "U64_FORMAT" frees; "U64_FORMAT" hits]", - U64_PRINTF_ARG(total), - freelists[i].cur_length, (int)freelists[i].alloc_size, - U64_PRINTF_ARG(freelists[i].n_alloc), - U64_PRINTF_ARG(freelists[i].n_free), - U64_PRINTF_ARG(freelists[i].n_hit)); - } - tor_log(severity, LD_MM, U64_FORMAT" allocations in non-freelist sizes", - U64_PRINTF_ARG(n_freelist_miss)); -#else - (void)severity; -#endif -} - -/** Magic value for buf_t.magic, to catch pointer errors. */ -#define BUFFER_MAGIC 0xB0FFF312u -/** A resizeable buffer, optimized for reading and writing. */ -struct buf_t { - uint32_t magic; /**< Magic cookie for debugging: Must be set to - * BUFFER_MAGIC. */ - size_t datalen; /**< How many bytes is this buffer holding right now? */ - size_t default_chunk_size; /**< Don't allocate any chunks smaller than - * this for this buffer. */ - chunk_t *head; /**< First chunk in the list, or NULL for none. */ - chunk_t *tail; /**< Last chunk in the list, or NULL for none. */ -}; - /** Collapse data from the first N chunks from <b>buf</b> into buf->head, * growing it as necessary, until buf->head has the first <b>bytes</b> bytes * of data from the buffer, or until buf->head has all the data in <b>buf</b>. - * - * If <b>nulterminate</b> is true, ensure that there is a 0 byte in - * buf->head->mem right after all the data. */ -static void -buf_pullup(buf_t *buf, size_t bytes, int nulterminate) + */ +STATIC void +buf_pullup(buf_t *buf, size_t bytes) { chunk_t *dest, *src; size_t capacity; @@ -387,17 +216,9 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) if (buf->datalen < bytes) bytes = buf->datalen; - if (nulterminate) { - capacity = bytes + 1; - if (buf->head->datalen >= bytes && CHUNK_REMAINING_CAPACITY(buf->head)) { - *CHUNK_WRITE_PTR(buf->head) = '\0'; - return; - } - } else { - capacity = bytes; - if (buf->head->datalen >= bytes) - return; - } + capacity = bytes; + if (buf->head->datalen >= bytes) + return; if (buf->head->memlen >= capacity) { /* We don't need to grow the first chunk, but we might need to repack it.*/ @@ -441,25 +262,25 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) } } - if (nulterminate) { - tor_assert(CHUNK_REMAINING_CAPACITY(buf->head)); - *CHUNK_WRITE_PTR(buf->head) = '\0'; - } - check(); } -/** Resize buf so it won't hold extra memory that we haven't been - * using lately. - */ +#ifdef TOR_UNIT_TESTS void -buf_shrink(buf_t *buf) +buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz) { - (void)buf; + if (!buf || !buf->head) { + *cp = NULL; + *sz = 0; + } else { + *cp = buf->head->data; + *sz = buf->head->datalen; + } } +#endif /** Remove the first <b>n</b> bytes from buf. */ -static INLINE void +static inline void buf_remove_from_front(buf_t *buf, size_t n) { tor_assert(buf->datalen >= n); @@ -503,6 +324,12 @@ buf_new(void) return buf; } +size_t +buf_get_default_chunk_size(const buf_t *buf) +{ + return buf->default_chunk_size; +} + /** Remove all data from <b>buf</b>. */ void buf_clear(buf_t *buf) @@ -517,8 +344,8 @@ buf_clear(buf_t *buf) } /** Return the number of bytes stored in <b>buf</b> */ -size_t -buf_datalen(const buf_t *buf) +MOCK_IMPL(size_t, +buf_datalen, (const buf_t *buf)) { return buf->datalen; } @@ -530,7 +357,7 @@ buf_allocation(const buf_t *buf) size_t total = 0; const chunk_t *chunk; for (chunk = buf->head; chunk; chunk = chunk->next) { - total += chunk->memlen; + total += CHUNK_ALLOC_SIZE(chunk->memlen); } return total; } @@ -563,6 +390,10 @@ static chunk_t * chunk_copy(const chunk_t *in_chunk) { chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen)); + total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen); +#ifdef DEBUG_CHUNK_ALLOC + newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen); +#endif newch->next = NULL; if (in_chunk->data) { off_t offset = in_chunk->data - in_chunk->mem; @@ -598,6 +429,7 @@ static chunk_t * buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) { chunk_t *chunk; + struct timeval now; if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) { chunk = chunk_new_with_alloc_size(buf->default_chunk_size); } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) { @@ -605,6 +437,10 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) } else { chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity)); } + + tor_gettimeofday_cached_monotonic(&now); + chunk->inserted_time = (uint32_t)tv_to_msec(&now); + if (buf->tail) { tor_assert(buf->head); buf->tail->next = chunk; @@ -617,11 +453,31 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) return chunk; } +/** Return the age of the oldest chunk in the buffer <b>buf</b>, in + * milliseconds. Requires the current time, in truncated milliseconds since + * the epoch, as its input <b>now</b>. + */ +uint32_t +buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now) +{ + if (buf->head) { + return now - buf->head->inserted_time; + } else { + return 0; + } +} + +size_t +buf_get_total_allocation(void) +{ + return total_bytes_allocated_in_chunks; +} + /** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, * and the number of bytes read otherwise. */ -static INLINE int +static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, int *reached_eof, int *socket_error) { @@ -657,7 +513,7 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, /** As read_to_chunk(), but return (negative) error code on error, blocking, * or TLS, and the number of bytes read otherwise. */ -static INLINE int +static inline int read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls, size_t at_most) { @@ -768,7 +624,7 @@ read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf) if (r < 0) return r; /* Error */ tor_assert(total_read+r < INT_MAX); - total_read += r; + total_read += r; if ((size_t)r < readlen) /* eof, block, or no more to read. */ break; } @@ -780,7 +636,7 @@ read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf) * the bytes written from *<b>buf_flushlen</b>. Return the number of bytes * written on success, 0 on blocking, -1 on failure. */ -static INLINE int +static inline int flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, size_t *buf_flushlen) { @@ -815,7 +671,7 @@ flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, * bytes written from *<b>buf_flushlen</b>. Return the number of bytes * written on success, and a TOR_TLS error code on failure or blocking. */ -static INLINE int +static inline int flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk, size_t sz, size_t *buf_flushlen) { @@ -966,7 +822,7 @@ write_to_buf(const char *string, size_t string_len, buf_t *buf) /** Helper: copy the first <b>string_len</b> bytes from <b>buf</b> * onto <b>string</b>. */ -static INLINE void +static inline void peek_from_buf(char *string, size_t string_len, const buf_t *buf) { chunk_t *chunk; @@ -1011,7 +867,7 @@ fetch_from_buf(char *string, size_t string_len, buf_t *buf) /** True iff the cell command <b>command</b> is one that implies a * variable-length cell in Tor link protocol <b>linkproto</b>. */ -static INLINE int +static inline int cell_command_is_var_length(uint8_t command, int linkproto) { /* If linkproto is v2 (2), CELL_VERSIONS is the only variable-length cells @@ -1252,7 +1108,7 @@ buf_find_pos_of_char(char ch, buf_pos_t *out) /** Advance <b>pos</b> by a single character, if there are any more characters * in the buffer. Returns 0 on success, -1 on failure. */ -static INLINE int +static inline int buf_pos_inc(buf_pos_t *pos) { ++pos->pos; @@ -1294,7 +1150,7 @@ buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n) /** Return the first position in <b>buf</b> at which the <b>n</b>-character * string <b>s</b> occurs, or -1 if it does not occur. */ -/*private*/ int +STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n) { buf_pos_t pos; @@ -1356,7 +1212,7 @@ fetch_from_buf_http(buf_t *buf, /* Okay, we have a full header. Make sure it all appears in the first * chunk. */ if ((int)buf->head->datalen < crlf_offset + 4) - buf_pullup(buf, crlf_offset+4, 0); + buf_pullup(buf, crlf_offset+4); headerlen = crlf_offset + 4; headers = buf->head->data; @@ -1604,7 +1460,7 @@ fetch_from_buf_socks(buf_t *buf, socks_request_t *req, do { n_drain = 0; - buf_pullup(buf, want_length, 0); + buf_pullup(buf, want_length); tor_assert(buf->head && buf->head->datalen >= 2); want_length = 0; @@ -1702,6 +1558,79 @@ fetch_from_evbuffer_socks(struct evbuffer *buf, socks_request_t *req, } #endif +/** The size of the header of an Extended ORPort message: 2 bytes for + * COMMAND, 2 bytes for BODYLEN */ +#define EXT_OR_CMD_HEADER_SIZE 4 + +/** Read <b>buf</b>, which should contain an Extended ORPort message + * from a transport proxy. If well-formed, create and populate + * <b>out</b> with the Extended ORport message. Return 0 if the + * buffer was incomplete, 1 if it was well-formed and -1 if we + * encountered an error while parsing it. */ +int +fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out) +{ + char hdr[EXT_OR_CMD_HEADER_SIZE]; + uint16_t len; + + check(); + if (buf->datalen < EXT_OR_CMD_HEADER_SIZE) + return 0; + peek_from_buf(hdr, sizeof(hdr), buf); + len = ntohs(get_uint16(hdr+2)); + if (buf->datalen < (unsigned)len + EXT_OR_CMD_HEADER_SIZE) + return 0; + *out = ext_or_cmd_new(len); + (*out)->cmd = ntohs(get_uint16(hdr)); + (*out)->len = len; + buf_remove_from_front(buf, EXT_OR_CMD_HEADER_SIZE); + fetch_from_buf((*out)->body, len, buf); + return 1; +} + +#ifdef USE_BUFFEREVENTS +/** Read <b>buf</b>, which should contain an Extended ORPort message + * from a transport proxy. If well-formed, create and populate + * <b>out</b> with the Extended ORport message. Return 0 if the + * buffer was incomplete, 1 if it was well-formed and -1 if we + * encountered an error while parsing it. */ +int +fetch_ext_or_command_from_evbuffer(struct evbuffer *buf, ext_or_cmd_t **out) +{ + char hdr[EXT_OR_CMD_HEADER_SIZE]; + uint16_t len; + size_t buf_len = evbuffer_get_length(buf); + + if (buf_len < EXT_OR_CMD_HEADER_SIZE) + return 0; + evbuffer_copyout(buf, hdr, EXT_OR_CMD_HEADER_SIZE); + len = ntohs(get_uint16(hdr+2)); + if (buf_len < (unsigned)len + EXT_OR_CMD_HEADER_SIZE) + return 0; + *out = ext_or_cmd_new(len); + (*out)->cmd = ntohs(get_uint16(hdr)); + (*out)->len = len; + evbuffer_drain(buf, EXT_OR_CMD_HEADER_SIZE); + evbuffer_remove(buf, (*out)->body, len); + return 1; +} +#endif + +/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and + * have Tor send it as error response to <b>req</b>. + */ +static void +socks_request_set_socks5_error(socks_request_t *req, + socks5_reply_status_t reason) +{ + req->replylen = 10; + memset(req->reply,0,10); + + req->reply[0] = 0x05; // VER field. + req->reply[1] = reason; // REP field. + req->reply[3] = 0x01; // ATYP field. +} + /** Implementation helper to implement fetch_from_*_socks. Instead of looking * at a buffer's contents, we look at the <b>datalen</b> bytes of data in * <b>data</b>. Instead of removing data from the buffer, we set @@ -1765,7 +1694,7 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, } *drain_out = 2u + usernamelen + 1u + passlen; req->got_auth = 1; - *want_length_out = 7; /* Minimal socks5 sommand. */ + *want_length_out = 7; /* Minimal socks5 command. */ return 0; } else if (req->auth_type == SOCKS_USER_PASS) { /* unknown version byte */ @@ -1837,6 +1766,8 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, req->command != SOCKS_COMMAND_RESOLVE && req->command != SOCKS_COMMAND_RESOLVE_PTR) { /* not a connect or resolve or a resolve_ptr? we don't support it. */ + socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED); + log_warn(LD_APP,"socks5: command %d not recognized. Rejecting.", req->command); return -1; @@ -1860,6 +1791,7 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1); if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) { + socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); log_warn(LD_APP, "socks5 IP takes %d bytes, which doesn't fit in %d. " "Rejecting.", @@ -1872,14 +1804,18 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, if (req->command != SOCKS_COMMAND_RESOLVE_PTR && !addressmap_have_mapping(req->address,0)) { log_unsafe_socks_warning(5, req->address, req->port, safe_socks); - if (safe_socks) + if (safe_socks) { + socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED); return -1; + } } return 1; } case 3: /* fqdn */ log_debug(LD_APP,"socks5: fqdn address type"); if (req->command == SOCKS_COMMAND_RESOLVE_PTR) { + socks_request_set_socks5_error(req, + SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED); log_warn(LD_APP, "socks5 received RESOLVE_PTR command with " "hostname type. Rejecting."); return -1; @@ -1890,6 +1826,7 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, return 0; /* not yet */ } if (len+1 > MAX_SOCKS_ADDR_LEN) { + socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); log_warn(LD_APP, "socks5 hostname is %d bytes, which doesn't fit in " "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN); @@ -1899,11 +1836,22 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, req->address[len] = 0; req->port = ntohs(get_uint16(data+5+len)); *drain_out = 5+len+2; - if (!tor_strisprint(req->address) || strchr(req->address,'\"')) { + + if (string_is_valid_ipv4_address(req->address) || + string_is_valid_ipv6_address(req->address)) { + log_unsafe_socks_warning(5,req->address,req->port,safe_socks); + + if (safe_socks) { + socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED); + return -1; + } + } else if (!string_is_valid_hostname(req->address)) { + socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); + log_warn(LD_PROTOCOL, "Your application (using socks5 to port %d) gave Tor " "a malformed hostname: %s. Rejecting the connection.", - req->port, escaped(req->address)); + req->port, escaped_safe_str_client(req->address)); return -1; } if (log_sockstype) @@ -1913,6 +1861,8 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, "necessary. This is good.", req->port); return 1; default: /* unsupported */ + socks_request_set_socks5_error(req, + SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED); log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.", (int) *(data+3)); return -1; @@ -1929,7 +1879,7 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, *want_length_out = SOCKS4_NETWORK_LEN; return 0; /* not yet */ } - // buf_pullup(buf, 1280, 0); + // buf_pullup(buf, 1280); req->command = (unsigned char) *(data+1); if (req->command != SOCKS_COMMAND_CONNECT && req->command != SOCKS_COMMAND_RESOLVE) { @@ -2020,7 +1970,7 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, log_warn(LD_PROTOCOL, "Your application (using socks4 to port %d) gave Tor " "a malformed hostname: %s. Rejecting the connection.", - req->port, escaped(req->address)); + req->port, escaped_safe_str_client(req->address)); return -1; } if (authend != authstart) { @@ -2097,7 +2047,7 @@ fetch_from_buf_socks_client(buf_t *buf, int state, char **reason) if (buf->datalen < 2) return 0; - buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN, 0); + buf_pullup(buf, MAX_SOCKS_MESSAGE_LEN); tor_assert(buf->head && buf->head->datalen >= 2); r = parse_socks_client((uint8_t*)buf->head->data, buf->head->datalen, @@ -2348,6 +2298,7 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, char *next; size_t old_avail, avail; int over = 0; + do { int need_new_chunk = 0; if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) { @@ -2510,23 +2461,3 @@ assert_buf_ok(buf_t *buf) } } -#ifdef ENABLE_BUF_FREELISTS -/** Log an error and exit if <b>fl</b> is corrupted. - */ -static void -assert_freelist_ok(chunk_freelist_t *fl) -{ - chunk_t *ch; - int n; - tor_assert(fl->alloc_size > 0); - n = 0; - for (ch = fl->head; ch; ch = ch->next) { - tor_assert(CHUNK_ALLOC_SIZE(ch->memlen) == fl->alloc_size); - ++n; - } - tor_assert(n == fl->cur_length); - tor_assert(n >= fl->lowest_length); - tor_assert(n <= fl->max_length); -} -#endif - diff --git a/src/or/buffers.h b/src/or/buffers.h index c947f0ba98..2b43ea14b1 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,19 +12,22 @@ #ifndef TOR_BUFFERS_H #define TOR_BUFFERS_H +#include "testsupport.h" + buf_t *buf_new(void); buf_t *buf_new_with_capacity(size_t size); +size_t buf_get_default_chunk_size(const buf_t *buf); void buf_free(buf_t *buf); void buf_clear(buf_t *buf); buf_t *buf_copy(const buf_t *buf); -void buf_shrink(buf_t *buf); -void buf_shrink_freelists(int free_all); -void buf_dump_freelist_sizes(int severity); -size_t buf_datalen(const buf_t *buf); +MOCK_DECL(size_t, buf_datalen, (const buf_t *buf)); size_t buf_allocation(const buf_t *buf); size_t buf_slack(const buf_t *buf); +uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now); +size_t buf_get_total_allocation(void); + int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error); int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); @@ -51,6 +54,8 @@ int fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len); int peek_buf_has_control0_command(buf_t *buf); +int fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out); + #ifdef USE_BUFFEREVENTS int fetch_var_cell_from_evbuffer(struct evbuffer *buf, var_cell_t **out, int linkproto); @@ -66,6 +71,8 @@ int peek_evbuffer_has_control0_command(struct evbuffer *buf); int write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, const char *data, size_t data_len, int done); +int fetch_ext_or_command_from_evbuffer(struct evbuffer *buf, + ext_or_cmd_t **out); #endif #ifdef USE_BUFFEREVENTS @@ -75,6 +82,8 @@ int write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, #define generic_buffer_get(b,buf,buflen) evbuffer_remove((b),(buf),(buflen)) #define generic_buffer_clear(b) evbuffer_drain((b), evbuffer_get_length((b))) #define generic_buffer_free(b) evbuffer_free((b)) +#define generic_buffer_fetch_ext_or_cmd(b, out) \ + fetch_ext_or_command_from_evbuffer((b), (out)) #else #define generic_buffer_new() buf_new() #define generic_buffer_len(b) buf_datalen((b)) @@ -82,6 +91,8 @@ int write_to_evbuffer_zlib(struct evbuffer *buf, tor_zlib_state_t *state, #define generic_buffer_get(b,buf,buflen) fetch_from_buf((buf),(buflen),(b)) #define generic_buffer_clear(b) buf_clear((b)) #define generic_buffer_free(b) buf_free((b)) +#define generic_buffer_fetch_ext_or_cmd(b, out) \ + fetch_ext_or_command_from_buf((b), (out)) #endif int generic_buffer_set_to_copy(generic_buffer_t **output, const generic_buffer_t *input); @@ -89,7 +100,38 @@ int generic_buffer_set_to_copy(generic_buffer_t **output, void assert_buf_ok(buf_t *buf); #ifdef BUFFERS_PRIVATE -int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +STATIC void buf_pullup(buf_t *buf, size_t bytes); +void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz); + +#define DEBUG_CHUNK_ALLOC +/** A single chunk on a buffer. */ +typedef struct chunk_t { + struct chunk_t *next; /**< The next chunk on the buffer. */ + size_t datalen; /**< The number of bytes stored in this chunk */ + size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ +#ifdef DEBUG_CHUNK_ALLOC + size_t DBG_alloc; +#endif + char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */ + uint32_t inserted_time; /**< Timestamp in truncated ms since epoch + * when this chunk was inserted. */ + char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in + * this chunk. */ +} chunk_t; + +/** Magic value for buf_t.magic, to catch pointer errors. */ +#define BUFFER_MAGIC 0xB0FFF312u +/** A resizeable buffer, optimized for reading and writing. */ +struct buf_t { + uint32_t magic; /**< Magic cookie for debugging: Must be set to + * BUFFER_MAGIC. */ + size_t datalen; /**< How many bytes is this buffer holding right now? */ + size_t default_chunk_size; /**< Don't allocate any chunks smaller than + * this for this buffer. */ + chunk_t *head; /**< First chunk in the list, or NULL for none. */ + chunk_t *tail; /**< Last chunk in the list, or NULL for none. */ +}; #endif #endif diff --git a/src/or/channel.c b/src/or/channel.c index 1270eace7d..5f69a0864b 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -1,9 +1,13 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file channel.c - * \brief OR-to-OR channel abstraction layer + * + * \brief OR/OP-to-OR channel abstraction layer. A channel's job is to + * transfer cells from Tor instance to Tor instance. + * Currently, there is only one implementation of the channel abstraction: in + * channeltls.c. **/ /* @@ -13,12 +17,16 @@ #define TOR_CHANNEL_INTERNAL_ +/* This one's for stuff only channel.c and the test suite should see */ +#define CHANNEL_PRIVATE_ + #include "or.h" #include "channel.h" #include "channeltls.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" +#include "config.h" #include "connection_or.h" /* For var_cell_free() */ #include "circuitmux.h" #include "entrynodes.h" @@ -28,29 +36,7 @@ #include "rephist.h" #include "router.h" #include "routerlist.h" - -/* Cell queue structure */ - -typedef struct cell_queue_entry_s cell_queue_entry_t; -struct cell_queue_entry_s { - TOR_SIMPLEQ_ENTRY(cell_queue_entry_s) next; - enum { - CELL_QUEUE_FIXED, - CELL_QUEUE_VAR, - CELL_QUEUE_PACKED - } type; - union { - struct { - cell_t *cell; - } fixed; - struct { - var_cell_t *var_cell; - } var; - struct { - packed_cell_t *packed_cell; - } packed; - } u; -}; +#include "scheduler.h" /* Global lists of channels */ @@ -74,6 +60,59 @@ static smartlist_t *finished_listeners = NULL; /* Counter for ID numbers */ static uint64_t n_channels_allocated = 0; +/* + * Channel global byte/cell counters, for statistics and for scheduler high + * /low-water marks. + */ + +/* + * Total number of cells ever given to any channel with the + * channel_write_*_cell() functions. + */ + +static uint64_t n_channel_cells_queued = 0; + +/* + * Total number of cells ever passed to a channel lower layer with the + * write_*_cell() methods. + */ + +static uint64_t n_channel_cells_passed_to_lower_layer = 0; + +/* + * Current number of cells in all channel queues; should be + * n_channel_cells_queued - n_channel_cells_passed_to_lower_layer. + */ + +static uint64_t n_channel_cells_in_queues = 0; + +/* + * Total number of bytes for all cells ever queued to a channel and + * counted in n_channel_cells_queued. + */ + +static uint64_t n_channel_bytes_queued = 0; + +/* + * Total number of bytes for all cells ever passed to a channel lower layer + * and counted in n_channel_cells_passed_to_lower_layer. + */ + +static uint64_t n_channel_bytes_passed_to_lower_layer = 0; + +/* + * Current number of bytes in all channel queues; should be + * n_channel_bytes_queued - n_channel_bytes_passed_to_lower_layer. + */ + +static uint64_t n_channel_bytes_in_queues = 0; + +/* + * Current total estimated queue size *including lower layer queues and + * transmit overhead* + */ + +STATIC uint64_t estimated_total_queue_size = 0; /* Digest->channel map * @@ -92,18 +131,13 @@ typedef struct channel_idmap_entry_s { TOR_LIST_HEAD(channel_list_s, channel_s) channel_list; } channel_idmap_entry_t; -static INLINE unsigned +static inline unsigned channel_idmap_hash(const channel_idmap_entry_t *ent) { - const unsigned *a = (const unsigned *)ent->digest; -#if SIZEOF_INT == 4 - return a[0] ^ a[1] ^ a[2] ^ a[3] ^ a[4]; -#elif SIZEOF_INT == 8 - return a[0] ^ a[1]; -#endif + return (unsigned) siphash24g(ent->digest, DIGEST_LEN); } -static INLINE int +static inline int channel_idmap_eq(const channel_idmap_entry_t *a, const channel_idmap_entry_t *b) { @@ -112,16 +146,21 @@ channel_idmap_eq(const channel_idmap_entry_t *a, HT_PROTOTYPE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, channel_idmap_eq); -HT_GENERATE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, - channel_idmap_eq, 0.5, tor_malloc, tor_realloc, tor_free_); +HT_GENERATE2(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, + channel_idmap_eq, 0.5, tor_reallocarray_, tor_free_); static cell_queue_entry_t * cell_queue_entry_dup(cell_queue_entry_t *q); -static void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off); +#if 0 static int cell_queue_entry_is_padding(cell_queue_entry_t *q); +#endif static cell_queue_entry_t * cell_queue_entry_new_fixed(cell_t *cell); static cell_queue_entry_t * cell_queue_entry_new_var(var_cell_t *var_cell); +static int is_destroy_cell(channel_t *chan, + const cell_queue_entry_t *q, circid_t *circid_out); + +static void channel_assert_counter_consistency(void); /* Functions to maintain the digest map */ static void channel_add_to_digest_map(channel_t *chan); @@ -140,6 +179,8 @@ channel_free_list(smartlist_t *channels, int mark_for_close); static void channel_listener_free_list(smartlist_t *channels, int mark_for_close); static void channel_listener_force_free(channel_listener_t *chan_l); +static size_t channel_get_cell_queue_entry_size(channel_t *chan, + cell_queue_entry_t *q); static void channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q); @@ -378,8 +419,7 @@ channel_register(channel_t *chan) smartlist_add(all_channels, chan); /* Is it finished? */ - if (chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) { + if (CHANNEL_FINISHED(chan)) { /* Put it in the finished list, creating it if necessary */ if (!finished_channels) finished_channels = smartlist_new(); smartlist_add(finished_channels, chan); @@ -388,7 +428,7 @@ channel_register(channel_t *chan) if (!active_channels) active_channels = smartlist_new(); smartlist_add(active_channels, chan); - if (chan->state != CHANNEL_STATE_CLOSING) { + if (!CHANNEL_IS_CLOSING(chan)) { /* It should have a digest set */ if (!tor_digest_is_zero(chan->identity_digest)) { /* Yeah, we're good, add it to the map */ @@ -423,8 +463,7 @@ channel_unregister(channel_t *chan) if (!(chan->registered)) return; /* Is it finished? */ - if (chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) { + if (CHANNEL_FINISHED(chan)) { /* Get it out of the finished list */ if (finished_channels) smartlist_remove(finished_channels, chan); } else { @@ -440,9 +479,7 @@ channel_unregister(channel_t *chan) /* Should it be in the digest map? */ if (!tor_digest_is_zero(chan->identity_digest) && - !(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { + !(CHANNEL_CONDEMNED(chan))) { /* Remove it */ channel_remove_from_digest_map(chan); } @@ -542,9 +579,7 @@ channel_add_to_digest_map(channel_t *chan) tor_assert(chan); /* Assert that the state makes sense */ - tor_assert(!(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)); + tor_assert(!CHANNEL_CONDEMNED(chan)); /* Assert that there is a digest */ tor_assert(!tor_digest_is_zero(chan->identity_digest)); @@ -729,10 +764,10 @@ channel_init(channel_t *chan) chan->global_identifier = n_channels_allocated++; /* Init timestamp */ - chan->timestamp_last_added_nonpadding = time(NULL); + chan->timestamp_last_had_circuits = time(NULL); - /* Init next_circ_id */ - chan->next_circ_id = crypto_rand_int(1 << 15); + /* Warn about exhausted circuit IDs no more than hourly. */ + chan->last_warned_circ_ids_exhausted.rate = 3600; /* Initialize queues. */ TOR_SIMPLEQ_INIT(&chan->incoming_queue); @@ -746,6 +781,9 @@ channel_init(channel_t *chan) /* It hasn't been open yet. */ chan->has_been_open = 0; + + /* Scheduler state is idle */ + chan->scheduler_state = SCHED_CHAN_IDLE; } /** @@ -779,8 +817,8 @@ channel_free(channel_t *chan) if (!chan) return; /* It must be closed or errored */ - tor_assert(chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + tor_assert(CHANNEL_FINISHED(chan)); + /* It must be deregistered */ tor_assert(!(chan->registered)); @@ -788,6 +826,9 @@ channel_free(channel_t *chan) "Freeing channel " U64_FORMAT " at %p", U64_PRINTF_ARG(chan->global_identifier), chan); + /* Get this one out of the scheduler */ + scheduler_release_channel(chan); + /* * Get rid of cmux policy before we do anything, so cmux policies don't * see channels in weird half-freed states. @@ -803,7 +844,8 @@ channel_free(channel_t *chan) /* Get rid of cmux */ if (chan->cmux) { - circuitmux_detach_all_circuits(chan->cmux); + circuitmux_detach_all_circuits(chan->cmux, NULL); + circuitmux_mark_destroyed_circids_usable(chan->cmux, chan); circuitmux_free(chan->cmux); chan->cmux = NULL; } @@ -862,6 +904,9 @@ channel_force_free(channel_t *chan) "Force-freeing channel " U64_FORMAT " at %p", U64_PRINTF_ARG(chan->global_identifier), chan); + /* Get this one out of the scheduler */ + scheduler_release_channel(chan); + /* * Get rid of cmux policy before we do anything, so cmux policies don't * see channels in weird half-freed states. @@ -987,9 +1032,7 @@ channel_get_cell_handler(channel_t *chan) { tor_assert(chan); - if (chan->state == CHANNEL_STATE_OPENING || - chan->state == CHANNEL_STATE_OPEN || - chan->state == CHANNEL_STATE_MAINT) + if (CHANNEL_CAN_HANDLE_CELLS(chan)) return chan->cell_handler; return NULL; @@ -1007,9 +1050,7 @@ channel_get_var_cell_handler(channel_t *chan) { tor_assert(chan); - if (chan->state == CHANNEL_STATE_OPENING || - chan->state == CHANNEL_STATE_OPEN || - chan->state == CHANNEL_STATE_MAINT) + if (CHANNEL_CAN_HANDLE_CELLS(chan)) return chan->var_cell_handler; return NULL; @@ -1032,9 +1073,7 @@ channel_set_cell_handlers(channel_t *chan, int try_again = 0; tor_assert(chan); - tor_assert(chan->state == CHANNEL_STATE_OPENING || - chan->state == CHANNEL_STATE_OPEN || - chan->state == CHANNEL_STATE_MAINT); + tor_assert(CHANNEL_CAN_HANDLE_CELLS(chan)); log_debug(LD_CHANNEL, "Setting cell_handler callback for channel %p to %p", @@ -1088,9 +1127,8 @@ channel_mark_for_close(channel_t *chan) tor_assert(chan->close != NULL); /* If it's already in CLOSING, CLOSED or ERROR, this is a no-op */ - if (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) return; + if (CHANNEL_CONDEMNED(chan)) + return; log_debug(LD_CHANNEL, "Closing channel %p (global ID " U64_FORMAT ") " @@ -1169,9 +1207,8 @@ channel_close_from_lower_layer(channel_t *chan) tor_assert(chan != NULL); /* If it's already in CLOSING, CLOSED or ERROR, this is a no-op */ - if (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) return; + if (CHANNEL_CONDEMNED(chan)) + return; log_debug(LD_CHANNEL, "Closing channel %p (global ID " U64_FORMAT ") " @@ -1229,9 +1266,8 @@ channel_close_for_error(channel_t *chan) tor_assert(chan != NULL); /* If it's already in CLOSING, CLOSED or ERROR, this is a no-op */ - if (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) return; + if (CHANNEL_CONDEMNED(chan)) + return; log_debug(LD_CHANNEL, "Closing channel %p due to lower-layer error", @@ -1287,18 +1323,16 @@ void channel_closed(channel_t *chan) { tor_assert(chan); - tor_assert(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + tor_assert(CHANNEL_CONDEMNED(chan)); /* No-op if already inactive */ - if (chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) return; + if (CHANNEL_FINISHED(chan)) + return; /* Inform any pending (not attached) circs that they should * give up. */ if (! chan->has_been_open) - circuit_n_chan_done(chan, 0); + circuit_n_chan_done(chan, 0, 0); /* Now close all the attached circuits on it. */ circuit_unlink_all_from_channel(chan, END_CIRC_REASON_CHANNEL_CLOSED); @@ -1356,10 +1390,7 @@ channel_clear_identity_digest(channel_t *chan) "global ID " U64_FORMAT, chan, U64_PRINTF_ARG(chan->global_identifier)); - state_not_in_map = - (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + state_not_in_map = CHANNEL_CONDEMNED(chan); if (!state_not_in_map && chan->registered && !tor_digest_is_zero(chan->identity_digest)) @@ -1392,10 +1423,8 @@ channel_set_identity_digest(channel_t *chan, identity_digest ? hex_str(identity_digest, DIGEST_LEN) : "(null)"); - state_not_in_map = - (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + state_not_in_map = CHANNEL_CONDEMNED(chan); + was_in_digest_map = !state_not_in_map && chan->registered && @@ -1445,10 +1474,7 @@ channel_clear_remote_end(channel_t *chan) "global ID " U64_FORMAT, chan, U64_PRINTF_ARG(chan->global_identifier)); - state_not_in_map = - (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + state_not_in_map = CHANNEL_CONDEMNED(chan); if (!state_not_in_map && chan->registered && !tor_digest_is_zero(chan->identity_digest)) @@ -1484,10 +1510,8 @@ channel_set_remote_end(channel_t *chan, identity_digest ? hex_str(identity_digest, DIGEST_LEN) : "(null)"); - state_not_in_map = - (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + state_not_in_map = CHANNEL_CONDEMNED(chan); + was_in_digest_map = !state_not_in_map && chan->registered && @@ -1547,7 +1571,7 @@ cell_queue_entry_dup(cell_queue_entry_t *q) * them) or not (we should free). */ -static void +STATIC void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off) { if (!q) return; @@ -1597,6 +1621,7 @@ cell_queue_entry_free(cell_queue_entry_t *q, int handed_off) tor_free(q); } +#if 0 /** * Check whether a cell queue entry is padding; this is a helper function * for channel_write_cell_queue_entry() @@ -1625,6 +1650,7 @@ cell_queue_entry_is_padding(cell_queue_entry_t *q) return 0; } +#endif /** * Allocate a new cell queue entry for a fixed-size cell @@ -1663,6 +1689,36 @@ cell_queue_entry_new_var(var_cell_t *var_cell) } /** + * Ask how big the cell contained in a cell_queue_entry_t is + */ + +static size_t +channel_get_cell_queue_entry_size(channel_t *chan, cell_queue_entry_t *q) +{ + size_t rv = 0; + + tor_assert(chan); + tor_assert(q); + + switch (q->type) { + case CELL_QUEUE_FIXED: + rv = get_cell_network_size(chan->wide_circ_ids); + break; + case CELL_QUEUE_VAR: + rv = get_var_cell_header_size(chan->wide_circ_ids) + + (q->u.var.var_cell ? q->u.var.var_cell->payload_len : 0); + break; + case CELL_QUEUE_PACKED: + rv = get_cell_network_size(chan->wide_circ_ids); + break; + default: + tor_assert(1); + } + + return rv; +} + +/** * Write to a channel based on a cell_queue_entry_t * * Given a cell_queue_entry_t filled out by the caller, try to send the cell @@ -1674,23 +1730,27 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) { int result = 0, sent = 0; cell_queue_entry_t *tmp = NULL; + size_t cell_bytes; tor_assert(chan); tor_assert(q); /* Assert that the state makes sense for a cell write */ - tor_assert(chan->state == CHANNEL_STATE_OPENING || - chan->state == CHANNEL_STATE_OPEN || - chan->state == CHANNEL_STATE_MAINT); + tor_assert(CHANNEL_CAN_HANDLE_CELLS(chan)); - /* Increment the timestamp unless it's padding */ - if (!cell_queue_entry_is_padding(q)) { - chan->timestamp_last_added_nonpadding = approx_time(); + { + circid_t circ_id; + if (is_destroy_cell(chan, q, &circ_id)) { + channel_note_destroy_not_pending(chan, circ_id); + } } + /* For statistical purposes, figure out how big this cell is */ + cell_bytes = channel_get_cell_queue_entry_size(chan, q); + /* Can we send it right out? If so, try */ if (TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue) && - chan->state == CHANNEL_STATE_OPEN) { + CHANNEL_IS_OPEN(chan)) { /* Pick the right write function for this cell type and save the result */ switch (q->type) { case CELL_QUEUE_FIXED: @@ -1721,6 +1781,13 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) channel_timestamp_drained(chan); /* Update the counter */ ++(chan->n_cells_xmitted); + chan->n_bytes_xmitted += cell_bytes; + /* Update global counters */ + ++n_channel_cells_queued; + ++n_channel_cells_passed_to_lower_layer; + n_channel_bytes_queued += cell_bytes; + n_channel_bytes_passed_to_lower_layer += cell_bytes; + channel_assert_counter_consistency(); } } @@ -1732,8 +1799,16 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) */ tmp = cell_queue_entry_dup(q); TOR_SIMPLEQ_INSERT_TAIL(&chan->outgoing_queue, tmp, next); + /* Update global counters */ + ++n_channel_cells_queued; + ++n_channel_cells_in_queues; + n_channel_bytes_queued += cell_bytes; + n_channel_bytes_in_queues += cell_bytes; + channel_assert_counter_consistency(); + /* Update channel queue size */ + chan->bytes_in_queue += cell_bytes; /* Try to process the queue? */ - if (chan->state == CHANNEL_STATE_OPEN) channel_flush_cells(chan); + if (CHANNEL_IS_OPEN(chan)) channel_flush_cells(chan); } } @@ -1754,7 +1829,7 @@ channel_write_cell(channel_t *chan, cell_t *cell) tor_assert(chan); tor_assert(cell); - if (chan->state == CHANNEL_STATE_CLOSING) { + if (CHANNEL_IS_CLOSING(chan)) { log_debug(LD_CHANNEL, "Discarding cell_t %p on closing channel %p with " "global ID "U64_FORMAT, cell, chan, U64_PRINTF_ARG(chan->global_identifier)); @@ -1770,6 +1845,9 @@ channel_write_cell(channel_t *chan, cell_t *cell) q.type = CELL_QUEUE_FIXED; q.u.fixed.cell = cell; channel_write_cell_queue_entry(chan, &q); + + /* Update the queue size estimate */ + channel_update_xmit_queue_size(chan); } /** @@ -1788,7 +1866,7 @@ channel_write_packed_cell(channel_t *chan, packed_cell_t *packed_cell) tor_assert(chan); tor_assert(packed_cell); - if (chan->state == CHANNEL_STATE_CLOSING) { + if (CHANNEL_IS_CLOSING(chan)) { log_debug(LD_CHANNEL, "Discarding packed_cell_t %p on closing channel %p " "with global ID "U64_FORMAT, packed_cell, chan, U64_PRINTF_ARG(chan->global_identifier)); @@ -1805,6 +1883,9 @@ channel_write_packed_cell(channel_t *chan, packed_cell_t *packed_cell) q.type = CELL_QUEUE_PACKED; q.u.packed.packed_cell = packed_cell; channel_write_cell_queue_entry(chan, &q); + + /* Update the queue size estimate */ + channel_update_xmit_queue_size(chan); } /** @@ -1824,7 +1905,7 @@ channel_write_var_cell(channel_t *chan, var_cell_t *var_cell) tor_assert(chan); tor_assert(var_cell); - if (chan->state == CHANNEL_STATE_CLOSING) { + if (CHANNEL_IS_CLOSING(chan)) { log_debug(LD_CHANNEL, "Discarding var_cell_t %p on closing channel %p " "with global ID "U64_FORMAT, var_cell, chan, U64_PRINTF_ARG(chan->global_identifier)); @@ -1841,6 +1922,9 @@ channel_write_var_cell(channel_t *chan, var_cell_t *var_cell) q.type = CELL_QUEUE_VAR; q.u.var.var_cell = var_cell; channel_write_cell_queue_entry(chan, &q); + + /* Update the queue size estimate */ + channel_update_xmit_queue_size(chan); } /** @@ -1936,6 +2020,41 @@ channel_change_state(channel_t *chan, channel_state_t to_state) } } + /* + * If we're going to a closed/closing state, we don't need scheduling any + * more; in CHANNEL_STATE_MAINT we can't accept writes. + */ + if (to_state == CHANNEL_STATE_CLOSING || + to_state == CHANNEL_STATE_CLOSED || + to_state == CHANNEL_STATE_ERROR) { + scheduler_release_channel(chan); + } else if (to_state == CHANNEL_STATE_MAINT) { + scheduler_channel_doesnt_want_writes(chan); + } + + /* + * If we're closing, this channel no longer counts toward the global + * estimated queue size; if we're open, it now does. + */ + if ((to_state == CHANNEL_STATE_CLOSING || + to_state == CHANNEL_STATE_CLOSED || + to_state == CHANNEL_STATE_ERROR) && + (from_state == CHANNEL_STATE_OPEN || + from_state == CHANNEL_STATE_MAINT)) { + estimated_total_queue_size -= chan->bytes_in_queue; + } + + /* + * If we're opening, this channel now does count toward the global + * estimated queue size. + */ + if ((to_state == CHANNEL_STATE_OPEN || + to_state == CHANNEL_STATE_MAINT) && + !(from_state == CHANNEL_STATE_OPEN || + from_state == CHANNEL_STATE_MAINT)) { + estimated_total_queue_size += chan->bytes_in_queue; + } + /* Tell circuits if we opened and stuff */ if (to_state == CHANNEL_STATE_OPEN) { channel_do_open_actions(chan); @@ -2051,12 +2170,13 @@ channel_listener_change_state(channel_listener_t *chan_l, #define MAX_CELLS_TO_GET_FROM_CIRCUITS_FOR_UNLIMITED 256 -ssize_t -channel_flush_some_cells(channel_t *chan, ssize_t num_cells) +MOCK_IMPL(ssize_t, +channel_flush_some_cells, (channel_t *chan, ssize_t num_cells)) { unsigned int unlimited = 0; ssize_t flushed = 0; int num_cells_from_circs, clamped_num_cells; + int q_len_before, q_len_after; tor_assert(chan); @@ -2064,7 +2184,7 @@ channel_flush_some_cells(channel_t *chan, ssize_t num_cells) if (!unlimited && num_cells <= flushed) goto done; /* If we aren't in CHANNEL_STATE_OPEN, nothing goes through */ - if (chan->state == CHANNEL_STATE_OPEN) { + if (CHANNEL_IS_OPEN(chan)) { /* Try to flush as much as we can that's already queued */ flushed += channel_flush_some_cells_from_outgoing_queue(chan, (unlimited ? -1 : num_cells - flushed)); @@ -2082,14 +2202,45 @@ channel_flush_some_cells(channel_t *chan, ssize_t num_cells) clamped_num_cells = (int)(num_cells - flushed); } } + + /* + * Keep track of the change in queue size; we have to count cells + * channel_flush_from_first_active_circuit() writes out directly, + * but not double-count ones we might get later in + * channel_flush_some_cells_from_outgoing_queue() + */ + q_len_before = chan_cell_queue_len(&(chan->outgoing_queue)); + /* Try to get more cells from any active circuits */ num_cells_from_circs = channel_flush_from_first_active_circuit( chan, clamped_num_cells); - /* If it claims we got some, process the queue again */ + q_len_after = chan_cell_queue_len(&(chan->outgoing_queue)); + + /* + * If it claims we got some, adjust the flushed counter and consider + * processing the queue again + */ if (num_cells_from_circs > 0) { - flushed += channel_flush_some_cells_from_outgoing_queue(chan, - (unlimited ? -1 : num_cells - flushed)); + /* + * Adjust flushed by the number of cells counted in + * num_cells_from_circs that didn't go to the cell queue. + */ + + if (q_len_after > q_len_before) { + num_cells_from_circs -= (q_len_after - q_len_before); + if (num_cells_from_circs < 0) num_cells_from_circs = 0; + } + + flushed += num_cells_from_circs; + + /* Now process the queue if necessary */ + + if ((q_len_after > q_len_before) && + (unlimited || (flushed < num_cells))) { + flushed += channel_flush_some_cells_from_outgoing_queue(chan, + (unlimited ? -1 : num_cells - flushed)); + } } } } @@ -2112,6 +2263,8 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, unsigned int unlimited = 0; ssize_t flushed = 0; cell_queue_entry_t *q = NULL; + size_t cell_size; + int free_q = 0, handed_off = 0; tor_assert(chan); tor_assert(chan->write_cell); @@ -2122,11 +2275,15 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, if (!unlimited && num_cells <= flushed) return 0; /* If we aren't in CHANNEL_STATE_OPEN, nothing goes through */ - if (chan->state == CHANNEL_STATE_OPEN) { + if (CHANNEL_IS_OPEN(chan)) { while ((unlimited || num_cells > flushed) && NULL != (q = TOR_SIMPLEQ_FIRST(&chan->outgoing_queue))) { + free_q = 0; + handed_off = 0; if (1) { + /* Figure out how big it is for statistical purposes */ + cell_size = channel_get_cell_queue_entry_size(chan, q); /* * Okay, we have a good queue entry, try to give it to the lower * layer. @@ -2139,8 +2296,9 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, ++flushed; channel_timestamp_xmit(chan); ++(chan->n_cells_xmitted); - cell_queue_entry_free(q, 1); - q = NULL; + chan->n_bytes_xmitted += cell_size; + free_q = 1; + handed_off = 1; } /* Else couldn't write it; leave it on the queue */ } else { @@ -2151,8 +2309,8 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, "(global ID " U64_FORMAT ").", chan, U64_PRINTF_ARG(chan->global_identifier)); /* Throw it away */ - cell_queue_entry_free(q, 0); - q = NULL; + free_q = 1; + handed_off = 0; } break; case CELL_QUEUE_PACKED: @@ -2162,8 +2320,9 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, ++flushed; channel_timestamp_xmit(chan); ++(chan->n_cells_xmitted); - cell_queue_entry_free(q, 1); - q = NULL; + chan->n_bytes_xmitted += cell_size; + free_q = 1; + handed_off = 1; } /* Else couldn't write it; leave it on the queue */ } else { @@ -2174,8 +2333,8 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, "(global ID " U64_FORMAT ").", chan, U64_PRINTF_ARG(chan->global_identifier)); /* Throw it away */ - cell_queue_entry_free(q, 0); - q = NULL; + free_q = 1; + handed_off = 0; } break; case CELL_QUEUE_VAR: @@ -2185,8 +2344,9 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, ++flushed; channel_timestamp_xmit(chan); ++(chan->n_cells_xmitted); - cell_queue_entry_free(q, 1); - q = NULL; + chan->n_bytes_xmitted += cell_size; + free_q = 1; + handed_off = 1; } /* Else couldn't write it; leave it on the queue */ } else { @@ -2197,8 +2357,8 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, "(global ID " U64_FORMAT ").", chan, U64_PRINTF_ARG(chan->global_identifier)); /* Throw it away */ - cell_queue_entry_free(q, 0); - q = NULL; + free_q = 1; + handed_off = 0; } break; default: @@ -2208,12 +2368,32 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, "(global ID " U64_FORMAT "; ignoring it." " Someone should fix this.", q->type, chan, U64_PRINTF_ARG(chan->global_identifier)); - cell_queue_entry_free(q, 0); - q = NULL; + free_q = 1; + handed_off = 0; } - /* if q got NULLed out, we used it and should remove the queue entry */ - if (!q) TOR_SIMPLEQ_REMOVE_HEAD(&chan->outgoing_queue, next); + /* + * if free_q is set, we used it and should remove the queue entry; + * we have to do the free down here so TOR_SIMPLEQ_REMOVE_HEAD isn't + * accessing freed memory + */ + if (free_q) { + TOR_SIMPLEQ_REMOVE_HEAD(&chan->outgoing_queue, next); + /* + * ...and we handed a cell off to the lower layer, so we should + * update the counters. + */ + ++n_channel_cells_passed_to_lower_layer; + --n_channel_cells_in_queues; + n_channel_bytes_passed_to_lower_layer += cell_size; + n_channel_bytes_in_queues -= cell_size; + channel_assert_counter_consistency(); + /* Update the channel's queue size too */ + chan->bytes_in_queue -= cell_size; + /* Finally, free q */ + cell_queue_entry_free(q, handed_off); + q = NULL; + } /* No cell removed from list, so we can't go on any further */ else break; } @@ -2225,6 +2405,9 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, channel_timestamp_drained(chan); } + /* Update the estimate queue size */ + channel_update_xmit_queue_size(chan); + return flushed; } @@ -2347,15 +2530,16 @@ void channel_do_open_actions(channel_t *chan) { tor_addr_t remote_addr; - int started_here, not_using = 0; + int started_here; time_t now = time(NULL); + int close_origin_circuits = 0; tor_assert(chan); started_here = channel_is_outgoing(chan); if (started_here) { - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); rep_hist_note_connect_succeeded(chan->identity_digest, now); if (entry_guard_register_connect_status( chan->identity_digest, 1, 0, now) < 0) { @@ -2365,22 +2549,27 @@ channel_do_open_actions(channel_t *chan) log_debug(LD_OR, "New entry guard was reachable, but closing this " "connection so we can retry the earlier entry guards."); - circuit_n_chan_done(chan, 0); - not_using = 1; + close_origin_circuits = 1; } router_set_status(chan->identity_digest, 1); } else { /* only report it to the geoip module if it's not a known router */ if (!router_get_by_id_digest(chan->identity_digest)) { if (channel_get_addr_if_possible(chan, &remote_addr)) { - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &remote_addr, + char *transport_name = NULL; + if (chan->get_transport_name(chan, &transport_name) < 0) + transport_name = NULL; + + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, + &remote_addr, transport_name, now); + tor_free(transport_name); } /* Otherwise the underlying transport can't tell us this, so skip it */ } } - if (!not_using) circuit_n_chan_done(chan, 1); + circuit_n_chan_done(chan, 1, close_origin_circuits); } /** @@ -2451,9 +2640,8 @@ channel_process_cells(channel_t *chan) { cell_queue_entry_t *q; tor_assert(chan); - tor_assert(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_MAINT || - chan->state == CHANNEL_STATE_OPEN); + tor_assert(CHANNEL_IS_CLOSING(chan) || CHANNEL_IS_MAINT(chan) || + CHANNEL_IS_OPEN(chan)); log_debug(LD_CHANNEL, "Processing as many incoming cells as we can for channel %p", @@ -2468,6 +2656,11 @@ channel_process_cells(channel_t *chan) /* * Process cells until we're done or find one we have no current handler * for. + * + * We must free the cells here after calling the handler, since custody + * of the buffer was given to the channel layer when they were queued; + * see comments on memory management in channel_queue_cell() and in + * channel_queue_var_cell() below. */ while (NULL != (q = TOR_SIMPLEQ_FIRST(&chan->incoming_queue))) { tor_assert(q); @@ -2485,6 +2678,7 @@ channel_process_cells(channel_t *chan) q->u.fixed.cell, chan, U64_PRINTF_ARG(chan->global_identifier)); chan->cell_handler(chan, q->u.fixed.cell); + tor_free(q->u.fixed.cell); tor_free(q); } else if (q->type == CELL_QUEUE_VAR && chan->var_cell_handler) { @@ -2497,6 +2691,7 @@ channel_process_cells(channel_t *chan) q->u.var.var_cell, chan, U64_PRINTF_ARG(chan->global_identifier)); chan->var_cell_handler(chan, q->u.var.var_cell); + tor_free(q->u.var.var_cell); tor_free(q); } else { /* Can't handle this one */ @@ -2517,10 +2712,11 @@ channel_queue_cell(channel_t *chan, cell_t *cell) { int need_to_queue = 0; cell_queue_entry_t *q; + cell_t *cell_copy = NULL; tor_assert(chan); tor_assert(cell); - tor_assert(chan->state == CHANNEL_STATE_OPEN); + tor_assert(CHANNEL_IS_OPEN(chan)); /* Do we need to queue it, or can we just call the handler right away? */ if (!(chan->cell_handler)) need_to_queue = 1; @@ -2530,8 +2726,9 @@ channel_queue_cell(channel_t *chan, cell_t *cell) /* Timestamp for receiving */ channel_timestamp_recv(chan); - /* Update the counter */ + /* Update the counters */ ++(chan->n_cells_recved); + chan->n_bytes_recved += get_cell_network_size(chan->wide_circ_ids); /* If we don't need to queue we can just call cell_handler */ if (!need_to_queue) { @@ -2543,8 +2740,19 @@ channel_queue_cell(channel_t *chan, cell_t *cell) U64_PRINTF_ARG(chan->global_identifier)); chan->cell_handler(chan, cell); } else { - /* Otherwise queue it and then process the queue if possible. */ - q = cell_queue_entry_new_fixed(cell); + /* + * Otherwise queue it and then process the queue if possible. + * + * We queue a copy, not the original pointer - it might have been on the + * stack in connection_or_process_cells_from_inbuf() (or another caller + * if we ever have a subclass other than channel_tls_t), or be freed + * there after we return. This is the uncommon case; the non-copying + * fast path occurs in the if (!need_to_queue) case above when the + * upper layer has installed cell handlers. + */ + cell_copy = tor_malloc_zero(sizeof(cell_t)); + memcpy(cell_copy, cell, sizeof(cell_t)); + q = cell_queue_entry_new_fixed(cell_copy); log_debug(LD_CHANNEL, "Queueing incoming cell_t %p for channel %p " "(global ID " U64_FORMAT ")", @@ -2570,10 +2778,11 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) { int need_to_queue = 0; cell_queue_entry_t *q; + var_cell_t *cell_copy = NULL; tor_assert(chan); tor_assert(var_cell); - tor_assert(chan->state == CHANNEL_STATE_OPEN); + tor_assert(CHANNEL_IS_OPEN(chan)); /* Do we need to queue it, or can we just call the handler right away? */ if (!(chan->var_cell_handler)) need_to_queue = 1; @@ -2585,6 +2794,8 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) /* Update the counter */ ++(chan->n_cells_recved); + chan->n_bytes_recved += get_var_cell_header_size(chan->wide_circ_ids) + + var_cell->payload_len; /* If we don't need to queue we can just call cell_handler */ if (!need_to_queue) { @@ -2596,8 +2807,18 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) U64_PRINTF_ARG(chan->global_identifier)); chan->var_cell_handler(chan, var_cell); } else { - /* Otherwise queue it and then process the queue if possible. */ - q = cell_queue_entry_new_var(var_cell); + /* + * Otherwise queue it and then process the queue if possible. + * + * We queue a copy, not the original pointer - it might have been on the + * stack in connection_or_process_cells_from_inbuf() (or another caller + * if we ever have a subclass other than channel_tls_t), or be freed + * there after we return. This is the uncommon case; the non-copying + * fast path occurs in the if (!need_to_queue) case above when the + * upper layer has installed cell handlers. + */ + cell_copy = var_cell_copy(var_cell); + q = cell_queue_entry_new_var(cell_copy); log_debug(LD_CHANNEL, "Queueing incoming var_cell_t %p for channel %p " "(global ID " U64_FORMAT ")", @@ -2611,6 +2832,67 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) } } +/** If <b>packed_cell</b> on <b>chan</b> is a destroy cell, then set + * *<b>circid_out</b> to its circuit ID, and return true. Otherwise, return + * false. */ +/* XXXX Move this function. */ +int +packed_cell_is_destroy(channel_t *chan, + const packed_cell_t *packed_cell, + circid_t *circid_out) +{ + if (chan->wide_circ_ids) { + if (packed_cell->body[4] == CELL_DESTROY) { + *circid_out = ntohl(get_uint32(packed_cell->body)); + return 1; + } + } else { + if (packed_cell->body[2] == CELL_DESTROY) { + *circid_out = ntohs(get_uint16(packed_cell->body)); + return 1; + } + } + return 0; +} + +/** + * Assert that the global channel stats counters are internally consistent + */ + +static void +channel_assert_counter_consistency(void) +{ + tor_assert(n_channel_cells_queued == + (n_channel_cells_in_queues + n_channel_cells_passed_to_lower_layer)); + tor_assert(n_channel_bytes_queued == + (n_channel_bytes_in_queues + n_channel_bytes_passed_to_lower_layer)); +} + +/* DOCDOC */ +static int +is_destroy_cell(channel_t *chan, + const cell_queue_entry_t *q, circid_t *circid_out) +{ + *circid_out = 0; + switch (q->type) { + case CELL_QUEUE_FIXED: + if (q->u.fixed.cell->command == CELL_DESTROY) { + *circid_out = q->u.fixed.cell->circ_id; + return 1; + } + break; + case CELL_QUEUE_VAR: + if (q->u.var.var_cell->command == CELL_DESTROY) { + *circid_out = q->u.var.var_cell->circ_id; + return 1; + } + break; + case CELL_QUEUE_PACKED: + return packed_cell_is_destroy(chan, q->u.packed.packed_cell, circid_out); + } + return 0; +} + /** * Send destroy cell on a channel * @@ -2622,25 +2904,25 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) int channel_send_destroy(circid_t circ_id, channel_t *chan, int reason) { - cell_t cell; - tor_assert(chan); + if (circ_id == 0) { + log_warn(LD_BUG, "Attempted to send a destroy cell for circID 0 " + "on a channel " U64_FORMAT " at %p in state %s (%d)", + U64_PRINTF_ARG(chan->global_identifier), + chan, channel_state_to_string(chan->state), + chan->state); + return 0; + } /* Check to make sure we can send on this channel first */ - if (!(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { - memset(&cell, 0, sizeof(cell_t)); - cell.circ_id = circ_id; - cell.command = CELL_DESTROY; - cell.payload[0] = (uint8_t) reason; + if (!CHANNEL_CONDEMNED(chan) && chan->cmux) { + channel_note_destroy_pending(chan, circ_id); + circuitmux_append_destroy_cell(chan, chan->cmux, circ_id, reason); log_debug(LD_OR, "Sending destroy (circID %u) on channel %p " "(global ID " U64_FORMAT ")", (unsigned)circ_id, chan, U64_PRINTF_ARG(chan->global_identifier)); - - channel_write_cell(chan, &cell); } else { log_warn(LD_BUG, "Someone called channel_send_destroy() for circID %u " @@ -2665,6 +2947,19 @@ channel_dumpstats(int severity) { if (all_channels && smartlist_len(all_channels) > 0) { tor_log(severity, LD_GENERAL, + "Channels have queued " U64_FORMAT " bytes in " U64_FORMAT " cells, " + "and handed " U64_FORMAT " bytes in " U64_FORMAT " cells to the lower" + " layer.", + U64_PRINTF_ARG(n_channel_bytes_queued), + U64_PRINTF_ARG(n_channel_cells_queued), + U64_PRINTF_ARG(n_channel_bytes_passed_to_lower_layer), + U64_PRINTF_ARG(n_channel_cells_passed_to_lower_layer)); + tor_log(severity, LD_GENERAL, + "There are currently " U64_FORMAT " bytes in " U64_FORMAT " cells " + "in channel queues.", + U64_PRINTF_ARG(n_channel_bytes_in_queues), + U64_PRINTF_ARG(n_channel_cells_in_queues)); + tor_log(severity, LD_GENERAL, "Dumping statistics about %d channels:", smartlist_len(all_channels)); tor_log(severity, LD_GENERAL, @@ -2806,13 +3101,12 @@ channel_free_list(smartlist_t *channels, int mark_for_close) channel_state_to_string(curr->state), curr->state); /* Detach circuits early so they can find the channel */ if (curr->cmux) { - circuitmux_detach_all_circuits(curr->cmux); + circuitmux_detach_all_circuits(curr->cmux, NULL); } + SMARTLIST_DEL_CURRENT(channels, curr); channel_unregister(curr); if (mark_for_close) { - if (!(curr->state == CHANNEL_STATE_CLOSING || - curr->state == CHANNEL_STATE_CLOSED || - curr->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(curr)) { channel_mark_for_close(curr); } channel_force_free(curr); @@ -3026,9 +3320,7 @@ channel_get_for_extend(const char *digest, tor_assert(tor_memeq(chan->identity_digest, digest, DIGEST_LEN)); - if (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) + if (CHANNEL_CONDEMNED(chan)) continue; /* Never return a channel on which the other end appears to be @@ -3038,7 +3330,7 @@ channel_get_for_extend(const char *digest, } /* Never return a non-open connection. */ - if (chan->state != CHANNEL_STATE_OPEN) { + if (!CHANNEL_IS_OPEN(chan)) { /* If the address matches, don't launch a new connection for this * circuit. */ if (channel_matches_target_addr_for_extend(chan, target_addr)) @@ -3138,7 +3430,7 @@ channel_listener_describe_transport(channel_listener_t *chan_l) /** * Return the number of entries in <b>queue</b> */ -static int +STATIC int chan_cell_queue_len(const chan_cell_queue_t *queue) { int r = 0; @@ -3154,8 +3446,8 @@ chan_cell_queue_len(const chan_cell_queue_t *queue) * Dump statistics for one channel to the log */ -void -channel_dump_statistics(channel_t *chan, int severity) +MOCK_IMPL(void, +channel_dump_statistics, (channel_t *chan, int severity)) { double avg, interval, age; time_t now = time(NULL); @@ -3224,9 +3516,9 @@ channel_dump_statistics(channel_t *chan, int severity) " is %s, and gives a canonical description of \"%s\" and an " "actual description of \"%s\"", U64_PRINTF_ARG(chan->global_identifier), - remote_addr_str, - channel_get_canonical_remote_descr(chan), - actual); + safe_str(remote_addr_str), + safe_str(channel_get_canonical_remote_descr(chan)), + safe_str(actual)); tor_free(remote_addr_str); tor_free(actual); } else { @@ -3298,7 +3590,7 @@ channel_dump_statistics(channel_t *chan, int severity) U64_PRINTF_ARG(chan->timestamp_recv), U64_PRINTF_ARG(now - chan->timestamp_recv)); tor_log(severity, LD_GENERAL, - " * Channel " U64_FORMAT " last trasmitted a cell " + " * Channel " U64_FORMAT " last transmitted a cell " "at " U64_FORMAT " (" U64_FORMAT " seconds ago)", U64_PRINTF_ARG(chan->global_identifier), U64_PRINTF_ARG(chan->timestamp_xmit), @@ -3307,12 +3599,22 @@ channel_dump_statistics(channel_t *chan, int severity) /* Describe counters and rates */ tor_log(severity, LD_GENERAL, " * Channel " U64_FORMAT " has received " - U64_FORMAT " cells and transmitted " U64_FORMAT, + U64_FORMAT " bytes in " U64_FORMAT " cells and transmitted " + U64_FORMAT " bytes in " U64_FORMAT " cells", U64_PRINTF_ARG(chan->global_identifier), + U64_PRINTF_ARG(chan->n_bytes_recved), U64_PRINTF_ARG(chan->n_cells_recved), + U64_PRINTF_ARG(chan->n_bytes_xmitted), U64_PRINTF_ARG(chan->n_cells_xmitted)); if (now > chan->timestamp_created && chan->timestamp_created > 0) { + if (chan->n_bytes_recved > 0) { + avg = (double)(chan->n_bytes_recved) / age; + tor_log(severity, LD_GENERAL, + " * Channel " U64_FORMAT " has averaged %f " + "bytes received per second", + U64_PRINTF_ARG(chan->global_identifier), avg); + } if (chan->n_cells_recved > 0) { avg = (double)(chan->n_cells_recved) / age; if (avg >= 1.0) { @@ -3328,6 +3630,13 @@ channel_dump_statistics(channel_t *chan, int severity) U64_PRINTF_ARG(chan->global_identifier), interval); } } + if (chan->n_bytes_xmitted > 0) { + avg = (double)(chan->n_bytes_xmitted) / age; + tor_log(severity, LD_GENERAL, + " * Channel " U64_FORMAT " has averaged %f " + "bytes transmitted per second", + U64_PRINTF_ARG(chan->global_identifier), avg); + } if (chan->n_cells_xmitted > 0) { avg = (double)(chan->n_cells_xmitted) / age; if (avg >= 1.0) { @@ -3698,6 +4007,23 @@ channel_mark_local(channel_t *chan) } /** + * Mark a channel as remote + * + * This internal-only function should be called by the lower layer if the + * channel is not to a local address but has previously been marked local. + * See channel_is_local() above or the description of the is_local bit in + * channel.h + */ + +void +channel_mark_remote(channel_t *chan) +{ + tor_assert(chan); + + chan->is_local = 0; +} + +/** * Test outgoing flag * * This function gets the outgoing flag; this is the inverse of the incoming @@ -3728,6 +4054,50 @@ channel_mark_outgoing(channel_t *chan) chan->is_incoming = 0; } +/************************ + * Flow control queries * + ***********************/ + +/* + * Get the latest estimate for the total queue size of all open channels + */ + +uint64_t +channel_get_global_queue_estimate(void) +{ + return estimated_total_queue_size; +} + +/* + * Estimate the number of writeable cells + * + * Ask the lower layer for an estimate of how many cells it can accept, and + * then subtract the length of our outgoing_queue, if any, to produce an + * estimate of the number of cells this channel can accept for writes. + */ + +int +channel_num_cells_writeable(channel_t *chan) +{ + int result; + + tor_assert(chan); + tor_assert(chan->num_cells_writeable); + + if (chan->state == CHANNEL_STATE_OPEN) { + /* Query lower layer */ + result = chan->num_cells_writeable(chan); + /* Subtract cell queue length, if any */ + result -= chan_cell_queue_len(&chan->outgoing_queue); + if (result < 0) result = 0; + } else { + /* No cells are writeable in any other state */ + result = 0; + } + + return result; +} + /********************* * Timestamp updates * ********************/ @@ -4096,10 +4466,10 @@ channel_num_circuits(channel_t *chan) * This is called when setting up a channel and replaces the old * connection_or_set_circid_type() */ -void -channel_set_circid_type(channel_t *chan, - crypto_pk_t *identity_rcvd, - int consider_identity) +MOCK_IMPL(void, +channel_set_circid_type,(channel_t *chan, + crypto_pk_t *identity_rcvd, + int consider_identity)) { int started_here; crypto_pk_t *our_identity; @@ -4130,3 +4500,87 @@ channel_set_circid_type(channel_t *chan, } } +/** + * Update the estimated number of bytes queued to transmit for this channel, + * and notify the scheduler. The estimate includes both the channel queue and + * the queue size reported by the lower layer, and an overhead estimate + * optionally provided by the lower layer. + */ + +void +channel_update_xmit_queue_size(channel_t *chan) +{ + uint64_t queued, adj; + double overhead; + + tor_assert(chan); + tor_assert(chan->num_bytes_queued); + + /* + * First, get the number of bytes we have queued without factoring in + * lower-layer overhead. + */ + queued = chan->num_bytes_queued(chan) + chan->bytes_in_queue; + /* Next, adjust by the overhead factor, if any is available */ + if (chan->get_overhead_estimate) { + overhead = chan->get_overhead_estimate(chan); + if (overhead >= 1.0f) { + queued *= overhead; + } else { + /* Ignore silly overhead factors */ + log_notice(LD_CHANNEL, "Ignoring silly overhead factor %f", overhead); + } + } + + /* Now, compare to the previous estimate */ + if (queued > chan->bytes_queued_for_xmit) { + adj = queued - chan->bytes_queued_for_xmit; + log_debug(LD_CHANNEL, + "Increasing queue size for channel " U64_FORMAT " by " U64_FORMAT + " from " U64_FORMAT " to " U64_FORMAT, + U64_PRINTF_ARG(chan->global_identifier), + U64_PRINTF_ARG(adj), + U64_PRINTF_ARG(chan->bytes_queued_for_xmit), + U64_PRINTF_ARG(queued)); + /* Update the channel's estimate */ + chan->bytes_queued_for_xmit = queued; + + /* Update the global queue size estimate if appropriate */ + if (chan->state == CHANNEL_STATE_OPEN || + chan->state == CHANNEL_STATE_MAINT) { + estimated_total_queue_size += adj; + log_debug(LD_CHANNEL, + "Increasing global queue size by " U64_FORMAT " for channel " + U64_FORMAT ", new size is " U64_FORMAT, + U64_PRINTF_ARG(adj), U64_PRINTF_ARG(chan->global_identifier), + U64_PRINTF_ARG(estimated_total_queue_size)); + /* Tell the scheduler we're increasing the queue size */ + scheduler_adjust_queue_size(chan, 1, adj); + } + } else if (queued < chan->bytes_queued_for_xmit) { + adj = chan->bytes_queued_for_xmit - queued; + log_debug(LD_CHANNEL, + "Decreasing queue size for channel " U64_FORMAT " by " U64_FORMAT + " from " U64_FORMAT " to " U64_FORMAT, + U64_PRINTF_ARG(chan->global_identifier), + U64_PRINTF_ARG(adj), + U64_PRINTF_ARG(chan->bytes_queued_for_xmit), + U64_PRINTF_ARG(queued)); + /* Update the channel's estimate */ + chan->bytes_queued_for_xmit = queued; + + /* Update the global queue size estimate if appropriate */ + if (chan->state == CHANNEL_STATE_OPEN || + chan->state == CHANNEL_STATE_MAINT) { + estimated_total_queue_size -= adj; + log_debug(LD_CHANNEL, + "Decreasing global queue size by " U64_FORMAT " for channel " + U64_FORMAT ", new size is " U64_FORMAT, + U64_PRINTF_ARG(adj), U64_PRINTF_ARG(chan->global_identifier), + U64_PRINTF_ARG(estimated_total_queue_size)); + /* Tell the scheduler we're decreasing the queue size */ + scheduler_adjust_queue_size(chan, -1, adj); + } + } +} + diff --git a/src/or/channel.h b/src/or/channel.h index 29ba40e326..129c0c2013 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -10,7 +10,6 @@ #define TOR_CHANNEL_H #include "or.h" -#include "tor_queue.h" #include "circuitmux.h" /* Channel handler function pointer typedefs */ @@ -22,7 +21,7 @@ struct cell_queue_entry_s; TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s) incoming_queue; typedef struct chan_cell_queue chan_cell_queue_t; -/* +/** * Channel struct; see the channel_t typedef in or.h. A channel is an * abstract interface for the OR-to-OR connection, similar to connection_or_t, * but without the strong coupling to the underlying TLS implementation. They @@ -32,18 +31,18 @@ typedef struct chan_cell_queue chan_cell_queue_t; */ struct channel_s { - /* Magic number for type-checking cast macros */ + /** Magic number for type-checking cast macros */ uint32_t magic; - /* Current channel state */ + /** Current channel state */ channel_state_t state; - /* Globally unique ID number for a channel over the lifetime of a Tor + /** Globally unique ID number for a channel over the lifetime of a Tor * process. */ uint64_t global_identifier; - /* Should we expect to see this channel in the channel lists? */ + /** Should we expect to see this channel in the channel lists? */ unsigned char registered:1; /** has this channel ever been open? */ @@ -58,27 +57,58 @@ struct channel_s { CHANNEL_CLOSE_FOR_ERROR } reason_for_closing; - /* Timestamps for both cell channels and listeners */ + /** State variable for use by the scheduler */ + enum { + /* + * The channel is not open, or it has a full output buffer but no queued + * cells. + */ + SCHED_CHAN_IDLE = 0, + /* + * The channel has space on its output buffer to write, but no queued + * cells. + */ + SCHED_CHAN_WAITING_FOR_CELLS, + /* + * The scheduler has queued cells but no output buffer space to write. + */ + SCHED_CHAN_WAITING_TO_WRITE, + /* + * The scheduler has both queued cells and output buffer space, and is + * eligible for the scheduler loop. + */ + SCHED_CHAN_PENDING + } scheduler_state; + + /** Heap index for use by the scheduler */ + int sched_heap_idx; + + /** Timestamps for both cell channels and listeners */ time_t timestamp_created; /* Channel created */ time_t timestamp_active; /* Any activity */ /* Methods implemented by the lower layer */ - /* Free a channel */ + /** Free a channel */ void (*free)(channel_t *); - /* Close an open channel */ + /** Close an open channel */ void (*close)(channel_t *); - /* Describe the transport subclass for this channel */ + /** Describe the transport subclass for this channel */ const char * (*describe_transport)(channel_t *); - /* Optional method to dump transport-specific statistics on the channel */ + /** Optional method to dump transport-specific statistics on the channel */ void (*dumpstats)(channel_t *, int); - /* Registered handlers for incoming cells */ + /** Registered handlers for incoming cells */ channel_cell_handler_fn_ptr cell_handler; channel_var_cell_handler_fn_ptr var_cell_handler; /* Methods implemented by the lower layer */ + /** + * Ask the lower layer for an estimate of the average overhead for + * transmissions on this channel. + */ + double (*get_overhead_estimate)(channel_t *); /* * Ask the underlying transport what the remote endpoint address is, in * a tor_addr_t. This is optional and subclasses may leave this NULL. @@ -87,79 +117,78 @@ struct channel_s { * available. */ int (*get_remote_addr)(channel_t *, tor_addr_t *); + int (*get_transport_name)(channel_t *chan, char **transport_out); + #define GRD_FLAG_ORIGINAL 1 #define GRD_FLAG_ADDR_ONLY 2 - /* + /** * Get a text description of the remote endpoint; canonicalized if the flag * GRD_FLAG_ORIGINAL is not set, or the one we originally connected * to/received from if it is. If GRD_FLAG_ADDR_ONLY is set, we return only * the original address. */ const char * (*get_remote_descr)(channel_t *, int); - /* Check if the lower layer has queued writes */ + /** Check if the lower layer has queued writes */ int (*has_queued_writes)(channel_t *); - /* + /** * If the second param is zero, ask the lower layer if this is * 'canonical', for a transport-specific definition of canonical; if * it is 1, ask if the answer to the preceding query is safe to rely * on. */ int (*is_canonical)(channel_t *, int); - /* Check if this channel matches a specified extend_info_t */ + /** Check if this channel matches a specified extend_info_t */ int (*matches_extend_info)(channel_t *, extend_info_t *); - /* Check if this channel matches a target address when extending */ + /** Check if this channel matches a target address when extending */ int (*matches_target)(channel_t *, const tor_addr_t *); + /* Ask the lower layer how many bytes it has queued but not yet sent */ + size_t (*num_bytes_queued)(channel_t *); + /* Ask the lower layer how many cells can be written */ + int (*num_cells_writeable)(channel_t *); /* Write a cell to an open channel */ int (*write_cell)(channel_t *, cell_t *); - /* Write a packed cell to an open channel */ + /** Write a packed cell to an open channel */ int (*write_packed_cell)(channel_t *, packed_cell_t *); - /* Write a variable-length cell to an open channel */ + /** Write a variable-length cell to an open channel */ int (*write_var_cell)(channel_t *, var_cell_t *); - /* + /** * Hash of the public RSA key for the other side's identity key, or * zeroes if the other side hasn't shown us a valid identity key. */ char identity_digest[DIGEST_LEN]; - /* Nickname of the OR on the other side, or NULL if none. */ + /** Nickname of the OR on the other side, or NULL if none. */ char *nickname; - /* + /** * Linked list of channels with the same identity digest, for the * digest->channel map */ TOR_LIST_ENTRY(channel_s) next_with_same_id; - /* List of incoming cells to handle */ + /** List of incoming cells to handle */ chan_cell_queue_t incoming_queue; - /* List of queued outgoing cells */ + /** List of queued outgoing cells */ chan_cell_queue_t outgoing_queue; - /* Circuit mux for circuits sending on this channel */ + /** Circuit mux for circuits sending on this channel */ circuitmux_t *cmux; - /* Circuit ID generation stuff for use by circuitbuild.c */ + /** Circuit ID generation stuff for use by circuitbuild.c */ - /* + /** * When we send CREATE cells along this connection, which half of the * space should we use? */ - ENUM_BF(circ_id_type_t) circ_id_type:2; - /** DOCDOC*/ + circ_id_type_bitfield_t circ_id_type:2; + /* DOCDOC */ unsigned wide_circ_ids:1; - /** Have we logged a warning about circID exhaustion on this channel? */ - unsigned warned_circ_ids_exhausted:1; - /* - * Which circ_id do we try to use next on this connection? This is - * always in the range 0..1<<15-1. - */ - circid_t next_circ_id; - /* For how many circuits are we n_chan? What about p_chan? */ + /** For how many circuits are we n_chan? What about p_chan? */ unsigned int num_n_circuits, num_p_circuits; - /* + /** * True iff this channel shouldn't get any new circs attached to it, * because the connection is too old, or because there's a better one. * More generally, this flag is used to note an unhealthy connection; @@ -183,14 +212,20 @@ struct channel_s { */ unsigned int is_local:1; + /** Have we logged a warning about circID exhaustion on this channel? + * If so, when? */ + ratelim_t last_warned_circ_ids_exhausted; + /** Channel timestamps for cell channels */ time_t timestamp_client; /* Client used this, according to relay.c */ time_t timestamp_drained; /* Output queue empty */ time_t timestamp_recv; /* Cell received from lower layer */ time_t timestamp_xmit; /* Cell sent to lower layer */ - /* Timestamp for relay.c */ - time_t timestamp_last_added_nonpadding; + /** Timestamp for run_connection_housekeeping(). We update this once a + * second when we run housekeeping and find a circuit on this channel, and + * whenever we add a circuit to the channel. */ + time_t timestamp_last_had_circuits; /** Unique ID for measuring direct network status requests;vtunneled ones * come over a circuit_t, which has a dirreq_id field as well, but is a @@ -198,8 +233,16 @@ struct channel_s { uint64_t dirreq_id; /** Channel counters for cell channels */ - uint64_t n_cells_recved; - uint64_t n_cells_xmitted; + uint64_t n_cells_recved, n_bytes_recved; + uint64_t n_cells_xmitted, n_bytes_xmitted; + + /** Our current contribution to the scheduler's total xmit queue */ + uint64_t bytes_queued_for_xmit; + + /** Number of bytes in this channel's cell queue; does not include + * lower-layer queueing. + */ + uint64_t bytes_in_queue; }; struct channel_listener_s { @@ -211,7 +254,7 @@ struct channel_listener_s { */ uint64_t global_identifier; - /* Should we expect to see this channel in the channel lists? */ + /** Should we expect to see this channel in the channel lists? */ unsigned char registered:1; /** Why did we close? @@ -223,31 +266,31 @@ struct channel_listener_s { CHANNEL_LISTENER_CLOSE_FOR_ERROR } reason_for_closing; - /* Timestamps for both cell channels and listeners */ + /** Timestamps for both cell channels and listeners */ time_t timestamp_created; /* Channel created */ time_t timestamp_active; /* Any activity */ /* Methods implemented by the lower layer */ - /* Free a channel */ + /** Free a channel */ void (*free)(channel_listener_t *); - /* Close an open channel */ + /** Close an open channel */ void (*close)(channel_listener_t *); - /* Describe the transport subclass for this channel */ + /** Describe the transport subclass for this channel */ const char * (*describe_transport)(channel_listener_t *); - /* Optional method to dump transport-specific statistics on the channel */ + /** Optional method to dump transport-specific statistics on the channel */ void (*dumpstats)(channel_listener_t *, int); - /* Registered listen handler to call on incoming connection */ + /** Registered listen handler to call on incoming connection */ channel_listener_fn_ptr listener; - /* List of pending incoming connections */ + /** List of pending incoming connections */ smartlist_t *incoming_list; - /* Timestamps for listeners */ + /** Timestamps for listeners */ time_t timestamp_accepted; - /* Counters for listeners */ + /** Counters for listeners */ uint64_t n_accepted; }; @@ -311,6 +354,36 @@ void channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol); #ifdef TOR_CHANNEL_INTERNAL_ +#ifdef CHANNEL_PRIVATE_ +/* Cell queue structure (here rather than channel.c for test suite use) */ + +typedef struct cell_queue_entry_s cell_queue_entry_t; +struct cell_queue_entry_s { + TOR_SIMPLEQ_ENTRY(cell_queue_entry_s) next; + enum { + CELL_QUEUE_FIXED, + CELL_QUEUE_VAR, + CELL_QUEUE_PACKED + } type; + union { + struct { + cell_t *cell; + } fixed; + struct { + var_cell_t *var_cell; + } var; + struct { + packed_cell_t *packed_cell; + } packed; + } u; +}; + +/* Cell queue functions for benefit of test suite */ +STATIC int chan_cell_queue_len(const chan_cell_queue_t *queue); + +STATIC void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off); +#endif + /* Channel operations for subclasses and internal use only */ /* Initialize a newly allocated channel - do this first in subclass @@ -349,6 +422,7 @@ void channel_clear_remote_end(channel_t *chan); void channel_mark_local(channel_t *chan); void channel_mark_incoming(channel_t *chan); void channel_mark_outgoing(channel_t *chan); +void channel_mark_remote(channel_t *chan); void channel_set_identity_digest(channel_t *chan, const char *identity_digest); void channel_set_remote_end(channel_t *chan, @@ -383,7 +457,8 @@ void channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell); void channel_flush_cells(channel_t *chan); /* Request from lower layer for more cells if available */ -ssize_t channel_flush_some_cells(channel_t *chan, ssize_t num_cells); +MOCK_DECL(ssize_t, channel_flush_some_cells, + (channel_t *chan, ssize_t num_cells)); /* Query if data available on this channel */ int channel_more_to_flush(channel_t *chan); @@ -430,11 +505,44 @@ channel_t * channel_find_by_remote_digest(const char *identity_digest); channel_t * channel_next_with_digest(channel_t *chan); /* + * Helper macros to lookup state of given channel. + */ + +#define CHANNEL_IS_CLOSED(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_CLOSED)) +#define CHANNEL_IS_OPENING(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_OPENING)) +#define CHANNEL_IS_OPEN(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_OPEN)) +#define CHANNEL_IS_MAINT(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_MAINT)) +#define CHANNEL_IS_CLOSING(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_CLOSING)) +#define CHANNEL_IS_ERROR(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_ERROR)) + +#define CHANNEL_FINISHED(chan) (CHANNEL_IS_CLOSED(chan) || \ + CHANNEL_IS_ERROR(chan)) + +#define CHANNEL_CONDEMNED(chan) (CHANNEL_IS_CLOSING(chan) || \ + CHANNEL_FINISHED(chan)) + +#define CHANNEL_CAN_HANDLE_CELLS(chan) (CHANNEL_IS_OPENING(chan) || \ + CHANNEL_IS_OPEN(chan) || \ + CHANNEL_IS_MAINT(chan)) + +static inline int +channel_is_in_state(channel_t *chan, channel_state_t state) +{ + return chan->state == state; +} + +/* * Metadata queries/updates */ const char * channel_describe_transport(channel_t *chan); -void channel_dump_statistics(channel_t *chan, int severity); +MOCK_DECL(void, channel_dump_statistics, (channel_t *chan, int severity)); void channel_dump_transport_statistics(channel_t *chan, int severity); const char * channel_get_actual_remote_descr(channel_t *chan); const char * channel_get_actual_remote_address(channel_t *chan); @@ -454,9 +562,11 @@ int channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info); int channel_matches_target_addr_for_extend(channel_t *chan, const tor_addr_t *target); unsigned int channel_num_circuits(channel_t *chan); -void channel_set_circid_type(channel_t *chan, crypto_pk_t *identity_rcvd, - int consider_identity); +MOCK_DECL(void,channel_set_circid_type,(channel_t *chan, + crypto_pk_t *identity_rcvd, + int consider_identity)); void channel_timestamp_client(channel_t *chan); +void channel_update_xmit_queue_size(channel_t *chan); const char * channel_listener_describe_transport(channel_listener_t *chan_l); void channel_listener_dump_statistics(channel_listener_t *chan_l, @@ -464,6 +574,10 @@ void channel_listener_dump_statistics(channel_listener_t *chan_l, void channel_listener_dump_transport_statistics(channel_listener_t *chan_l, int severity); +/* Flow control queries */ +uint64_t channel_get_global_queue_estimate(void); +int channel_num_cells_writeable(channel_t *chan); + /* Timestamp queries */ time_t channel_when_created(channel_t *chan); time_t channel_when_last_active(channel_t *chan); @@ -482,5 +596,9 @@ uint64_t channel_count_xmitted(channel_t *chan); uint64_t channel_listener_count_accepted(channel_listener_t *chan_l); +int packed_cell_is_destroy(channel_t *chan, + const packed_cell_t *packed_cell, + circid_t *circid_out); + #endif diff --git a/src/or/channeltls.c b/src/or/channeltls.c index d5428c1abd..c65af5d040 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -1,9 +1,11 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file channeltls.c - * \brief channel_t concrete subclass using or_connection_t + * + * \brief A concrete subclass of channel_t using or_connection_t to transfer + * cells between Tor instances. **/ /* @@ -13,6 +15,8 @@ #define TOR_CHANNEL_INTERNAL_ +#define CHANNELTLS_PRIVATE + #include "or.h" #include "channel.h" #include "channeltls.h" @@ -22,9 +26,12 @@ #include "connection.h" #include "connection_or.h" #include "control.h" +#include "link_handshake.h" #include "relay.h" +#include "rephist.h" #include "router.h" #include "routerlist.h" +#include "scheduler.h" /** How many CELL_PADDING cells have we received, ever? */ uint64_t stats_n_padding_cells_processed = 0; @@ -46,16 +53,16 @@ uint64_t stats_n_authorize_cells_processed = 0; /** Active listener, if any */ channel_listener_t *channel_tls_listener = NULL; -/* Utility function declarations */ -static void channel_tls_common_init(channel_tls_t *tlschan); - /* channel_tls_t method declarations */ static void channel_tls_close_method(channel_t *chan); static const char * channel_tls_describe_transport_method(channel_t *chan); static void channel_tls_free_method(channel_t *chan); +static double channel_tls_get_overhead_estimate_method(channel_t *chan); static int channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out); +static int +channel_tls_get_transport_name_method(channel_t *chan, char **transport_out); static const char * channel_tls_get_remote_descr_method(channel_t *chan, int flags); static int channel_tls_has_queued_writes_method(channel_t *chan); @@ -65,6 +72,8 @@ channel_tls_matches_extend_info_method(channel_t *chan, extend_info_t *extend_info); static int channel_tls_matches_target_method(channel_t *chan, const tor_addr_t *target); +static int channel_tls_num_cells_writeable_method(channel_t *chan); +static size_t channel_tls_num_bytes_queued_method(channel_t *chan); static int channel_tls_write_cell_method(channel_t *chan, cell_t *cell); static int channel_tls_write_packed_cell_method(channel_t *chan, @@ -85,12 +94,6 @@ static void channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *tlschan); static void channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *tlschan); -static void channel_tls_process_certs_cell(var_cell_t *cell, - channel_tls_t *tlschan); -static void channel_tls_process_auth_challenge_cell(var_cell_t *cell, - channel_tls_t *tlschan); -static void channel_tls_process_authenticate_cell(var_cell_t *cell, - channel_tls_t *tlschan); static int command_allowed_before_handshake(uint8_t command); static int enter_v3_handshake_with_cell(var_cell_t *cell, channel_tls_t *tlschan); @@ -100,7 +103,7 @@ static int enter_v3_handshake_with_cell(var_cell_t *cell, * and channel_tls_handle_incoming(). */ -static void +STATIC void channel_tls_common_init(channel_tls_t *tlschan) { channel_t *chan; @@ -114,12 +117,16 @@ channel_tls_common_init(channel_tls_t *tlschan) chan->close = channel_tls_close_method; chan->describe_transport = channel_tls_describe_transport_method; chan->free = channel_tls_free_method; + chan->get_overhead_estimate = channel_tls_get_overhead_estimate_method; chan->get_remote_addr = channel_tls_get_remote_addr_method; chan->get_remote_descr = channel_tls_get_remote_descr_method; + chan->get_transport_name = channel_tls_get_transport_name_method; chan->has_queued_writes = channel_tls_has_queued_writes_method; chan->is_canonical = channel_tls_is_canonical_method; chan->matches_extend_info = channel_tls_matches_extend_info_method; chan->matches_target = channel_tls_matches_target_method; + chan->num_bytes_queued = channel_tls_num_bytes_queued_method; + chan->num_cells_writeable = channel_tls_num_cells_writeable_method; chan->write_cell = channel_tls_write_cell_method; chan->write_packed_cell = channel_tls_write_packed_cell_method; chan->write_var_cell = channel_tls_write_var_cell_method; @@ -153,7 +160,18 @@ channel_tls_connect(const tor_addr_t *addr, uint16_t port, tlschan, U64_PRINTF_ARG(chan->global_identifier)); - if (is_local_addr(addr)) channel_mark_local(chan); + if (is_local_addr(addr)) { + log_debug(LD_CHANNEL, + "Marking new outgoing channel " U64_FORMAT " at %p as local", + U64_PRINTF_ARG(chan->global_identifier), chan); + channel_mark_local(chan); + } else { + log_debug(LD_CHANNEL, + "Marking new outgoing channel " U64_FORMAT " at %p as remote", + U64_PRINTF_ARG(chan->global_identifier), chan); + channel_mark_remote(chan); + } + channel_mark_outgoing(chan); /* Set up or_connection stuff */ @@ -283,11 +301,22 @@ channel_tls_handle_incoming(or_connection_t *orconn) tlschan->conn = orconn; orconn->chan = tlschan; - if (is_local_addr(&(TO_CONN(orconn)->addr))) channel_mark_local(chan); + if (is_local_addr(&(TO_CONN(orconn)->addr))) { + log_debug(LD_CHANNEL, + "Marking new incoming channel " U64_FORMAT " at %p as local", + U64_PRINTF_ARG(chan->global_identifier), chan); + channel_mark_local(chan); + } else { + log_debug(LD_CHANNEL, + "Marking new incoming channel " U64_FORMAT " at %p as remote", + U64_PRINTF_ARG(chan->global_identifier), chan); + channel_mark_remote(chan); + } + channel_mark_incoming(chan); - /* If we got one, we should register it */ - if (chan) channel_register(chan); + /* Register it */ + channel_register(chan); return chan; } @@ -410,6 +439,40 @@ channel_tls_free_method(channel_t *chan) } /** + * Get an estimate of the average TLS overhead for the upper layer + */ + +static double +channel_tls_get_overhead_estimate_method(channel_t *chan) +{ + double overhead = 1.0f; + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(tlschan->conn); + + /* Just return 1.0f if we don't have sensible data */ + if (tlschan->conn->bytes_xmitted > 0 && + tlschan->conn->bytes_xmitted_by_tls >= + tlschan->conn->bytes_xmitted) { + overhead = ((double)(tlschan->conn->bytes_xmitted_by_tls)) / + ((double)(tlschan->conn->bytes_xmitted)); + + /* + * Never estimate more than 2.0; otherwise we get silly large estimates + * at the very start of a new TLS connection. + */ + if (overhead > 2.0f) overhead = 2.0f; + } + + log_debug(LD_CHANNEL, + "Estimated overhead ratio for TLS chan " U64_FORMAT " is %f", + U64_PRINTF_ARG(chan->global_identifier), overhead); + + return overhead; +} + +/** * Get the remote address of a channel_tls_t * * This implements the get_remote_addr method for channel_tls_t; copy the @@ -435,6 +498,30 @@ channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out) } /** + * Get the name of the pluggable transport used by a channel_tls_t. + * + * This implements the get_transport_name for channel_tls_t. If the + * channel uses a pluggable transport, copy its name to + * <b>transport_out</b> and return 0. If the channel did not use a + * pluggable transport, return -1. */ + +static int +channel_tls_get_transport_name_method(channel_t *chan, char **transport_out) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(transport_out); + tor_assert(tlschan->conn); + + if (!tlschan->conn->ext_or_transport) + return -1; + + *transport_out = tor_strdup(tlschan->conn->ext_or_transport); + return 0; +} + +/** * Get endpoint description of a channel_tls_t * * This implements the get_remote_descr method for channel_tls_t; it returns @@ -624,6 +711,53 @@ channel_tls_matches_target_method(channel_t *chan, } /** + * Tell the upper layer how many bytes we have queued and not yet + * sent. + */ + +static size_t +channel_tls_num_bytes_queued_method(channel_t *chan) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(tlschan->conn); + + return connection_get_outbuf_len(TO_CONN(tlschan->conn)); +} + +/** + * Tell the upper layer how many cells we can accept to write + * + * This implements the num_cells_writeable method for channel_tls_t; it + * returns an estimate of the number of cells we can accept with + * channel_tls_write_*_cell(). + */ + +static int +channel_tls_num_cells_writeable_method(channel_t *chan) +{ + size_t outbuf_len; + ssize_t n; + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + size_t cell_network_size; + + tor_assert(tlschan); + tor_assert(tlschan->conn); + + cell_network_size = get_cell_network_size(tlschan->conn->wide_circ_ids); + outbuf_len = connection_get_outbuf_len(TO_CONN(tlschan->conn)); + /* Get the number of cells */ + n = CEIL_DIV(OR_CONN_HIGHWATER - outbuf_len, cell_network_size); + if (n < 0) n = 0; +#if SIZEOF_SIZE_T > SIZEOF_INT + if (n > INT_MAX) n = INT_MAX; +#endif + + return (int)n; +} + +/** * Write a cell to a channel_tls_t * * This implements the write_cell method for channel_tls_t; given a @@ -798,18 +932,18 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, tor_assert(conn); tor_assert(conn->chan == chan); tor_assert(chan->conn == conn); - /* -Werror appeasement */ - tor_assert(old_state == old_state); + /* Shut the compiler up without triggering -Wtautological-compare */ + (void)old_state; base_chan = TLS_CHAN_TO_BASE(chan); - /* Make sure the base connection state makes sense - shouldn't be error, - * closed or listening. */ + /* Make sure the base connection state makes sense - shouldn't be error + * or closed. */ - tor_assert(base_chan->state == CHANNEL_STATE_OPENING || - base_chan->state == CHANNEL_STATE_OPEN || - base_chan->state == CHANNEL_STATE_MAINT || - base_chan->state == CHANNEL_STATE_CLOSING); + tor_assert(CHANNEL_IS_OPENING(base_chan) || + CHANNEL_IS_OPEN(base_chan) || + CHANNEL_IS_MAINT(base_chan) || + CHANNEL_IS_CLOSING(base_chan)); /* Did we just go to state open? */ if (state == OR_CONN_STATE_OPEN) { @@ -818,69 +952,21 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, * CHANNEL_STATE_MAINT on this. */ channel_change_state(base_chan, CHANNEL_STATE_OPEN); + /* We might have just become writeable; check and tell the scheduler */ + if (connection_or_num_cells_writeable(conn) > 0) { + scheduler_channel_wants_writes(base_chan); + } } else { /* * Not open, so from CHANNEL_STATE_OPEN we go to CHANNEL_STATE_MAINT, * otherwise no change. */ - if (base_chan->state == CHANNEL_STATE_OPEN) { + if (CHANNEL_IS_OPEN(base_chan)) { channel_change_state(base_chan, CHANNEL_STATE_MAINT); } } } -/** - * Flush cells from a channel_tls_t - * - * Try to flush up to about num_cells cells, and return how many we flushed. - */ - -ssize_t -channel_tls_flush_some_cells(channel_tls_t *chan, ssize_t num_cells) -{ - ssize_t flushed = 0; - - tor_assert(chan); - - if (flushed >= num_cells) goto done; - - /* - * If channel_tls_t ever buffers anything below the channel_t layer, flush - * that first here. - */ - - flushed += channel_flush_some_cells(TLS_CHAN_TO_BASE(chan), - num_cells - flushed); - - /* - * If channel_tls_t ever buffers anything below the channel_t layer, check - * how much we actually got and push it on down here. - */ - - done: - return flushed; -} - -/** - * Check if a channel_tls_t has anything to flush - * - * Return true if there is any more to flush on this channel (cells in queue - * or active circuits). - */ - -int -channel_tls_more_to_flush(channel_tls_t *chan) -{ - tor_assert(chan); - - /* - * If channel_tls_t ever buffers anything below channel_t, the - * check for that should go here first. - */ - - return channel_more_to_flush(TLS_CHAN_TO_BASE(chan)); -} - #ifdef KEEP_TIMING_STATS /** @@ -925,6 +1011,11 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time, * for cell types specific to the handshake for this transport protocol and * handles them, and queues all other cells to the channel_t layer, which * eventually will hand them off to command.c. + * + * The channel layer itself decides whether the cell should be queued or + * can be handed off immediately to the upper-layer code. It is responsible + * for copying in the case that it queues; we merely pass pointers through + * which we get from connection_or_process_cells_from_inbuf(). */ void @@ -1022,6 +1113,12 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn) * related and live below the channel_t layer, so no variable-length * cells ever get delivered in the current implementation, but I've left * the mechanism in place for future use. + * + * If we were handing them off to the upper layer, the channel_t queueing + * code would be responsible for memory management, and we'd just be passing + * pointers through from connection_or_process_cells_from_inbuf(). That + * caller always frees them after this function returns, so this function + * should never free var_cell. */ void @@ -1182,6 +1279,44 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn) } /** + * Update channel marks after connection_or.c has changed an address + * + * This is called from connection_or_init_conn_from_address() after the + * connection's _base.addr or real_addr fields have potentially been changed + * so we can recalculate the local mark. Notably, this happens when incoming + * connections are reverse-proxied and we only learn the real address of the + * remote router by looking it up in the consensus after we finish the + * handshake and know an authenticated identity digest. + */ + +void +channel_tls_update_marks(or_connection_t *conn) +{ + channel_t *chan = NULL; + + tor_assert(conn); + tor_assert(conn->chan); + + chan = TLS_CHAN_TO_BASE(conn->chan); + + if (is_local_addr(&(TO_CONN(conn)->addr))) { + if (!channel_is_local(chan)) { + log_debug(LD_CHANNEL, + "Marking channel " U64_FORMAT " at %p as local", + U64_PRINTF_ARG(chan->global_identifier), chan); + channel_mark_local(chan); + } + } else { + if (channel_is_local(chan)) { + log_debug(LD_CHANNEL, + "Marking channel " U64_FORMAT " at %p as remote", + U64_PRINTF_ARG(chan->global_identifier), chan); + channel_mark_remote(chan); + } + } +} + +/** * Check if this cell type is allowed before the handshake is finished * * Return true if <b>command</b> is a cell command that's allowed to start a @@ -1255,13 +1390,20 @@ static void channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) { int highest_supported_version = 0; - const uint8_t *cp, *end; int started_here = 0; tor_assert(cell); tor_assert(chan); tor_assert(chan->conn); + if ((cell->payload_len % 2) == 1) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Received a VERSION cell with odd payload length %d; " + "closing connection.",cell->payload_len); + connection_or_close_for_error(chan->conn, 0); + return; + } + started_here = connection_or_nonopen_was_started_here(chan->conn); if (chan->conn->link_proto != 0 || @@ -1287,11 +1429,15 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) } tor_assert(chan->conn->handshake_state); - end = cell->payload + cell->payload_len; - for (cp = cell->payload; cp+1 < end; cp += 2) { - uint16_t v = ntohs(get_uint16(cp)); - if (is_or_protocol_version_known(v) && v > highest_supported_version) - highest_supported_version = v; + + { + int i; + const uint8_t *cp = cell->payload; + for (i = 0; i < cell->payload_len / 2; ++i, cp += 2) { + uint16_t v = ntohs(get_uint16(cp)); + if (is_or_protocol_version_known(v) && v > highest_supported_version) + highest_supported_version = v; + } } if (!highest_supported_version) { log_fn(LOG_PROTOCOL_WARN, LD_OR, @@ -1325,6 +1471,8 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) return; } + rep_hist_note_negotiated_link_proto(highest_supported_version, started_here); + chan->conn->link_proto = highest_supported_version; chan->conn->handshake_state->received_versions = 1; @@ -1489,12 +1637,14 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) my_addr_ptr = (uint8_t*) cell->payload + 6; end = cell->payload + CELL_PAYLOAD_SIZE; cp = cell->payload + 6 + my_addr_len; - if (cp >= end) { - log_fn(LOG_PROTOCOL_WARN, LD_OR, - "Addresses too long in netinfo cell; closing connection."); - connection_or_close_for_error(chan->conn, 0); - return; - } else if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { + + /* We used to check: + * if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) { + * + * This is actually never going to happen, since my_addr_len is at most 255, + * and CELL_PAYLOAD_LEN - 6 is 503. So we know that cp is < end. */ + + if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); @@ -1514,7 +1664,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) return; } if (tor_addr_eq(&addr, &(chan->conn->real_addr))) { - chan->conn->is_canonical = 1; + connection_or_set_canonical(chan->conn, 1); break; } cp = next; @@ -1526,30 +1676,9 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) #define NETINFO_NOTICE_SKEW 3600 if (labs(apparent_skew) > NETINFO_NOTICE_SKEW && router_get_by_id_digest(chan->conn->identity_digest)) { - char dbuf[64]; - int severity; - /*XXXX be smarter about when everybody says we are skewed. */ - if (router_digest_is_trusted_dir(chan->conn->identity_digest)) - severity = LOG_WARN; - else - severity = LOG_INFO; - format_time_interval(dbuf, sizeof(dbuf), apparent_skew); - log_fn(severity, LD_GENERAL, - "Received NETINFO cell with skewed time from " - "server at %s:%d. It seems that our clock is %s by %s, or " - "that theirs is %s. Tor requires an accurate clock to work: " - "please check your time and date settings.", - chan->conn->base_.address, - (int)(chan->conn->base_.port), - apparent_skew > 0 ? "ahead" : "behind", - dbuf, - apparent_skew > 0 ? "behind" : "ahead"); - if (severity == LOG_WARN) /* only tell the controller if an authority */ - control_event_general_status(LOG_WARN, - "CLOCK_SKEW SKEW=%ld SOURCE=OR:%s:%d", - apparent_skew, - chan->conn->base_.address, - chan->conn->base_.port); + int trusted = router_digest_is_trusted_dir(chan->conn->identity_digest); + clock_skew_warning(TO_CONN(chan->conn), apparent_skew, trusted, LD_GENERAL, + "NETINFO cell", "OR"); } /* XXX maybe act on my_apparent_addr, if the source is sufficiently @@ -1604,16 +1733,17 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) * If it's the server side, wait for an AUTHENTICATE cell. */ -static void +STATIC void channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan) { - tor_cert_t *link_cert = NULL; - tor_cert_t *id_cert = NULL; - tor_cert_t *auth_cert = NULL; - uint8_t *ptr; +#define MAX_CERT_TYPE_WANTED OR_CERT_TYPE_AUTH_1024 + tor_x509_cert_t *certs[MAX_CERT_TYPE_WANTED + 1]; int n_certs, i; + certs_cell_t *cc = NULL; + int send_netinfo = 0; + memset(certs, 0, sizeof(certs)); tor_assert(cell); tor_assert(chan); tor_assert(chan->conn); @@ -1643,59 +1773,41 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan) if (cell->circ_id) ERR("It had a nonzero circuit ID"); - n_certs = cell->payload[0]; - ptr = cell->payload + 1; + if (certs_cell_parse(&cc, cell->payload, cell->payload_len) < 0) + ERR("It couldn't be parsed."); + + n_certs = cc->n_certs; + for (i = 0; i < n_certs; ++i) { - uint8_t cert_type; - uint16_t cert_len; - if (ptr + 3 > cell->payload + cell->payload_len) { - goto truncated; - } - cert_type = *ptr; - cert_len = ntohs(get_uint16(ptr+1)); - if (ptr + 3 + cert_len > cell->payload + cell->payload_len) { - goto truncated; - } - if (cert_type == OR_CERT_TYPE_TLS_LINK || - cert_type == OR_CERT_TYPE_ID_1024 || - cert_type == OR_CERT_TYPE_AUTH_1024) { - tor_cert_t *cert = tor_cert_decode(ptr + 3, cert_len); - if (!cert) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Received undecodable certificate in CERTS cell from %s:%d", - safe_str(chan->conn->base_.address), - chan->conn->base_.port); + certs_cell_cert_t *c = certs_cell_get_certs(cc, i); + + uint16_t cert_type = c->cert_type; + uint16_t cert_len = c->cert_len; + uint8_t *cert_body = certs_cell_cert_getarray_body(c); + + if (cert_type > MAX_CERT_TYPE_WANTED) + continue; + + tor_x509_cert_t *cert = tor_x509_cert_decode(cert_body, cert_len); + if (!cert) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received undecodable certificate in CERTS cell from %s:%d", + safe_str(chan->conn->base_.address), + chan->conn->base_.port); + } else { + if (certs[cert_type]) { + tor_x509_cert_free(cert); + ERR("Duplicate x509 certificate"); } else { - if (cert_type == OR_CERT_TYPE_TLS_LINK) { - if (link_cert) { - tor_cert_free(cert); - ERR("Too many TLS_LINK certificates"); - } - link_cert = cert; - } else if (cert_type == OR_CERT_TYPE_ID_1024) { - if (id_cert) { - tor_cert_free(cert); - ERR("Too many ID_1024 certificates"); - } - id_cert = cert; - } else if (cert_type == OR_CERT_TYPE_AUTH_1024) { - if (auth_cert) { - tor_cert_free(cert); - ERR("Too many AUTH_1024 certificates"); - } - auth_cert = cert; - } else { - tor_cert_free(cert); - } + certs[cert_type] = cert; } } - ptr += 3 + cert_len; - continue; - - truncated: - ERR("It ends in the middle of a certificate"); } + tor_x509_cert_t *id_cert = certs[OR_CERT_TYPE_ID_1024]; + tor_x509_cert_t *auth_cert = certs[OR_CERT_TYPE_AUTH_1024]; + tor_x509_cert_t *link_cert = certs[OR_CERT_TYPE_TLS_LINK]; + if (chan->conn->handshake_state->started_here) { int severity; if (! (id_cert && link_cert)) @@ -1720,7 +1832,8 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan) chan->conn->handshake_state->authenticated = 1; { - const digests_t *id_digests = tor_cert_get_id_digests(id_cert); + const common_digests_t *id_digests = + tor_x509_cert_get_id_digests(id_cert); crypto_pk_t *identity_rcvd; if (!id_digests) ERR("Couldn't compute digests for key in ID cert"); @@ -1744,7 +1857,7 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan) safe_str(chan->conn->base_.address), chan->conn->base_.port); chan->conn->handshake_state->id_cert = id_cert; - id_cert = NULL; + certs[OR_CERT_TYPE_ID_1024] = NULL; if (!public_server_mode(get_options())) { /* If we initiated the connection and we are not a public server, we @@ -1771,7 +1884,7 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan) chan->conn->handshake_state->id_cert = id_cert; chan->conn->handshake_state->auth_cert = auth_cert; - id_cert = auth_cert = NULL; + certs[OR_CERT_TYPE_ID_1024] = certs[OR_CERT_TYPE_AUTH_1024] = NULL; } chan->conn->handshake_state->received_certs_cell = 1; @@ -1785,9 +1898,10 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan) } err: - tor_cert_free(id_cert); - tor_cert_free(link_cert); - tor_cert_free(auth_cert); + for (unsigned i = 0; i < ARRAY_LENGTH(certs); ++i) { + tor_x509_cert_free(certs[i]); + } + certs_cell_free(cc); #undef ERR } @@ -1802,11 +1916,11 @@ channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *chan) * want to authenticate, send an AUTHENTICATE cell and then a NETINFO cell. */ -static void +STATIC void channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan) { int n_types, i, use_type = -1; - uint8_t *cp; + auth_challenge_cell_t *ac = NULL; tor_assert(cell); tor_assert(chan); @@ -1819,7 +1933,7 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan) safe_str(chan->conn->base_.address), \ chan->conn->base_.port, (s)); \ connection_or_close_for_error(chan->conn, 0); \ - return; \ + goto done; \ } while (0) if (chan->conn->base_.state != OR_CONN_STATE_OR_HANDSHAKING_V3) @@ -1832,19 +1946,17 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan) ERR("We already received one"); if (!(chan->conn->handshake_state->received_certs_cell)) ERR("We haven't gotten a CERTS cell yet"); - if (cell->payload_len < OR_AUTH_CHALLENGE_LEN + 2) - ERR("It was too short"); if (cell->circ_id) ERR("It had a nonzero circuit ID"); - n_types = ntohs(get_uint16(cell->payload + OR_AUTH_CHALLENGE_LEN)); - if (cell->payload_len < OR_AUTH_CHALLENGE_LEN + 2 + 2*n_types) - ERR("It looks truncated"); + if (auth_challenge_cell_parse(&ac, cell->payload, cell->payload_len) < 0) + ERR("It was not well-formed."); + + n_types = ac->n_methods; /* Now see if there is an authentication type we can use */ - cp = cell->payload+OR_AUTH_CHALLENGE_LEN + 2; - for (i = 0; i < n_types; ++i, cp += 2) { - uint16_t authtype = ntohs(get_uint16(cp)); + for (i = 0; i < n_types; ++i) { + uint16_t authtype = auth_challenge_cell_get_methods(ac, i); if (authtype == AUTHTYPE_RSA_SHA256_TLSSECRET) use_type = authtype; } @@ -1855,7 +1967,7 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan) /* If we're not a public server then we don't want to authenticate on a connection we originated, and we already sent a NETINFO cell when we got the CERTS cell. We have nothing more to do. */ - return; + goto done; } if (use_type >= 0) { @@ -1869,7 +1981,7 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan) log_warn(LD_OR, "Couldn't send authenticate cell"); connection_or_close_for_error(chan->conn, 0); - return; + goto done; } } else { log_info(LD_OR, @@ -1882,9 +1994,12 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan) if (connection_or_send_netinfo(chan->conn) < 0) { log_warn(LD_OR, "Couldn't send netinfo cell"); connection_or_close_for_error(chan->conn, 0); - return; + goto done; } + done: + auth_challenge_cell_free(ac); + #undef ERR } @@ -1898,10 +2013,10 @@ channel_tls_process_auth_challenge_cell(var_cell_t *cell, channel_tls_t *chan) * the identity of the router on the other side of the connection. */ -static void +STATIC void channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan) { - uint8_t expected[V3_AUTH_FIXED_PART_LEN]; + uint8_t expected[V3_AUTH_FIXED_PART_LEN+256]; const uint8_t *auth; int authlen; @@ -1957,11 +2072,13 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan) if (authlen < V3_AUTH_BODY_LEN + 1) ERR("Authenticator was too short"); - if (connection_or_compute_authenticate_cell_body( - chan->conn, expected, sizeof(expected), NULL, 1) < 0) + ssize_t bodylen = + connection_or_compute_authenticate_cell_body( + chan->conn, expected, sizeof(expected), NULL, 1); + if (bodylen < 0 || bodylen != V3_AUTH_FIXED_PART_LEN) ERR("Couldn't compute expected AUTHENTICATE cell body"); - if (tor_memneq(expected, auth, sizeof(expected))) + if (tor_memneq(expected, auth, bodylen)) ERR("Some field in the AUTHENTICATE cell body was not as expected"); { @@ -2006,8 +2123,8 @@ channel_tls_process_authenticate_cell(var_cell_t *cell, channel_tls_t *chan) { crypto_pk_t *identity_rcvd = tor_tls_cert_get_key(chan->conn->handshake_state->id_cert); - const digests_t *id_digests = - tor_cert_get_id_digests(chan->conn->handshake_state->id_cert); + const common_digests_t *id_digests = + tor_x509_cert_get_id_digests(chan->conn->handshake_state->id_cert); /* This must exist; we checked key type when reading the cert. */ tor_assert(id_digests); diff --git a/src/or/channeltls.h b/src/or/channeltls.h index b4a7e2beac..a4d9c7a095 100644 --- a/src/or/channeltls.h +++ b/src/or/channeltls.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -40,8 +40,6 @@ channel_t * channel_tls_to_base(channel_tls_t *tlschan); channel_tls_t * channel_tls_from_base(channel_t *chan); /* Things for connection_or.c to call back into */ -ssize_t channel_tls_flush_some_cells(channel_tls_t *chan, ssize_t num_cells); -int channel_tls_more_to_flush(channel_tls_t *chan); void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn); void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, or_connection_t *conn, @@ -49,9 +47,20 @@ void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, uint8_t state); void channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn); +void channel_tls_update_marks(or_connection_t *conn); /* Cleanup at shutdown */ void channel_tls_free_all(void); +#ifdef CHANNELTLS_PRIVATE +STATIC void channel_tls_process_certs_cell(var_cell_t *cell, + channel_tls_t *tlschan); +STATIC void channel_tls_process_auth_challenge_cell(var_cell_t *cell, + channel_tls_t *tlschan); +STATIC void channel_tls_common_init(channel_tls_t *tlschan); +STATIC void channel_tls_process_authenticate_cell(var_cell_t *cell, + channel_tls_t *tlschan); +#endif + #endif diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c new file mode 100644 index 0000000000..552947eba2 --- /dev/null +++ b/src/or/circpathbias.c @@ -0,0 +1,1547 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circpathbias.c + * + * \brief Code to track success/failure rates of circuits built through + * different tor nodes, in an attempt to detect attacks where + * an attacker deliberately causes circuits to fail until the client + * choses a path they like. + */ + +#include "or.h" +#include "channel.h" +#include "circpathbias.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "circuitstats.h" +#include "connection_edge.h" +#include "config.h" +#include "entrynodes.h" +#include "networkstatus.h" +#include "relay.h" + +static void pathbias_count_successful_close(origin_circuit_t *circ); +static void pathbias_count_collapse(origin_circuit_t *circ); +static void pathbias_count_use_failed(origin_circuit_t *circ); +static void pathbias_measure_use_rate(entry_guard_t *guard); +static void pathbias_measure_close_rate(entry_guard_t *guard); +static void pathbias_scale_use_rates(entry_guard_t *guard); +static void pathbias_scale_close_rates(entry_guard_t *guard); +static int entry_guard_inc_circ_attempt_count(entry_guard_t *guard); + +/** Increment the number of times we successfully extended a circuit to + * <b>guard</b>, first checking if the failure rate is high enough that + * we should eliminate the guard. Return -1 if the guard looks no good; + * return 0 if the guard looks fine. + */ +static int +entry_guard_inc_circ_attempt_count(entry_guard_t *guard) +{ + entry_guards_changed(); + + pathbias_measure_close_rate(guard); + + if (guard->path_bias_disabled) + return -1; + + pathbias_scale_close_rates(guard); + guard->circ_attempts++; + + log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + return 0; +} + +/** The minimum number of circuit attempts before we start + * thinking about warning about path bias and dropping guards */ +static int +pathbias_get_min_circs(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_MIN_CIRC 150 + if (options->PathBiasCircThreshold >= 5) + return options->PathBiasCircThreshold; + else + return networkstatus_get_param(NULL, "pb_mincircs", + DFLT_PATH_BIAS_MIN_CIRC, + 5, INT32_MAX); +} + +/** The circuit success rate below which we issue a notice */ +static double +pathbias_get_notice_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_NOTICE_PCT 70 + if (options->PathBiasNoticeRate >= 0.0) + return options->PathBiasNoticeRate; + else + return networkstatus_get_param(NULL, "pb_noticepct", + DFLT_PATH_BIAS_NOTICE_PCT, 0, 100)/100.0; +} + +/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/** The circuit success rate below which we issue a warn */ +static double +pathbias_get_warn_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_WARN_PCT 50 + if (options->PathBiasWarnRate >= 0.0) + return options->PathBiasWarnRate; + else + return networkstatus_get_param(NULL, "pb_warnpct", + DFLT_PATH_BIAS_WARN_PCT, 0, 100)/100.0; +} + +/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/** + * The extreme rate is the rate at which we would drop the guard, + * if pb_dropguard is also set. Otherwise we just warn. + */ +double +pathbias_get_extreme_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_EXTREME_PCT 30 + if (options->PathBiasExtremeRate >= 0.0) + return options->PathBiasExtremeRate; + else + return networkstatus_get_param(NULL, "pb_extremepct", + DFLT_PATH_BIAS_EXTREME_PCT, 0, 100)/100.0; +} + +/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/** + * If 1, we actually disable use of guards that fall below + * the extreme_pct. + */ +int +pathbias_get_dropguards(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_DROP_GUARDS 0 + if (options->PathBiasDropGuards >= 0) + return options->PathBiasDropGuards; + else + return networkstatus_get_param(NULL, "pb_dropguards", + DFLT_PATH_BIAS_DROP_GUARDS, 0, 1); +} + +/** + * This is the number of circuits at which we scale our + * counts by mult_factor/scale_factor. Note, this count is + * not exact, as we only perform the scaling in the event + * of no integer truncation. + */ +static int +pathbias_get_scale_threshold(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_SCALE_THRESHOLD 300 + if (options->PathBiasScaleThreshold >= 10) + return options->PathBiasScaleThreshold; + else + return networkstatus_get_param(NULL, "pb_scalecircs", + DFLT_PATH_BIAS_SCALE_THRESHOLD, 10, + INT32_MAX); +} + +/** + * Compute the path bias scaling ratio from the consensus + * parameters pb_multfactor/pb_scalefactor. + * + * Returns a value in (0, 1.0] which we multiply our pathbias + * counts with to scale them down. + */ +static double +pathbias_get_scale_ratio(const or_options_t *options) +{ + /* + * The scale factor is the denominator for our scaling + * of circuit counts for our path bias window. + * + * Note that our use of doubles for the path bias state + * file means that powers of 2 work best here. + */ + int denominator = networkstatus_get_param(NULL, "pb_scalefactor", + 2, 2, INT32_MAX); + (void) options; + /** + * The mult factor is the numerator for our scaling + * of circuit counts for our path bias window. It + * allows us to scale by fractions. + */ + return networkstatus_get_param(NULL, "pb_multfactor", + 1, 1, denominator)/((double)denominator); +} + +/** The minimum number of circuit usage attempts before we start + * thinking about warning about path use bias and dropping guards */ +static int +pathbias_get_min_use(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_MIN_USE 20 + if (options->PathBiasUseThreshold >= 3) + return options->PathBiasUseThreshold; + else + return networkstatus_get_param(NULL, "pb_minuse", + DFLT_PATH_BIAS_MIN_USE, + 3, INT32_MAX); +} + +/** The circuit use success rate below which we issue a notice */ +static double +pathbias_get_notice_use_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_NOTICE_USE_PCT 80 + if (options->PathBiasNoticeUseRate >= 0.0) + return options->PathBiasNoticeUseRate; + else + return networkstatus_get_param(NULL, "pb_noticeusepct", + DFLT_PATH_BIAS_NOTICE_USE_PCT, + 0, 100)/100.0; +} + +/** + * The extreme use rate is the rate at which we would drop the guard, + * if pb_dropguard is also set. Otherwise we just warn. + */ +double +pathbias_get_extreme_use_rate(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_EXTREME_USE_PCT 60 + if (options->PathBiasExtremeUseRate >= 0.0) + return options->PathBiasExtremeUseRate; + else + return networkstatus_get_param(NULL, "pb_extremeusepct", + DFLT_PATH_BIAS_EXTREME_USE_PCT, + 0, 100)/100.0; +} + +/** + * This is the number of circuits at which we scale our + * use counts by mult_factor/scale_factor. Note, this count is + * not exact, as we only perform the scaling in the event + * of no integer truncation. + */ +static int +pathbias_get_scale_use_threshold(const or_options_t *options) +{ +#define DFLT_PATH_BIAS_SCALE_USE_THRESHOLD 100 + if (options->PathBiasScaleUseThreshold >= 10) + return options->PathBiasScaleUseThreshold; + else + return networkstatus_get_param(NULL, "pb_scaleuse", + DFLT_PATH_BIAS_SCALE_USE_THRESHOLD, + 10, INT32_MAX); +} + +/** + * Convert a Guard's path state to string. + */ +const char * +pathbias_state_to_string(path_state_t state) +{ + switch (state) { + case PATH_STATE_NEW_CIRC: + return "new"; + case PATH_STATE_BUILD_ATTEMPTED: + return "build attempted"; + case PATH_STATE_BUILD_SUCCEEDED: + return "build succeeded"; + case PATH_STATE_USE_ATTEMPTED: + return "use attempted"; + case PATH_STATE_USE_SUCCEEDED: + return "use succeeded"; + case PATH_STATE_USE_FAILED: + return "use failed"; + case PATH_STATE_ALREADY_COUNTED: + return "already counted"; + } + + return "unknown"; +} + +/** + * This function decides if a circuit has progressed far enough to count + * as a circuit "attempt". As long as end-to-end tagging is possible, + * we assume the adversary will use it over hop-to-hop failure. Therefore, + * we only need to account bias for the last hop. This should make us + * much more resilient to ambient circuit failure, and also make that + * failure easier to measure (we only need to measure Exit failure rates). + */ +static int +pathbias_is_new_circ_attempt(origin_circuit_t *circ) +{ +#define N2N_TAGGING_IS_POSSIBLE +#ifdef N2N_TAGGING_IS_POSSIBLE + /* cpath is a circular list. We want circs with more than one hop, + * and the second hop must be waiting for keys still (it's just + * about to get them). */ + return circ->cpath && + circ->cpath->next != circ->cpath && + circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS; +#else + /* If tagging attacks are no longer possible, we probably want to + * count bias from the first hop. However, one could argue that + * timing-based tagging is still more useful than per-hop failure. + * In which case, we'd never want to use this. + */ + return circ->cpath && + circ->cpath->state == CPATH_STATE_AWAITING_KEYS; +#endif +} + +/** + * Decide if the path bias code should count a circuit. + * + * @returns 1 if we should count it, 0 otherwise. + */ +static int +pathbias_should_count(origin_circuit_t *circ) +{ +#define PATHBIAS_COUNT_INTERVAL (600) + static ratelim_t count_limit = + RATELIM_INIT(PATHBIAS_COUNT_INTERVAL); + char *rate_msg = NULL; + + /* We can't do path bias accounting without entry guards. + * Testing and controller circuits also have no guards. + * + * We also don't count server-side rends, because their + * endpoint could be chosen maliciously. + * Similarly, we can't count client-side intro attempts, + * because clients can be manipulated into connecting to + * malicious intro points. */ + if (get_options()->UseEntryGuards == 0 || + circ->base_.purpose == CIRCUIT_PURPOSE_TESTING || + circ->base_.purpose == CIRCUIT_PURPOSE_CONTROLLER || + circ->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND || + circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED || + (circ->base_.purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && + circ->base_.purpose <= CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)) { + + /* Check to see if the shouldcount result has changed due to a + * unexpected purpose change that would affect our results. + * + * The reason we check the path state too here is because for the + * cannibalized versions of these purposes, we count them as successful + * before their purpose change. + */ + if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED + && circ->path_state != PATH_STATE_ALREADY_COUNTED) { + log_info(LD_BUG, + "Circuit %d is now being ignored despite being counted " + "in the past. Purpose is %s, path state is %s", + circ->global_identifier, + circuit_purpose_to_string(circ->base_.purpose), + pathbias_state_to_string(circ->path_state)); + } + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; + return 0; + } + + /* Completely ignore one hop circuits */ + if (circ->build_state->onehop_tunnel || + circ->build_state->desired_path_len == 1) { + /* Check for inconsistency */ + if (circ->build_state->desired_path_len != 1 || + !circ->build_state->onehop_tunnel) { + if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) { + log_info(LD_BUG, + "One-hop circuit has length %d. Path state is %s. " + "Circuit is a %s currently %s.%s", + circ->build_state->desired_path_len, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + tor_fragile_assert(); + } + + /* Check to see if the shouldcount result has changed due to a + * unexpected change that would affect our results */ + if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED) { + log_info(LD_BUG, + "One-hop circuit %d is now being ignored despite being counted " + "in the past. Purpose is %s, path state is %s", + circ->global_identifier, + circuit_purpose_to_string(circ->base_.purpose), + pathbias_state_to_string(circ->path_state)); + } + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; + return 0; + } + + /* Check to see if the shouldcount result has changed due to a + * unexpected purpose change that would affect our results */ + if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) { + log_info(LD_BUG, + "Circuit %d is now being counted despite being ignored " + "in the past. Purpose is %s, path state is %s", + circ->global_identifier, + circuit_purpose_to_string(circ->base_.purpose), + pathbias_state_to_string(circ->path_state)); + } + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED; + + return 1; +} + +/** + * Check our circuit state to see if this is a successful circuit attempt. + * If so, record it in the current guard's path bias circ_attempt count. + * + * Also check for several potential error cases for bug #6475. + */ +int +pathbias_count_build_attempt(origin_circuit_t *circ) +{ +#define CIRC_ATTEMPT_NOTICE_INTERVAL (600) + static ratelim_t circ_attempt_notice_limit = + RATELIM_INIT(CIRC_ATTEMPT_NOTICE_INTERVAL); + char *rate_msg = NULL; + + if (!pathbias_should_count(circ)) { + return 0; + } + + if (pathbias_is_new_circ_attempt(circ)) { + /* Help track down the real cause of bug #6475: */ + if (circ->has_opened && circ->path_state != PATH_STATE_BUILD_ATTEMPTED) { + if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Opened circuit is in strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + + /* Don't re-count cannibalized circs.. */ + if (!circ->has_opened) { + entry_guard_t *guard = NULL; + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } else if (circ->base_.n_chan) { + guard = + entry_guard_get_by_id_digest(circ->base_.n_chan->identity_digest); + } + + if (guard) { + if (circ->path_state == PATH_STATE_NEW_CIRC) { + circ->path_state = PATH_STATE_BUILD_ATTEMPTED; + + if (entry_guard_inc_circ_attempt_count(guard) < 0) { + /* Bogus guard; we already warned. */ + return -END_CIRC_REASON_TORPROTOCOL; + } + } else { + if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Unopened circuit has strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } else { + if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, + approx_time()))) { + log_info(LD_CIRC, + "Unopened circuit has no known guard. " + "Circuit is a %s currently %s.%s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } + } + + return 0; +} + +/** + * Check our circuit state to see if this is a successful circuit + * completion. If so, record it in the current guard's path bias + * success count. + * + * Also check for several potential error cases for bug #6475. + */ +void +pathbias_count_build_success(origin_circuit_t *circ) +{ +#define SUCCESS_NOTICE_INTERVAL (600) + static ratelim_t success_notice_limit = + RATELIM_INIT(SUCCESS_NOTICE_INTERVAL); + char *rate_msg = NULL; + entry_guard_t *guard = NULL; + + if (!pathbias_should_count(circ)) { + return; + } + + /* Don't count cannibalized/reused circs for path bias + * "build" success, since they get counted under "use" success. */ + if (!circ->has_opened) { + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + if (circ->path_state == PATH_STATE_BUILD_ATTEMPTED) { + circ->path_state = PATH_STATE_BUILD_SUCCEEDED; + guard->circ_successes++; + entry_guards_changed(); + + log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } else { + if ((rate_msg = rate_limit_log(&success_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Succeeded circuit is in strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + + if (guard->circ_attempts < guard->circ_successes) { + log_notice(LD_BUG, "Unexpectedly high successes counts (%f/%f) " + "for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + if ((rate_msg = rate_limit_log(&success_notice_limit, + approx_time()))) { + log_info(LD_CIRC, + "Completed circuit has no known guard. " + "Circuit is a %s currently %s.%s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } else { + if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { + if ((rate_msg = rate_limit_log(&success_notice_limit, + approx_time()))) { + log_info(LD_BUG, + "Opened circuit is in strange path state %s. " + "Circuit is a %s currently %s.%s", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state), + rate_msg); + tor_free(rate_msg); + } + } + } +} + +/** + * Record an attempt to use a circuit. Changes the circuit's + * path state and update its guard's usage counter. + * + * Used for path bias usage accounting. + */ +void +pathbias_count_use_attempt(origin_circuit_t *circ) +{ + entry_guard_t *guard; + + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { + log_notice(LD_BUG, + "Used circuit is in strange path state %s. " + "Circuit is a %s currently %s.", + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } else if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + if (guard) { + pathbias_measure_use_rate(guard); + pathbias_scale_use_rates(guard); + guard->use_attempts++; + entry_guards_changed(); + + log_debug(LD_CIRC, + "Marked circuit %d (%f/%f) as used for guard %s ($%s).", + circ->global_identifier, + guard->use_successes, guard->use_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } + + circ->path_state = PATH_STATE_USE_ATTEMPTED; + } else { + /* Harmless but educational log message */ + log_info(LD_CIRC, + "Used circuit %d is already in path state %s. " + "Circuit is a %s currently %s.", + circ->global_identifier, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } + + return; +} + +/** + * Check the circuit's path state is appropriate and mark it as + * successfully used. Used for path bias usage accounting. + * + * We don't actually increment the guard's counters until + * pathbias_check_close(), because the circuit can still transition + * back to PATH_STATE_USE_ATTEMPTED if a stream fails later (this + * is done so we can probe the circuit for liveness at close). + */ +void +pathbias_mark_use_success(origin_circuit_t *circ) +{ + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { + log_notice(LD_BUG, + "Used circuit %d is in strange path state %s. " + "Circuit is a %s currently %s.", + circ->global_identifier, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + + pathbias_count_use_attempt(circ); + } + + /* We don't do any accounting at the guard until actual circuit close */ + circ->path_state = PATH_STATE_USE_SUCCEEDED; + + return; +} + +/** + * If a stream ever detatches from a circuit in a retriable way, + * we need to mark this circuit as still needing either another + * successful stream, or in need of a probe. + * + * An adversary could let the first stream request succeed (ie the + * resolve), but then tag and timeout the remainder (via cell + * dropping), forcing them on new circuits. + * + * Rolling back the state will cause us to probe such circuits, which + * should lead to probe failures in the event of such tagging due to + * either unrecognized cells coming in while we wait for the probe, + * or the cipher state getting out of sync in the case of dropped cells. + */ +void +pathbias_mark_use_rollback(origin_circuit_t *circ) +{ + if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { + log_info(LD_CIRC, + "Rolling back pathbias use state to 'attempted' for detached " + "circuit %d", circ->global_identifier); + circ->path_state = PATH_STATE_USE_ATTEMPTED; + } +} + +/** + * Actually count a circuit success towards a guard's usage counters + * if the path state is appropriate. + */ +static void +pathbias_count_use_success(origin_circuit_t *circ) +{ + entry_guard_t *guard; + + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->path_state != PATH_STATE_USE_SUCCEEDED) { + log_notice(LD_BUG, + "Successfully used circuit %d is in strange path state %s. " + "Circuit is a %s currently %s.", + circ->global_identifier, + pathbias_state_to_string(circ->path_state), + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } else { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + if (guard) { + guard->use_successes++; + entry_guards_changed(); + + if (guard->use_attempts < guard->use_successes) { + log_notice(LD_BUG, "Unexpectedly high use successes counts (%f/%f) " + "for guard %s=%s", + guard->use_successes, guard->use_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + } + + log_debug(LD_CIRC, + "Marked circuit %d (%f/%f) as used successfully for guard " + "%s ($%s).", + circ->global_identifier, guard->use_successes, + guard->use_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + } + } + + return; +} + +/** + * Send a probe down a circuit that the client attempted to use, + * but for which the stream timed out/failed. The probe is a + * RELAY_BEGIN cell with a 0.a.b.c destination address, which + * the exit will reject and reply back, echoing that address. + * + * The reason for such probes is because it is possible to bias + * a user's paths simply by causing timeouts, and these timeouts + * are not possible to differentiate from unresponsive servers. + * + * The probe is sent at the end of the circuit lifetime for two + * reasons: to prevent cryptographic taggers from being able to + * drop cells to cause timeouts, and to prevent easy recognition + * of probes before any real client traffic happens. + * + * Returns -1 if we couldn't probe, 0 otherwise. + */ +static int +pathbias_send_usable_probe(circuit_t *circ) +{ + /* Based on connection_ap_handshake_send_begin() */ + char payload[CELL_PAYLOAD_SIZE]; + int payload_len; + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + crypt_path_t *cpath_layer = NULL; + char *probe_nonce = NULL; + + tor_assert(ocirc); + + cpath_layer = ocirc->cpath->prev; + + if (cpath_layer->state != CPATH_STATE_OPEN) { + /* This can happen for cannibalized circuits. Their + * last hop isn't yet open */ + log_info(LD_CIRC, + "Got pathbias probe request for unopened circuit %d. " + "Opened %d, len %d", ocirc->global_identifier, + ocirc->has_opened, ocirc->build_state->desired_path_len); + return -1; + } + + /* We already went down this road. */ + if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING && + ocirc->pathbias_probe_id) { + log_info(LD_CIRC, + "Got pathbias probe request for circuit %d with " + "outstanding probe", ocirc->global_identifier); + return -1; + } + + /* Can't probe if the channel isn't open */ + if (circ->n_chan == NULL || + (!CHANNEL_IS_OPEN(circ->n_chan) + && !CHANNEL_IS_MAINT(circ->n_chan))) { + log_info(LD_CIRC, + "Skipping pathbias probe for circuit %d: Channel is not open.", + ocirc->global_identifier); + return -1; + } + + circuit_change_purpose(circ, CIRCUIT_PURPOSE_PATH_BIAS_TESTING); + + /* Update timestamp for when circuit_expire_building() should kill us */ + tor_gettimeofday(&circ->timestamp_began); + + /* Generate a random address for the nonce */ + crypto_rand((char*)ô->pathbias_probe_nonce, + sizeof(ocirc->pathbias_probe_nonce)); + ocirc->pathbias_probe_nonce &= 0x00ffffff; + probe_nonce = tor_dup_ip(ocirc->pathbias_probe_nonce); + + tor_snprintf(payload,RELAY_PAYLOAD_SIZE, "%s:25", probe_nonce); + payload_len = (int)strlen(payload)+1; + + // XXX: need this? Can we assume ipv4 will always be supported? + // If not, how do we tell? + //if (payload_len <= RELAY_PAYLOAD_SIZE - 4 && edge_conn->begincell_flags) { + // set_uint32(payload + payload_len, htonl(edge_conn->begincell_flags)); + // payload_len += 4; + //} + + /* Generate+Store stream id, make sure it's non-zero */ + ocirc->pathbias_probe_id = get_unique_stream_id_by_circ(ocirc); + + if (ocirc->pathbias_probe_id==0) { + log_warn(LD_CIRC, + "Ran out of stream IDs on circuit %u during " + "pathbias probe attempt.", ocirc->global_identifier); + tor_free(probe_nonce); + return -1; + } + + log_info(LD_CIRC, + "Sending pathbias testing cell to %s:25 on stream %d for circ %d.", + probe_nonce, ocirc->pathbias_probe_id, ocirc->global_identifier); + tor_free(probe_nonce); + + /* Send a test relay cell */ + if (relay_send_command_from_edge(ocirc->pathbias_probe_id, circ, + RELAY_COMMAND_BEGIN, payload, + payload_len, cpath_layer) < 0) { + log_notice(LD_CIRC, + "Failed to send pathbias probe cell on circuit %d.", + ocirc->global_identifier); + return -1; + } + + /* Mark it freshly dirty so it doesn't get expired in the meantime */ + circ->timestamp_dirty = time(NULL); + + return 0; +} + +/** + * Check the response to a pathbias probe, to ensure the + * cell is recognized and the nonce and other probe + * characteristics are as expected. + * + * If the response is valid, return 0. Otherwise return < 0. + */ +int +pathbias_check_probe_response(circuit_t *circ, const cell_t *cell) +{ + /* Based on connection_edge_process_relay_cell() */ + relay_header_t rh; + int reason; + uint32_t ipv4_host; + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + + tor_assert(cell); + tor_assert(ocirc); + tor_assert(circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING); + + relay_header_unpack(&rh, cell->payload); + + reason = rh.length > 0 ? + get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC; + + if (rh.command == RELAY_COMMAND_END && + reason == END_STREAM_REASON_EXITPOLICY && + ocirc->pathbias_probe_id == rh.stream_id) { + + /* Check length+extract host: It is in network order after the reason code. + * See connection_edge_end(). */ + if (rh.length < 9) { /* reason+ipv4+dns_ttl */ + log_notice(LD_PROTOCOL, + "Short path bias probe response length field (%d).", rh.length); + return - END_CIRC_REASON_TORPROTOCOL; + } + + ipv4_host = ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+1)); + + /* Check nonce */ + if (ipv4_host == ocirc->pathbias_probe_nonce) { + pathbias_mark_use_success(ocirc); + circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + log_info(LD_CIRC, + "Got valid path bias probe back for circ %d, stream %d.", + ocirc->global_identifier, ocirc->pathbias_probe_id); + return 0; + } else { + log_notice(LD_CIRC, + "Got strange probe value 0x%x vs 0x%x back for circ %d, " + "stream %d.", ipv4_host, ocirc->pathbias_probe_nonce, + ocirc->global_identifier, ocirc->pathbias_probe_id); + return -1; + } + } + log_info(LD_CIRC, + "Got another cell back back on pathbias probe circuit %d: " + "Command: %d, Reason: %d, Stream-id: %d", + ocirc->global_identifier, rh.command, reason, rh.stream_id); + return -1; +} + +/** + * Check if a circuit was used and/or closed successfully. + * + * If we attempted to use the circuit to carry a stream but failed + * for whatever reason, or if the circuit mysteriously died before + * we could attach any streams, record these two cases. + * + * If we *have* successfully used the circuit, or it appears to + * have been closed by us locally, count it as a success. + * + * Returns 0 if we're done making decisions with the circ, + * or -1 if we want to probe it first. + */ +int +pathbias_check_close(origin_circuit_t *ocirc, int reason) +{ + circuit_t *circ = ô->base_; + + if (!pathbias_should_count(ocirc)) { + return 0; + } + + switch (ocirc->path_state) { + /* If the circuit was closed after building, but before use, we need + * to ensure we were the ones who tried to close it (and not a remote + * actor). */ + case PATH_STATE_BUILD_SUCCEEDED: + if (reason & END_CIRC_REASON_FLAG_REMOTE) { + /* Remote circ close reasons on an unused circuit all could be bias */ + log_info(LD_CIRC, + "Circuit %d remote-closed without successful use for reason %d. " + "Circuit purpose %d currently %d,%s. Len %d.", + ocirc->global_identifier, + reason, circ->purpose, ocirc->has_opened, + circuit_state_to_string(circ->state), + ocirc->build_state->desired_path_len); + pathbias_count_collapse(ocirc); + } else if ((reason & ~END_CIRC_REASON_FLAG_REMOTE) + == END_CIRC_REASON_CHANNEL_CLOSED && + circ->n_chan && + circ->n_chan->reason_for_closing + != CHANNEL_CLOSE_REQUESTED) { + /* If we didn't close the channel ourselves, it could be bias */ + /* XXX: Only count bias if the network is live? + * What about clock jumps/suspends? */ + log_info(LD_CIRC, + "Circuit %d's channel closed without successful use for reason " + "%d, channel reason %d. Circuit purpose %d currently %d,%s. Len " + "%d.", ocirc->global_identifier, + reason, circ->n_chan->reason_for_closing, + circ->purpose, ocirc->has_opened, + circuit_state_to_string(circ->state), + ocirc->build_state->desired_path_len); + pathbias_count_collapse(ocirc); + } else { + pathbias_count_successful_close(ocirc); + } + break; + + /* If we tried to use a circuit but failed, we should probe it to ensure + * it has not been tampered with. */ + case PATH_STATE_USE_ATTEMPTED: + /* XXX: Only probe and/or count failure if the network is live? + * What about clock jumps/suspends? */ + if (pathbias_send_usable_probe(circ) == 0) + return -1; + else + pathbias_count_use_failed(ocirc); + + /* Any circuit where there were attempted streams but no successful + * streams could be bias */ + log_info(LD_CIRC, + "Circuit %d closed without successful use for reason %d. " + "Circuit purpose %d currently %d,%s. Len %d.", + ocirc->global_identifier, + reason, circ->purpose, ocirc->has_opened, + circuit_state_to_string(circ->state), + ocirc->build_state->desired_path_len); + break; + + case PATH_STATE_USE_SUCCEEDED: + pathbias_count_successful_close(ocirc); + pathbias_count_use_success(ocirc); + break; + + case PATH_STATE_USE_FAILED: + pathbias_count_use_failed(ocirc); + break; + + case PATH_STATE_NEW_CIRC: + case PATH_STATE_BUILD_ATTEMPTED: + case PATH_STATE_ALREADY_COUNTED: + default: + // Other states are uninteresting. No stats to count. + break; + } + + ocirc->path_state = PATH_STATE_ALREADY_COUNTED; + + return 0; +} + +/** + * Count a successfully closed circuit. + */ +static void +pathbias_count_successful_close(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + /* In the long run: circuit_success ~= successful_circuit_close + + * circ_failure + stream_failure */ + guard->successful_circuits_closed++; + entry_guards_changed(); + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + log_info(LD_CIRC, + "Successfully closed circuit has no known guard. " + "Circuit is a %s currently %s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } +} + +/** + * Count a circuit that fails after it is built, but before it can + * carry any traffic. + * + * This is needed because there are ways to destroy a + * circuit after it has successfully completed. Right now, this is + * used for purely informational/debugging purposes. + */ +static void +pathbias_count_collapse(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + guard->collapsed_circuits++; + entry_guards_changed(); + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + log_info(LD_CIRC, + "Destroyed circuit has no known guard. " + "Circuit is a %s currently %s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } +} + +/** + * Count a known failed circuit (because we could not probe it). + * + * This counter is informational. + */ +static void +pathbias_count_use_failed(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + if (!pathbias_should_count(circ)) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + guard->unusable_circuits++; + entry_guards_changed(); + } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { + /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to + * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. + * No need to log that case. */ + /* XXX note cut-and-paste code in this function compared to nearby + * functions. Would be nice to refactor. -RD */ + log_info(LD_CIRC, + "Stream-failing circuit has no known guard. " + "Circuit is a %s currently %s", + circuit_purpose_to_string(circ->base_.purpose), + circuit_state_to_string(circ->base_.state)); + } +} + +/** + * Count timeouts for path bias log messages. + * + * These counts are purely informational. + */ +void +pathbias_count_timeout(origin_circuit_t *circ) +{ + entry_guard_t *guard = NULL; + + if (!pathbias_should_count(circ)) { + return; + } + + /* For hidden service circs, they can actually be used + * successfully and then time out later (because + * the other side declines to use them). */ + if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { + return; + } + + if (circ->cpath && circ->cpath->extend_info) { + guard = entry_guard_get_by_id_digest( + circ->cpath->extend_info->identity_digest); + } + + if (guard) { + guard->timeouts++; + entry_guards_changed(); + } +} + +/** + * Helper function to count all of the currently opened circuits + * for a guard that are in a given path state range. The state + * range is inclusive on both ends. + */ +static int +pathbias_count_circs_in_states(entry_guard_t *guard, + path_state_t from, + path_state_t to) +{ + int open_circuits = 0; + + /* Count currently open circuits. Give them the benefit of the doubt. */ + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + origin_circuit_t *ocirc = NULL; + if (!CIRCUIT_IS_ORIGIN(circ) || /* didn't originate here */ + circ->marked_for_close) /* already counted */ + continue; + + ocirc = TO_ORIGIN_CIRCUIT(circ); + + if (!ocirc->cpath || !ocirc->cpath->extend_info) + continue; + + if (ocirc->path_state >= from && + ocirc->path_state <= to && + pathbias_should_count(ocirc) && + fast_memeq(guard->identity, + ocirc->cpath->extend_info->identity_digest, + DIGEST_LEN)) { + log_debug(LD_CIRC, "Found opened circuit %d in path_state %s", + ocirc->global_identifier, + pathbias_state_to_string(ocirc->path_state)); + open_circuits++; + } + } + SMARTLIST_FOREACH_END(circ); + + return open_circuits; +} + +/** + * Return the number of circuits counted as successfully closed for + * this guard. + * + * Also add in the currently open circuits to give them the benefit + * of the doubt. + */ +double +pathbias_get_close_success_count(entry_guard_t *guard) +{ + return guard->successful_circuits_closed + + pathbias_count_circs_in_states(guard, + PATH_STATE_BUILD_SUCCEEDED, + PATH_STATE_USE_SUCCEEDED); +} + +/** + * Return the number of circuits counted as successfully used + * this guard. + * + * Also add in the currently open circuits that we are attempting + * to use to give them the benefit of the doubt. + */ +double +pathbias_get_use_success_count(entry_guard_t *guard) +{ + return guard->use_successes + + pathbias_count_circs_in_states(guard, + PATH_STATE_USE_ATTEMPTED, + PATH_STATE_USE_SUCCEEDED); +} + +/** + * Check the path bias use rate against our consensus parameter limits. + * + * Emits a log message if the use success rates are too low. + * + * If pathbias_get_dropguards() is set, we also disable the use of + * very failure prone guards. + */ +static void +pathbias_measure_use_rate(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + if (guard->use_attempts > pathbias_get_min_use(options)) { + /* Note: We rely on the < comparison here to allow us to set a 0 + * rate and disable the feature entirely. If refactoring, don't + * change to <= */ + if (pathbias_get_use_success_count(guard)/guard->use_attempts + < pathbias_get_extreme_use_rate(options)) { + /* Dropping is currently disabled by default. */ + if (pathbias_get_dropguards(options)) { + if (!guard->path_bias_disabled) { + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing to carry an extremely large " + "amount of stream on its circuits. " + "To avoid potential route manipulation attacks, Tor has " + "disabled use of this guard. " + "Use counts are %ld/%ld. Success counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + guard->path_bias_disabled = 1; + guard->bad_since = approx_time(); + entry_guards_changed(); + return; + } + } else if (!guard->path_bias_use_extreme) { + guard->path_bias_use_extreme = 1; + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing to carry an extremely large " + "amount of streams on its circuits. " + "This could indicate a route manipulation attack, network " + "overload, bad local network connectivity, or a bug. " + "Use counts are %ld/%ld. Success counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } else if (pathbias_get_use_success_count(guard)/guard->use_attempts + < pathbias_get_notice_use_rate(options)) { + if (!guard->path_bias_use_noticed) { + guard->path_bias_use_noticed = 1; + log_notice(LD_CIRC, + "Your Guard %s ($%s) is failing to carry more streams on its " + "circuits than usual. " + "Most likely this means the Tor network is overloaded " + "or your network connection is poor. " + "Use counts are %ld/%ld. Success counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } + } +} + +/** + * Check the path bias circuit close status rates against our consensus + * parameter limits. + * + * Emits a log message if the use success rates are too low. + * + * If pathbias_get_dropguards() is set, we also disable the use of + * very failure prone guards. + * + * XXX: This function shares similar log messages and checks to + * pathbias_measure_use_rate(). It may be possible to combine them + * eventually, especially if we can ever remove the need for 3 + * levels of closure warns (if the overall circuit failure rate + * goes down with ntor). One way to do so would be to multiply + * the build rate with the use rate to get an idea of the total + * fraction of the total network paths the user is able to use. + * See ticket #8159. + */ +static void +pathbias_measure_close_rate(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + if (guard->circ_attempts > pathbias_get_min_circs(options)) { + /* Note: We rely on the < comparison here to allow us to set a 0 + * rate and disable the feature entirely. If refactoring, don't + * change to <= */ + if (pathbias_get_close_success_count(guard)/guard->circ_attempts + < pathbias_get_extreme_rate(options)) { + /* Dropping is currently disabled by default. */ + if (pathbias_get_dropguards(options)) { + if (!guard->path_bias_disabled) { + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing an extremely large " + "amount of circuits. " + "To avoid potential route manipulation attacks, Tor has " + "disabled use of this guard. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + guard->path_bias_disabled = 1; + guard->bad_since = approx_time(); + entry_guards_changed(); + return; + } + } else if (!guard->path_bias_extreme) { + guard->path_bias_extreme = 1; + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing an extremely large " + "amount of circuits. " + "This could indicate a route manipulation attack, " + "extreme network overload, or a bug. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts + < pathbias_get_warn_rate(options)) { + if (!guard->path_bias_warned) { + guard->path_bias_warned = 1; + log_warn(LD_CIRC, + "Your Guard %s ($%s) is failing a very large " + "amount of circuits. " + "Most likely this means the Tor network is " + "overloaded, but it could also mean an attack against " + "you or potentially the guard itself. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts + < pathbias_get_notice_rate(options)) { + if (!guard->path_bias_noticed) { + guard->path_bias_noticed = 1; + log_notice(LD_CIRC, + "Your Guard %s ($%s) is failing more circuits than " + "usual. " + "Most likely this means the Tor network is overloaded. " + "Success counts are %ld/%ld. Use counts are %ld/%ld. " + "%ld circuits completed, %ld were unusable, %ld collapsed, " + "and %ld timed out. " + "For reference, your timeout cutoff is %ld seconds.", + guard->nickname, hex_str(guard->identity, DIGEST_LEN), + tor_lround(pathbias_get_close_success_count(guard)), + tor_lround(guard->circ_attempts), + tor_lround(pathbias_get_use_success_count(guard)), + tor_lround(guard->use_attempts), + tor_lround(guard->circ_successes), + tor_lround(guard->unusable_circuits), + tor_lround(guard->collapsed_circuits), + tor_lround(guard->timeouts), + tor_lround(get_circuit_build_close_time_ms()/1000)); + } + } + } +} + +/** + * This function scales the path bias use rates if we have + * more data than the scaling threshold. This allows us to + * be more sensitive to recent measurements. + * + * XXX: The attempt count transfer stuff here might be done + * better by keeping separate pending counters that get + * transfered at circuit close. See ticket #8160. + */ +static void +pathbias_scale_close_rates(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + /* If we get a ton of circuits, just scale everything down */ + if (guard->circ_attempts > pathbias_get_scale_threshold(options)) { + double scale_ratio = pathbias_get_scale_ratio(options); + int opened_attempts = pathbias_count_circs_in_states(guard, + PATH_STATE_BUILD_ATTEMPTED, PATH_STATE_BUILD_ATTEMPTED); + int opened_built = pathbias_count_circs_in_states(guard, + PATH_STATE_BUILD_SUCCEEDED, + PATH_STATE_USE_FAILED); + /* Verify that the counts are sane before and after scaling */ + int counts_are_sane = (guard->circ_attempts >= guard->circ_successes); + + guard->circ_attempts -= (opened_attempts+opened_built); + guard->circ_successes -= opened_built; + + guard->circ_attempts *= scale_ratio; + guard->circ_successes *= scale_ratio; + guard->timeouts *= scale_ratio; + guard->successful_circuits_closed *= scale_ratio; + guard->collapsed_circuits *= scale_ratio; + guard->unusable_circuits *= scale_ratio; + + guard->circ_attempts += (opened_attempts+opened_built); + guard->circ_successes += opened_built; + + entry_guards_changed(); + + log_info(LD_CIRC, + "Scaled pathbias counts to (%f,%f)/%f (%d/%d open) for guard " + "%s ($%s)", + guard->circ_successes, guard->successful_circuits_closed, + guard->circ_attempts, opened_built, opened_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + + /* Have the counts just become invalid by this scaling attempt? */ + if (counts_are_sane && guard->circ_attempts < guard->circ_successes) { + log_notice(LD_BUG, + "Scaling has mangled pathbias counts to %f/%f (%d/%d open) " + "for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, opened_built, + opened_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + } + } +} + +/** + * This function scales the path bias circuit close rates if we have + * more data than the scaling threshold. This allows us to be more + * sensitive to recent measurements. + * + * XXX: The attempt count transfer stuff here might be done + * better by keeping separate pending counters that get + * transfered at circuit close. See ticket #8160. + */ +void +pathbias_scale_use_rates(entry_guard_t *guard) +{ + const or_options_t *options = get_options(); + + /* If we get a ton of circuits, just scale everything down */ + if (guard->use_attempts > pathbias_get_scale_use_threshold(options)) { + double scale_ratio = pathbias_get_scale_ratio(options); + int opened_attempts = pathbias_count_circs_in_states(guard, + PATH_STATE_USE_ATTEMPTED, PATH_STATE_USE_SUCCEEDED); + /* Verify that the counts are sane before and after scaling */ + int counts_are_sane = (guard->use_attempts >= guard->use_successes); + + guard->use_attempts -= opened_attempts; + + guard->use_attempts *= scale_ratio; + guard->use_successes *= scale_ratio; + + guard->use_attempts += opened_attempts; + + log_info(LD_CIRC, + "Scaled pathbias use counts to %f/%f (%d open) for guard %s ($%s)", + guard->use_successes, guard->use_attempts, opened_attempts, + guard->nickname, hex_str(guard->identity, DIGEST_LEN)); + + /* Have the counts just become invalid by this scaling attempt? */ + if (counts_are_sane && guard->use_attempts < guard->use_successes) { + log_notice(LD_BUG, + "Scaling has mangled pathbias usage counts to %f/%f " + "(%d open) for guard %s ($%s)", + guard->circ_successes, guard->circ_attempts, + opened_attempts, guard->nickname, + hex_str(guard->identity, DIGEST_LEN)); + } + + entry_guards_changed(); + } +} + diff --git a/src/or/circpathbias.h b/src/or/circpathbias.h new file mode 100644 index 0000000000..ce76689d5f --- /dev/null +++ b/src/or/circpathbias.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitbuild.h + * \brief Header file for circuitbuild.c. + **/ + +#ifndef TOR_CIRCPATHBIAS_H +#define TOR_CIRCPATHBIAS_H + +double pathbias_get_extreme_rate(const or_options_t *options); +double pathbias_get_extreme_use_rate(const or_options_t *options); +int pathbias_get_dropguards(const or_options_t *options); +void pathbias_count_timeout(origin_circuit_t *circ); +void pathbias_count_build_success(origin_circuit_t *circ); +int pathbias_count_build_attempt(origin_circuit_t *circ); +int pathbias_check_close(origin_circuit_t *circ, int reason); +int pathbias_check_probe_response(circuit_t *circ, const cell_t *cell); +void pathbias_count_use_attempt(origin_circuit_t *circ); +void pathbias_mark_use_success(origin_circuit_t *circ); +void pathbias_mark_use_rollback(origin_circuit_t *circ); +const char *pathbias_state_to_string(path_state_t state); + +#endif + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 4603de071f..28d286cd72 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -1,16 +1,22 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file circuitbuild.c - * \brief The actual details of building circuits. + * + * \brief Implements the details of building circuits (by chosing paths, + * constructing/sending create/extend cells, and so on). **/ +#define CIRCUITBUILD_PRIVATE + #include "or.h" #include "channel.h" +#include "circpathbias.h" +#define CIRCUITBUILD_PRIVATE #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" @@ -40,19 +46,11 @@ #include "routerparse.h" #include "routerset.h" #include "crypto.h" -#include "connection_edge.h" #ifndef MIN #define MIN(a,b) ((a)<(b)?(a):(b)) #endif -/********* START VARIABLES **********/ - -/** A global list of all circuits at this hop. */ -extern circuit_t *global_circuitlist; - -/********* END VARIABLES ************/ - static channel_t * channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port, const char *id_digest); @@ -64,17 +62,7 @@ static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath); static int onion_extend_cpath(origin_circuit_t *circ); static int count_acceptable_nodes(smartlist_t *routers); static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); -static int entry_guard_inc_circ_attempt_count(entry_guard_t *guard); -static void pathbias_count_build_success(origin_circuit_t *circ); -static void pathbias_count_successful_close(origin_circuit_t *circ); -static void pathbias_count_collapse(origin_circuit_t *circ); -static void pathbias_count_use_failed(origin_circuit_t *circ); -static void pathbias_measure_use_rate(entry_guard_t *guard); -static void pathbias_measure_close_rate(entry_guard_t *guard); -static void pathbias_scale_use_rates(entry_guard_t *guard); -#ifdef CURVE25519_ENABLED static int circuits_can_use_ntor(void); -#endif /** This function tries to get a channel to the specified endpoint, * and then calls command_setup_channel() to give it the right @@ -92,18 +80,29 @@ channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port, return chan; } -/** Iterate over values of circ_id, starting from conn-\>next_circ_id, - * and with the high bit specified by conn-\>circ_id_type, until we get - * a circ_id that is not in use by any other circuit on that conn. +/** Search for a value for circ_id that we can use on <b>chan</b> for an + * outbound circuit, until we get a circ_id that is not in use by any other + * circuit on that conn. * * Return it, or 0 if can't get a unique circ_id. */ -static circid_t +STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan) { +/* This number is chosen somewhat arbitrarily; see comment below for more + * info. When the space is 80% full, it gives a one-in-a-million failure + * chance; when the space is 90% full, it gives a one-in-850 chance; and when + * the space is 95% full, it gives a one-in-26 failure chance. That seems + * okay, though you could make a case IMO for anything between N=32 and + * N=256. */ +#define MAX_CIRCID_ATTEMPTS 64 + int in_use; + unsigned n_with_circ = 0, n_pending_destroy = 0, n_weird_pending_destroy = 0; circid_t test_circ_id; circid_t attempts=0; - circid_t high_bit, max_range; + circid_t high_bit, max_range, mask; + int64_t pending_destroy_time_total = 0; + int64_t pending_destroy_time_max = 0; tor_assert(chan); @@ -113,32 +112,108 @@ get_unique_circ_id_by_chan(channel_t *chan) "a client with no identity."); return 0; } - max_range = (chan->wide_circ_ids) ? (1u<<31) : (1u<<15); + max_range = (chan->wide_circ_ids) ? (1u<<31) : (1u<<15); + mask = max_range - 1; high_bit = (chan->circ_id_type == CIRC_ID_TYPE_HIGHER) ? max_range : 0; do { - /* Sequentially iterate over test_circ_id=1...max_range until we find a - * circID such that (high_bit|test_circ_id) is not already used. */ - test_circ_id = chan->next_circ_id++; - if (test_circ_id == 0 || test_circ_id >= max_range) { - test_circ_id = 1; - chan->next_circ_id = 2; - } - if (++attempts > max_range) { - /* Make sure we don't loop forever if all circ_id's are used. This - * matters because it's an external DoS opportunity. + if (++attempts > MAX_CIRCID_ATTEMPTS) { + /* Make sure we don't loop forever because all circuit IDs are used. + * + * Once, we would try until we had tried every possible circuit ID. But + * that's quite expensive. Instead, we try MAX_CIRCID_ATTEMPTS random + * circuit IDs, and then give up. + * + * This potentially causes us to give up early if our circuit ID space + * is nearly full. If we have N circuit IDs in use, then we will reject + * a new circuit with probability (N / max_range) ^ MAX_CIRCID_ATTEMPTS. + * This means that in practice, a few percent of our circuit ID capacity + * will go unused. + * + * The alternative here, though, is to do a linear search over the + * whole circuit ID space every time we extend a circuit, which is + * not so great either. */ - if (! chan->warned_circ_ids_exhausted) { - chan->warned_circ_ids_exhausted = 1; - log_warn(LD_CIRC,"No unused circIDs found on channel %s wide " + int64_t queued_destroys; + char *m = rate_limit_log(&chan->last_warned_circ_ids_exhausted, + approx_time()); + if (m == NULL) + return 0; /* This message has been rate-limited away. */ + if (n_pending_destroy) + pending_destroy_time_total /= n_pending_destroy; + log_warn(LD_CIRC,"No unused circIDs found on channel %s wide " "circID support, with %u inbound and %u outbound circuits. " - "Failing a circuit.", + "Found %u circuit IDs in use by circuits, and %u with " + "pending destroy cells. (%u of those were marked bogusly.) " + "The ones with pending destroy cells " + "have been marked unusable for an average of %ld seconds " + "and a maximum of %ld seconds. This channel is %ld seconds " + "old. Failing a circuit.%s", chan->wide_circ_ids ? "with" : "without", - chan->num_p_circuits, chan->num_n_circuits); + chan->num_p_circuits, chan->num_n_circuits, + n_with_circ, n_pending_destroy, n_weird_pending_destroy, + (long)pending_destroy_time_total, + (long)pending_destroy_time_max, + (long)(approx_time() - chan->timestamp_created), + m); + tor_free(m); + + if (!chan->cmux) { + /* This warning should be impossible. */ + log_warn(LD_BUG, " This channel somehow has no cmux on it!"); + return 0; } + + /* analysis so far on 12184 suggests that we're running out of circuit + IDs because it looks like we have too many pending destroy + cells. Let's see how many we really have pending. + */ + queued_destroys = circuitmux_count_queued_destroy_cells(chan, + chan->cmux); + + log_warn(LD_CIRC, " Circuitmux on this channel has %u circuits, " + "of which %u are active. It says it has "I64_FORMAT + " destroy cells queued.", + circuitmux_num_circuits(chan->cmux), + circuitmux_num_active_circuits(chan->cmux), + I64_PRINTF_ARG(queued_destroys)); + + /* Change this into "if (1)" in order to get more information about + * possible failure modes here. You'll need to know how to use gdb with + * Tor: this will make Tor exit with an assertion failure if the cmux is + * corrupt. */ + if (0) + circuitmux_assert_okay(chan->cmux); + + channel_dump_statistics(chan, LOG_WARN); + return 0; } + + do { + crypto_rand((char*) &test_circ_id, sizeof(test_circ_id)); + test_circ_id &= mask; + } while (test_circ_id == 0); + test_circ_id |= high_bit; - } while (circuit_id_in_use_on_channel(test_circ_id, chan)); + + in_use = circuit_id_in_use_on_channel(test_circ_id, chan); + if (in_use == 1) + ++n_with_circ; + else if (in_use == 2) { + time_t since_when; + ++n_pending_destroy; + since_when = + circuit_id_when_marked_unusable_on_channel(test_circ_id, chan); + if (since_when) { + time_t waiting = approx_time() - since_when; + pending_destroy_time_total += waiting; + if (waiting > pending_destroy_time_max) + pending_destroy_time_max = waiting; + } else { + ++n_weird_pending_destroy; + } + } + } while (in_use); return test_circ_id; } @@ -294,14 +369,13 @@ circuit_rep_hist_note_result(origin_circuit_t *circ) } while (hop!=circ->cpath); } -#ifdef CURVE25519_ENABLED /** Return 1 iff at least one node in circ's cpath supports ntor. */ static int circuit_cpath_supports_ntor(const origin_circuit_t *circ) { - crypt_path_t *head = circ->cpath, *cpath = circ->cpath; + crypt_path_t *head, *cpath; - cpath = head; + cpath = head = circ->cpath; do { if (cpath->extend_info && !tor_mem_is_zero( @@ -314,9 +388,6 @@ circuit_cpath_supports_ntor(const origin_circuit_t *circ) return 0; } -#else -#define circuit_cpath_supports_ntor(circ) 0 -#endif /** Pick all the entries in our cpath. Stop and return 0 when we're * happy, or return -1 if an error occurs. */ @@ -324,11 +395,7 @@ static int onion_populate_cpath(origin_circuit_t *circ) { int n_tries = 0; -#ifdef CURVE25519_ENABLED const int using_ntor = circuits_can_use_ntor(); -#else - const int using_ntor = 0; -#endif #define MAX_POPULATE_ATTEMPTS 32 @@ -428,11 +495,26 @@ circuit_handle_first_hop(origin_circuit_t *circ) int err_reason = 0; const char *msg = NULL; int should_launch = 0; + const or_options_t *options = get_options(); firsthop = onion_next_hop_in_cpath(circ->cpath); tor_assert(firsthop); tor_assert(firsthop->extend_info); + /* Some bridges are on private addresses. Others pass a dummy private + * address to the pluggable transport, which ignores it. + * Deny the connection if: + * - the address is internal, and + * - we're not connecting to a configured bridge, and + * - we're not configured to allow extends to private addresses. */ + if (tor_addr_is_internal(&firsthop->extend_info->addr, 0) && + !extend_info_is_a_configured_bridge(firsthop->extend_info) && + !options->ExtendAllowPrivateAddresses) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Client asked me to connect directly to a private address"); + return -END_CIRC_REASON_TORPROTOCOL; + } + /* now see if we're already connected to the first OR in 'route' */ log_debug(LD_CIRC,"Looking for firsthop '%s'", fmt_addrport(&firsthop->extend_info->addr, @@ -475,6 +557,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) log_debug(LD_CIRC,"Conn open. Delivering first onion skin."); if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) { log_info(LD_CIRC,"circuit_send_next_onion_skin failed."); + circ->base_.n_chan = NULL; return err_reason; } } @@ -485,9 +568,13 @@ circuit_handle_first_hop(origin_circuit_t *circ) * open and get them to send their create cells forward. * * Status is 1 if connect succeeded, or 0 if connect failed. + * + * Close_origin_circuits is 1 if we should close all the origin circuits + * through this channel, or 0 otherwise. (This happens when we want to retry + * an older guard.) */ void -circuit_n_chan_done(channel_t *chan, int status) +circuit_n_chan_done(channel_t *chan, int status, int close_origin_circuits) { smartlist_t *pending_circs; int err_reason = 0; @@ -525,6 +612,11 @@ circuit_n_chan_done(channel_t *chan, int status) circuit_mark_for_close(circ, END_CIRC_REASON_CHANNEL_CLOSED); continue; } + if (close_origin_circuits && CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC,"Channel deprecated for origin circs; closing circ."); + circuit_mark_for_close(circ, END_CIRC_REASON_CHANNEL_CLOSED); + continue; + } log_debug(LD_CIRC, "Found circ, sending create cell."); /* circuit_deliver_create_cell will set n_circ_id and add us to * chan_circuid_circuit_map, so we don't need to call @@ -583,19 +675,21 @@ circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, id = get_unique_circ_id_by_chan(circ->n_chan); if (!id) { - log_warn(LD_CIRC,"failed to get unique circID."); - return -1; + static ratelim_t circid_warning_limit = RATELIM_INIT(9600); + log_fn_ratelim(&circid_warning_limit, LOG_WARN, LD_CIRC, + "failed to get unique circID."); + goto error; } - log_debug(LD_CIRC,"Chosen circID %u.", (unsigned)id); - circuit_set_n_circid_chan(circ, id, circ->n_chan); memset(&cell, 0, sizeof(cell_t)); r = relayed ? create_cell_format_relayed(&cell, create_cell) : create_cell_format(&cell, create_cell); if (r < 0) { log_warn(LD_CIRC,"Couldn't format create cell"); - return -1; + goto error; } + log_debug(LD_CIRC,"Chosen circID %u.", (unsigned)id); + circuit_set_n_circid_chan(circ, id, circ->n_chan); cell.circ_id = circ->n_circ_id; append_cell_to_circuit_queue(circ, circ->n_chan, &cell, @@ -604,7 +698,7 @@ circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, if (CIRCUIT_IS_ORIGIN(circ)) { /* Update began timestamp for circuits starting their first hop */ if (TO_ORIGIN_CIRCUIT(circ)->cpath->state == CPATH_STATE_CLOSED) { - if (circ->n_chan->state != CHANNEL_STATE_OPEN) { + if (!CHANNEL_IS_OPEN(circ->n_chan)) { log_warn(LD_CIRC, "Got first hop for a circuit without an opened channel. " "State: %s.", channel_state_to_string(circ->n_chan->state)); @@ -619,6 +713,9 @@ circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, } return 0; + error: + circ->n_chan = NULL; + return -1; } /** We've decided to start our reachability testing. If all @@ -628,33 +725,36 @@ int inform_testing_reachability(void) { char dirbuf[128]; + char *address; const routerinfo_t *me = router_get_my_routerinfo(); if (!me) return 0; + address = tor_dup_ip(me->addr); control_event_server_status(LOG_NOTICE, "CHECKING_REACHABILITY ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); if (me->dir_port) { tor_snprintf(dirbuf, sizeof(dirbuf), " and DirPort %s:%d", - me->address, me->dir_port); + address, me->dir_port); control_event_server_status(LOG_NOTICE, "CHECKING_REACHABILITY DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); } log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", - me->address, me->or_port, + address, me->or_port, me->dir_port ? dirbuf : "", me->dir_port ? "are" : "is", TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT/60); + tor_free(address); return 1; } /** Return true iff we should send a create_fast cell to start building a given * circuit */ -static INLINE int +static inline int should_use_create_fast_for_circuit(origin_circuit_t *circ) { const or_options_t *options = get_options(); @@ -689,7 +789,6 @@ circuit_timeout_want_to_count_circ(origin_circuit_t *circ) && circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN; } -#ifdef CURVE25519_ENABLED /** Return true if the ntor handshake is enabled in the configuration, or if * it's been set to "auto" in the configuration and it's enabled in the * consensus. */ @@ -701,7 +800,6 @@ circuits_can_use_ntor(void) return options->UseNTorHandshake; return networkstatus_get_param(NULL, "UseNTorHandshake", 0, 0, 1); } -#endif /** Decide whether to use a TAP or ntor handshake for connecting to <b>ei</b> * directly, and set *<b>cell_type_out</b> and *<b>handshake_type_out</b> @@ -711,7 +809,6 @@ circuit_pick_create_handshake(uint8_t *cell_type_out, uint16_t *handshake_type_out, const extend_info_t *ei) { -#ifdef CURVE25519_ENABLED if (!tor_mem_is_zero((const char*)ei->curve25519_onion_key.public_key, CURVE25519_PUBKEY_LEN) && circuits_can_use_ntor()) { @@ -719,9 +816,6 @@ circuit_pick_create_handshake(uint8_t *cell_type_out, *handshake_type_out = ONION_HANDSHAKE_TYPE_NTOR; return; } -#else - (void) ei; -#endif *cell_type_out = CELL_CREATE; *handshake_type_out = ONION_HANDSHAKE_TYPE_TAP; @@ -844,20 +938,24 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) * it off at, we probably had a suspend event along this codepath, * and we should discard the value. */ - if (timediff < 0 || timediff > 2*circ_times.close_ms+1000) { + if (timediff < 0 || + timediff > 2*get_circuit_build_close_time_ms()+1000) { log_notice(LD_CIRC, "Strange value for circuit build time: %ldmsec. " "Assuming clock jump. Purpose %d (%s)", timediff, circ->base_.purpose, circuit_purpose_to_string(circ->base_.purpose)); } else if (!circuit_build_times_disabled()) { /* Only count circuit times if the network is live */ - if (circuit_build_times_network_check_live(&circ_times)) { - circuit_build_times_add_time(&circ_times, (build_time_t)timediff); - circuit_build_times_set_timeout(&circ_times); + if (circuit_build_times_network_check_live( + get_circuit_build_times())) { + circuit_build_times_add_time(get_circuit_build_times_mutable(), + (build_time_t)timediff); + circuit_build_times_set_timeout(get_circuit_build_times_mutable()); } if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - circuit_build_times_network_circ_success(&circ_times); + circuit_build_times_network_circ_success( + get_circuit_build_times_mutable()); } } } @@ -872,17 +970,21 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) circuit_rep_hist_note_result(circ); circuit_has_opened(circ); /* do other actions as necessary */ - if (!can_complete_circuit && !circ->build_state->onehop_tunnel) { + if (!have_completed_a_circuit() && !circ->build_state->onehop_tunnel) { const or_options_t *options = get_options(); - can_complete_circuit=1; + note_that_we_completed_a_circuit(); /* FFFF Log a count of known routers here */ log_notice(LD_GENERAL, "Tor has successfully opened a circuit. " "Looks like client functionality is working."); - control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0); + if (control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0) == 0) { + log_notice(LD_GENERAL, + "Tor has successfully opened a circuit. " + "Looks like client functionality is working."); + } control_event_client_status(LOG_NOTICE, "CIRCUIT_ESTABLISHED"); clear_broken_connection_map(1); - if (server_mode(options) && !check_whether_orport_reachable()) { + if (server_mode(options) && !check_whether_orport_reachable(options)) { inform_testing_reachability(); consider_testing_reachability(1, 1); } @@ -962,11 +1064,16 @@ circuit_note_clock_jumped(int seconds_elapsed) seconds_elapsed >=0 ? "forward" : "backward"); control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%d", seconds_elapsed); - can_complete_circuit=0; /* so it'll log when it works again */ + /* so we log when it works again */ + note_that_we_maybe_cant_complete_circuits(); control_event_client_status(severity, "CIRCUIT_NOT_ESTABLISHED REASON=%s", "CLOCK_JUMPED"); circuit_mark_all_unused_circs(); circuit_mark_all_dirty_circs_as_unusable(); + if (seconds_elapsed < 0) { + /* Restart all the timers in case we jumped a long way into the past. */ + reset_all_main_loop_timers(); + } } /** Take the 'extend' <b>cell</b>, pull out addr/port plus the onion @@ -1152,1516 +1259,6 @@ circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, return 0; } -/** The minimum number of circuit attempts before we start - * thinking about warning about path bias and dropping guards */ -static int -pathbias_get_min_circs(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_MIN_CIRC 150 - if (options->PathBiasCircThreshold >= 5) - return options->PathBiasCircThreshold; - else - return networkstatus_get_param(NULL, "pb_mincircs", - DFLT_PATH_BIAS_MIN_CIRC, - 5, INT32_MAX); -} - -/** The circuit success rate below which we issue a notice */ -static double -pathbias_get_notice_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_NOTICE_PCT 70 - if (options->PathBiasNoticeRate >= 0.0) - return options->PathBiasNoticeRate; - else - return networkstatus_get_param(NULL, "pb_noticepct", - DFLT_PATH_BIAS_NOTICE_PCT, 0, 100)/100.0; -} - -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ -/** The circuit success rate below which we issue a warn */ -static double -pathbias_get_warn_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_WARN_PCT 50 - if (options->PathBiasWarnRate >= 0.0) - return options->PathBiasWarnRate; - else - return networkstatus_get_param(NULL, "pb_warnpct", - DFLT_PATH_BIAS_WARN_PCT, 0, 100)/100.0; -} - -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ -/** - * The extreme rate is the rate at which we would drop the guard, - * if pb_dropguard is also set. Otherwise we just warn. - */ -double -pathbias_get_extreme_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_EXTREME_PCT 30 - if (options->PathBiasExtremeRate >= 0.0) - return options->PathBiasExtremeRate; - else - return networkstatus_get_param(NULL, "pb_extremepct", - DFLT_PATH_BIAS_EXTREME_PCT, 0, 100)/100.0; -} - -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ -/** - * If 1, we actually disable use of guards that fall below - * the extreme_pct. - */ -int -pathbias_get_dropguards(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_DROP_GUARDS 0 - if (options->PathBiasDropGuards >= 0) - return options->PathBiasDropGuards; - else - return networkstatus_get_param(NULL, "pb_dropguards", - DFLT_PATH_BIAS_DROP_GUARDS, 0, 1); -} - -/** - * This is the number of circuits at which we scale our - * counts by mult_factor/scale_factor. Note, this count is - * not exact, as we only perform the scaling in the event - * of no integer truncation. - */ -static int -pathbias_get_scale_threshold(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_SCALE_THRESHOLD 300 - if (options->PathBiasScaleThreshold >= 10) - return options->PathBiasScaleThreshold; - else - return networkstatus_get_param(NULL, "pb_scalecircs", - DFLT_PATH_BIAS_SCALE_THRESHOLD, 10, - INT32_MAX); -} - -/** - * Compute the path bias scaling ratio from the consensus - * parameters pb_multfactor/pb_scalefactor. - * - * Returns a value in (0, 1.0] which we multiply our pathbias - * counts with to scale them down. - */ -static double -pathbias_get_scale_ratio(const or_options_t *options) -{ - /* - * The scale factor is the denominator for our scaling - * of circuit counts for our path bias window. - * - * Note that our use of doubles for the path bias state - * file means that powers of 2 work best here. - */ - int denominator = networkstatus_get_param(NULL, "pb_scalefactor", - 2, 2, INT32_MAX); - (void) options; - /** - * The mult factor is the numerator for our scaling - * of circuit counts for our path bias window. It - * allows us to scale by fractions. - */ - return networkstatus_get_param(NULL, "pb_multfactor", - 1, 1, denominator)/((double)denominator); -} - -/** The minimum number of circuit usage attempts before we start - * thinking about warning about path use bias and dropping guards */ -static int -pathbias_get_min_use(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_MIN_USE 20 - if (options->PathBiasUseThreshold >= 3) - return options->PathBiasUseThreshold; - else - return networkstatus_get_param(NULL, "pb_minuse", - DFLT_PATH_BIAS_MIN_USE, - 3, INT32_MAX); -} - -/** The circuit use success rate below which we issue a notice */ -static double -pathbias_get_notice_use_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_NOTICE_USE_PCT 80 - if (options->PathBiasNoticeUseRate >= 0.0) - return options->PathBiasNoticeUseRate; - else - return networkstatus_get_param(NULL, "pb_noticeusepct", - DFLT_PATH_BIAS_NOTICE_USE_PCT, - 0, 100)/100.0; -} - -/** - * The extreme use rate is the rate at which we would drop the guard, - * if pb_dropguard is also set. Otherwise we just warn. - */ -double -pathbias_get_extreme_use_rate(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_EXTREME_USE_PCT 60 - if (options->PathBiasExtremeUseRate >= 0.0) - return options->PathBiasExtremeUseRate; - else - return networkstatus_get_param(NULL, "pb_extremeusepct", - DFLT_PATH_BIAS_EXTREME_USE_PCT, - 0, 100)/100.0; -} - -/** - * This is the number of circuits at which we scale our - * use counts by mult_factor/scale_factor. Note, this count is - * not exact, as we only perform the scaling in the event - * of no integer truncation. - */ -static int -pathbias_get_scale_use_threshold(const or_options_t *options) -{ -#define DFLT_PATH_BIAS_SCALE_USE_THRESHOLD 100 - if (options->PathBiasScaleUseThreshold >= 10) - return options->PathBiasScaleUseThreshold; - else - return networkstatus_get_param(NULL, "pb_scaleuse", - DFLT_PATH_BIAS_SCALE_USE_THRESHOLD, - 10, INT32_MAX); -} - -/** - * Convert a Guard's path state to string. - */ -const char * -pathbias_state_to_string(path_state_t state) -{ - switch (state) { - case PATH_STATE_NEW_CIRC: - return "new"; - case PATH_STATE_BUILD_ATTEMPTED: - return "build attempted"; - case PATH_STATE_BUILD_SUCCEEDED: - return "build succeeded"; - case PATH_STATE_USE_ATTEMPTED: - return "use attempted"; - case PATH_STATE_USE_SUCCEEDED: - return "use succeeded"; - case PATH_STATE_USE_FAILED: - return "use failed"; - case PATH_STATE_ALREADY_COUNTED: - return "already counted"; - } - - return "unknown"; -} - -/** - * This function decides if a circuit has progressed far enough to count - * as a circuit "attempt". As long as end-to-end tagging is possible, - * we assume the adversary will use it over hop-to-hop failure. Therefore, - * we only need to account bias for the last hop. This should make us - * much more resilient to ambient circuit failure, and also make that - * failure easier to measure (we only need to measure Exit failure rates). - */ -static int -pathbias_is_new_circ_attempt(origin_circuit_t *circ) -{ -#define N2N_TAGGING_IS_POSSIBLE -#ifdef N2N_TAGGING_IS_POSSIBLE - /* cpath is a circular list. We want circs with more than one hop, - * and the second hop must be waiting for keys still (it's just - * about to get them). */ - return circ->cpath && - circ->cpath->next != circ->cpath && - circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS; -#else - /* If tagging attacks are no longer possible, we probably want to - * count bias from the first hop. However, one could argue that - * timing-based tagging is still more useful than per-hop failure. - * In which case, we'd never want to use this. - */ - return circ->cpath && - circ->cpath->state == CPATH_STATE_AWAITING_KEYS; -#endif -} - -/** - * Decide if the path bias code should count a circuit. - * - * @returns 1 if we should count it, 0 otherwise. - */ -static int -pathbias_should_count(origin_circuit_t *circ) -{ -#define PATHBIAS_COUNT_INTERVAL (600) - static ratelim_t count_limit = - RATELIM_INIT(PATHBIAS_COUNT_INTERVAL); - char *rate_msg = NULL; - - /* We can't do path bias accounting without entry guards. - * Testing and controller circuits also have no guards. - * - * We also don't count server-side rends, because their - * endpoint could be chosen maliciously. - * Similarly, we can't count client-side intro attempts, - * because clients can be manipulated into connecting to - * malicious intro points. */ - if (get_options()->UseEntryGuards == 0 || - circ->base_.purpose == CIRCUIT_PURPOSE_TESTING || - circ->base_.purpose == CIRCUIT_PURPOSE_CONTROLLER || - circ->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND || - circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED || - (circ->base_.purpose >= CIRCUIT_PURPOSE_C_INTRODUCING && - circ->base_.purpose <= CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)) { - - /* Check to see if the shouldcount result has changed due to a - * unexpected purpose change that would affect our results. - * - * The reason we check the path state too here is because for the - * cannibalized versions of these purposes, we count them as successful - * before their purpose change. - */ - if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED - && circ->path_state != PATH_STATE_ALREADY_COUNTED) { - log_info(LD_BUG, - "Circuit %d is now being ignored despite being counted " - "in the past. Purpose is %s, path state is %s", - circ->global_identifier, - circuit_purpose_to_string(circ->base_.purpose), - pathbias_state_to_string(circ->path_state)); - } - circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; - return 0; - } - - /* Completely ignore one hop circuits */ - if (circ->build_state->onehop_tunnel || - circ->build_state->desired_path_len == 1) { - /* Check for inconsistency */ - if (circ->build_state->desired_path_len != 1 || - !circ->build_state->onehop_tunnel) { - if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) { - log_info(LD_BUG, - "One-hop circuit has length %d. Path state is %s. " - "Circuit is a %s currently %s.%s", - circ->build_state->desired_path_len, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - tor_fragile_assert(); - } - - /* Check to see if the shouldcount result has changed due to a - * unexpected change that would affect our results */ - if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_COUNTED) { - log_info(LD_BUG, - "One-hop circuit %d is now being ignored despite being counted " - "in the past. Purpose is %s, path state is %s", - circ->global_identifier, - circuit_purpose_to_string(circ->base_.purpose), - pathbias_state_to_string(circ->path_state)); - } - circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; - return 0; - } - - /* Check to see if the shouldcount result has changed due to a - * unexpected purpose change that would affect our results */ - if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) { - log_info(LD_BUG, - "Circuit %d is now being counted despite being ignored " - "in the past. Purpose is %s, path state is %s", - circ->global_identifier, - circuit_purpose_to_string(circ->base_.purpose), - pathbias_state_to_string(circ->path_state)); - } - circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED; - - return 1; -} - -/** - * Check our circuit state to see if this is a successful circuit attempt. - * If so, record it in the current guard's path bias circ_attempt count. - * - * Also check for several potential error cases for bug #6475. - */ -static int -pathbias_count_build_attempt(origin_circuit_t *circ) -{ -#define CIRC_ATTEMPT_NOTICE_INTERVAL (600) - static ratelim_t circ_attempt_notice_limit = - RATELIM_INIT(CIRC_ATTEMPT_NOTICE_INTERVAL); - char *rate_msg = NULL; - - if (!pathbias_should_count(circ)) { - return 0; - } - - if (pathbias_is_new_circ_attempt(circ)) { - /* Help track down the real cause of bug #6475: */ - if (circ->has_opened && circ->path_state != PATH_STATE_BUILD_ATTEMPTED) { - if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Opened circuit is in strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - - /* Don't re-count cannibalized circs.. */ - if (!circ->has_opened) { - entry_guard_t *guard = NULL; - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } else if (circ->base_.n_chan) { - guard = - entry_guard_get_by_id_digest(circ->base_.n_chan->identity_digest); - } - - if (guard) { - if (circ->path_state == PATH_STATE_NEW_CIRC) { - circ->path_state = PATH_STATE_BUILD_ATTEMPTED; - - if (entry_guard_inc_circ_attempt_count(guard) < 0) { - /* Bogus guard; we already warned. */ - return -END_CIRC_REASON_TORPROTOCOL; - } - } else { - if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Unopened circuit has strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } else { - if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit, - approx_time()))) { - log_info(LD_CIRC, - "Unopened circuit has no known guard. " - "Circuit is a %s currently %s.%s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } - } - - return 0; -} - -/** - * Check our circuit state to see if this is a successful circuit - * completion. If so, record it in the current guard's path bias - * success count. - * - * Also check for several potential error cases for bug #6475. - */ -static void -pathbias_count_build_success(origin_circuit_t *circ) -{ -#define SUCCESS_NOTICE_INTERVAL (600) - static ratelim_t success_notice_limit = - RATELIM_INIT(SUCCESS_NOTICE_INTERVAL); - char *rate_msg = NULL; - entry_guard_t *guard = NULL; - - if (!pathbias_should_count(circ)) { - return; - } - - /* Don't count cannibalized/reused circs for path bias - * "build" success, since they get counted under "use" success. */ - if (!circ->has_opened) { - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - if (circ->path_state == PATH_STATE_BUILD_ATTEMPTED) { - circ->path_state = PATH_STATE_BUILD_SUCCEEDED; - guard->circ_successes++; - entry_guards_changed(); - - log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } else { - if ((rate_msg = rate_limit_log(&success_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Succeeded circuit is in strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - - if (guard->circ_attempts < guard->circ_successes) { - log_notice(LD_BUG, "Unexpectedly high successes counts (%f/%f) " - "for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - if ((rate_msg = rate_limit_log(&success_notice_limit, - approx_time()))) { - log_info(LD_CIRC, - "Completed circuit has no known guard. " - "Circuit is a %s currently %s.%s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } else { - if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { - if ((rate_msg = rate_limit_log(&success_notice_limit, - approx_time()))) { - log_info(LD_BUG, - "Opened circuit is in strange path state %s. " - "Circuit is a %s currently %s.%s", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state), - rate_msg); - tor_free(rate_msg); - } - } - } -} - -/** - * Record an attempt to use a circuit. Changes the circuit's - * path state and update its guard's usage counter. - * - * Used for path bias usage accounting. - */ -void -pathbias_count_use_attempt(origin_circuit_t *circ) -{ - entry_guard_t *guard; - - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) { - log_notice(LD_BUG, - "Used circuit is in strange path state %s. " - "Circuit is a %s currently %s.", - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } else if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - if (guard) { - pathbias_measure_use_rate(guard); - pathbias_scale_use_rates(guard); - guard->use_attempts++; - entry_guards_changed(); - - log_debug(LD_CIRC, - "Marked circuit %d (%f/%f) as used for guard %s ($%s).", - circ->global_identifier, - guard->use_successes, guard->use_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } - - circ->path_state = PATH_STATE_USE_ATTEMPTED; - } else { - /* Harmless but educational log message */ - log_info(LD_CIRC, - "Used circuit %d is already in path state %s. " - "Circuit is a %s currently %s.", - circ->global_identifier, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } - - return; -} - -/** - * Check the circuit's path state is appropriate and mark it as - * successfully used. Used for path bias usage accounting. - * - * We don't actually increment the guard's counters until - * pathbias_check_close(), because the circuit can still transition - * back to PATH_STATE_USE_ATTEMPTED if a stream fails later (this - * is done so we can probe the circuit for liveness at close). - */ -void -pathbias_mark_use_success(origin_circuit_t *circ) -{ - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->path_state < PATH_STATE_USE_ATTEMPTED) { - log_notice(LD_BUG, - "Used circuit %d is in strange path state %s. " - "Circuit is a %s currently %s.", - circ->global_identifier, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - - pathbias_count_use_attempt(circ); - } - - /* We don't do any accounting at the guard until actual circuit close */ - circ->path_state = PATH_STATE_USE_SUCCEEDED; - - return; -} - -/** - * If a stream ever detatches from a circuit in a retriable way, - * we need to mark this circuit as still needing either another - * successful stream, or in need of a probe. - * - * An adversary could let the first stream request succeed (ie the - * resolve), but then tag and timeout the remainder (via cell - * dropping), forcing them on new circuits. - * - * Rolling back the state will cause us to probe such circuits, which - * should lead to probe failures in the event of such tagging due to - * either unrecognized cells coming in while we wait for the probe, - * or the cipher state getting out of sync in the case of dropped cells. - */ -void -pathbias_mark_use_rollback(origin_circuit_t *circ) -{ - if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { - log_info(LD_CIRC, - "Rolling back pathbias use state to 'attempted' for detached " - "circuit %d", circ->global_identifier); - circ->path_state = PATH_STATE_USE_ATTEMPTED; - } -} - -/** - * Actually count a circuit success towards a guard's usage counters - * if the path state is appropriate. - */ -static void -pathbias_count_use_success(origin_circuit_t *circ) -{ - entry_guard_t *guard; - - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->path_state != PATH_STATE_USE_SUCCEEDED) { - log_notice(LD_BUG, - "Successfully used circuit %d is in strange path state %s. " - "Circuit is a %s currently %s.", - circ->global_identifier, - pathbias_state_to_string(circ->path_state), - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } else { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - if (guard) { - guard->use_successes++; - entry_guards_changed(); - - if (guard->use_attempts < guard->use_successes) { - log_notice(LD_BUG, "Unexpectedly high use successes counts (%f/%f) " - "for guard %s=%s", - guard->use_successes, guard->use_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - } - - log_debug(LD_CIRC, - "Marked circuit %d (%f/%f) as used successfully for guard " - "%s ($%s).", - circ->global_identifier, guard->use_successes, - guard->use_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - } - } - - return; -} - -/** - * Send a probe down a circuit that the client attempted to use, - * but for which the stream timed out/failed. The probe is a - * RELAY_BEGIN cell with a 0.a.b.c destination address, which - * the exit will reject and reply back, echoing that address. - * - * The reason for such probes is because it is possible to bias - * a user's paths simply by causing timeouts, and these timeouts - * are not possible to differentiate from unresponsive servers. - * - * The probe is sent at the end of the circuit lifetime for two - * reasons: to prevent cryptographic taggers from being able to - * drop cells to cause timeouts, and to prevent easy recognition - * of probes before any real client traffic happens. - * - * Returns -1 if we couldn't probe, 0 otherwise. - */ -static int -pathbias_send_usable_probe(circuit_t *circ) -{ - /* Based on connection_ap_handshake_send_begin() */ - char payload[CELL_PAYLOAD_SIZE]; - int payload_len; - origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); - crypt_path_t *cpath_layer = NULL; - char *probe_nonce = NULL; - - tor_assert(ocirc); - - cpath_layer = ocirc->cpath->prev; - - if (cpath_layer->state != CPATH_STATE_OPEN) { - /* This can happen for cannibalized circuits. Their - * last hop isn't yet open */ - log_info(LD_CIRC, - "Got pathbias probe request for unopened circuit %d. " - "Opened %d, len %d", ocirc->global_identifier, - ocirc->has_opened, ocirc->build_state->desired_path_len); - return -1; - } - - /* We already went down this road. */ - if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING && - ocirc->pathbias_probe_id) { - log_info(LD_CIRC, - "Got pathbias probe request for circuit %d with " - "outstanding probe", ocirc->global_identifier); - return -1; - } - - /* Can't probe if the channel isn't open */ - if (circ->n_chan == NULL || - (circ->n_chan->state != CHANNEL_STATE_OPEN - && circ->n_chan->state != CHANNEL_STATE_MAINT)) { - log_info(LD_CIRC, - "Skipping pathbias probe for circuit %d: Channel is not open.", - ocirc->global_identifier); - return -1; - } - - circuit_change_purpose(circ, CIRCUIT_PURPOSE_PATH_BIAS_TESTING); - - /* Update timestamp for when circuit_expire_building() should kill us */ - tor_gettimeofday(&circ->timestamp_began); - - /* Generate a random address for the nonce */ - crypto_rand((char*)ô->pathbias_probe_nonce, - sizeof(ocirc->pathbias_probe_nonce)); - ocirc->pathbias_probe_nonce &= 0x00ffffff; - probe_nonce = tor_dup_ip(ocirc->pathbias_probe_nonce); - - tor_snprintf(payload,RELAY_PAYLOAD_SIZE, "%s:25", probe_nonce); - payload_len = (int)strlen(payload)+1; - - // XXX: need this? Can we assume ipv4 will always be supported? - // If not, how do we tell? - //if (payload_len <= RELAY_PAYLOAD_SIZE - 4 && edge_conn->begincell_flags) { - // set_uint32(payload + payload_len, htonl(edge_conn->begincell_flags)); - // payload_len += 4; - //} - - /* Generate+Store stream id, make sure it's non-zero */ - ocirc->pathbias_probe_id = get_unique_stream_id_by_circ(ocirc); - - if (ocirc->pathbias_probe_id==0) { - log_warn(LD_CIRC, - "Ran out of stream IDs on circuit %u during " - "pathbias probe attempt.", ocirc->global_identifier); - tor_free(probe_nonce); - return -1; - } - - log_info(LD_CIRC, - "Sending pathbias testing cell to %s:25 on stream %d for circ %d.", - probe_nonce, ocirc->pathbias_probe_id, ocirc->global_identifier); - tor_free(probe_nonce); - - /* Send a test relay cell */ - if (relay_send_command_from_edge(ocirc->pathbias_probe_id, circ, - RELAY_COMMAND_BEGIN, payload, - payload_len, cpath_layer) < 0) { - log_notice(LD_CIRC, - "Failed to send pathbias probe cell on circuit %d.", - ocirc->global_identifier); - return -1; - } - - /* Mark it freshly dirty so it doesn't get expired in the meantime */ - circ->timestamp_dirty = time(NULL); - - return 0; -} - -/** - * Check the response to a pathbias probe, to ensure the - * cell is recognized and the nonce and other probe - * characteristics are as expected. - * - * If the response is valid, return 0. Otherwise return < 0. - */ -int -pathbias_check_probe_response(circuit_t *circ, const cell_t *cell) -{ - /* Based on connection_edge_process_relay_cell() */ - relay_header_t rh; - int reason; - uint32_t ipv4_host; - origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); - - tor_assert(cell); - tor_assert(ocirc); - tor_assert(circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING); - - relay_header_unpack(&rh, cell->payload); - - reason = rh.length > 0 ? - get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC; - - if (rh.command == RELAY_COMMAND_END && - reason == END_STREAM_REASON_EXITPOLICY && - ocirc->pathbias_probe_id == rh.stream_id) { - - /* Check length+extract host: It is in network order after the reason code. - * See connection_edge_end(). */ - if (rh.length < 9) { /* reason+ipv4+dns_ttl */ - log_notice(LD_PROTOCOL, - "Short path bias probe response length field (%d).", rh.length); - return - END_CIRC_REASON_TORPROTOCOL; - } - - ipv4_host = ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+1)); - - /* Check nonce */ - if (ipv4_host == ocirc->pathbias_probe_nonce) { - pathbias_mark_use_success(ocirc); - circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); - log_info(LD_CIRC, - "Got valid path bias probe back for circ %d, stream %d.", - ocirc->global_identifier, ocirc->pathbias_probe_id); - return 0; - } else { - log_notice(LD_CIRC, - "Got strange probe value 0x%x vs 0x%x back for circ %d, " - "stream %d.", ipv4_host, ocirc->pathbias_probe_nonce, - ocirc->global_identifier, ocirc->pathbias_probe_id); - return -1; - } - } - log_info(LD_CIRC, - "Got another cell back back on pathbias probe circuit %d: " - "Command: %d, Reason: %d, Stream-id: %d", - ocirc->global_identifier, rh.command, reason, rh.stream_id); - return -1; -} - -/** - * Check if a circuit was used and/or closed successfully. - * - * If we attempted to use the circuit to carry a stream but failed - * for whatever reason, or if the circuit mysteriously died before - * we could attach any streams, record these two cases. - * - * If we *have* successfully used the circuit, or it appears to - * have been closed by us locally, count it as a success. - * - * Returns 0 if we're done making decisions with the circ, - * or -1 if we want to probe it first. - */ -int -pathbias_check_close(origin_circuit_t *ocirc, int reason) -{ - circuit_t *circ = ô->base_; - - if (!pathbias_should_count(ocirc)) { - return 0; - } - - switch (ocirc->path_state) { - /* If the circuit was closed after building, but before use, we need - * to ensure we were the ones who tried to close it (and not a remote - * actor). */ - case PATH_STATE_BUILD_SUCCEEDED: - if (reason & END_CIRC_REASON_FLAG_REMOTE) { - /* Remote circ close reasons on an unused circuit all could be bias */ - log_info(LD_CIRC, - "Circuit %d remote-closed without successful use for reason %d. " - "Circuit purpose %d currently %d,%s. Len %d.", - ocirc->global_identifier, - reason, circ->purpose, ocirc->has_opened, - circuit_state_to_string(circ->state), - ocirc->build_state->desired_path_len); - pathbias_count_collapse(ocirc); - } else if ((reason & ~END_CIRC_REASON_FLAG_REMOTE) - == END_CIRC_REASON_CHANNEL_CLOSED && - circ->n_chan && - circ->n_chan->reason_for_closing - != CHANNEL_CLOSE_REQUESTED) { - /* If we didn't close the channel ourselves, it could be bias */ - /* XXX: Only count bias if the network is live? - * What about clock jumps/suspends? */ - log_info(LD_CIRC, - "Circuit %d's channel closed without successful use for reason " - "%d, channel reason %d. Circuit purpose %d currently %d,%s. Len " - "%d.", ocirc->global_identifier, - reason, circ->n_chan->reason_for_closing, - circ->purpose, ocirc->has_opened, - circuit_state_to_string(circ->state), - ocirc->build_state->desired_path_len); - pathbias_count_collapse(ocirc); - } else { - pathbias_count_successful_close(ocirc); - } - break; - - /* If we tried to use a circuit but failed, we should probe it to ensure - * it has not been tampered with. */ - case PATH_STATE_USE_ATTEMPTED: - /* XXX: Only probe and/or count failure if the network is live? - * What about clock jumps/suspends? */ - if (pathbias_send_usable_probe(circ) == 0) - return -1; - else - pathbias_count_use_failed(ocirc); - - /* Any circuit where there were attempted streams but no successful - * streams could be bias */ - log_info(LD_CIRC, - "Circuit %d closed without successful use for reason %d. " - "Circuit purpose %d currently %d,%s. Len %d.", - ocirc->global_identifier, - reason, circ->purpose, ocirc->has_opened, - circuit_state_to_string(circ->state), - ocirc->build_state->desired_path_len); - break; - - case PATH_STATE_USE_SUCCEEDED: - pathbias_count_successful_close(ocirc); - pathbias_count_use_success(ocirc); - break; - - case PATH_STATE_USE_FAILED: - pathbias_count_use_failed(ocirc); - break; - - case PATH_STATE_NEW_CIRC: - case PATH_STATE_BUILD_ATTEMPTED: - case PATH_STATE_ALREADY_COUNTED: - default: - // Other states are uninteresting. No stats to count. - break; - } - - ocirc->path_state = PATH_STATE_ALREADY_COUNTED; - - return 0; -} - -/** - * Count a successfully closed circuit. - */ -static void -pathbias_count_successful_close(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - /* In the long run: circuit_success ~= successful_circuit_close + - * circ_failure + stream_failure */ - guard->successful_circuits_closed++; - entry_guards_changed(); - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - log_info(LD_CIRC, - "Successfully closed circuit has no known guard. " - "Circuit is a %s currently %s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } -} - -/** - * Count a circuit that fails after it is built, but before it can - * carry any traffic. - * - * This is needed because there are ways to destroy a - * circuit after it has successfully completed. Right now, this is - * used for purely informational/debugging purposes. - */ -static void -pathbias_count_collapse(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - guard->collapsed_circuits++; - entry_guards_changed(); - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - log_info(LD_CIRC, - "Destroyed circuit has no known guard. " - "Circuit is a %s currently %s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } -} - -/** - * Count a known failed circuit (because we could not probe it). - * - * This counter is informational. - */ -static void -pathbias_count_use_failed(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - if (!pathbias_should_count(circ)) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - guard->unusable_circuits++; - entry_guards_changed(); - } else if (circ->base_.purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { - /* In rare cases, CIRCUIT_PURPOSE_TESTING can get converted to - * CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT and have no guards here. - * No need to log that case. */ - /* XXX note cut-and-paste code in this function compared to nearby - * functions. Would be nice to refactor. -RD */ - log_info(LD_CIRC, - "Stream-failing circuit has no known guard. " - "Circuit is a %s currently %s", - circuit_purpose_to_string(circ->base_.purpose), - circuit_state_to_string(circ->base_.state)); - } -} - -/** - * Count timeouts for path bias log messages. - * - * These counts are purely informational. - */ -void -pathbias_count_timeout(origin_circuit_t *circ) -{ - entry_guard_t *guard = NULL; - - if (!pathbias_should_count(circ)) { - return; - } - - /* For hidden service circs, they can actually be used - * successfully and then time out later (because - * the other side declines to use them). */ - if (circ->path_state == PATH_STATE_USE_SUCCEEDED) { - return; - } - - if (circ->cpath && circ->cpath->extend_info) { - guard = entry_guard_get_by_id_digest( - circ->cpath->extend_info->identity_digest); - } - - if (guard) { - guard->timeouts++; - entry_guards_changed(); - } -} - -/** - * Helper function to count all of the currently opened circuits - * for a guard that are in a given path state range. The state - * range is inclusive on both ends. - */ -static int -pathbias_count_circs_in_states(entry_guard_t *guard, - path_state_t from, - path_state_t to) -{ - circuit_t *circ; - int open_circuits = 0; - - /* Count currently open circuits. Give them the benefit of the doubt. */ - for (circ = global_circuitlist; circ; circ = circ->next) { - origin_circuit_t *ocirc = NULL; - if (!CIRCUIT_IS_ORIGIN(circ) || /* didn't originate here */ - circ->marked_for_close) /* already counted */ - continue; - - ocirc = TO_ORIGIN_CIRCUIT(circ); - - if (!ocirc->cpath || !ocirc->cpath->extend_info) - continue; - - if (ocirc->path_state >= from && - ocirc->path_state <= to && - pathbias_should_count(ocirc) && - fast_memeq(guard->identity, - ocirc->cpath->extend_info->identity_digest, - DIGEST_LEN)) { - log_debug(LD_CIRC, "Found opened circuit %d in path_state %s", - ocirc->global_identifier, - pathbias_state_to_string(ocirc->path_state)); - open_circuits++; - } - } - - return open_circuits; -} - -/** - * Return the number of circuits counted as successfully closed for - * this guard. - * - * Also add in the currently open circuits to give them the benefit - * of the doubt. - */ -double -pathbias_get_close_success_count(entry_guard_t *guard) -{ - return guard->successful_circuits_closed + - pathbias_count_circs_in_states(guard, - PATH_STATE_BUILD_SUCCEEDED, - PATH_STATE_USE_SUCCEEDED); -} - -/** - * Return the number of circuits counted as successfully used - * this guard. - * - * Also add in the currently open circuits that we are attempting - * to use to give them the benefit of the doubt. - */ -double -pathbias_get_use_success_count(entry_guard_t *guard) -{ - return guard->use_successes + - pathbias_count_circs_in_states(guard, - PATH_STATE_USE_ATTEMPTED, - PATH_STATE_USE_SUCCEEDED); -} - -/** - * Check the path bias use rate against our consensus parameter limits. - * - * Emits a log message if the use success rates are too low. - * - * If pathbias_get_dropguards() is set, we also disable the use of - * very failure prone guards. - */ -static void -pathbias_measure_use_rate(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - if (guard->use_attempts > pathbias_get_min_use(options)) { - /* Note: We rely on the < comparison here to allow us to set a 0 - * rate and disable the feature entirely. If refactoring, don't - * change to <= */ - if (pathbias_get_use_success_count(guard)/guard->use_attempts - < pathbias_get_extreme_use_rate(options)) { - /* Dropping is currently disabled by default. */ - if (pathbias_get_dropguards(options)) { - if (!guard->path_bias_disabled) { - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing to carry an extremely large " - "amount of stream on its circuits. " - "To avoid potential route manipulation attacks, Tor has " - "disabled use of this guard. " - "Use counts are %ld/%ld. Success counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - guard->path_bias_disabled = 1; - guard->bad_since = approx_time(); - entry_guards_changed(); - return; - } - } else if (!guard->path_bias_use_extreme) { - guard->path_bias_use_extreme = 1; - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing to carry an extremely large " - "amount of streams on its circuits. " - "This could indicate a route manipulation attack, network " - "overload, bad local network connectivity, or a bug. " - "Use counts are %ld/%ld. Success counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } else if (pathbias_get_use_success_count(guard)/guard->use_attempts - < pathbias_get_notice_use_rate(options)) { - if (!guard->path_bias_use_noticed) { - guard->path_bias_use_noticed = 1; - log_notice(LD_CIRC, - "Your Guard %s ($%s) is failing to carry more streams on its " - "circuits than usual. " - "Most likely this means the Tor network is overloaded " - "or your network connection is poor. " - "Use counts are %ld/%ld. Success counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } - } -} - -/** - * Check the path bias circuit close status rates against our consensus - * parameter limits. - * - * Emits a log message if the use success rates are too low. - * - * If pathbias_get_dropguards() is set, we also disable the use of - * very failure prone guards. - * - * XXX: This function shares similar log messages and checks to - * pathbias_measure_use_rate(). It may be possible to combine them - * eventually, especially if we can ever remove the need for 3 - * levels of closure warns (if the overall circuit failure rate - * goes down with ntor). One way to do so would be to multiply - * the build rate with the use rate to get an idea of the total - * fraction of the total network paths the user is able to use. - * See ticket #8159. - */ -static void -pathbias_measure_close_rate(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - if (guard->circ_attempts > pathbias_get_min_circs(options)) { - /* Note: We rely on the < comparison here to allow us to set a 0 - * rate and disable the feature entirely. If refactoring, don't - * change to <= */ - if (pathbias_get_close_success_count(guard)/guard->circ_attempts - < pathbias_get_extreme_rate(options)) { - /* Dropping is currently disabled by default. */ - if (pathbias_get_dropguards(options)) { - if (!guard->path_bias_disabled) { - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing an extremely large " - "amount of circuits. " - "To avoid potential route manipulation attacks, Tor has " - "disabled use of this guard. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - guard->path_bias_disabled = 1; - guard->bad_since = approx_time(); - entry_guards_changed(); - return; - } - } else if (!guard->path_bias_extreme) { - guard->path_bias_extreme = 1; - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing an extremely large " - "amount of circuits. " - "This could indicate a route manipulation attack, " - "extreme network overload, or a bug. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts - < pathbias_get_warn_rate(options)) { - if (!guard->path_bias_warned) { - guard->path_bias_warned = 1; - log_warn(LD_CIRC, - "Your Guard %s ($%s) is failing a very large " - "amount of circuits. " - "Most likely this means the Tor network is " - "overloaded, but it could also mean an attack against " - "you or potentially the guard itself. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } else if (pathbias_get_close_success_count(guard)/guard->circ_attempts - < pathbias_get_notice_rate(options)) { - if (!guard->path_bias_noticed) { - guard->path_bias_noticed = 1; - log_notice(LD_CIRC, - "Your Guard %s ($%s) is failing more circuits than " - "usual. " - "Most likely this means the Tor network is overloaded. " - "Success counts are %ld/%ld. Use counts are %ld/%ld. " - "%ld circuits completed, %ld were unusable, %ld collapsed, " - "and %ld timed out. " - "For reference, your timeout cutoff is %ld seconds.", - guard->nickname, hex_str(guard->identity, DIGEST_LEN), - tor_lround(pathbias_get_close_success_count(guard)), - tor_lround(guard->circ_attempts), - tor_lround(pathbias_get_use_success_count(guard)), - tor_lround(guard->use_attempts), - tor_lround(guard->circ_successes), - tor_lround(guard->unusable_circuits), - tor_lround(guard->collapsed_circuits), - tor_lround(guard->timeouts), - tor_lround(circ_times.close_ms/1000)); - } - } - } -} - -/** - * This function scales the path bias use rates if we have - * more data than the scaling threshold. This allows us to - * be more sensitive to recent measurements. - * - * XXX: The attempt count transfer stuff here might be done - * better by keeping separate pending counters that get - * transfered at circuit close. See ticket #8160. - */ -static void -pathbias_scale_close_rates(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - /* If we get a ton of circuits, just scale everything down */ - if (guard->circ_attempts > pathbias_get_scale_threshold(options)) { - double scale_ratio = pathbias_get_scale_ratio(options); - int opened_attempts = pathbias_count_circs_in_states(guard, - PATH_STATE_BUILD_ATTEMPTED, PATH_STATE_BUILD_ATTEMPTED); - int opened_built = pathbias_count_circs_in_states(guard, - PATH_STATE_BUILD_SUCCEEDED, - PATH_STATE_USE_FAILED); - /* Verify that the counts are sane before and after scaling */ - int counts_are_sane = (guard->circ_attempts >= guard->circ_successes); - - guard->circ_attempts -= (opened_attempts+opened_built); - guard->circ_successes -= opened_built; - - guard->circ_attempts *= scale_ratio; - guard->circ_successes *= scale_ratio; - guard->timeouts *= scale_ratio; - guard->successful_circuits_closed *= scale_ratio; - guard->collapsed_circuits *= scale_ratio; - guard->unusable_circuits *= scale_ratio; - - guard->circ_attempts += (opened_attempts+opened_built); - guard->circ_successes += opened_built; - - entry_guards_changed(); - - log_info(LD_CIRC, - "Scaled pathbias counts to (%f,%f)/%f (%d/%d open) for guard " - "%s ($%s)", - guard->circ_successes, guard->successful_circuits_closed, - guard->circ_attempts, opened_built, opened_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - - /* Have the counts just become invalid by this scaling attempt? */ - if (counts_are_sane && guard->circ_attempts < guard->circ_successes) { - log_notice(LD_BUG, - "Scaling has mangled pathbias counts to %f/%f (%d/%d open) " - "for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, opened_built, - opened_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - } - } -} - -/** - * This function scales the path bias circuit close rates if we have - * more data than the scaling threshold. This allows us to be more - * sensitive to recent measurements. - * - * XXX: The attempt count transfer stuff here might be done - * better by keeping separate pending counters that get - * transfered at circuit close. See ticket #8160. - */ -void -pathbias_scale_use_rates(entry_guard_t *guard) -{ - const or_options_t *options = get_options(); - - /* If we get a ton of circuits, just scale everything down */ - if (guard->use_attempts > pathbias_get_scale_use_threshold(options)) { - double scale_ratio = pathbias_get_scale_ratio(options); - int opened_attempts = pathbias_count_circs_in_states(guard, - PATH_STATE_USE_ATTEMPTED, PATH_STATE_USE_SUCCEEDED); - /* Verify that the counts are sane before and after scaling */ - int counts_are_sane = (guard->use_attempts >= guard->use_successes); - - guard->use_attempts -= opened_attempts; - - guard->use_attempts *= scale_ratio; - guard->use_successes *= scale_ratio; - - guard->use_attempts += opened_attempts; - - log_info(LD_CIRC, - "Scaled pathbias use counts to %f/%f (%d open) for guard %s ($%s)", - guard->use_successes, guard->use_attempts, opened_attempts, - guard->nickname, hex_str(guard->identity, DIGEST_LEN)); - - /* Have the counts just become invalid by this scaling attempt? */ - if (counts_are_sane && guard->use_attempts < guard->use_successes) { - log_notice(LD_BUG, - "Scaling has mangled pathbias usage counts to %f/%f " - "(%d open) for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, - opened_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - } - - entry_guards_changed(); - } -} - -/** Increment the number of times we successfully extended a circuit to - * <b>guard</b>, first checking if the failure rate is high enough that - * we should eliminate the guard. Return -1 if the guard looks no good; - * return 0 if the guard looks fine. - */ -static int -entry_guard_inc_circ_attempt_count(entry_guard_t *guard) -{ - entry_guards_changed(); - - pathbias_measure_close_rate(guard); - - if (guard->path_bias_disabled) - return -1; - - pathbias_scale_close_rates(guard); - guard->circ_attempts++; - - log_info(LD_CIRC, "Got success count %f/%f for guard %s ($%s)", - guard->circ_successes, guard->circ_attempts, guard->nickname, - hex_str(guard->identity, DIGEST_LEN)); - return 0; -} - /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>. * (The body of <b>reply</b> varies depending on what sort of handshake * this is.) @@ -2679,8 +1276,10 @@ circuit_finish_handshake(origin_circuit_t *circ, crypt_path_t *hop; int rv; - if ((rv = pathbias_count_build_attempt(circ)) < 0) + if ((rv = pathbias_count_build_attempt(circ)) < 0) { + log_warn(LD_CIRC, "pathbias_count_build_attempt failed: %d", rv); return rv; + } if (circ->cpath->state == CPATH_STATE_AWAITING_KEYS) { hop = circ->cpath; @@ -2694,12 +1293,15 @@ circuit_finish_handshake(origin_circuit_t *circ, tor_assert(hop->state == CPATH_STATE_AWAITING_KEYS); { + const char *msg = NULL; if (onion_skin_client_handshake(hop->handshake_state.tag, &hop->handshake_state, reply->reply, reply->handshake_len, (uint8_t*)keys, sizeof(keys), - (uint8_t*)hop->rend_circ_nonce) < 0) { - log_warn(LD_CIRC,"onion_skin_client_handshake failed."); + (uint8_t*)hop->rend_circ_nonce, + &msg) < 0) { + if (msg) + log_warn(LD_CIRC,"onion_skin_client_handshake failed: %s", msg); return -END_CIRC_REASON_TORPROTOCOL; } } @@ -2815,8 +1417,13 @@ onionskin_answer(or_circuit_t *circ, log_debug(LD_CIRC,"Finished sending '%s' cell.", circ->is_first_hop ? "created_fast" : "created"); - if (!channel_is_local(circ->p_chan) && - !channel_is_outgoing(circ->p_chan)) { + /* Ignore the local bit when ExtendAllowPrivateAddresses is set: + * it violates the assumption that private addresses are local. + * Also, many test networks run on local addresses, and + * TestingTorNetwork sets ExtendAllowPrivateAddresses. */ + if ((!channel_is_local(circ->p_chan) + || get_options()->ExtendAllowPrivateAddresses) + && !channel_is_outgoing(circ->p_chan)) { /* record that we could process create cells from a non-local conn * that we didn't initiate; presumably this means that create cells * can reach us too. */ @@ -2830,11 +1437,7 @@ onionskin_answer(or_circuit_t *circ, * number of endpoints that would give something away about our destination. * * If the routerlist <b>nodes</b> doesn't have enough routers - * to handle the desired path length, return as large a path length as - * is feasible, except if it's less than 2, in which case return -1. - * XXX ^^ I think this behavior is a hold-over from back when we had only a - * few relays in the network, and certainly back before guards existed. - * We should very likely get rid of it. -RD + * to handle the desired path length, return -1. */ static int new_route_len(uint8_t purpose, extend_info_t *exit, smartlist_t *nodes) @@ -2855,19 +1458,13 @@ new_route_len(uint8_t purpose, extend_info_t *exit, smartlist_t *nodes) log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).", routelen, num_acceptable_routers, smartlist_len(nodes)); - if (num_acceptable_routers < 2) { + if (num_acceptable_routers < routelen) { log_info(LD_CIRC, - "Not enough acceptable routers (%d). Discarding this circuit.", - num_acceptable_routers); + "Not enough acceptable routers (%d/%d). Discarding this circuit.", + num_acceptable_routers, routelen); return -1; } - if (num_acceptable_routers < routelen) { - log_info(LD_CIRC,"Not enough routers: cutting routelen from %d to %d.", - routelen, num_acceptable_routers); - routelen = num_acceptable_routers; - } - return routelen; } @@ -2997,7 +1594,7 @@ choose_good_exit_server_general(int need_uptime, int need_capacity) * -1 means "Don't use this router at all." */ the_nodes = nodelist_get_list(); - n_supported = tor_malloc(sizeof(int)*smartlist_len(the_nodes)); + n_supported = tor_calloc(smartlist_len(the_nodes), sizeof(int)); SMARTLIST_FOREACH_BEGIN(the_nodes, const node_t *, node) { const int i = node_sl_idx; if (router_digest_is_me(node->identity)) { @@ -3168,6 +1765,87 @@ choose_good_exit_server_general(int need_uptime, int need_capacity) return NULL; } +#if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS) +/* The config option Tor2webRendezvousPoints has been set and we need + * to pick an RP out of that set. Make sure that the RP we choose is + * alive, and return it. Return NULL if no usable RP could be found in + * Tor2webRendezvousPoints. */ +STATIC const node_t * +pick_tor2web_rendezvous_node(router_crn_flags_t flags, + const or_options_t *options) +{ + const node_t *rp_node = NULL; + const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; + const int need_desc = (flags & CRN_NEED_DESC) != 0; + const int pref_addr = (flags & CRN_PREF_ADDR) != 0; + const int direct_conn = (flags & CRN_DIRECT_CONN) != 0; + + smartlist_t *whitelisted_live_rps = smartlist_new(); + smartlist_t *all_live_nodes = smartlist_new(); + + tor_assert(options->Tor2webRendezvousPoints); + + /* Add all running nodes to all_live_nodes */ + router_add_running_nodes_to_smartlist(all_live_nodes, + allow_invalid, + 0, 0, 0, + need_desc, + pref_addr, + direct_conn); + + /* Filter all_live_nodes to only add live *and* whitelisted RPs to + * the list whitelisted_live_rps. */ + SMARTLIST_FOREACH_BEGIN(all_live_nodes, node_t *, live_node) { + if (routerset_contains_node(options->Tor2webRendezvousPoints, live_node)) { + smartlist_add(whitelisted_live_rps, live_node); + } + } SMARTLIST_FOREACH_END(live_node); + + /* Honor ExcludeNodes */ + if (options->ExcludeNodes) { + routerset_subtract_nodes(whitelisted_live_rps, options->ExcludeNodes); + } + + /* Now pick randomly amongst the whitelisted RPs. No need to waste time + doing bandwidth load balancing, for most use cases + 'whitelisted_live_rps' contains a single OR anyway. */ + rp_node = smartlist_choose(whitelisted_live_rps); + + if (!rp_node) { + log_warn(LD_REND, "Could not find a Rendezvous Point that suits " + "the purposes of Tor2webRendezvousPoints. Choosing random one."); + } + + smartlist_free(whitelisted_live_rps); + smartlist_free(all_live_nodes); + + return rp_node; +} +#endif + +/* Pick a Rendezvous Point for our HS circuits according to <b>flags</b>. */ +static const node_t * +pick_rendezvous_node(router_crn_flags_t flags) +{ + const or_options_t *options = get_options(); + + if (options->AllowInvalid_ & ALLOW_INVALID_RENDEZVOUS) + flags |= CRN_ALLOW_INVALID; + +#ifdef ENABLE_TOR2WEB_MODE + /* The user wants us to pick specific RPs. */ + if (options->Tor2webRendezvousPoints) { + const node_t *tor2web_rp = pick_tor2web_rendezvous_node(flags, options); + if (tor2web_rp) { + return tor2web_rp; + } + /* Else, if no tor2web RP was found, fall back to choosing a random node */ + } +#endif + + return router_choose_random_node(NULL, options->ExcludeNodes, flags); +} + /** Return a pointer to a suitable router to be the exit node for the * circuit of purpose <b>purpose</b> that we're about to build (or NULL * if no router is suitable). @@ -3198,9 +1876,13 @@ choose_good_exit_server(uint8_t purpose, else return choose_good_exit_server_general(need_uptime,need_capacity); case CIRCUIT_PURPOSE_C_ESTABLISH_REND: - if (options->AllowInvalid_ & ALLOW_INVALID_RENDEZVOUS) - flags |= CRN_ALLOW_INVALID; - return router_choose_random_node(NULL, options->ExcludeNodes, flags); + { + /* Pick a new RP */ + const node_t *rendezvous_node = pick_rendezvous_node(flags); + log_info(LD_REND, "Picked new RP: %s", + safe_str_client(node_describe(rendezvous_node))); + return rendezvous_node; + } } log_warn(LD_BUG,"Unhandled purpose %d", purpose); tor_fragile_assert(); @@ -3310,7 +1992,7 @@ onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit) choose_good_exit_server(circ->base_.purpose, state->need_uptime, state->need_capacity, state->is_internal); if (!node) { - log_warn(LD_CIRC,"failed to choose an exit server"); + log_warn(LD_CIRC,"Failed to choose an exit server"); return -1; } exit = extend_info_from_node(node, 0); @@ -3437,7 +2119,8 @@ choose_good_middle_server(uint8_t purpose, tor_assert(CIRCUIT_PURPOSE_MIN_ <= purpose && purpose <= CIRCUIT_PURPOSE_MAX_); - log_debug(LD_CIRC, "Contemplating intermediate hop: random choice."); + log_debug(LD_CIRC, "Contemplating intermediate hop %d: random choice.", + cur_len); excluded = smartlist_new(); if ((r = build_state_get_exit_node(state))) { nodelist_add_node_and_family(excluded, r); @@ -3474,7 +2157,10 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) const node_t *choice; smartlist_t *excluded; const or_options_t *options = get_options(); - router_crn_flags_t flags = CRN_NEED_GUARD|CRN_NEED_DESC; + /* If possible, choose an entry server with a preferred address, + * otherwise, choose one with an allowed address */ + router_crn_flags_t flags = (CRN_NEED_GUARD|CRN_NEED_DESC|CRN_PREF_ADDR| + CRN_DIRECT_CONN); const node_t *node; if (state && options->UseEntryGuards && @@ -3491,17 +2177,18 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) * family. */ nodelist_add_node_and_family(excluded, node); } - if (firewall_is_fascist_or()) { - /* Exclude all ORs that we can't reach through our firewall */ - smartlist_t *nodes = nodelist_get_list(); - SMARTLIST_FOREACH(nodes, const node_t *, node, { - if (!fascist_firewall_allows_node(node)) - smartlist_add(excluded, (void*)node); - }); - } - /* and exclude current entry guards and their families, if applicable */ + /* and exclude current entry guards and their families, + * unless we're in a test network, and excluding guards + * would exclude all nodes (i.e. we're in an incredibly small tor network, + * or we're using TestingAuthVoteGuard *). + * This is an incomplete fix, but is no worse than the previous behaviour, + * and only applies to minimal, testing tor networks + * (so it's no less secure) */ /*XXXX025 use the using_as_guard flag to accomplish this.*/ - if (options->UseEntryGuards) { + if (options->UseEntryGuards + && (!options->TestingTorNetwork || + smartlist_len(nodelist_get_list()) > smartlist_len(get_entry_guards()) + )) { SMARTLIST_FOREACH(get_entry_guards(), const entry_guard_t *, entry, { if ((node = node_get_by_id(entry->identity))) { @@ -3568,9 +2255,11 @@ onion_extend_cpath(origin_circuit_t *circ) if (r) { /* If we're a client, use the preferred address rather than the primary address, for potentially connecting to an IPv6 OR - port. */ - info = extend_info_from_node(r, server_mode(get_options()) == 0); - tor_assert(info); + port. Servers always want the primary (IPv4) address. */ + int client = (server_mode(get_options()) == 0); + info = extend_info_from_node(r, client); + /* Clients can fail to find an allowed address */ + tor_assert(info || client); } } else { const node_t *r = @@ -3631,13 +2320,9 @@ extend_info_new(const char *nickname, const char *digest, strlcpy(info->nickname, nickname, sizeof(info->nickname)); if (onion_key) info->onion_key = crypto_pk_dup_key(onion_key); -#ifdef CURVE25519_ENABLED if (curve25519_key) memcpy(&info->curve25519_onion_key, curve25519_key, sizeof(curve25519_public_key_t)); -#else - (void)curve25519_key; -#endif tor_addr_copy(&info->addr, addr); info->port = port; return info; @@ -3649,33 +2334,43 @@ extend_info_new(const char *nickname, const char *digest, * <b>for_direct_connect</b> is true, in which case the preferred * address is used instead. May return NULL if there is not enough * info about <b>node</b> to extend to it--for example, if there is no - * routerinfo_t or microdesc_t. + * routerinfo_t or microdesc_t, or if for_direct_connect is true and none of + * the node's addresses are allowed by tor's firewall and IP version config. **/ extend_info_t * extend_info_from_node(const node_t *node, int for_direct_connect) { tor_addr_port_t ap; + int valid_addr = 0; if (node->ri == NULL && (node->rs == NULL || node->md == NULL)) return NULL; + /* Choose a preferred address first, but fall back to an allowed address. + * choose_address returns 1 on success, but get_prim_orport returns 0. */ if (for_direct_connect) - node_get_pref_orport(node, &ap); + valid_addr = fascist_firewall_choose_address_node(node, + FIREWALL_OR_CONNECTION, + 0, &ap); else - node_get_prim_orport(node, &ap); + valid_addr = !node_get_prim_orport(node, &ap); - log_debug(LD_CIRC, "using %s for %s", - fmt_addrport(&ap.addr, ap.port), - node->ri ? node->ri->nickname : node->rs->nickname); + if (valid_addr) + log_debug(LD_CIRC, "using %s for %s", + fmt_addrport(&ap.addr, ap.port), + node->ri ? node->ri->nickname : node->rs->nickname); + else + log_warn(LD_CIRC, "Could not choose valid address for %s", + node->ri ? node->ri->nickname : node->rs->nickname); - if (node->ri) + if (valid_addr && node->ri) return extend_info_new(node->ri->nickname, node->identity, node->ri->onion_pkey, node->ri->onion_curve25519_pkey, &ap.addr, ap.port); - else if (node->rs && node->md) + else if (valid_addr && node->rs && node->md) return extend_info_new(node->rs->nickname, node->identity, node->md->onion_pkey, @@ -3736,3 +2431,20 @@ build_state_get_exit_nickname(cpath_build_state_t *state) return state->chosen_exit->nickname; } +/** Return true iff the given address can be used to extend to. */ +int +extend_info_addr_is_allowed(const tor_addr_t *addr) +{ + tor_assert(addr); + + /* Check if we have a private address and if we can extend to it. */ + if ((tor_addr_is_internal(addr, 0) || tor_addr_is_multicast(addr)) && + !get_options()->ExtendAllowPrivateAddresses) { + goto disallow; + } + /* Allowed! */ + return 1; + disallow: + return 0; +} + diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index a3091707e8..7f5fd511a9 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,7 +22,8 @@ origin_circuit_t *circuit_establish_circuit(uint8_t purpose, extend_info_t *exit, int flags); int circuit_handle_first_hop(origin_circuit_t *circ); -void circuit_n_chan_done(channel_t *chan, int status); +void circuit_n_chan_done(channel_t *chan, int status, + int close_origin_circuits); int inform_testing_reachability(void); int circuit_timeout_want_to_count_circ(origin_circuit_t *circ); int circuit_send_next_onion_skin(origin_circuit_t *circ); @@ -52,21 +53,21 @@ extend_info_t *extend_info_new(const char *nickname, const char *digest, extend_info_t *extend_info_from_node(const node_t *r, int for_direct_connect); extend_info_t *extend_info_dup(extend_info_t *info); void extend_info_free(extend_info_t *info); +int extend_info_addr_is_allowed(const tor_addr_t *addr); const node_t *build_state_get_exit_node(cpath_build_state_t *state); const char *build_state_get_exit_nickname(cpath_build_state_t *state); const node_t *choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state); -double pathbias_get_extreme_rate(const or_options_t *options); -double pathbias_get_extreme_use_rate(const or_options_t *options); -int pathbias_get_dropguards(const or_options_t *options); -void pathbias_count_timeout(origin_circuit_t *circ); -int pathbias_check_close(origin_circuit_t *circ, int reason); -int pathbias_check_probe_response(circuit_t *circ, const cell_t *cell); -void pathbias_count_use_attempt(origin_circuit_t *circ); -void pathbias_mark_use_success(origin_circuit_t *circ); -void pathbias_mark_use_rollback(origin_circuit_t *circ); -const char *pathbias_state_to_string(path_state_t state); + +#ifdef CIRCUITBUILD_PRIVATE +STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); +#if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS) +STATIC const node_t *pick_tor2web_rendezvous_node(router_crn_flags_t flags, + const or_options_t *options); +#endif + +#endif #endif diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index c7b15e40ba..d7dbfe5744 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1,16 +1,18 @@ /* Copyright 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file circuitlist.c - * \brief Manage the global circuit list. + * + * \brief Manage the global circuit list, and looking up circuits within it. **/ - +#define CIRCUITLIST_PRIVATE #include "or.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -20,6 +22,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "main.h" #include "networkstatus.h" #include "nodelist.h" #include "onion.h" @@ -31,20 +34,28 @@ #include "rephist.h" #include "routerlist.h" #include "routerset.h" + #include "ht.h" /********* START VARIABLES **********/ /** A global list of all circuits at this hop. */ -circuit_t *global_circuitlist=NULL; +static smartlist_t *global_circuitlist = NULL; /** A list of all the circuits in CIRCUIT_STATE_CHAN_WAIT. */ static smartlist_t *circuits_pending_chans = NULL; -static void circuit_free(circuit_t *circ); -static void circuit_free_cpath(crypt_path_t *cpath); +/** A list of all the circuits that have been marked with + * circuit_mark_for_close and which are waiting for circuit_about_to_free. */ +static smartlist_t *circuits_pending_close = NULL; + static void circuit_free_cpath_node(crypt_path_t *victim); static void cpath_ref_decref(crypt_path_reference_t *cpath_ref); +//static void circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ, +// const uint8_t *token); +static void circuit_clear_rend_token(or_circuit_t *circ); +static void circuit_about_to_free_atexit(circuit_t *circ); +static void circuit_about_to_free(circuit_t *circ); /********* END VARIABLES ************/ @@ -55,12 +66,14 @@ typedef struct chan_circid_circuit_map_t { channel_t *chan; circid_t circ_id; circuit_t *circuit; + /* For debugging 12184: when was this placeholder item added? */ + time_t made_placeholder_at; } chan_circid_circuit_map_t; /** Helper for hash tables: compare the channel and circuit ID for a and * b, and return less than, equal to, or greater than zero appropriately. */ -static INLINE int +static inline int chan_circid_entries_eq_(chan_circid_circuit_map_t *a, chan_circid_circuit_map_t *b) { @@ -69,10 +82,18 @@ chan_circid_entries_eq_(chan_circid_circuit_map_t *a, /** Helper: return a hash based on circuit ID and the pointer value of * chan in <b>a</b>. */ -static INLINE unsigned int +static inline unsigned int chan_circid_entry_hash_(chan_circid_circuit_map_t *a) { - return ((unsigned)a->circ_id) ^ (unsigned)(uintptr_t)(a->chan); + /* Try to squeze the siphash input into 8 bytes to save any extra siphash + * rounds. This hash function is in the critical path. */ + uintptr_t chan = (uintptr_t) (void*) a->chan; + uint32_t array[2]; + array[0] = a->circ_id; + /* The low bits of the channel pointer are uninteresting, since the channel + * is a pretty big structure. */ + array[1] = (uint32_t) (chan >> 6); + return (unsigned) siphash24g(array, sizeof(array)); } /** Map from [chan,circid] to circuit. */ @@ -80,9 +101,9 @@ static HT_HEAD(chan_circid_map, chan_circid_circuit_map_t) chan_circid_map = HT_INITIALIZER(); HT_PROTOTYPE(chan_circid_map, chan_circid_circuit_map_t, node, chan_circid_entry_hash_, chan_circid_entries_eq_) -HT_GENERATE(chan_circid_map, chan_circid_circuit_map_t, node, - chan_circid_entry_hash_, chan_circid_entries_eq_, 0.6, - malloc, realloc, free) +HT_GENERATE2(chan_circid_map, chan_circid_circuit_map_t, node, + chan_circid_entry_hash_, chan_circid_entries_eq_, 0.6, + tor_reallocarray_, tor_free_) /** The most recently returned entry from circuit_get_by_circid_chan; * used to improve performance when many cells arrive in a row from the @@ -172,6 +193,7 @@ circuit_set_circid_chan_helper(circuit_t *circ, int direction, found = HT_FIND(chan_circid_map, &chan_circid_map, &search); if (found) { found->circuit = circ; + found->made_placeholder_at = 0; } else { found = tor_malloc_zero(sizeof(chan_circid_circuit_map_t)); found->circ_id = id; @@ -207,18 +229,129 @@ circuit_set_circid_chan_helper(circuit_t *circ, int direction, } } +/** Mark that circuit id <b>id</b> shouldn't be used on channel <b>chan</b>, + * even if there is no circuit on the channel. We use this to keep the + * circuit id from getting re-used while we have queued but not yet sent + * a destroy cell. */ +void +channel_mark_circid_unusable(channel_t *chan, circid_t id) +{ + chan_circid_circuit_map_t search; + chan_circid_circuit_map_t *ent; + + /* See if there's an entry there. That wouldn't be good. */ + memset(&search, 0, sizeof(search)); + search.chan = chan; + search.circ_id = id; + ent = HT_FIND(chan_circid_map, &chan_circid_map, &search); + + if (ent && ent->circuit) { + /* we have a problem. */ + log_warn(LD_BUG, "Tried to mark %u unusable on %p, but there was already " + "a circuit there.", (unsigned)id, chan); + } else if (ent) { + /* It's already marked. */ + if (!ent->made_placeholder_at) + ent->made_placeholder_at = approx_time(); + } else { + ent = tor_malloc_zero(sizeof(chan_circid_circuit_map_t)); + ent->chan = chan; + ent->circ_id = id; + /* leave circuit at NULL. */ + ent->made_placeholder_at = approx_time(); + HT_INSERT(chan_circid_map, &chan_circid_map, ent); + } +} + +/** Mark that a circuit id <b>id</b> can be used again on <b>chan</b>. + * We use this to re-enable the circuit ID after we've sent a destroy cell. + */ +void +channel_mark_circid_usable(channel_t *chan, circid_t id) +{ + chan_circid_circuit_map_t search; + chan_circid_circuit_map_t *ent; + + /* See if there's an entry there. That wouldn't be good. */ + memset(&search, 0, sizeof(search)); + search.chan = chan; + search.circ_id = id; + ent = HT_REMOVE(chan_circid_map, &chan_circid_map, &search); + if (ent && ent->circuit) { + log_warn(LD_BUG, "Tried to mark %u usable on %p, but there was already " + "a circuit there.", (unsigned)id, chan); + return; + } + if (_last_circid_chan_ent == ent) + _last_circid_chan_ent = NULL; + tor_free(ent); +} + +/** Called to indicate that a DESTROY is pending on <b>chan</b> with + * circuit ID <b>id</b>, but hasn't been sent yet. */ +void +channel_note_destroy_pending(channel_t *chan, circid_t id) +{ + circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan); + if (circ) { + if (circ->n_chan == chan && circ->n_circ_id == id) { + circ->n_delete_pending = 1; + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + if (orcirc->p_chan == chan && orcirc->p_circ_id == id) { + circ->p_delete_pending = 1; + } + } + return; + } + channel_mark_circid_unusable(chan, id); +} + +/** Called to indicate that a DESTROY is no longer pending on <b>chan</b> with + * circuit ID <b>id</b> -- typically, because it has been sent. */ +MOCK_IMPL(void, channel_note_destroy_not_pending, + (channel_t *chan, circid_t id)) +{ + circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan); + if (circ) { + if (circ->n_chan == chan && circ->n_circ_id == id) { + circ->n_delete_pending = 0; + } else { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + if (orcirc->p_chan == chan && orcirc->p_circ_id == id) { + circ->p_delete_pending = 0; + } + } + /* XXXX this shouldn't happen; log a bug here. */ + return; + } + channel_mark_circid_usable(chan, id); +} + /** Set the p_conn field of a circuit <b>circ</b>, along * with the corresponding circuit ID, and add the circuit as appropriate * to the (chan,id)-\>circuit map. */ void -circuit_set_p_circid_chan(or_circuit_t *circ, circid_t id, +circuit_set_p_circid_chan(or_circuit_t *or_circ, circid_t id, channel_t *chan) { - circuit_set_circid_chan_helper(TO_CIRCUIT(circ), CELL_DIRECTION_IN, - id, chan); + circuit_t *circ = TO_CIRCUIT(or_circ); + channel_t *old_chan = or_circ->p_chan; + circid_t old_id = or_circ->p_circ_id; - if (chan) - tor_assert(bool_eq(circ->p_chan_cells.n, circ->next_active_on_p_chan)); + circuit_set_circid_chan_helper(circ, CELL_DIRECTION_IN, id, chan); + + if (chan) { + tor_assert(bool_eq(or_circ->p_chan_cells.n, + or_circ->next_active_on_p_chan)); + + chan->timestamp_last_had_circuits = approx_time(); + } + + if (circ->p_delete_pending && old_chan) { + channel_mark_circid_unusable(old_chan, old_id); + circ->p_delete_pending = 0; + } } /** Set the n_conn field of a circuit <b>circ</b>, along @@ -228,10 +361,21 @@ void circuit_set_n_circid_chan(circuit_t *circ, circid_t id, channel_t *chan) { + channel_t *old_chan = circ->n_chan; + circid_t old_id = circ->n_circ_id; + circuit_set_circid_chan_helper(circ, CELL_DIRECTION_OUT, id, chan); - if (chan) + if (chan) { tor_assert(bool_eq(circ->n_chan_cells.n, circ->next_active_on_n_chan)); + + chan->timestamp_last_had_circuits = approx_time(); + } + + if (circ->n_delete_pending && old_chan) { + channel_mark_circid_unusable(old_chan, old_id); + circ->n_delete_pending = 0; + } } /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing @@ -257,21 +401,6 @@ circuit_set_state(circuit_t *circ, uint8_t state) circ->state = state; } -/** Add <b>circ</b> to the global list of circuits. This is called only from - * within circuit_new. - */ -static void -circuit_add(circuit_t *circ) -{ - if (!global_circuitlist) { /* first one */ - global_circuitlist = circ; - circ->next = NULL; - } else { - circ->next = global_circuitlist; - global_circuitlist = circ; - } -} - /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for * the given connection. */ void @@ -329,32 +458,35 @@ circuit_count_pending_on_channel(channel_t *chan) void circuit_close_all_marked(void) { - circuit_t *tmp,*m; + if (circuits_pending_close == NULL) + return; - while (global_circuitlist && global_circuitlist->marked_for_close) { - tmp = global_circuitlist->next; - circuit_free(global_circuitlist); - global_circuitlist = tmp; - } + smartlist_t *lst = circuit_get_global_list(); + SMARTLIST_FOREACH_BEGIN(circuits_pending_close, circuit_t *, circ) { + tor_assert(circ->marked_for_close); - tmp = global_circuitlist; - while (tmp && tmp->next) { - if (tmp->next->marked_for_close) { - m = tmp->next->next; - circuit_free(tmp->next); - tmp->next = m; - /* Need to check new tmp->next; don't advance tmp. */ - } else { - /* Advance tmp. */ - tmp = tmp->next; + /* Remove it from the circuit list. */ + int idx = circ->global_circuitlist_idx; + smartlist_del(lst, idx); + if (idx < smartlist_len(lst)) { + circuit_t *replacement = smartlist_get(lst, idx); + replacement->global_circuitlist_idx = idx; } - } + circ->global_circuitlist_idx = -1; + + circuit_about_to_free(circ); + circuit_free(circ); + } SMARTLIST_FOREACH_END(circ); + + smartlist_clear(circuits_pending_close); } /** Return the head of the global linked list of circuits. */ -circuit_t * -circuit_get_global_list_(void) +MOCK_IMPL(smartlist_t *, +circuit_get_global_list,(void)) { + if (NULL == global_circuitlist) + global_circuitlist = smartlist_new(); return global_circuitlist; } @@ -570,8 +702,10 @@ init_circuit_base(circuit_t *circ) circ->package_window = circuit_initial_package_window(); circ->deliver_window = CIRCWINDOW_START; + cell_queue_init(&circ->n_chan_cells); - circuit_add(circ); + smartlist_add(circuit_get_global_list(), circ); + circ->global_circuitlist_idx = smartlist_len(circuit_get_global_list()) - 1; } /** Allocate space for a new circuit, initializing with <b>p_circ_id</b> @@ -595,13 +729,13 @@ origin_circuit_new(void) init_circuit_base(TO_CIRCUIT(circ)); - circ_times.last_circ_at = approx_time(); + circuit_build_times_update_last_circ(get_circuit_build_times_mutable()); return circ; } -/** Allocate a new or_circuit_t, connected to <b>p_conn</b> as - * <b>p_circ_id</b>. If <b>p_conn</b> is NULL, the circuit is unattached. */ +/** Allocate a new or_circuit_t, connected to <b>p_chan</b> as + * <b>p_circ_id</b>. If <b>p_chan</b> is NULL, the circuit is unattached. */ or_circuit_t * or_circuit_new(circid_t p_circ_id, channel_t *p_chan) { @@ -615,22 +749,38 @@ or_circuit_new(circid_t p_circ_id, channel_t *p_chan) circuit_set_p_circid_chan(circ, p_circ_id, p_chan); circ->remaining_relay_early_cells = MAX_RELAY_EARLY_CELLS_PER_CIRCUIT; + cell_queue_init(&circ->p_chan_cells); init_circuit_base(TO_CIRCUIT(circ)); return circ; } +/** Free all storage held in circ->testing_cell_stats */ +void +circuit_clear_testing_cell_stats(circuit_t *circ) +{ + if (!circ || !circ->testing_cell_stats) + return; + SMARTLIST_FOREACH(circ->testing_cell_stats, testing_cell_stats_entry_t *, + ent, tor_free(ent)); + smartlist_free(circ->testing_cell_stats); + circ->testing_cell_stats = NULL; +} + /** Deallocate space associated with circ. */ -static void +STATIC void circuit_free(circuit_t *circ) { void *mem; size_t memlen; + int should_free = 1; if (!circ) return; + circuit_clear_testing_cell_stats(circ); + if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); mem = ocirc; @@ -643,7 +793,7 @@ circuit_free(circuit_t *circ) } tor_free(ocirc->build_state); - circuit_free_cpath(ocirc->cpath); + circuit_clear_cpath(ocirc); crypto_pk_free(ocirc->intro_key); rend_data_free(ocirc->rend_data); @@ -667,11 +817,15 @@ circuit_free(circuit_t *circ) memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); + should_free = (ocirc->workqueue_entry == NULL); + crypto_cipher_free(ocirc->p_crypto); crypto_digest_free(ocirc->p_digest); crypto_cipher_free(ocirc->n_crypto); crypto_digest_free(ocirc->n_digest); + circuit_clear_rend_token(ocirc); + if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; tor_assert(other->base_.magic == OR_CIRCUIT_MAGIC); @@ -689,6 +843,17 @@ circuit_free(circuit_t *circ) extend_info_free(circ->n_hop); tor_free(circ->n_chan_create_cell); + if (circ->global_circuitlist_idx != -1) { + int idx = circ->global_circuitlist_idx; + circuit_t *c2 = smartlist_get(global_circuitlist, idx); + tor_assert(c2 == circ); + smartlist_del(global_circuitlist, idx); + if (idx < smartlist_len(global_circuitlist)) { + c2 = smartlist_get(global_circuitlist, idx); + c2->global_circuitlist_idx = idx; + } + } + /* Remove from map. */ circuit_set_n_circid_chan(circ, 0, NULL); @@ -696,15 +861,28 @@ circuit_free(circuit_t *circ) * "active" checks will be violated. */ cell_queue_clear(&circ->n_chan_cells); - memwipe(mem, 0xAA, memlen); /* poison memory */ - tor_free(mem); + if (should_free) { + memwipe(mem, 0xAA, memlen); /* poison memory */ + tor_free(mem); + } else { + /* If we made it here, this is an or_circuit_t that still has a pending + * cpuworker request which we weren't able to cancel. Instead, set up + * the magic value so that when the reply comes back, we'll know to discard + * the reply and free this structure. + */ + memwipe(mem, 0xAA, memlen); + circ->magic = DEAD_CIRCUIT_MAGIC; + } } -/** Deallocate space associated with the linked list <b>cpath</b>. */ -static void -circuit_free_cpath(crypt_path_t *cpath) +/** Deallocate the linked list circ-><b>cpath</b>, and remove the cpath from + * <b>circ</b>. */ +void +circuit_clear_cpath(origin_circuit_t *circ) { - crypt_path_t *victim, *head=cpath; + crypt_path_t *victim, *head, *cpath; + + head = cpath = circ->cpath; if (!cpath) return; @@ -718,13 +896,7 @@ circuit_free_cpath(crypt_path_t *cpath) } circuit_free_cpath_node(cpath); -} -/** Remove all the items in the cpath on <b>circ</b>.*/ -void -circuit_clear_cpath(origin_circuit_t *circ) -{ - circuit_free_cpath(circ->cpath); circ->cpath = NULL; } @@ -732,11 +904,11 @@ circuit_clear_cpath(origin_circuit_t *circ) void circuit_free_all(void) { - circuit_t *next; - while (global_circuitlist) { - next = global_circuitlist->next; - if (! CIRCUIT_IS_ORIGIN(global_circuitlist)) { - or_circuit_t *or_circ = TO_OR_CIRCUIT(global_circuitlist); + smartlist_t *lst = circuit_get_global_list(); + + SMARTLIST_FOREACH_BEGIN(lst, circuit_t *, tmp) { + if (! CIRCUIT_IS_ORIGIN(tmp)) { + or_circuit_t *or_circ = TO_OR_CIRCUIT(tmp); while (or_circ->resolving_streams) { edge_connection_t *next_conn; next_conn = or_circ->resolving_streams->next_stream; @@ -744,13 +916,33 @@ circuit_free_all(void) or_circ->resolving_streams = next_conn; } } - circuit_free(global_circuitlist); - global_circuitlist = next; - } + tmp->global_circuitlist_idx = -1; + circuit_about_to_free_atexit(tmp); + circuit_free(tmp); + SMARTLIST_DEL_CURRENT(lst, tmp); + } SMARTLIST_FOREACH_END(tmp); + + smartlist_free(lst); + global_circuitlist = NULL; smartlist_free(circuits_pending_chans); circuits_pending_chans = NULL; + smartlist_free(circuits_pending_close); + circuits_pending_close = NULL; + + { + chan_circid_circuit_map_t **elt, **next, *c; + for (elt = HT_START(chan_circid_map, &chan_circid_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(chan_circid_map, &chan_circid_map, elt); + + tor_assert(c->circuit == NULL); + tor_free(c); + } + } HT_CLEAR(chan_circid_map, &chan_circid_map); } @@ -812,10 +1004,9 @@ circuit_dump_conn_details(int severity, void circuit_dump_by_conn(connection_t *conn, int severity) { - circuit_t *circ; edge_connection_t *tmpconn; - for (circ = global_circuitlist; circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { circid_t n_circ_id = circ->n_circ_id, p_circ_id = 0; if (circ->marked_for_close) { @@ -846,72 +1037,7 @@ circuit_dump_by_conn(connection_t *conn, int severity) } } } -} - -/** A helper function for circuit_dump_by_chan() below. Log a bunch - * of information about circuit <b>circ</b>. - */ -static void -circuit_dump_chan_details(int severity, - circuit_t *circ, - channel_t *chan, - const char *type, - circid_t this_circid, - circid_t other_circid) -{ - tor_log(severity, LD_CIRC, "Conn %p has %s circuit: circID %u " - "(other side %u), state %d (%s), born %ld:", - chan, type, (unsigned)this_circid, (unsigned)other_circid, circ->state, - circuit_state_to_string(circ->state), - (long)circ->timestamp_began.tv_sec); - if (CIRCUIT_IS_ORIGIN(circ)) { /* circ starts at this node */ - circuit_log_path(severity, LD_CIRC, TO_ORIGIN_CIRCUIT(circ)); - } -} - -/** Log, at severity <b>severity</b>, information about each circuit - * that is connected to <b>chan</b>. - */ -void -circuit_dump_by_chan(channel_t *chan, int severity) -{ - circuit_t *circ; - - tor_assert(chan); - - for (circ = global_circuitlist; circ; circ = circ->next) { - circid_t n_circ_id = circ->n_circ_id, p_circ_id = 0; - - if (circ->marked_for_close) { - continue; - } - - if (!CIRCUIT_IS_ORIGIN(circ)) { - p_circ_id = TO_OR_CIRCUIT(circ)->p_circ_id; - } - - if (! CIRCUIT_IS_ORIGIN(circ) && TO_OR_CIRCUIT(circ)->p_chan && - TO_OR_CIRCUIT(circ)->p_chan == chan) { - circuit_dump_chan_details(severity, circ, chan, "App-ward", - p_circ_id, n_circ_id); - } - - if (circ->n_chan && circ->n_chan == chan) { - circuit_dump_chan_details(severity, circ, chan, "Exit-ward", - n_circ_id, p_circ_id); - } - - if (!circ->n_chan && circ->n_hop && - channel_matches_extend_info(chan, circ->n_hop) && - tor_memeq(chan->identity_digest, - circ->n_hop->identity_digest, DIGEST_LEN)) { - circuit_dump_chan_details(severity, circ, chan, - (circ->state == CIRCUIT_STATE_OPEN && - !CIRCUIT_IS_ORIGIN(circ)) ? - "Endpoint" : "Pending", - n_circ_id, p_circ_id); - } - } + SMARTLIST_FOREACH_END(circ); } /** Return the circuit whose global ID is <b>id</b>, or NULL if no @@ -919,8 +1045,7 @@ circuit_dump_by_chan(channel_t *chan, int severity) origin_circuit_t * circuit_get_by_global_id(uint32_t id) { - circuit_t *circ; - for (circ=global_circuitlist;circ;circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (CIRCUIT_IS_ORIGIN(circ) && TO_ORIGIN_CIRCUIT(circ)->global_identifier == id) { if (circ->marked_for_close) @@ -929,6 +1054,7 @@ circuit_get_by_global_id(uint32_t id) return TO_ORIGIN_CIRCUIT(circ); } } + SMARTLIST_FOREACH_END(circ); return NULL; } @@ -936,9 +1062,13 @@ circuit_get_by_global_id(uint32_t id) * - circ-\>n_circ_id or circ-\>p_circ_id is equal to <b>circ_id</b>, and * - circ is attached to <b>chan</b>, either as p_chan or n_chan. * Return NULL if no such circuit exists. + * + * If <b>found_entry_out</b> is provided, set it to true if we have a + * placeholder entry for circid/chan, and leave it unset otherwise. */ -static INLINE circuit_t * -circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan) +static inline circuit_t * +circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan, + int *found_entry_out) { chan_circid_circuit_map_t search; chan_circid_circuit_map_t *found; @@ -959,21 +1089,27 @@ circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan) " circ_id %u, channel ID " U64_FORMAT " (%p)", found->circuit, (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier), chan); + if (found_entry_out) + *found_entry_out = 1; return found->circuit; } log_debug(LD_CIRC, - "circuit_get_by_circid_channel_impl() found nothing for" + "circuit_get_by_circid_channel_impl() found %s for" " circ_id %u, channel ID " U64_FORMAT " (%p)", + found ? "placeholder" : "nothing", (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier), chan); + if (found_entry_out) + *found_entry_out = found ? 1 : 0; + return NULL; /* The rest of this checks for bugs. Disabled by default. */ /* We comment it out because coverity complains otherwise. { circuit_t *circ; - for (circ=global_circuitlist;circ;circ = circ->next) { + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { if (! CIRCUIT_IS_ORIGIN(circ)) { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); if (or_circ->p_chan == chan && or_circ->p_circ_id == circ_id) { @@ -1001,7 +1137,7 @@ circuit_get_by_circid_channel_impl(circid_t circ_id, channel_t *chan) circuit_t * circuit_get_by_circid_channel(circid_t circ_id, channel_t *chan) { - circuit_t *circ = circuit_get_by_circid_channel_impl(circ_id, chan); + circuit_t *circ = circuit_get_by_circid_channel_impl(circ_id, chan, NULL); if (!circ || circ->marked_for_close) return NULL; else @@ -1017,15 +1153,45 @@ circuit_t * circuit_get_by_circid_channel_even_if_marked(circid_t circ_id, channel_t *chan) { - return circuit_get_by_circid_channel_impl(circ_id, chan); + return circuit_get_by_circid_channel_impl(circ_id, chan, NULL); } /** Return true iff the circuit ID <b>circ_id</b> is currently used by a - * circuit, marked or not, on <b>chan</b>. */ + * circuit, marked or not, on <b>chan</b>, or if the circ ID is reserved until + * a queued destroy cell can be sent. + * + * (Return 1 if the circuit is present, marked or not; Return 2 + * if the circuit ID is pending a destroy.) + **/ int circuit_id_in_use_on_channel(circid_t circ_id, channel_t *chan) { - return circuit_get_by_circid_channel_impl(circ_id, chan) != NULL; + int found = 0; + if (circuit_get_by_circid_channel_impl(circ_id, chan, &found) != NULL) + return 1; + if (found) + return 2; + return 0; +} + +/** Helper for debugging 12184. Returns the time since which 'circ_id' has + * been marked unusable on 'chan'. */ +time_t +circuit_id_when_marked_unusable_on_channel(circid_t circ_id, channel_t *chan) +{ + chan_circid_circuit_map_t search; + chan_circid_circuit_map_t *found; + + memset(&search, 0, sizeof(search)); + search.circ_id = circ_id; + search.chan = chan; + + found = HT_FIND(chan_circid_map, &chan_circid_map, &search); + + if (! found || found->circuit) + return 0; + + return found->made_placeholder_at; } /** Return the circuit that a given edge connection is using. */ @@ -1049,13 +1215,59 @@ circuit_get_by_edge_conn(edge_connection_t *conn) void circuit_unlink_all_from_channel(channel_t *chan, int reason) { - circuit_t *circ; + smartlist_t *detached = smartlist_new(); + +/* #define DEBUG_CIRCUIT_UNLINK_ALL */ + + channel_unlink_all_circuits(chan, detached); + +#ifdef DEBUG_CIRCUIT_UNLINK_ALL + { + smartlist_t *detached_2 = smartlist_new(); + int mismatch = 0, badlen = 0; + + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + if (circ->n_chan == chan || + (!CIRCUIT_IS_ORIGIN(circ) && + TO_OR_CIRCUIT(circ)->p_chan == chan)) { + smartlist_add(detached_2, circ); + } + } + SMARTLIST_FOREACH_END(circ); + + if (smartlist_len(detached) != smartlist_len(detached_2)) { + log_warn(LD_BUG, "List of detached circuits had the wrong length! " + "(got %d, should have gotten %d)", + (int)smartlist_len(detached), + (int)smartlist_len(detached_2)); + badlen = 1; + } + smartlist_sort_pointers(detached); + smartlist_sort_pointers(detached_2); - channel_unlink_all_circuits(chan); + SMARTLIST_FOREACH(detached, circuit_t *, c, + if (c != smartlist_get(detached_2, c_sl_idx)) + mismatch = 1; + ); + + if (mismatch) + log_warn(LD_BUG, "Mismatch in list of detached circuits."); + + if (badlen || mismatch) { + smartlist_free(detached); + detached = detached_2; + } else { + log_notice(LD_CIRC, "List of %d circuits was as expected.", + (int)smartlist_len(detached)); + smartlist_free(detached_2); + } + } +#endif - for (circ = global_circuitlist; circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(detached, circuit_t *, circ) { int mark = 0; if (circ->n_chan == chan) { + circuit_set_n_circid_chan(circ, 0, NULL); mark = 1; @@ -1071,9 +1283,16 @@ circuit_unlink_all_from_channel(channel_t *chan, int reason) mark = 1; } } - if (mark && !circ->marked_for_close) + if (!mark) { + log_warn(LD_BUG, "Circuit on detached list which I had no reason " + "to mark"); + continue; + } + if (!circ->marked_for_close) circuit_mark_for_close(circ, reason); - } + } SMARTLIST_FOREACH_END(circ); + + smartlist_free(detached); } /** Return a circ such that @@ -1088,9 +1307,7 @@ circuit_unlink_all_from_channel(channel_t *chan, int reason) origin_circuit_t * circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data) { - circuit_t *circ; - - for (circ = global_circuitlist; circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->marked_for_close && circ->purpose == CIRCUIT_PURPOSE_C_REND_READY) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); @@ -1103,6 +1320,7 @@ circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data) return ocirc; } } + SMARTLIST_FOREACH_END(circ); return NULL; } @@ -1115,14 +1333,17 @@ origin_circuit_t * circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, const char *digest, uint8_t purpose) { - circuit_t *circ; + int idx; + smartlist_t *lst = circuit_get_global_list(); tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(purpose)); if (start == NULL) - circ = global_circuitlist; + idx = 0; else - circ = TO_CIRCUIT(start)->next; + idx = TO_CIRCUIT(start)->global_circuitlist_idx + 1; + + for ( ; idx < smartlist_len(lst); ++idx) { + circuit_t *circ = smartlist_get(lst, idx); - for ( ; circ; circ = circ->next) { if (circ->marked_for_close) continue; if (circ->purpose != purpose) @@ -1137,43 +1358,175 @@ circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, return NULL; } -/** Return the first OR circuit in the global list whose purpose is - * <b>purpose</b>, and whose rend_token is the <b>len</b>-byte - * <b>token</b>. */ +/** Map from rendezvous cookie to or_circuit_t */ +static digestmap_t *rend_cookie_map = NULL; + +/** Map from introduction point digest to or_circuit_t */ +static digestmap_t *intro_digest_map = NULL; + +/** Return the OR circuit whose purpose is <b>purpose</b>, and whose + * rend_token is the REND_TOKEN_LEN-byte <b>token</b>. If <b>is_rend_circ</b>, + * look for rendezvous point circuits; otherwise look for introduction point + * circuits. */ static or_circuit_t * -circuit_get_by_rend_token_and_purpose(uint8_t purpose, const char *token, - size_t len) +circuit_get_by_rend_token_and_purpose(uint8_t purpose, int is_rend_circ, + const char *token) { - circuit_t *circ; - for (circ = global_circuitlist; circ; circ = circ->next) { - if (! circ->marked_for_close && - circ->purpose == purpose && - tor_memeq(TO_OR_CIRCUIT(circ)->rend_token, token, len)) - return TO_OR_CIRCUIT(circ); + or_circuit_t *circ; + digestmap_t *map = is_rend_circ ? rend_cookie_map : intro_digest_map; + + if (!map) + return NULL; + + circ = digestmap_get(map, token); + if (!circ || + circ->base_.purpose != purpose || + circ->base_.marked_for_close) + return NULL; + + if (!circ->rendinfo) { + char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN)); + log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned a " + "circuit with no rendinfo set.", + safe_str(t), is_rend_circ); + tor_free(t); + return NULL; } - return NULL; + + if (! bool_eq(circ->rendinfo->is_rend_circ, is_rend_circ) || + tor_memneq(circ->rendinfo->rend_token, token, REND_TOKEN_LEN)) { + char *t = tor_strdup(hex_str(token, REND_TOKEN_LEN)); + log_warn(LD_BUG, "Wanted a circuit with %s:%d, but lookup returned %s:%d", + safe_str(t), is_rend_circ, + safe_str(hex_str(circ->rendinfo->rend_token, REND_TOKEN_LEN)), + (int)circ->rendinfo->is_rend_circ); + tor_free(t); + return NULL; + } + + return circ; +} + +/** Clear the rendezvous cookie or introduction point key digest that's + * configured on <b>circ</b>, if any, and remove it from any such maps. */ +static void +circuit_clear_rend_token(or_circuit_t *circ) +{ + or_circuit_t *found_circ; + digestmap_t *map; + + if (!circ || !circ->rendinfo) + return; + + map = circ->rendinfo->is_rend_circ ? rend_cookie_map : intro_digest_map; + + if (!map) { + log_warn(LD_BUG, "Tried to clear rend token on circuit, but found no map"); + return; + } + + found_circ = digestmap_get(map, circ->rendinfo->rend_token); + if (found_circ == circ) { + /* Great, this is the right one. */ + digestmap_remove(map, circ->rendinfo->rend_token); + } else if (found_circ) { + log_warn(LD_BUG, "Tried to clear rend token on circuit, but " + "it was already replaced in the map."); + } else { + log_warn(LD_BUG, "Tried to clear rend token on circuit, but " + "it not in the map at all."); + } + + tor_free(circ->rendinfo); /* Sets it to NULL too */ +} + +/** Set the rendezvous cookie (if is_rend_circ), or the introduction point + * digest (if ! is_rend_circ) of <b>circ</b> to the REND_TOKEN_LEN-byte value + * in <b>token</b>, and add it to the appropriate map. If it previously had a + * token, clear it. If another circuit previously had the same + * cookie/intro-digest, mark that circuit and remove it from the map. */ +static void +circuit_set_rend_token(or_circuit_t *circ, int is_rend_circ, + const uint8_t *token) +{ + digestmap_t **map_p, *map; + or_circuit_t *found_circ; + + /* Find the right map, creating it as needed */ + map_p = is_rend_circ ? &rend_cookie_map : &intro_digest_map; + + if (!*map_p) + *map_p = digestmap_new(); + + map = *map_p; + + /* If this circuit already has a token, we need to remove that. */ + if (circ->rendinfo) + circuit_clear_rend_token(circ); + + if (token == NULL) { + /* We were only trying to remove this token, not set a new one. */ + return; + } + + found_circ = digestmap_get(map, (const char *)token); + if (found_circ) { + tor_assert(found_circ != circ); + circuit_clear_rend_token(found_circ); + if (! found_circ->base_.marked_for_close) { + circuit_mark_for_close(TO_CIRCUIT(found_circ), END_CIRC_REASON_FINISHED); + if (is_rend_circ) { + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Duplicate rendezvous cookie (%s...) used on two circuits", + hex_str((const char*)token, 4)); /* only log first 4 chars */ + } + } + } + + /* Now set up the rendinfo */ + circ->rendinfo = tor_malloc(sizeof(*circ->rendinfo)); + memcpy(circ->rendinfo->rend_token, token, REND_TOKEN_LEN); + circ->rendinfo->is_rend_circ = is_rend_circ ? 1 : 0; + + digestmap_set(map, (const char *)token, circ); } /** Return the circuit waiting for a rendezvous with the provided cookie. * Return NULL if no such circuit is found. */ or_circuit_t * -circuit_get_rendezvous(const char *cookie) +circuit_get_rendezvous(const uint8_t *cookie) { return circuit_get_by_rend_token_and_purpose( CIRCUIT_PURPOSE_REND_POINT_WAITING, - cookie, REND_COOKIE_LEN); + 1, (const char*)cookie); } /** Return the circuit waiting for intro cells of the given digest. * Return NULL if no such circuit is found. */ or_circuit_t * -circuit_get_intro_point(const char *digest) +circuit_get_intro_point(const uint8_t *digest) { return circuit_get_by_rend_token_and_purpose( - CIRCUIT_PURPOSE_INTRO_POINT, digest, - DIGEST_LEN); + CIRCUIT_PURPOSE_INTRO_POINT, 0, + (const char *)digest); +} + +/** Set the rendezvous cookie of <b>circ</b> to <b>cookie</b>. If another + * circuit previously had that cookie, mark it. */ +void +circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie) +{ + circuit_set_rend_token(circ, 1, cookie); +} + +/** Set the intro point key digest of <b>circ</b> to <b>cookie</b>. If another + * circuit previously had that intro point digest, mark it. */ +void +circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest) +{ + circuit_set_rend_token(circ, 0, digest); } /** Return a circuit that is open, is CIRCUIT_PURPOSE_C_GENERAL, @@ -1191,7 +1544,6 @@ origin_circuit_t * circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int flags) { - circuit_t *circ_; origin_circuit_t *best=NULL; int need_uptime = (flags & CIRCLAUNCH_NEED_UPTIME) != 0; int need_capacity = (flags & CIRCLAUNCH_NEED_CAPACITY) != 0; @@ -1207,7 +1559,7 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, "capacity %d, internal %d", purpose, need_uptime, need_capacity, internal); - for (circ_=global_circuitlist; circ_; circ_ = circ_->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) { if (CIRCUIT_IS_ORIGIN(circ_) && circ_->state == CIRCUIT_STATE_OPEN && !circ_->marked_for_close && @@ -1229,7 +1581,7 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, do { const node_t *ri2; if (tor_memeq(hop->extend_info->identity_digest, - info->identity_digest, DIGEST_LEN)) + info->identity_digest, DIGEST_LEN)) goto next; if (ri1 && (ri2 = node_get_by_id(hop->extend_info->identity_digest)) @@ -1257,6 +1609,7 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, } } } + SMARTLIST_FOREACH_END(circ_); return best; } @@ -1296,14 +1649,13 @@ circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum) void circuit_mark_all_unused_circs(void) { - circuit_t *circ; - - for (circ=global_circuitlist; circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && !circ->timestamp_dirty) circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); } + SMARTLIST_FOREACH_END(circ); } /** Go through the circuitlist; for each circuit that starts at us @@ -1316,15 +1668,14 @@ circuit_mark_all_unused_circs(void) void circuit_mark_all_dirty_circs_as_unusable(void) { - circuit_t *circ; - - for (circ=global_circuitlist; circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->timestamp_dirty) { mark_circuit_unusable_for_new_conns(TO_ORIGIN_CIRCUIT(circ)); } } + SMARTLIST_FOREACH_END(circ); } /** Mark <b>circ</b> to be closed next time we call @@ -1344,9 +1695,9 @@ circuit_mark_all_dirty_circs_as_unusable(void) * - If circ->rend_splice is set (we are the midpoint of a joined * rendezvous stream), then mark the other circuit to close as well. */ -void -circuit_mark_for_close_(circuit_t *circ, int reason, int line, - const char *file) +MOCK_IMPL(void, +circuit_mark_for_close_, (circuit_t *circ, int reason, int line, + const char *file)) { int orig_reason = reason; /* Passed to the controller */ assert_circuit_ok(circ); @@ -1388,6 +1739,65 @@ circuit_mark_for_close_(circuit_t *circ, int reason, int line, reason = END_CIRC_REASON_NONE; } + circ->marked_for_close = line; + circ->marked_for_close_file = file; + circ->marked_for_close_reason = reason; + circ->marked_for_close_orig_reason = orig_reason; + + if (!CIRCUIT_IS_ORIGIN(circ)) { + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + if (or_circ->rend_splice) { + if (!or_circ->rend_splice->base_.marked_for_close) { + /* do this after marking this circuit, to avoid infinite recursion. */ + circuit_mark_for_close(TO_CIRCUIT(or_circ->rend_splice), reason); + } + or_circ->rend_splice = NULL; + } + } + + if (circuits_pending_close == NULL) + circuits_pending_close = smartlist_new(); + + smartlist_add(circuits_pending_close, circ); +} + +/** Called immediately before freeing a marked circuit <b>circ</b> from + * circuit_free_all() while shutting down Tor; this is a safe-at-shutdown + * version of circuit_about_to_free(). It's important that it at least + * do circuitmux_detach_circuit() when appropriate. + */ +static void +circuit_about_to_free_atexit(circuit_t *circ) +{ + + if (circ->n_chan) { + circuit_clear_cell_queue(circ, circ->n_chan); + circuitmux_detach_circuit(circ->n_chan->cmux, circ); + circuit_set_n_circid_chan(circ, 0, NULL); + } + + if (! CIRCUIT_IS_ORIGIN(circ)) { + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + + if (or_circ->p_chan) { + circuit_clear_cell_queue(circ, or_circ->p_chan); + circuitmux_detach_circuit(or_circ->p_chan->cmux, circ); + circuit_set_p_circid_chan(or_circ, 0, NULL); + } + } +} + +/** Called immediately before freeing a marked circuit <b>circ</b>. + * Disconnects the circuit from other data structures, launches events + * as appropriate, and performs other housekeeping. + */ +static void +circuit_about_to_free(circuit_t *circ) +{ + + int reason = circ->marked_for_close_reason; + int orig_reason = circ->marked_for_close_orig_reason; + if (circ->state == CIRCUIT_STATE_ONIONSKIN_PENDING) { onion_pending_remove(TO_OR_CIRCUIT(circ)); } @@ -1411,45 +1821,51 @@ circuit_mark_for_close_(circuit_t *circ, int reason, int line, (circ->state == CIRCUIT_STATE_OPEN)?CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED, orig_reason); } + if (circ->purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); int timed_out = (reason == END_CIRC_REASON_TIMEOUT); tor_assert(circ->state == CIRCUIT_STATE_OPEN); tor_assert(ocirc->build_state->chosen_exit); tor_assert(ocirc->rend_data); - /* treat this like getting a nack from it */ - log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s", - safe_str_client(ocirc->rend_data->onion_address), - safe_str_client(build_state_get_exit_nickname(ocirc->build_state)), - timed_out ? "Recording timeout." : "Removing from descriptor."); - rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit, - ocirc->rend_data, - timed_out ? - INTRO_POINT_FAILURE_TIMEOUT : - INTRO_POINT_FAILURE_GENERIC); + if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT) { + /* treat this like getting a nack from it */ + log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s", + safe_str_client(ocirc->rend_data->onion_address), + safe_str_client(build_state_get_exit_nickname(ocirc->build_state)), + timed_out ? "Recording timeout." : "Removing from descriptor."); + rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit, + ocirc->rend_data, + timed_out ? + INTRO_POINT_FAILURE_TIMEOUT : + INTRO_POINT_FAILURE_GENERIC); + } } else if (circ->purpose == CIRCUIT_PURPOSE_C_INTRODUCING && reason != END_CIRC_REASON_TIMEOUT) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); if (ocirc->build_state->chosen_exit && ocirc->rend_data) { - log_info(LD_REND, "Failed intro circ %s to %s " - "(building circuit to intro point). " - "Marking intro point as possibly unreachable.", - safe_str_client(ocirc->rend_data->onion_address), - safe_str_client(build_state_get_exit_nickname(ocirc->build_state))); - rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit, - ocirc->rend_data, - INTRO_POINT_FAILURE_UNREACHABLE); + if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT) { + log_info(LD_REND, "Failed intro circ %s to %s " + "(building circuit to intro point). " + "Marking intro point as possibly unreachable.", + safe_str_client(ocirc->rend_data->onion_address), + safe_str_client(build_state_get_exit_nickname( + ocirc->build_state))); + rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit, + ocirc->rend_data, + INTRO_POINT_FAILURE_UNREACHABLE); + } } } + if (circ->n_chan) { circuit_clear_cell_queue(circ, circ->n_chan); /* Only send destroy if the channel isn't closing anyway */ - if (!(circ->n_chan->state == CHANNEL_STATE_CLOSING || - circ->n_chan->state == CHANNEL_STATE_CLOSED || - circ->n_chan->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(circ->n_chan)) { channel_send_destroy(circ->n_circ_id, circ->n_chan, reason); } circuitmux_detach_circuit(circ->n_chan->cmux, circ); + circuit_set_n_circid_chan(circ, 0, NULL); } if (! CIRCUIT_IS_ORIGIN(circ)) { @@ -1477,12 +1893,11 @@ circuit_mark_for_close_(circuit_t *circ, int reason, int line, if (or_circ->p_chan) { circuit_clear_cell_queue(circ, or_circ->p_chan); /* Only send destroy if the channel isn't closing anyway */ - if (!(or_circ->p_chan->state == CHANNEL_STATE_CLOSING || - or_circ->p_chan->state == CHANNEL_STATE_CLOSED || - or_circ->p_chan->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(or_circ->p_chan)) { channel_send_destroy(or_circ->p_circ_id, or_circ->p_chan, reason); } circuitmux_detach_circuit(or_circ->p_chan->cmux, circ); + circuit_set_p_circid_chan(or_circ, 0, NULL); } } else { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); @@ -1491,20 +1906,6 @@ circuit_mark_for_close_(circuit_t *circ, int reason, int line, connection_edge_destroy(circ->n_circ_id, conn); ocirc->p_streams = NULL; } - - circ->marked_for_close = line; - circ->marked_for_close_file = file; - - if (!CIRCUIT_IS_ORIGIN(circ)) { - or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); - if (or_circ->rend_splice) { - if (!or_circ->rend_splice->base_.marked_for_close) { - /* do this after marking this circuit, to avoid infinite recursion. */ - circuit_mark_for_close(TO_CIRCUIT(or_circ->rend_splice), reason); - } - or_circ->rend_splice = NULL; - } - } } /** Given a marked circuit <b>circ</b>, aggressively free its cell queues to @@ -1517,12 +1918,69 @@ marked_circuit_free_cells(circuit_t *circ) return; } cell_queue_clear(&circ->n_chan_cells); - if (! CIRCUIT_IS_ORIGIN(circ)) - cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_chan_cells); + if (circ->n_mux) + circuitmux_clear_num_cells(circ->n_mux, circ); + if (! CIRCUIT_IS_ORIGIN(circ)) { + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + cell_queue_clear(&orcirc->p_chan_cells); + if (orcirc->p_mux) + circuitmux_clear_num_cells(orcirc->p_mux, circ); + } } -/** Return the number of cells used by the circuit <b>c</b>'s cell queues. */ static size_t +single_conn_free_bytes(connection_t *conn) +{ + size_t result = 0; + if (conn->inbuf) { + result += buf_allocation(conn->inbuf); + buf_clear(conn->inbuf); + } + if (conn->outbuf) { + result += buf_allocation(conn->outbuf); + buf_clear(conn->outbuf); + } + if (conn->type == CONN_TYPE_DIR) { + dir_connection_t *dir_conn = TO_DIR_CONN(conn); + if (dir_conn->zlib_state) { + result += tor_zlib_state_size(dir_conn->zlib_state); + tor_zlib_free(dir_conn->zlib_state); + dir_conn->zlib_state = NULL; + } + } + return result; +} + +/** Aggressively free buffer contents on all the buffers of all streams in the + * list starting at <b>stream</b>. Return the number of bytes recovered. */ +static size_t +marked_circuit_streams_free_bytes(edge_connection_t *stream) +{ + size_t result = 0; + for ( ; stream; stream = stream->next_stream) { + connection_t *conn = TO_CONN(stream); + result += single_conn_free_bytes(conn); + if (conn->linked_conn) { + result += single_conn_free_bytes(conn->linked_conn); + } + } + return result; +} + +/** Aggressively free buffer contents on all the buffers of all streams on + * circuit <b>c</b>. Return the number of bytes recovered. */ +static size_t +marked_circuit_free_stream_bytes(circuit_t *c) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return marked_circuit_streams_free_bytes(TO_ORIGIN_CIRCUIT(c)->p_streams); + } else { + return marked_circuit_streams_free_bytes(TO_OR_CIRCUIT(c)->n_streams); + } +} + +/** Return the number of cells used by the circuit <b>c</b>'s cell queues. */ +STATIC size_t n_cells_in_circ_queues(const circuit_t *c) { size_t n = c->n_chan_cells.n; @@ -1541,17 +1999,19 @@ n_cells_in_circ_queues(const circuit_t *c) * This function will return incorrect results if the oldest cell queued on * the circuit is older than 2**32 msec (about 49 days) old. */ -static uint32_t +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) { uint32_t age = 0; - if (c->n_chan_cells.head) - age = now - c->n_chan_cells.head->inserted_time; + packed_cell_t *cell; + + if (NULL != (cell = TOR_SIMPLEQ_FIRST(&c->n_chan_cells.head))) + age = now - cell->inserted_time; if (! CIRCUIT_IS_ORIGIN(c)) { - const or_circuit_t *orcirc = TO_OR_CIRCUIT((circuit_t*)c); - if (orcirc->p_chan_cells.head) { - uint32_t age2 = now - orcirc->p_chan_cells.head->inserted_time; + const or_circuit_t *orcirc = CONST_TO_OR_CIRCUIT(c); + if (NULL != (cell = TOR_SIMPLEQ_FIRST(&orcirc->p_chan_cells.head))) { + uint32_t age2 = now - cell->inserted_time; if (age2 > age) return age2; } @@ -1559,20 +2019,107 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) return age; } -/** Temporary variable for circuits_compare_by_oldest_queued_cell_ This is a - * kludge to work around the fact that qsort doesn't provide a way for - * comparison functions to take an extra argument. */ -static uint32_t circcomp_now_tmp; +/** Return the age in milliseconds of the oldest buffer chunk on <b>conn</b>, + * where age is taken in milliseconds before the time <b>now</b> (in truncated + * milliseconds since the epoch). If the connection has no data, treat + * it as having age zero. + **/ +static uint32_t +conn_get_buffer_age(const connection_t *conn, uint32_t now) +{ + uint32_t age = 0, age2; + if (conn->outbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); + if (age2 > age) + age = age2; + } + if (conn->inbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); + if (age2 > age) + age = age2; + } + return age; +} + +/** Return the age in milliseconds of the oldest buffer chunk on any stream in + * the linked list <b>stream</b>, where age is taken in milliseconds before + * the time <b>now</b> (in truncated milliseconds since the epoch). */ +static uint32_t +circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) +{ + uint32_t age = 0, age2; + for (; stream; stream = stream->next_stream) { + const connection_t *conn = TO_CONN(stream); + age2 = conn_get_buffer_age(conn, now); + if (age2 > age) + age = age2; + if (conn->linked_conn) { + age2 = conn_get_buffer_age(conn->linked_conn, now); + if (age2 > age) + age = age2; + } + } + return age; +} + +/** Return the age in milliseconds of the oldest buffer chunk on any stream + * attached to the circuit <b>c</b>, where age is taken in milliseconds before + * the time <b>now</b> (in truncated milliseconds since the epoch). */ +STATIC uint32_t +circuit_max_queued_data_age(const circuit_t *c, uint32_t now) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return circuit_get_streams_max_data_age( + CONST_TO_ORIGIN_CIRCUIT(c)->p_streams, now); + } else { + return circuit_get_streams_max_data_age( + CONST_TO_OR_CIRCUIT(c)->n_streams, now); + } +} + +/** Return the age of the oldest cell or stream buffer chunk on the circuit + * <b>c</b>, where age is taken in milliseconds before the time <b>now</b> (in + * truncated milliseconds since the epoch). */ +STATIC uint32_t +circuit_max_queued_item_age(const circuit_t *c, uint32_t now) +{ + uint32_t cell_age = circuit_max_queued_cell_age(c, now); + uint32_t data_age = circuit_max_queued_data_age(c, now); + if (cell_age > data_age) + return cell_age; + else + return data_age; +} -/** Helper to sort a list of circuit_t by age of oldest cell, in descending - * order. Requires that circcomp_now_tmp is set correctly. */ +/** Helper to sort a list of circuit_t by age of oldest item, in descending + * order. */ static int -circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) +circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) { const circuit_t *a = *a_; const circuit_t *b = *b_; - uint32_t age_a = circuit_max_queued_cell_age(a, circcomp_now_tmp); - uint32_t age_b = circuit_max_queued_cell_age(b, circcomp_now_tmp); + uint32_t age_a = a->age_tmp; + uint32_t age_b = b->age_tmp; + + if (age_a < age_b) + return 1; + else if (age_a == age_b) + return 0; + else + return -1; +} + +static uint32_t now_ms_for_buf_cmp; + +/** Helper to sort a list of circuit_t by age of oldest item, in descending + * order. */ +static int +conns_compare_by_buffer_age_(const void **a_, const void **b_) +{ + const connection_t *a = *a_; + const connection_t *b = *b_; + time_t age_a = conn_get_buffer_age(a, now_ms_for_buf_cmp); + time_t age_b = conn_get_buffer_age(b, now_ms_for_buf_cmp); if (age_a < age_b) return 1; @@ -1582,69 +2129,117 @@ circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) return -1; } -#define FRACTION_OF_CELLS_TO_RETAIN_ON_OOM 0.90 +#define FRACTION_OF_DATA_TO_RETAIN_ON_OOM 0.90 /** We're out of memory for cells, having allocated <b>current_allocation</b> * bytes' worth. Kill the 'worst' circuits until we're under - * FRACTION_OF_CIRCS_TO_RETAIN_ON_OOM of our maximum usage. */ + * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. */ void circuits_handle_oom(size_t current_allocation) { - /* Let's hope there's enough slack space for this allocation here... */ - smartlist_t *circlist = smartlist_new(); - circuit_t *circ; - size_t n_cells_removed=0, n_cells_to_remove; + smartlist_t *circlist; + smartlist_t *connection_array = get_connection_array(); + int conn_idx; + size_t mem_to_recover; + size_t mem_recovered=0; int n_circuits_killed=0; + int n_dirconns_killed=0; struct timeval now; + uint32_t now_ms; log_notice(LD_GENERAL, "We're low on memory. Killing circuits with " "over-long queues. (This behavior is controlled by " - "MaxMemInCellQueues.)"); + "MaxMemInQueues.)"); { - size_t mem_target = (size_t)(get_options()->MaxMemInCellQueues * - FRACTION_OF_CELLS_TO_RETAIN_ON_OOM); - size_t mem_to_recover; + size_t mem_target = (size_t)(get_options()->MaxMemInQueues * + FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) return; mem_to_recover = current_allocation - mem_target; - n_cells_to_remove = CEIL_DIV(mem_to_recover, packed_cell_mem_cost()); } - /* This algorithm itself assumes that you've got enough memory slack - * to actually run it. */ - for (circ = global_circuitlist; circ; circ = circ->next) - smartlist_add(circlist, circ); + tor_gettimeofday_cached_monotonic(&now); + now_ms = (uint32_t)tv_to_msec(&now); - /* Set circcomp_now_tmp so that the sort can work. */ - tor_gettimeofday_cached(&now); - circcomp_now_tmp = (uint32_t)tv_to_msec(&now); + circlist = circuit_get_global_list(); + SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { + circ->age_tmp = circuit_max_queued_item_age(circ, now_ms); + } SMARTLIST_FOREACH_END(circ); /* This is O(n log n); there are faster algorithms we could use instead. * Let's hope this doesn't happen enough to be in the critical path. */ - smartlist_sort(circlist, circuits_compare_by_oldest_queued_cell_); + smartlist_sort(circlist, circuits_compare_by_oldest_queued_item_); - /* Okay, now the worst circuits are at the front of the list. Let's mark - * them, and reclaim their storage aggressively. */ + /* Fix up the indices before we run into trouble */ SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { - size_t n = n_cells_in_circ_queues(circ); + circ->global_circuitlist_idx = circ_sl_idx; + } SMARTLIST_FOREACH_END(circ); + + /* Now sort the connection array ... */ + now_ms_for_buf_cmp = now_ms; + smartlist_sort(connection_array, conns_compare_by_buffer_age_); + now_ms_for_buf_cmp = 0; + + /* Fix up the connection array to its new order. */ + SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) { + conn->conn_array_index = conn_sl_idx; + } SMARTLIST_FOREACH_END(conn); + + /* Okay, now the worst circuits and connections are at the front of their + * respective lists. Let's mark them, and reclaim their storage + * aggressively. */ + conn_idx = 0; + SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { + size_t n; + size_t freed; + + /* Free storage in any non-linked directory connections that have buffered + * data older than this circuit. */ + while (conn_idx < smartlist_len(connection_array)) { + connection_t *conn = smartlist_get(connection_array, conn_idx); + uint32_t conn_age = conn_get_buffer_age(conn, now_ms); + if (conn_age < circ->age_tmp) { + break; + } + if (conn->type == CONN_TYPE_DIR && conn->linked_conn == NULL) { + if (!conn->marked_for_close) + connection_mark_for_close(conn); + mem_recovered += single_conn_free_bytes(conn); + + ++n_dirconns_killed; + + if (mem_recovered >= mem_to_recover) + goto done_recovering_mem; + } + ++conn_idx; + } + + /* Now, kill the circuit. */ + n = n_cells_in_circ_queues(circ); if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } marked_circuit_free_cells(circ); + freed = marked_circuit_free_stream_bytes(circ); ++n_circuits_killed; - n_cells_removed += n; - if (n_cells_removed >= n_cells_to_remove) - break; - } SMARTLIST_FOREACH_END(circ); - clean_cell_pool(); /* In case this helps. */ + mem_recovered += n * packed_cell_mem_cost(); + mem_recovered += freed; + + if (mem_recovered >= mem_to_recover) + goto done_recovering_mem; + } SMARTLIST_FOREACH_END(circ); - log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits.", - U64_PRINTF_ARG(n_cells_removed * packed_cell_mem_cost()), - n_circuits_killed); + done_recovering_mem: - smartlist_free(circlist); + log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; " + "%d circuits remain alive. Also killed %d non-linked directory " + "connections.", + U64_PRINTF_ARG(mem_recovered), + n_circuits_killed, + smartlist_len(circlist) - n_circuits_killed, + n_dirconns_killed); } /** Verify that cpath layer <b>cp</b> has all of its invariants @@ -1716,15 +2311,10 @@ assert_circuit_ok(const circuit_t *c) tor_assert(c->purpose >= CIRCUIT_PURPOSE_MIN_ && c->purpose <= CIRCUIT_PURPOSE_MAX_); - { - /* Having a separate variable for this pleases GCC 4.2 in ways I hope I - * never understand. -NM. */ - circuit_t *nonconst_circ = (circuit_t*) c; - if (CIRCUIT_IS_ORIGIN(c)) - origin_circ = TO_ORIGIN_CIRCUIT(nonconst_circ); - else - or_circ = TO_OR_CIRCUIT(nonconst_circ); - } + if (CIRCUIT_IS_ORIGIN(c)) + origin_circ = CONST_TO_ORIGIN_CIRCUIT(c); + else + or_circ = CONST_TO_OR_CIRCUIT(c); if (c->n_chan) { tor_assert(!c->n_hop); @@ -1733,15 +2323,16 @@ assert_circuit_ok(const circuit_t *c) /* We use the _impl variant here to make sure we don't fail on marked * circuits, which would not be returned by the regular function. */ circuit_t *c2 = circuit_get_by_circid_channel_impl(c->n_circ_id, - c->n_chan); + c->n_chan, NULL); tor_assert(c == c2); } } if (or_circ && or_circ->p_chan) { if (or_circ->p_circ_id) { /* ibid */ - circuit_t *c2 = circuit_get_by_circid_channel_impl(or_circ->p_circ_id, - or_circ->p_chan); + circuit_t *c2 = + circuit_get_by_circid_channel_impl(or_circ->p_circ_id, + or_circ->p_chan, NULL); tor_assert(c == c2); } } diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h index acc4b81fcd..2707b426ab 100644 --- a/src/or/circuitlist.h +++ b/src/or/circuitlist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,17 +12,22 @@ #ifndef TOR_CIRCUITLIST_H #define TOR_CIRCUITLIST_H -circuit_t * circuit_get_global_list_(void); +#include "testsupport.h" + +MOCK_DECL(smartlist_t *, circuit_get_global_list, (void)); const char *circuit_state_to_string(int state); const char *circuit_purpose_to_controller_string(uint8_t purpose); const char *circuit_purpose_to_controller_hs_state_string(uint8_t purpose); const char *circuit_purpose_to_string(uint8_t purpose); void circuit_dump_by_conn(connection_t *conn, int severity); -void circuit_dump_by_chan(channel_t *chan, int severity); void circuit_set_p_circid_chan(or_circuit_t *circ, circid_t id, channel_t *chan); void circuit_set_n_circid_chan(circuit_t *circ, circid_t id, channel_t *chan); +void channel_mark_circid_unusable(channel_t *chan, circid_t id); +void channel_mark_circid_usable(channel_t *chan, circid_t id); +time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id, + channel_t *chan); void circuit_set_state(circuit_t *circ, uint8_t state); void circuit_close_all_marked(void); int32_t circuit_initial_package_window(void); @@ -41,14 +46,16 @@ origin_circuit_t *circuit_get_ready_rend_circ_by_rend_data( const rend_data_t *rend_data); origin_circuit_t *circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, const char *digest, uint8_t purpose); -or_circuit_t *circuit_get_rendezvous(const char *cookie); -or_circuit_t *circuit_get_intro_point(const char *digest); +or_circuit_t *circuit_get_rendezvous(const uint8_t *cookie); +or_circuit_t *circuit_get_intro_point(const uint8_t *digest); +void circuit_set_rendezvous_cookie(or_circuit_t *circ, const uint8_t *cookie); +void circuit_set_intro_point_digest(or_circuit_t *circ, const uint8_t *digest); origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int flags); void circuit_mark_all_unused_circs(void); void circuit_mark_all_dirty_circs_as_unusable(void); -void circuit_mark_for_close_(circuit_t *circ, int reason, - int line, const char *file); +MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason, + int line, const char *file)); int circuit_get_cpath_len(origin_circuit_t *circ); void circuit_clear_cpath(origin_circuit_t *circ); crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); @@ -64,5 +71,19 @@ void assert_circuit_ok(const circuit_t *c); void circuit_free_all(void); void circuits_handle_oom(size_t current_allocation); +void circuit_clear_testing_cell_stats(circuit_t *circ); + +void channel_note_destroy_pending(channel_t *chan, circid_t id); +MOCK_DECL(void, channel_note_destroy_not_pending, + (channel_t *chan, circid_t id)); + +#ifdef CIRCUITLIST_PRIVATE +STATIC void circuit_free(circuit_t *circ); +STATIC size_t n_cells_in_circ_queues(const circuit_t *c); +STATIC uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now); +#endif + #endif diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c index 545cfd0650..cc1c4cd401 100644 --- a/src/or/circuitmux.c +++ b/src/or/circuitmux.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -10,6 +10,7 @@ #include "channel.h" #include "circuitlist.h" #include "circuitmux.h" +#include "relay.h" /* * Private typedefs for circuitmux.c @@ -115,6 +116,22 @@ struct circuitmux_s { */ struct circuit_t *active_circuits_head, *active_circuits_tail; + /** List of queued destroy cells */ + cell_queue_t destroy_cell_queue; + /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit + * returned the destroy queue. Used to force alternation between + * destroy/non-destroy cells. + * + * XXXX There is no reason to think that alternating is a particularly good + * approach -- it's just designed to prevent destroys from starving other + * cells completely. + */ + unsigned int last_cell_was_destroy : 1; + /** Destroy counter: increment this when a destroy gets queued, decrement + * when we unqueue it, so we can test to make sure they don't starve. + */ + int64_t destroy_ctr; + /* * Circuitmux policy; if this is non-NULL, it can override the built- * in round-robin active circuits behavior. This is how EWMA works in @@ -169,10 +186,10 @@ struct chanid_circid_muxinfo_t { * Static function declarations */ -static INLINE int +static inline int chanid_circid_entries_eq(chanid_circid_muxinfo_t *a, chanid_circid_muxinfo_t *b); -static INLINE unsigned int +static inline unsigned int chanid_circid_entry_hash(chanid_circid_muxinfo_t *a); static chanid_circid_muxinfo_t * circuitmux_find_map_entry(circuitmux_t *cmux, circuit_t *circ); @@ -182,17 +199,22 @@ circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ, static void circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ, cell_direction_t direction); -static INLINE void +static inline void circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ, cell_direction_t direction); -static INLINE circuit_t ** +static inline circuit_t ** circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ); -static INLINE circuit_t ** +static inline circuit_t ** circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ); static void circuitmux_assert_okay_pass_one(circuitmux_t *cmux); static void circuitmux_assert_okay_pass_two(circuitmux_t *cmux); static void circuitmux_assert_okay_pass_three(circuitmux_t *cmux); +/* Static global variables */ + +/** Count the destroy balance to debug destroy queue logic */ +static int64_t global_destroy_ctr = 0; + /* Function definitions */ /** @@ -204,7 +226,7 @@ static void circuitmux_assert_okay_pass_three(circuitmux_t *cmux); * used by circuitmux_notify_xmit_cells(). */ -static INLINE void +static inline void circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ, cell_direction_t direction) { @@ -284,7 +306,7 @@ circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ, circuitmux_assert_okay_paranoid(cmux); } -static INLINE circuit_t ** +static inline circuit_t ** circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ) { tor_assert(cmux); @@ -297,7 +319,7 @@ circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ) } } -static INLINE circuit_t ** +static inline circuit_t ** circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ) { tor_assert(cmux); @@ -316,7 +338,7 @@ circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ) * than zero appropriately. */ -static INLINE int +static inline int chanid_circid_entries_eq(chanid_circid_muxinfo_t *a, chanid_circid_muxinfo_t *b) { @@ -327,7 +349,7 @@ chanid_circid_entries_eq(chanid_circid_muxinfo_t *a, * Helper: return a hash based on circuit ID and channel ID in a. */ -static INLINE unsigned int +static inline unsigned int chanid_circid_entry_hash(chanid_circid_muxinfo_t *a) { return (((unsigned int)(a->circ_id) << 8) ^ @@ -341,9 +363,9 @@ HT_HEAD(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t); /* Emit a bunch of hash table stuff */ HT_PROTOTYPE(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node, chanid_circid_entry_hash, chanid_circid_entries_eq); -HT_GENERATE(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node, - chanid_circid_entry_hash, chanid_circid_entries_eq, 0.6, - malloc, realloc, free); +HT_GENERATE2(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node, + chanid_circid_entry_hash, chanid_circid_entries_eq, 0.6, + tor_reallocarray_, tor_free_) /* * Circuitmux alloc/free functions @@ -361,16 +383,20 @@ circuitmux_alloc(void) rv = tor_malloc_zero(sizeof(*rv)); rv->chanid_circid_map = tor_malloc_zero(sizeof(*( rv->chanid_circid_map))); HT_INIT(chanid_circid_muxinfo_map, rv->chanid_circid_map); + cell_queue_init(&rv->destroy_cell_queue); return rv; } /** * Detach all circuits from a circuitmux (use before circuitmux_free()) + * + * If <b>detached_out</b> is non-NULL, add every detached circuit_t to + * detached_out. */ void -circuitmux_detach_all_circuits(circuitmux_t *cmux) +circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) { chanid_circid_muxinfo_t **i = NULL, *to_remove; channel_t *chan = NULL; @@ -386,7 +412,11 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map); while (i) { to_remove = *i; - if (to_remove) { + + if (! to_remove) { + log_warn(LD_BUG, "Somehow, an HT iterator gave us a NULL pointer."); + break; + } else { /* Find a channel and circuit */ chan = channel_find_by_global_id(to_remove->chan_id); if (chan) { @@ -407,6 +437,9 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) /* Clear n_mux */ circ->n_mux = NULL; + + if (detached_out) + smartlist_add(detached_out, circ); } else if (circ->magic == OR_CIRCUIT_MAGIC) { /* * Update active_circuits et al.; this does policy notifies, so @@ -422,6 +455,9 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) * so clear p_mux. */ TO_OR_CIRCUIT(circ)->p_mux = NULL; + + if (detached_out) + smartlist_add(detached_out, circ); } else { /* Complain and move on */ log_warn(LD_CIRC, @@ -476,6 +512,31 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux) cmux->n_cells = 0; } +/** Reclaim all circuit IDs currently marked as unusable on <b>chan</b> because + * of pending destroy cells in <b>cmux</b>. + * + * This function must be called AFTER circuits are unlinked from the (channel, + * circuid-id) map with circuit_unlink_all_from_channel(), but before calling + * circuitmux_free(). + */ +void +circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux, channel_t *chan) +{ + packed_cell_t *cell; + int n_bad = 0; + TOR_SIMPLEQ_FOREACH(cell, &cmux->destroy_cell_queue.head, next) { + circid_t circid = 0; + if (packed_cell_is_destroy(chan, cell, &circid)) { + channel_mark_circid_usable(chan, circid); + } else { + ++n_bad; + } + } + if (n_bad) + log_warn(LD_BUG, "%d cell(s) on destroy queue did not look like a " + "DESTROY cell.", n_bad); +} + /** * Free a circuitmux_t; the circuits must be detached first with * circuitmux_detach_all_circuits(). @@ -508,6 +569,30 @@ circuitmux_free(circuitmux_t *cmux) tor_free(cmux->chanid_circid_map); } + /* + * We're throwing away some destroys; log the counter and + * adjust the global counter by the queue size. + */ + if (cmux->destroy_cell_queue.n > 0) { + cmux->destroy_ctr -= cmux->destroy_cell_queue.n; + global_destroy_ctr -= cmux->destroy_cell_queue.n; + log_debug(LD_CIRC, + "Freeing cmux at %p with %u queued destroys; the last cmux " + "destroy balance was "I64_FORMAT", global is "I64_FORMAT, + cmux, cmux->destroy_cell_queue.n, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); + } else { + log_debug(LD_CIRC, + "Freeing cmux at %p with no queued destroys, the cmux destroy " + "balance was "I64_FORMAT", global is "I64_FORMAT, + cmux, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); + } + + cell_queue_clear(&cmux->destroy_cell_queue); + tor_free(cmux); } @@ -536,8 +621,8 @@ circuitmux_clear_policy(circuitmux_t *cmux) * Return the policy currently installed on a circuitmux_t */ -const circuitmux_policy_t * -circuitmux_get_policy(circuitmux_t *cmux) +MOCK_IMPL(const circuitmux_policy_t *, +circuitmux_get_policy, (circuitmux_t *cmux)) { tor_assert(cmux); @@ -811,12 +896,12 @@ circuitmux_num_cells_for_circuit(circuitmux_t *cmux, circuit_t *circ) * Query total number of available cells on a circuitmux */ -unsigned int -circuitmux_num_cells(circuitmux_t *cmux) +MOCK_IMPL(unsigned int, +circuitmux_num_cells, (circuitmux_t *cmux)) { tor_assert(cmux); - return cmux->n_cells; + return cmux->n_cells + cmux->destroy_cell_queue.n; } /** @@ -851,9 +936,9 @@ circuitmux_num_circuits(circuitmux_t *cmux) * Attach a circuit to a circuitmux, for the specified direction. */ -void -circuitmux_attach_circuit(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction) +MOCK_IMPL(void, +circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, + cell_direction_t direction)) { channel_t *chan = NULL; uint64_t channel_id; @@ -1000,15 +1085,18 @@ circuitmux_attach_circuit(circuitmux_t *cmux, circuit_t *circ, * no-op if not attached. */ -void -circuitmux_detach_circuit(circuitmux_t *cmux, circuit_t *circ) +MOCK_IMPL(void, +circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) { chanid_circid_muxinfo_t search, *hashent = NULL; /* * Use this to keep track of whether we found it for n_chan or * p_chan for consistency checking. + * + * The 0 initializer is not a valid cell_direction_t value. + * We assert that it has been replaced with a valid value before it is used. */ - cell_direction_t last_searched_direction; + cell_direction_t last_searched_direction = 0; tor_assert(cmux); tor_assert(cmux->chanid_circid_map); @@ -1038,6 +1126,9 @@ circuitmux_detach_circuit(circuitmux_t *cmux, circuit_t *circ) } } + tor_assert(last_searched_direction == CELL_DIRECTION_OUT + || last_searched_direction == CELL_DIRECTION_IN); + /* * If hashent isn't NULL, we have a circuit to detach; don't remove it from * the map until later of circuitmux_make_circuit_inactive() breaks. @@ -1368,16 +1459,36 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, /** * Pick a circuit to send from, using the active circuits list or a * circuitmux policy if one is available. This is called from channel.c. + * + * If we would rather send a destroy cell, return NULL and set + * *<b>destroy_queue_out</b> to the destroy queue. + * + * If we have nothing to send, set *<b>destroy_queue_out</b> to NULL and + * return NULL. */ circuit_t * -circuitmux_get_first_active_circuit(circuitmux_t *cmux) +circuitmux_get_first_active_circuit(circuitmux_t *cmux, + cell_queue_t **destroy_queue_out) { circuit_t *circ = NULL; tor_assert(cmux); + tor_assert(destroy_queue_out); + + *destroy_queue_out = NULL; + + if (cmux->destroy_cell_queue.n && + (!cmux->last_cell_was_destroy || cmux->n_active_circuits == 0)) { + /* We have destroy cells to send, and either we just sent a relay cell, + * or we have no relay cells to send. */ - if (cmux->n_active_circuits > 0) { + /* XXXX We should let the cmux policy have some say in this eventually. */ + /* XXXX Alternating is not a terribly brilliant approach here. */ + *destroy_queue_out = &cmux->destroy_cell_queue; + + cmux->last_cell_was_destroy = 1; + } else if (cmux->n_active_circuits > 0) { /* We also must have a cell available for this to be the case */ tor_assert(cmux->n_cells > 0); /* Do we have a policy-provided circuit selector? */ @@ -1389,7 +1500,11 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux) tor_assert(cmux->active_circuits_head); circ = cmux->active_circuits_head; } - } else tor_assert(cmux->n_cells == 0); + cmux->last_cell_was_destroy = 0; + } else { + tor_assert(cmux->n_cells == 0); + tor_assert(cmux->destroy_cell_queue.n == 0); + } return circ; } @@ -1463,6 +1578,26 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, circuitmux_assert_okay_paranoid(cmux); } +/** + * Notify the circuitmux that a destroy was sent, so we can update + * the counter. + */ + +void +circuitmux_notify_xmit_destroy(circuitmux_t *cmux) +{ + tor_assert(cmux); + + --(cmux->destroy_ctr); + --(global_destroy_ctr); + log_debug(LD_CIRC, + "Cmux at %p sent a destroy, cmux counter is now "I64_FORMAT", " + "global counter is now "I64_FORMAT, + cmux, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); +} + /* * Circuitmux consistency checking assertions */ @@ -1743,3 +1878,124 @@ circuitmux_assert_okay_pass_three(circuitmux_t *cmux) } } +/*DOCDOC */ +void +circuitmux_append_destroy_cell(channel_t *chan, + circuitmux_t *cmux, + circid_t circ_id, + uint8_t reason) +{ + cell_t cell; + memset(&cell, 0, sizeof(cell_t)); + cell.circ_id = circ_id; + cell.command = CELL_DESTROY; + cell.payload[0] = (uint8_t) reason; + + cell_queue_append_packed_copy(NULL, &cmux->destroy_cell_queue, 0, &cell, + chan->wide_circ_ids, 0); + + /* Destroy entering the queue, update counters */ + ++(cmux->destroy_ctr); + ++global_destroy_ctr; + log_debug(LD_CIRC, + "Cmux at %p queued a destroy for circ %u, cmux counter is now " + I64_FORMAT", global counter is now "I64_FORMAT, + cmux, circ_id, + I64_PRINTF_ARG(cmux->destroy_ctr), + I64_PRINTF_ARG(global_destroy_ctr)); + + /* XXXX Duplicate code from append_cell_to_circuit_queue */ + if (!channel_has_queued_writes(chan)) { + /* There is no data at all waiting to be sent on the outbuf. Add a + * cell, so that we can notice when it gets flushed, flushed_some can + * get called, and we can start putting more data onto the buffer then. + */ + log_debug(LD_GENERAL, "Primed a buffer."); + channel_flush_from_first_active_circuit(chan, 1); + } +} + +/*DOCDOC; for debugging 12184. This runs slowly. */ +int64_t +circuitmux_count_queued_destroy_cells(const channel_t *chan, + const circuitmux_t *cmux) +{ + int64_t n_destroy_cells = cmux->destroy_ctr; + int64_t destroy_queue_size = cmux->destroy_cell_queue.n; + + int64_t manual_total = 0; + int64_t manual_total_in_map = 0; + packed_cell_t *cell; + + TOR_SIMPLEQ_FOREACH(cell, &cmux->destroy_cell_queue.head, next) { + circid_t id; + ++manual_total; + + id = packed_cell_get_circid(cell, chan->wide_circ_ids); + if (circuit_id_in_use_on_channel(id, (channel_t*)chan)) + ++manual_total_in_map; + } + + if (n_destroy_cells != destroy_queue_size || + n_destroy_cells != manual_total || + n_destroy_cells != manual_total_in_map) { + log_warn(LD_BUG, " Discrepancy in counts for queued destroy cells on " + "circuitmux. n="I64_FORMAT". queue_size="I64_FORMAT". " + "manual_total="I64_FORMAT". manual_total_in_map="I64_FORMAT".", + I64_PRINTF_ARG(n_destroy_cells), + I64_PRINTF_ARG(destroy_queue_size), + I64_PRINTF_ARG(manual_total), + I64_PRINTF_ARG(manual_total_in_map)); + } + + return n_destroy_cells; +} + +/** + * Compare cmuxes to see which is more preferred; return < 0 if + * cmux_1 has higher priority (i.e., cmux_1 < cmux_2 in the scheduler's + * sort order), > 0 if cmux_2 has higher priority, or 0 if they are + * equally preferred. + * + * If the cmuxes have different cmux policies or the policy does not + * support the cmp_cmux method, return 0. + */ + +MOCK_IMPL(int, +circuitmux_compare_muxes, (circuitmux_t *cmux_1, circuitmux_t *cmux_2)) +{ + const circuitmux_policy_t *policy; + + tor_assert(cmux_1); + tor_assert(cmux_2); + + if (cmux_1 == cmux_2) { + /* Equivalent because they're the same cmux */ + return 0; + } + + if (cmux_1->policy && cmux_2->policy) { + if (cmux_1->policy == cmux_2->policy) { + policy = cmux_1->policy; + + if (policy->cmp_cmux) { + /* Okay, we can compare! */ + return policy->cmp_cmux(cmux_1, cmux_1->policy_data, + cmux_2, cmux_2->policy_data); + } else { + /* + * Equivalent because the policy doesn't know how to compare between + * muxes. + */ + return 0; + } + } else { + /* Equivalent because they have different policies */ + return 0; + } + } else { + /* Equivalent because one or both are missing a policy */ + return 0; + } +} + diff --git a/src/or/circuitmux.h b/src/or/circuitmux.h index 25644ffab7..00745ac4a1 100644 --- a/src/or/circuitmux.h +++ b/src/or/circuitmux.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -10,6 +10,7 @@ #define TOR_CIRCUITMUX_H #include "or.h" +#include "testsupport.h" typedef struct circuitmux_policy_s circuitmux_policy_t; typedef struct circuitmux_policy_data_s circuitmux_policy_data_t; @@ -56,6 +57,9 @@ struct circuitmux_policy_s { /* Choose a circuit */ circuit_t * (*pick_active_circuit)(circuitmux_t *cmux, circuitmux_policy_data_t *pol_data); + /* Optional: channel comparator for use by the scheduler */ + int (*cmp_cmux)(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1, + circuitmux_t *cmux_2, circuitmux_policy_data_t *pol_data_2); }; /* @@ -98,12 +102,14 @@ void circuitmux_assert_okay(circuitmux_t *cmux); /* Create/destroy */ circuitmux_t * circuitmux_alloc(void); -void circuitmux_detach_all_circuits(circuitmux_t *cmux); +void circuitmux_detach_all_circuits(circuitmux_t *cmux, + smartlist_t *detached_out); void circuitmux_free(circuitmux_t *cmux); /* Policy control */ void circuitmux_clear_policy(circuitmux_t *cmux); -const circuitmux_policy_t * circuitmux_get_policy(circuitmux_t *cmux); +MOCK_DECL(const circuitmux_policy_t *, + circuitmux_get_policy, (circuitmux_t *cmux)); void circuitmux_set_policy(circuitmux_t *cmux, const circuitmux_policy_t *pol); @@ -115,22 +121,40 @@ int circuitmux_is_circuit_attached(circuitmux_t *cmux, circuit_t *circ); int circuitmux_is_circuit_active(circuitmux_t *cmux, circuit_t *circ); unsigned int circuitmux_num_cells_for_circuit(circuitmux_t *cmux, circuit_t *circ); -unsigned int circuitmux_num_cells(circuitmux_t *cmux); +MOCK_DECL(unsigned int, circuitmux_num_cells, (circuitmux_t *cmux)); unsigned int circuitmux_num_circuits(circuitmux_t *cmux); unsigned int circuitmux_num_active_circuits(circuitmux_t *cmux); +/* Debuging interface - slow. */ +int64_t circuitmux_count_queued_destroy_cells(const channel_t *chan, + const circuitmux_t *cmux); + /* Channel interface */ -circuit_t * circuitmux_get_first_active_circuit(circuitmux_t *cmux); +circuit_t * circuitmux_get_first_active_circuit(circuitmux_t *cmux, + cell_queue_t **destroy_queue_out); void circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, unsigned int n_cells); +void circuitmux_notify_xmit_destroy(circuitmux_t *cmux); /* Circuit interface */ -void circuitmux_attach_circuit(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction); -void circuitmux_detach_circuit(circuitmux_t *cmux, circuit_t *circ); +MOCK_DECL(void, circuitmux_attach_circuit, (circuitmux_t *cmux, + circuit_t *circ, + cell_direction_t direction)); +MOCK_DECL(void, circuitmux_detach_circuit, + (circuitmux_t *cmux, circuit_t *circ)); void circuitmux_clear_num_cells(circuitmux_t *cmux, circuit_t *circ); void circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, unsigned int n_cells); +void circuitmux_append_destroy_cell(channel_t *chan, + circuitmux_t *cmux, circid_t circ_id, + uint8_t reason); +void circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux, + channel_t *chan); + +/* Optional interchannel comparisons for scheduling */ +MOCK_DECL(int, circuitmux_compare_muxes, + (circuitmux_t *cmux_1, circuitmux_t *cmux_2)); + #endif /* TOR_CIRCUITMUX_H */ diff --git a/src/or/circuitmux_ewma.c b/src/or/circuitmux_ewma.c index 3f37d7b9a0..b784a140ac 100644 --- a/src/or/circuitmux_ewma.c +++ b/src/or/circuitmux_ewma.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -115,7 +115,7 @@ TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *); * if the cast is impossible. */ -static INLINE ewma_policy_data_t * +static inline ewma_policy_data_t * TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol) { if (!pol) return NULL; @@ -130,7 +130,7 @@ TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol) * and assert if the cast is impossible. */ -static INLINE ewma_policy_circ_data_t * +static inline ewma_policy_circ_data_t * TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol) { if (!pol) return NULL; @@ -147,7 +147,7 @@ static int compare_cell_ewma_counts(const void *p1, const void *p2); static unsigned cell_ewma_tick_from_timeval(const struct timeval *now, double *remainder_out); static circuit_t * cell_ewma_to_circuit(cell_ewma_t *ewma); -static INLINE double get_scale_factor(unsigned from_tick, unsigned to_tick); +static inline double get_scale_factor(unsigned from_tick, unsigned to_tick); static cell_ewma_t * pop_first_cell_ewma(ewma_policy_data_t *pol); static void remove_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma); static void scale_single_cell_ewma(cell_ewma_t *ewma, unsigned cur_tick); @@ -187,6 +187,9 @@ ewma_notify_xmit_cells(circuitmux_t *cmux, static circuit_t * ewma_pick_active_circuit(circuitmux_t *cmux, circuitmux_policy_data_t *pol_data); +static int +ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1, + circuitmux_t *cmux_2, circuitmux_policy_data_t *pol_data_2); /*** EWMA global variables ***/ @@ -209,7 +212,8 @@ circuitmux_policy_t ewma_policy = { /*.notify_circ_inactive =*/ ewma_notify_circ_inactive, /*.notify_set_n_cells =*/ NULL, /* EWMA doesn't need this */ /*.notify_xmit_cells =*/ ewma_notify_xmit_cells, - /*.pick_active_circuit =*/ ewma_pick_active_circuit + /*.pick_active_circuit =*/ ewma_pick_active_circuit, + /*.cmp_cmux =*/ ewma_cmp_cmux }; /*** EWMA method implementations using the below EWMA helper functions ***/ @@ -273,8 +277,8 @@ ewma_alloc_circ_data(circuitmux_t *cmux, tor_assert(circ); tor_assert(direction == CELL_DIRECTION_OUT || direction == CELL_DIRECTION_IN); - /* Shut the compiler up */ - tor_assert(cell_count == cell_count); + /* Shut the compiler up without triggering -Wtautological-compare */ + (void)cell_count; cdata = tor_malloc_zero(sizeof(*cdata)); cdata->base_.magic = EWMA_POL_CIRC_DATA_MAGIC; @@ -453,6 +457,58 @@ ewma_pick_active_circuit(circuitmux_t *cmux, return circ; } +/** + * Compare two EWMA cmuxes, and return -1, 0 or 1 to indicate which should + * be more preferred - see circuitmux_compare_muxes() of circuitmux.c. + */ + +static int +ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1, + circuitmux_t *cmux_2, circuitmux_policy_data_t *pol_data_2) +{ + ewma_policy_data_t *p1 = NULL, *p2 = NULL; + cell_ewma_t *ce1 = NULL, *ce2 = NULL; + + tor_assert(cmux_1); + tor_assert(pol_data_1); + tor_assert(cmux_2); + tor_assert(pol_data_2); + + p1 = TO_EWMA_POL_DATA(pol_data_1); + p2 = TO_EWMA_POL_DATA(pol_data_1); + + if (p1 != p2) { + /* Get the head cell_ewma_t from each queue */ + if (smartlist_len(p1->active_circuit_pqueue) > 0) { + ce1 = smartlist_get(p1->active_circuit_pqueue, 0); + } + + if (smartlist_len(p2->active_circuit_pqueue) > 0) { + ce2 = smartlist_get(p2->active_circuit_pqueue, 0); + } + + /* Got both of them? */ + if (ce1 != NULL && ce2 != NULL) { + /* Pick whichever one has the better best circuit */ + return compare_cell_ewma_counts(ce1, ce2); + } else { + if (ce1 != NULL ) { + /* We only have a circuit on cmux_1, so prefer it */ + return -1; + } else if (ce2 != NULL) { + /* We only have a circuit on cmux_2, so prefer it */ + return 1; + } else { + /* No circuits at all; no preference */ + return 0; + } + } + } else { + /* We got identical params */ + return 0; + } +} + /** Helper for sorting cell_ewma_t values in their priority queue. */ static int compare_cell_ewma_counts(const void *p1, const void *p2) @@ -588,7 +644,7 @@ cell_ewma_set_scale_factor(const or_options_t *options, /** Return the multiplier necessary to convert the value of a cell sent in * 'from_tick' to one sent in 'to_tick'. */ -static INLINE double +static inline double get_scale_factor(unsigned from_tick, unsigned to_tick) { /* This math can wrap around, but that's okay: unsigned overflow is diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h index a512745c77..58aac1e196 100644 --- a/src/or/circuitmux_ewma.h +++ b/src/or/circuitmux_ewma.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c index 1d7812bf2b..9ac2d565b5 100644 --- a/src/or/circuitstats.c +++ b/src/or/circuitstats.c @@ -1,9 +1,16 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * \file circuitstats.c + * + * \brief Maintains and analyzes statistics about circuit built times, so we + * can tell how long we may need to wait for a fast circuit to be constructed. + */ + #define CIRCUITSTATS_PRIVATE #include "or.h" @@ -12,12 +19,17 @@ #include "config.h" #include "confparse.h" #include "control.h" +#include "main.h" #include "networkstatus.h" #include "statefile.h" #undef log #include <math.h> +static void cbt_control_event_buildtimeout_set( + const circuit_build_times_t *cbt, + buildtimeout_set_event_t type); + #define CBT_BIN_TO_MS(bin) ((bin)*CBT_BIN_WIDTH + (CBT_BIN_WIDTH/2)) /** Global list of circuit build times */ @@ -26,12 +38,46 @@ // vary in their own latency. The downside of this is that guards // can change frequently, so we'd be building a lot more circuits // most likely. -/* XXXX024 Make this static; add accessor functions. */ -circuit_build_times_t circ_times; +static circuit_build_times_t circ_times; +#ifdef TOR_UNIT_TESTS /** If set, we're running the unit tests: we should avoid clobbering * our state file or accessing get_options() or get_or_state() */ static int unit_tests = 0; +#else +#define unit_tests 0 +#endif + +/** Return a pointer to the data structure describing our current circuit + * build time history and computations. */ +const circuit_build_times_t * +get_circuit_build_times(void) +{ + return &circ_times; +} + +/** As get_circuit_build_times, but return a mutable pointer. */ +circuit_build_times_t * +get_circuit_build_times_mutable(void) +{ + return &circ_times; +} + +/** Return the time to wait before actually closing an under-construction, in + * milliseconds. */ +double +get_circuit_build_close_time_ms(void) +{ + return circ_times.close_ms; +} + +/** Return the time to wait before giving up on an under-construction circuit, + * in milliseconds. */ +double +get_circuit_build_timeout_ms(void) +{ + return circ_times.timeout_ms; +} /** * This function decides if CBT learning should be disabled. It returns @@ -56,18 +102,22 @@ circuit_build_times_disabled(void) if (consensus_disabled || config_disabled || dirauth_disabled || state_disabled) { +#if 0 log_debug(LD_CIRC, "CircuitBuildTime learning is disabled. " "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d", consensus_disabled, config_disabled, dirauth_disabled, state_disabled); +#endif return 1; } else { +#if 0 log_debug(LD_CIRC, "CircuitBuildTime learning is not disabled. " "Consensus=%d, Config=%d, AuthDir=%d, StateFile=%d", consensus_disabled, config_disabled, dirauth_disabled, state_disabled); +#endif return 0; } } @@ -154,7 +204,7 @@ circuit_build_times_min_circs_to_observe(void) /** Return true iff <b>cbt</b> has recorded enough build times that we * want to start acting on the timeout it implies. */ int -circuit_build_times_enough_to_compute(circuit_build_times_t *cbt) +circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt) { return cbt->total_build_times >= circuit_build_times_min_circs_to_observe(); } @@ -361,7 +411,7 @@ circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, * distress anyway, so memory correctness here is paramount over * doing acrobatics to preserve the array. */ - recent_circs = tor_malloc_zero(sizeof(int8_t)*num); + recent_circs = tor_calloc(num, sizeof(int8_t)); if (cbt->liveness.timeouts_after_firsthop && cbt->liveness.num_recent_circs > 0) { memcpy(recent_circs, cbt->liveness.timeouts_after_firsthop, @@ -438,7 +488,7 @@ circuit_build_times_get_initial_timeout(void) * Leave estimated parameters, timeout and network liveness intact * for future use. */ -void +STATIC void circuit_build_times_reset(circuit_build_times_t *cbt) { memset(cbt->circuit_build_times, 0, sizeof(cbt->circuit_build_times)); @@ -465,13 +515,13 @@ circuit_build_times_init(circuit_build_times_t *cbt) cbt->liveness.num_recent_circs = circuit_build_times_recent_circuit_count(NULL); cbt->liveness.timeouts_after_firsthop = - tor_malloc_zero(sizeof(int8_t)*cbt->liveness.num_recent_circs); + tor_calloc(cbt->liveness.num_recent_circs, sizeof(int8_t)); } else { cbt->liveness.num_recent_circs = 0; cbt->liveness.timeouts_after_firsthop = NULL; } cbt->close_ms = cbt->timeout_ms = circuit_build_times_get_initial_timeout(); - control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + cbt_control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); } /** @@ -557,7 +607,7 @@ circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time) * Return maximum circuit build time */ static build_time_t -circuit_build_times_max(circuit_build_times_t *cbt) +circuit_build_times_max(const circuit_build_times_t *cbt) { int i = 0; build_time_t max_build_time = 0; @@ -598,7 +648,7 @@ circuit_build_times_min(circuit_build_times_t *cbt) * The return value must be freed by the caller. */ static uint32_t * -circuit_build_times_create_histogram(circuit_build_times_t *cbt, +circuit_build_times_create_histogram(const circuit_build_times_t *cbt, build_time_t *nbins) { uint32_t *histogram; @@ -606,7 +656,7 @@ circuit_build_times_create_histogram(circuit_build_times_t *cbt, int i, c; *nbins = 1 + (max_build_time / CBT_BIN_WIDTH); - histogram = tor_malloc_zero(*nbins * sizeof(build_time_t)); + histogram = tor_calloc(*nbins, sizeof(build_time_t)); // calculate histogram for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { @@ -648,7 +698,7 @@ circuit_build_times_get_xm(circuit_build_times_t *cbt) if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) num_modes = 1; - nth_max_bin = (build_time_t*)tor_malloc_zero(num_modes*sizeof(build_time_t)); + nth_max_bin = tor_calloc(num_modes, sizeof(build_time_t)); /* Determine the N most common build times */ for (i = 0; i < nbins; i++) { @@ -688,7 +738,7 @@ circuit_build_times_get_xm(circuit_build_times_t *cbt) * the or_state_t state structure. */ void -circuit_build_times_update_state(circuit_build_times_t *cbt, +circuit_build_times_update_state(const circuit_build_times_t *cbt, or_state_t *state) { uint32_t *histogram; @@ -830,7 +880,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, } /* build_time_t 0 means uninitialized */ - loaded_times = tor_malloc_zero(sizeof(build_time_t)*state->TotalBuildTimes); + loaded_times = tor_calloc(state->TotalBuildTimes, sizeof(build_time_t)); for (line = state->BuildtimeHistogram; line; line = line->next) { smartlist_t *args = smartlist_new(); @@ -949,7 +999,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, * an acceptable approximation because we are only concerned with the * accuracy of the CDF of the tail. */ -int +STATIC int circuit_build_times_update_alpha(circuit_build_times_t *cbt) { build_time_t *x=cbt->circuit_build_times; @@ -1031,9 +1081,9 @@ circuit_build_times_update_alpha(circuit_build_times_t *cbt) * random_sample_from_Pareto_distribution * That's right. I'll cite wikipedia all day long. * - * Return value is in milliseconds. + * Return value is in milliseconds, clamped to INT32_MAX. */ -double +STATIC double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, double quantile) { @@ -1042,7 +1092,21 @@ circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, tor_assert(1.0-quantile > 0); tor_assert(cbt->Xm > 0); - ret = cbt->Xm/pow(1.0-quantile,1.0/cbt->alpha); + /* If either alpha or p are 0, we would divide by zero, yielding an + * infinite (double) result; which would be clamped to INT32_MAX. + * Instead, initialise ret to INT32_MAX, and skip over these + * potentially illegal/trapping divides by zero. + */ + ret = INT32_MAX; + + if (cbt->alpha > 0) { + double p; + p = pow(1.0-quantile,1.0/cbt->alpha); + if (p > 0) { + ret = cbt->Xm/p; + } + } + if (ret > INT32_MAX) { ret = INT32_MAX; } @@ -1050,6 +1114,7 @@ circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, return ret; } +#ifdef TOR_UNIT_TESTS /** Pareto CDF */ double circuit_build_times_cdf(circuit_build_times_t *cbt, double x) @@ -1060,7 +1125,9 @@ circuit_build_times_cdf(circuit_build_times_t *cbt, double x) tor_assert(0 <= ret && ret <= 1.0); return ret; } +#endif +#ifdef TOR_UNIT_TESTS /** * Generate a synthetic time using our distribution parameters. * @@ -1093,7 +1160,9 @@ circuit_build_times_generate_sample(circuit_build_times_t *cbt, tor_assert(ret > 0); return ret; } +#endif +#ifdef TOR_UNIT_TESTS /** * Estimate an initial alpha parameter by solving the quantile * function with a quantile point and a specific timeout value. @@ -1114,12 +1183,13 @@ circuit_build_times_initial_alpha(circuit_build_times_t *cbt, (tor_mathlog(cbt->Xm)-tor_mathlog(timeout_ms)); tor_assert(cbt->alpha > 0); } +#endif /** * Returns true if we need circuits to be built */ int -circuit_build_times_needs_circuits(circuit_build_times_t *cbt) +circuit_build_times_needs_circuits(const circuit_build_times_t *cbt) { /* Return true if < MIN_CIRCUITS_TO_OBSERVE */ return !circuit_build_times_enough_to_compute(cbt); @@ -1130,13 +1200,19 @@ circuit_build_times_needs_circuits(circuit_build_times_t *cbt) * right now. */ int -circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt) +circuit_build_times_needs_circuits_now(const circuit_build_times_t *cbt) { return circuit_build_times_needs_circuits(cbt) && approx_time()-cbt->last_circ_at > circuit_build_times_test_frequency(); } /** + * How long should we be unreachable before we think we need to check if + * our published IP address has changed. + */ +#define CIRCUIT_TIMEOUT_BEFORE_RECHECK_IP (60*3) + +/** * Called to indicate that the network showed some signs of liveness, * i.e. we received a cell. * @@ -1151,15 +1227,21 @@ circuit_build_times_network_is_live(circuit_build_times_t *cbt) { time_t now = approx_time(); if (cbt->liveness.nonlive_timeouts > 0) { + time_t time_since_live = now - cbt->liveness.network_last_live; log_notice(LD_CIRC, "Tor now sees network activity. Restoring circuit build " "timeout recording. Network was down for %d seconds " "during %d circuit attempts.", - (int)(now - cbt->liveness.network_last_live), + (int)time_since_live, cbt->liveness.nonlive_timeouts); + if (time_since_live > CIRCUIT_TIMEOUT_BEFORE_RECHECK_IP) + reschedule_descriptor_update_check(); } cbt->liveness.network_last_live = now; cbt->liveness.nonlive_timeouts = 0; + + /* Tell control.c */ + control_event_network_liveness_update(1); } /** @@ -1244,6 +1326,9 @@ circuit_build_times_network_close(circuit_build_times_t *cbt, "Tor has not observed any network activity for the past %d " "seconds. Disabling circuit build timeout recording.", (int)(now - cbt->liveness.network_last_live)); + + /* Tell control.c */ + control_event_network_liveness_update(0); } else { log_info(LD_CIRC, "Got non-live timeout. Current count is: %d", @@ -1263,7 +1348,7 @@ circuit_build_times_network_close(circuit_build_times_t *cbt, * in the case of recent liveness changes. */ int -circuit_build_times_network_check_live(circuit_build_times_t *cbt) +circuit_build_times_network_check_live(const circuit_build_times_t *cbt) { if (cbt->liveness.nonlive_timeouts > 0) { return 0; @@ -1282,7 +1367,7 @@ circuit_build_times_network_check_live(circuit_build_times_t *cbt) * to restart the process of building test circuits and estimating a * new timeout. */ -int +STATIC int circuit_build_times_network_check_changed(circuit_build_times_t *cbt) { int total_build_times = cbt->total_build_times; @@ -1313,10 +1398,11 @@ circuit_build_times_network_check_changed(circuit_build_times_t *cbt) } cbt->liveness.after_firsthop_idx = 0; +#define MAX_TIMEOUT ((int32_t) (INT32_MAX/2)) /* Check to see if this has happened before. If so, double the timeout * to give people on abysmally bad network connections a shot at access */ if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) { - if (cbt->timeout_ms > INT32_MAX/2 || cbt->close_ms > INT32_MAX/2) { + if (cbt->timeout_ms > MAX_TIMEOUT || cbt->close_ms > MAX_TIMEOUT) { log_warn(LD_CIRC, "Insanely large circuit build timeout value. " "(timeout = %fmsec, close = %fmsec)", cbt->timeout_ms, cbt->close_ms); @@ -1328,8 +1414,9 @@ circuit_build_times_network_check_changed(circuit_build_times_t *cbt) cbt->close_ms = cbt->timeout_ms = circuit_build_times_get_initial_timeout(); } +#undef MAX_TIMEOUT - control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); + cbt_control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); log_notice(LD_CIRC, "Your network connection speed appears to have changed. Resetting " @@ -1511,7 +1598,7 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt) } } - control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); + cbt_control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_COMPUTED); timeout_rate = circuit_build_times_timeout_rate(cbt); @@ -1546,6 +1633,8 @@ circuit_build_times_set_timeout(circuit_build_times_t *cbt) cbt->total_build_times); } } + +#ifdef TOR_UNIT_TESTS /** Make a note that we're running unit tests (rather than running Tor * itself), so we avoid clobbering our state file. */ void @@ -1553,4 +1642,46 @@ circuitbuild_running_unit_tests(void) { unit_tests = 1; } +#endif + +void +circuit_build_times_update_last_circ(circuit_build_times_t *cbt) +{ + cbt->last_circ_at = approx_time(); +} + +static void +cbt_control_event_buildtimeout_set(const circuit_build_times_t *cbt, + buildtimeout_set_event_t type) +{ + char *args = NULL; + double qnt; + + switch (type) { + case BUILDTIMEOUT_SET_EVENT_RESET: + case BUILDTIMEOUT_SET_EVENT_SUSPENDED: + case BUILDTIMEOUT_SET_EVENT_DISCARD: + qnt = 1.0; + break; + case BUILDTIMEOUT_SET_EVENT_COMPUTED: + case BUILDTIMEOUT_SET_EVENT_RESUME: + default: + qnt = circuit_build_times_quantile_cutoff(); + break; + } + + tor_asprintf(&args, "TOTAL_TIMES=%lu " + "TIMEOUT_MS=%lu XM=%lu ALPHA=%f CUTOFF_QUANTILE=%f " + "TIMEOUT_RATE=%f CLOSE_MS=%lu CLOSE_RATE=%f", + (unsigned long)cbt->total_build_times, + (unsigned long)cbt->timeout_ms, + (unsigned long)cbt->Xm, cbt->alpha, qnt, + circuit_build_times_timeout_rate(cbt), + (unsigned long)cbt->close_ms, + circuit_build_times_close_rate(cbt)); + + control_event_buildtimeout_set(type, args); + + tor_free(args); +} diff --git a/src/or/circuitstats.h b/src/or/circuitstats.h index 87dce99f4f..72b160983f 100644 --- a/src/or/circuitstats.h +++ b/src/or/circuitstats.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,11 +12,14 @@ #ifndef TOR_CIRCUITSTATS_H #define TOR_CIRCUITSTATS_H -extern circuit_build_times_t circ_times; +const circuit_build_times_t *get_circuit_build_times(void); +circuit_build_times_t *get_circuit_build_times_mutable(void); +double get_circuit_build_close_time_ms(void); +double get_circuit_build_timeout_ms(void); int circuit_build_times_disabled(void); -int circuit_build_times_enough_to_compute(circuit_build_times_t *cbt); -void circuit_build_times_update_state(circuit_build_times_t *cbt, +int circuit_build_times_enough_to_compute(const circuit_build_times_t *cbt); +void circuit_build_times_update_state(const circuit_build_times_t *cbt, or_state_t *state); int circuit_build_times_parse_state(circuit_build_times_t *cbt, or_state_t *state); @@ -27,9 +30,9 @@ int circuit_build_times_count_close(circuit_build_times_t *cbt, void circuit_build_times_set_timeout(circuit_build_times_t *cbt); int circuit_build_times_add_time(circuit_build_times_t *cbt, build_time_t time); -int circuit_build_times_needs_circuits(circuit_build_times_t *cbt); +int circuit_build_times_needs_circuits(const circuit_build_times_t *cbt); -int circuit_build_times_needs_circuits_now(circuit_build_times_t *cbt); +int circuit_build_times_needs_circuits_now(const circuit_build_times_t *cbt); void circuit_build_times_init(circuit_build_times_t *cbt); void circuit_build_times_free_timeouts(circuit_build_times_t *cbt); void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, @@ -37,29 +40,59 @@ void circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, double circuit_build_times_timeout_rate(const circuit_build_times_t *cbt); double circuit_build_times_close_rate(const circuit_build_times_t *cbt); +void circuit_build_times_update_last_circ(circuit_build_times_t *cbt); + #ifdef CIRCUITSTATS_PRIVATE -double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, +STATIC double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, double quantile); +STATIC int circuit_build_times_update_alpha(circuit_build_times_t *cbt); +STATIC void circuit_build_times_reset(circuit_build_times_t *cbt); + +/* Network liveness functions */ +STATIC int circuit_build_times_network_check_changed( + circuit_build_times_t *cbt); +#endif + +#ifdef TOR_UNIT_TESTS build_time_t circuit_build_times_generate_sample(circuit_build_times_t *cbt, double q_lo, double q_hi); +double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); void circuit_build_times_initial_alpha(circuit_build_times_t *cbt, double quantile, double time_ms); -int circuit_build_times_update_alpha(circuit_build_times_t *cbt); -double circuit_build_times_cdf(circuit_build_times_t *cbt, double x); void circuitbuild_running_unit_tests(void); -void circuit_build_times_reset(circuit_build_times_t *cbt); - -/* Network liveness functions */ -int circuit_build_times_network_check_changed(circuit_build_times_t *cbt); #endif /* Network liveness functions */ void circuit_build_times_network_is_live(circuit_build_times_t *cbt); -int circuit_build_times_network_check_live(circuit_build_times_t *cbt); +int circuit_build_times_network_check_live(const circuit_build_times_t *cbt); void circuit_build_times_network_circ_success(circuit_build_times_t *cbt); -/* DOCDOC circuit_build_times_get_bw_scale */ -int circuit_build_times_get_bw_scale(networkstatus_t *ns); +#ifdef CIRCUITSTATS_PRIVATE +/** Structure for circuit build times history */ +struct circuit_build_times_s { + /** The circular array of recorded build times in milliseconds */ + build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; + /** Current index in the circuit_build_times circular array */ + int build_times_idx; + /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ + int total_build_times; + /** Information about the state of our local network connection */ + network_liveness_t liveness; + /** Last time we built a circuit. Used to decide to build new test circs */ + time_t last_circ_at; + /** "Minimum" value of our pareto distribution (actually mode) */ + build_time_t Xm; + /** alpha exponent for pareto dist. */ + double alpha; + /** Have we computed a timeout? */ + int have_computed_timeout; + /** The exact value for that timeout in milliseconds. Stored as a double + * to maintain precision from calculations to and from quantile value. */ + double timeout_ms; + /** How long we wait before actually closing the circuit. */ + double close_ms; +}; +#endif #endif diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 06a51a04a2..2c724dee05 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,6 +12,7 @@ #include "or.h" #include "addressmap.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" @@ -31,12 +32,6 @@ #include "router.h" #include "routerlist.h" -/********* START VARIABLES **********/ - -extern circuit_t *global_circuitlist; /* from circuitlist.c */ - -/********* END VARIABLES ************/ - static void circuit_expire_old_circuits_clientside(void); static void circuit_increment_failure_count(void); @@ -97,7 +92,7 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ, /* decide if this circ is suitable for this conn */ /* for rend circs, circ->cpath->prev is not the last router in the - * circuit, it's the magical extra bob hop. so just check the nickname + * circuit, it's the magical extra service hop. so just check the nickname * of the one we meant to finish at. */ build_state = origin_circ->build_state; @@ -205,7 +200,7 @@ circuit_is_better(const origin_circuit_t *oa, const origin_circuit_t *ob, return 1; } else { if (a->timestamp_dirty || - timercmp(&a->timestamp_began, &b->timestamp_began, >)) + timercmp(&a->timestamp_began, &b->timestamp_began, OP_GT)) return 1; if (ob->build_state->is_internal) /* XXX023 what the heck is this internal thing doing here. I @@ -273,7 +268,6 @@ circuit_get_best(const entry_connection_t *conn, int must_be_open, uint8_t purpose, int need_uptime, int need_internal) { - circuit_t *circ; origin_circuit_t *best=NULL; struct timeval now; int intro_going_on_but_too_old = 0; @@ -286,7 +280,7 @@ circuit_get_best(const entry_connection_t *conn, tor_gettimeofday(&now); - for (circ=global_circuitlist;circ;circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { origin_circuit_t *origin_circ; if (!CIRCUIT_IS_ORIGIN(circ)) continue; @@ -301,7 +295,7 @@ circuit_get_best(const entry_connection_t *conn, } if (!circuit_is_acceptable(origin_circ,conn,must_be_open,purpose, - need_uptime,need_internal,now.tv_sec)) + need_uptime,need_internal, (time_t)now.tv_sec)) continue; /* now this is an acceptable circ to hand back. but that doesn't @@ -310,6 +304,7 @@ circuit_get_best(const entry_connection_t *conn, if (!best || circuit_is_better(origin_circ,best,conn)) best = origin_circ; } + SMARTLIST_FOREACH_END(circ); if (!best && intro_going_on_but_too_old) log_info(LD_REND|LD_CIRC, "There is an intro circuit being created " @@ -323,11 +318,9 @@ circuit_get_best(const entry_connection_t *conn, static int count_pending_general_client_circuits(void) { - const circuit_t *circ; - int count = 0; - for (circ = global_circuitlist; circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (circ->marked_for_close || circ->state == CIRCUIT_STATE_OPEN || circ->purpose != CIRCUIT_PURPOSE_C_GENERAL || @@ -336,6 +329,7 @@ count_pending_general_client_circuits(void) ++count; } + SMARTLIST_FOREACH_END(circ); return count; } @@ -375,7 +369,6 @@ circuit_conforms_to_options(const origin_circuit_t *circ, void circuit_expire_building(void) { - circuit_t *victim, *next_circ = global_circuitlist; /* circ_times.timeout_ms and circ_times.close_ms are from * circuit_build_times_get_initial_timeout() if we haven't computed * custom timeouts yet */ @@ -393,10 +386,9 @@ circuit_expire_building(void) * we want to be more lenient with timeouts, in case the * user has relocated and/or changed network connections. * See bug #3443. */ - while (next_circ) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, next_circ) { if (!CIRCUIT_IS_ORIGIN(next_circ) || /* didn't originate here */ next_circ->marked_for_close) { /* don't mess with marked circs */ - next_circ = next_circ->next; continue; } @@ -408,9 +400,7 @@ circuit_expire_building(void) any_opened_circs = 1; break; } - next_circ = next_circ->next; - } - next_circ = global_circuitlist; + } SMARTLIST_FOREACH_END(next_circ); #define SET_CUTOFF(target, msec) do { \ long ms = tor_lround(msec); \ @@ -451,12 +441,12 @@ circuit_expire_building(void) * RTTs = 4a + 3b + 2c * RTTs = 9h */ - SET_CUTOFF(general_cutoff, circ_times.timeout_ms); - SET_CUTOFF(begindir_cutoff, circ_times.timeout_ms); + SET_CUTOFF(general_cutoff, get_circuit_build_timeout_ms()); + SET_CUTOFF(begindir_cutoff, get_circuit_build_timeout_ms()); /* > 3hop circs seem to have a 1.0 second delay on their cannibalized * 4th hop. */ - SET_CUTOFF(fourhop_cutoff, circ_times.timeout_ms * (10/6.0) + 1000); + SET_CUTOFF(fourhop_cutoff, get_circuit_build_timeout_ms() * (10/6.0) + 1000); /* CIRCUIT_PURPOSE_C_ESTABLISH_REND behaves more like a RELAY cell. * Use the stream cutoff (more or less). */ @@ -465,26 +455,24 @@ circuit_expire_building(void) /* Be lenient with cannibalized circs. They already survived the official * CBT, and they're usually not performance-critical. */ SET_CUTOFF(cannibalized_cutoff, - MAX(circ_times.close_ms*(4/6.0), + MAX(get_circuit_build_close_time_ms()*(4/6.0), options->CircuitStreamTimeout * 1000) + 1000); /* Intro circs have an extra round trip (and are also 4 hops long) */ - SET_CUTOFF(c_intro_cutoff, circ_times.timeout_ms * (14/6.0) + 1000); + SET_CUTOFF(c_intro_cutoff, get_circuit_build_timeout_ms() * (14/6.0) + 1000); /* Server intro circs have an extra round trip */ - SET_CUTOFF(s_intro_cutoff, circ_times.timeout_ms * (9/6.0) + 1000); + SET_CUTOFF(s_intro_cutoff, get_circuit_build_timeout_ms() * (9/6.0) + 1000); - SET_CUTOFF(close_cutoff, circ_times.close_ms); - SET_CUTOFF(extremely_old_cutoff, circ_times.close_ms*2 + 1000); + SET_CUTOFF(close_cutoff, get_circuit_build_close_time_ms()); + SET_CUTOFF(extremely_old_cutoff, get_circuit_build_close_time_ms()*2 + 1000); SET_CUTOFF(hs_extremely_old_cutoff, - MAX(circ_times.close_ms*2 + 1000, + MAX(get_circuit_build_close_time_ms()*2 + 1000, options->SocksTimeout * 1000)); - while (next_circ) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *,victim) { struct timeval cutoff; - victim = next_circ; - next_circ = next_circ->next; if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */ victim->marked_for_close) /* don't mess with marked circs */ continue; @@ -526,7 +514,7 @@ circuit_expire_building(void) if (TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) cutoff = hs_extremely_old_cutoff; - if (timercmp(&victim->timestamp_began, &cutoff, >)) + if (timercmp(&victim->timestamp_began, &cutoff, OP_GT)) continue; /* it's still young, leave it alone */ /* We need to double-check the opened state here because @@ -536,7 +524,7 @@ circuit_expire_building(void) * aren't either. */ if (!any_opened_circs && victim->state != CIRCUIT_STATE_OPEN) { /* It's still young enough that we wouldn't close it, right? */ - if (timercmp(&victim->timestamp_began, &close_cutoff, >)) { + if (timercmp(&victim->timestamp_began, &close_cutoff, OP_GT)) { if (!TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout) { int first_hop_succeeded = TO_ORIGIN_CIRCUIT(victim)->cpath->state == CPATH_STATE_OPEN; @@ -546,7 +534,9 @@ circuit_expire_building(void) "%d guards are live.", TO_ORIGIN_CIRCUIT(victim)->global_identifier, circuit_purpose_to_string(victim->purpose), - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len, + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1, circuit_state_to_string(victim->state), channel_state_to_string(victim->n_chan->state), num_live_entry_guards(0)); @@ -555,12 +545,14 @@ circuit_expire_building(void) * was a timeout, and the timeout value needs to reset if we * see enough of them. Note this means we also need to avoid * double-counting below, too. */ - circuit_build_times_count_timeout(&circ_times, first_hop_succeeded); + circuit_build_times_count_timeout(get_circuit_build_times_mutable(), + first_hop_succeeded); TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout = 1; } continue; } else { static ratelim_t relax_timeout_limit = RATELIM_INIT(3600); + const double build_close_ms = get_circuit_build_close_time_ms(); log_fn_ratelim(&relax_timeout_limit, LOG_NOTICE, LD_CIRC, "No circuits are opened. Relaxed timeout for circuit %d " "(a %s %d-hop circuit in state %s with channel state %s) to " @@ -568,10 +560,13 @@ circuit_expire_building(void) "anyway. %d guards are live.", TO_ORIGIN_CIRCUIT(victim)->global_identifier, circuit_purpose_to_string(victim->purpose), - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len, + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1, circuit_state_to_string(victim->state), channel_state_to_string(victim->n_chan->state), - (long)circ_times.close_ms, num_live_entry_guards(0)); + (long)build_close_ms, + num_live_entry_guards(0)); } } @@ -651,7 +646,7 @@ circuit_expire_building(void) } if (circuit_timeout_want_to_count_circ(TO_ORIGIN_CIRCUIT(victim)) && - circuit_build_times_enough_to_compute(&circ_times)) { + circuit_build_times_enough_to_compute(get_circuit_build_times())) { /* Circuits are allowed to last longer for measurement. * Switch their purpose and wait. */ if (victim->purpose != CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) { @@ -665,8 +660,9 @@ circuit_expire_building(void) * have a timeout. We also want to avoid double-counting * already "relaxed" circuits, which are counted above. */ if (!TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout) { - circuit_build_times_count_timeout(&circ_times, - first_hop_succeeded); + circuit_build_times_count_timeout( + get_circuit_build_times_mutable(), + first_hop_succeeded); } continue; } @@ -676,17 +672,18 @@ circuit_expire_building(void) * it off at, we probably had a suspend event along this codepath, * and we should discard the value. */ - if (timercmp(&victim->timestamp_began, &extremely_old_cutoff, <)) { + if (timercmp(&victim->timestamp_began, &extremely_old_cutoff, OP_LT)) { log_notice(LD_CIRC, "Extremely large value for circuit build timeout: %lds. " "Assuming clock jump. Purpose %d (%s)", (long)(now.tv_sec - victim->timestamp_began.tv_sec), victim->purpose, circuit_purpose_to_string(victim->purpose)); - } else if (circuit_build_times_count_close(&circ_times, - first_hop_succeeded, - victim->timestamp_created.tv_sec)) { - circuit_build_times_set_timeout(&circ_times); + } else if (circuit_build_times_count_close( + get_circuit_build_times_mutable(), + first_hop_succeeded, + (time_t)victim->timestamp_created.tv_sec)) { + circuit_build_times_set_timeout(get_circuit_build_times_mutable()); } } } @@ -711,7 +708,8 @@ circuit_expire_building(void) * and we have tried to send an INTRODUCE1 cell specifying it. * Thus, if the pending_final_cpath field *is* NULL, then we * want to not spare it. */ - if (TO_ORIGIN_CIRCUIT(victim)->build_state->pending_final_cpath == + if (TO_ORIGIN_CIRCUIT(victim)->build_state && + TO_ORIGIN_CIRCUIT(victim)->build_state->pending_final_cpath == NULL) break; /* fallthrough! */ @@ -750,23 +748,27 @@ circuit_expire_building(void) if (victim->n_chan) log_info(LD_CIRC, - "Abandoning circ %u %s:%d (state %d,%d:%s, purpose %d, " + "Abandoning circ %u %s:%u (state %d,%d:%s, purpose %d, " "len %d)", TO_ORIGIN_CIRCUIT(victim)->global_identifier, channel_get_canonical_remote_descr(victim->n_chan), (unsigned)victim->n_circ_id, TO_ORIGIN_CIRCUIT(victim)->has_opened, victim->state, circuit_state_to_string(victim->state), victim->purpose, - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len); + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1); else log_info(LD_CIRC, - "Abandoning circ %u %d (state %d,%d:%s, purpose %d, len %d)", + "Abandoning circ %u %u (state %d,%d:%s, purpose %d, len %d)", TO_ORIGIN_CIRCUIT(victim)->global_identifier, (unsigned)victim->n_circ_id, TO_ORIGIN_CIRCUIT(victim)->has_opened, victim->state, circuit_state_to_string(victim->state), victim->purpose, - TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len); + TO_ORIGIN_CIRCUIT(victim)->build_state ? + TO_ORIGIN_CIRCUIT(victim)->build_state->desired_path_len : + -1); circuit_log_path(LOG_INFO,LD_CIRC,TO_ORIGIN_CIRCUIT(victim)); if (victim->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) @@ -775,7 +777,138 @@ circuit_expire_building(void) circuit_mark_for_close(victim, END_CIRC_REASON_TIMEOUT); pathbias_count_timeout(TO_ORIGIN_CIRCUIT(victim)); + } SMARTLIST_FOREACH_END(victim); +} + +/** For debugging #8387: track when we last called + * circuit_expire_old_circuits_clientside. */ +static time_t last_expired_clientside_circuits = 0; + +/** + * As a diagnostic for bug 8387, log information about how many one-hop + * circuits we have around that have been there for at least <b>age</b> + * seconds. Log a few of them. + */ +void +circuit_log_ancient_one_hop_circuits(int age) +{ +#define MAX_ANCIENT_ONEHOP_CIRCUITS_TO_LOG 10 + time_t now = time(NULL); + time_t cutoff = now - age; + int n_found = 0; + smartlist_t *log_these = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + const origin_circuit_t *ocirc; + if (! CIRCUIT_IS_ORIGIN(circ)) + continue; + if (circ->timestamp_created.tv_sec >= cutoff) + continue; + ocirc = CONST_TO_ORIGIN_CIRCUIT(circ); + + if (ocirc->build_state && ocirc->build_state->onehop_tunnel) { + ++n_found; + + if (smartlist_len(log_these) < MAX_ANCIENT_ONEHOP_CIRCUITS_TO_LOG) + smartlist_add(log_these, (origin_circuit_t*) ocirc); + } } + SMARTLIST_FOREACH_END(circ); + + if (n_found == 0) + goto done; + + log_notice(LD_HEARTBEAT, + "Diagnostic for issue 8387: Found %d one-hop circuits more " + "than %d seconds old! Logging %d...", + n_found, age, smartlist_len(log_these)); + + SMARTLIST_FOREACH_BEGIN(log_these, const origin_circuit_t *, ocirc) { + char created[ISO_TIME_LEN+1]; + int stream_num; + const edge_connection_t *conn; + char *dirty = NULL; + const circuit_t *circ = TO_CIRCUIT(ocirc); + + format_local_iso_time(created, + (time_t)circ->timestamp_created.tv_sec); + + if (circ->timestamp_dirty) { + char dirty_since[ISO_TIME_LEN+1]; + format_local_iso_time(dirty_since, circ->timestamp_dirty); + + tor_asprintf(&dirty, "Dirty since %s (%ld seconds vs %ld-second cutoff)", + dirty_since, (long)(now - circ->timestamp_dirty), + (long) get_options()->MaxCircuitDirtiness); + } else { + dirty = tor_strdup("Not marked dirty"); + } + + log_notice(LD_HEARTBEAT, " #%d created at %s. %s, %s. %s for close. " + "Package window: %d. " + "%s for new conns. %s.", + ocirc_sl_idx, + created, + circuit_state_to_string(circ->state), + circuit_purpose_to_string(circ->purpose), + circ->marked_for_close ? "Marked" : "Not marked", + circ->package_window, + ocirc->unusable_for_new_conns ? "Not usable" : "usable", + dirty); + tor_free(dirty); + + stream_num = 0; + for (conn = ocirc->p_streams; conn; conn = conn->next_stream) { + const connection_t *c = TO_CONN(conn); + char stream_created[ISO_TIME_LEN+1]; + if (++stream_num >= 5) + break; + + format_local_iso_time(stream_created, c->timestamp_created); + + log_notice(LD_HEARTBEAT, " Stream#%d created at %s. " + "%s conn in state %s. " + "It is %slinked and %sreading from a linked connection %p. " + "Package window %d. " + "%s for close (%s:%d). Hold-open is %sset. " + "Has %ssent RELAY_END. %s on circuit.", + stream_num, + stream_created, + conn_type_to_string(c->type), + conn_state_to_string(c->type, c->state), + c->linked ? "" : "not ", + c->reading_from_linked_conn ? "": "not", + c->linked_conn, + conn->package_window, + c->marked_for_close ? "Marked" : "Not marked", + c->marked_for_close_file ? c->marked_for_close_file : "--", + c->marked_for_close, + c->hold_open_until_flushed ? "" : "not ", + conn->edge_has_sent_end ? "" : "not ", + conn->edge_blocked_on_circ ? "Blocked" : "Not blocked"); + if (! c->linked_conn) + continue; + + c = c->linked_conn; + + log_notice(LD_HEARTBEAT, " Linked to %s connection in state %s " + "(Purpose %d). %s for close (%s:%d). Hold-open is %sset. ", + conn_type_to_string(c->type), + conn_state_to_string(c->type, c->state), + c->purpose, + c->marked_for_close ? "Marked" : "Not marked", + c->marked_for_close_file ? c->marked_for_close_file : "--", + c->marked_for_close, + c->hold_open_until_flushed ? "" : "not "); + } + } SMARTLIST_FOREACH_END(ocirc); + + log_notice(LD_HEARTBEAT, "It has been %ld seconds since I last called " + "circuit_expire_old_circuits_clientside().", + (long)(now - last_expired_clientside_circuits)); + + done: + smartlist_free(log_these); } /** Remove any elements in <b>needed_ports</b> that are handled by an @@ -810,7 +943,6 @@ int circuit_stream_is_being_handled(entry_connection_t *conn, uint16_t port, int min) { - circuit_t *circ; const node_t *exitnode; int num=0; time_t now = time(NULL); @@ -818,7 +950,7 @@ circuit_stream_is_being_handled(entry_connection_t *conn, get_options()->LongLivedPorts, conn ? conn->socks_request->port : port); - for (circ=global_circuitlist;circ;circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->purpose == CIRCUIT_PURPOSE_C_GENERAL && @@ -848,6 +980,7 @@ circuit_stream_is_being_handled(entry_connection_t *conn, } } } + SMARTLIST_FOREACH_END(circ); return 0; } @@ -861,7 +994,6 @@ circuit_stream_is_being_handled(entry_connection_t *conn, static void circuit_predict_and_launch_new(void) { - circuit_t *circ; int num=0, num_internal=0, num_uptime_internal=0; int hidserv_needs_uptime=0, hidserv_needs_capacity=1; int port_needs_uptime=0, port_needs_capacity=1; @@ -869,7 +1001,7 @@ circuit_predict_and_launch_new(void) int flags = 0; /* First, count how many of each type of circuit we have already. */ - for (circ=global_circuitlist;circ;circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { cpath_build_state_t *build_state; origin_circuit_t *origin_circ; if (!CIRCUIT_IS_ORIGIN(circ)) @@ -892,6 +1024,7 @@ circuit_predict_and_launch_new(void) if (build_state->need_uptime && build_state->is_internal) num_uptime_internal++; } + SMARTLIST_FOREACH_END(circ); /* If that's enough, then stop now. */ if (num >= MAX_UNUSED_OPEN_CIRCUITS) @@ -899,9 +1032,11 @@ circuit_predict_and_launch_new(void) /* Second, see if we need any more exit circuits. */ /* check if we know of a port that's been requested recently - * and no circuit is currently available that can handle it. */ + * and no circuit is currently available that can handle it. + * Exits (obviously) require an exit circuit. */ if (!circuit_all_predicted_ports_handled(now, &port_needs_uptime, - &port_needs_capacity)) { + &port_needs_capacity) + && router_have_consensus_path() == CONSENSUS_PATH_EXIT) { if (port_needs_uptime) flags |= CIRCLAUNCH_NEED_UPTIME; if (port_needs_capacity) @@ -913,8 +1048,10 @@ circuit_predict_and_launch_new(void) return; } - /* Third, see if we need any more hidden service (server) circuits. */ - if (num_rend_services() && num_uptime_internal < 3) { + /* Third, see if we need any more hidden service (server) circuits. + * HS servers only need an internal circuit. */ + if (num_rend_services() && num_uptime_internal < 3 + && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) { flags = (CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL); log_info(LD_CIRC, @@ -925,11 +1062,13 @@ circuit_predict_and_launch_new(void) return; } - /* Fourth, see if we need any more hidden service (client) circuits. */ + /* Fourth, see if we need any more hidden service (client) circuits. + * HS clients only need an internal circuit. */ if (rep_hist_get_predicted_internal(now, &hidserv_needs_uptime, &hidserv_needs_capacity) && ((num_uptime_internal<2 && hidserv_needs_uptime) || - num_internal<2)) { + num_internal<2) + && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) { if (hidserv_needs_uptime) flags |= CIRCLAUNCH_NEED_UPTIME; if (hidserv_needs_capacity) @@ -946,15 +1085,23 @@ circuit_predict_and_launch_new(void) /* Finally, check to see if we still need more circuits to learn * a good build timeout. But if we're close to our max number we * want, don't do another -- we want to leave a few slots open so - * we can still build circuits preemptively as needed. */ - if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && - ! circuit_build_times_disabled() && - circuit_build_times_needs_circuits_now(&circ_times)) { - flags = CIRCLAUNCH_NEED_CAPACITY; - log_info(LD_CIRC, - "Have %d clean circs need another buildtime test circ.", num); - circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags); - return; + * we can still build circuits preemptively as needed. + * XXXX make the assumption that build timeout streams should be + * created whenever we can build internal circuits. */ + if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) { + if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && + ! circuit_build_times_disabled() && + circuit_build_times_needs_circuits_now(get_circuit_build_times())) { + flags = CIRCLAUNCH_NEED_CAPACITY; + /* if there are no exits in the consensus, make timeout + * circuits internal */ + if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL) + flags |= CIRCLAUNCH_IS_INTERNAL; + log_info(LD_CIRC, + "Have %d clean circs need another buildtime test circ.", num); + circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags); + return; + } } } @@ -969,23 +1116,48 @@ circuit_predict_and_launch_new(void) void circuit_build_needed_circs(time_t now) { - static time_t time_to_new_circuit = 0; const or_options_t *options = get_options(); - /* launch a new circ for any pending streams that need one */ - connection_ap_attach_pending(); + /* launch a new circ for any pending streams that need one + * XXXX make the assumption that (some) AP streams (i.e. HS clients) + * don't require an exit circuit, review in #13814. + * This allows HSs to function in a consensus without exits. */ + if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) + connection_ap_rescan_and_attach_pending(); + + /* make sure any hidden services have enough intro points + * HS intro point streams only require an internal circuit */ + if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) + rend_consider_services_intro_points(); + + circuit_expire_old_circs_as_needed(now); - /* make sure any hidden services have enough intro points */ - rend_services_introduce(); + if (!options->DisablePredictedCircuits) + circuit_predict_and_launch_new(); +} - if (time_to_new_circuit < now) { +/** + * Called once a second either directly or from + * circuit_build_needed_circs(). As appropriate (once per NewCircuitPeriod) + * resets failure counts and expires old circuits. + */ +void +circuit_expire_old_circs_as_needed(time_t now) +{ + static time_t time_to_expire_and_reset = 0; + + if (time_to_expire_and_reset < now) { circuit_reset_failure_count(1); - time_to_new_circuit = now + options->NewCircuitPeriod; + time_to_expire_and_reset = now + get_options()->NewCircuitPeriod; if (proxy_mode(get_options())) addressmap_clean(now); circuit_expire_old_circuits_clientside(); #if 0 /* disable for now, until predict-and-launch-new can cull leftovers */ + + /* If we ever re-enable, this has to move into + * circuit_build_needed_circs */ + circ = circuit_get_youngest_clean_open(CIRCUIT_PURPOSE_C_GENERAL); if (get_options()->RunTesting && circ && @@ -995,8 +1167,6 @@ circuit_build_needed_circs(time_t now) } #endif } - if (!options->DisablePredictedCircuits) - circuit_predict_and_launch_new(); } /** If the stream <b>conn</b> is a member of any of the linked @@ -1019,17 +1189,31 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *origin_circ = TO_ORIGIN_CIRCUIT(circ); + int removed = 0; if (conn == origin_circ->p_streams) { origin_circ->p_streams = conn->next_stream; - return; + removed = 1; + } else { + for (prevconn = origin_circ->p_streams; + prevconn && prevconn->next_stream && prevconn->next_stream != conn; + prevconn = prevconn->next_stream) + ; + if (prevconn && prevconn->next_stream) { + prevconn->next_stream = conn->next_stream; + removed = 1; + } } + if (removed) { + log_debug(LD_APP, "Removing stream %d from circ %u", + conn->stream_id, (unsigned)circ->n_circ_id); - for (prevconn = origin_circ->p_streams; - prevconn && prevconn->next_stream && prevconn->next_stream != conn; - prevconn = prevconn->next_stream) - ; - if (prevconn && prevconn->next_stream) { - prevconn->next_stream = conn->next_stream; + /* If the stream was removed, and it was a rend stream, decrement the + * number of streams on the circuit associated with the rend service. + */ + if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) { + tor_assert(origin_circ->rend_data); + origin_circ->rend_data->nr_streams--; + } return; } } else { @@ -1078,14 +1262,14 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) static void circuit_expire_old_circuits_clientside(void) { - circuit_t *circ; struct timeval cutoff, now; tor_gettimeofday(&now); cutoff = now; + last_expired_clientside_circuits = now.tv_sec; if (! circuit_build_times_disabled() && - circuit_build_times_needs_circuits(&circ_times)) { + circuit_build_times_needs_circuits(get_circuit_build_times())) { /* Circuits should be shorter lived if we need more of them * for learning a good build timeout */ cutoff.tv_sec -= IDLE_TIMEOUT_WHILE_LEARNING; @@ -1093,7 +1277,7 @@ circuit_expire_old_circuits_clientside(void) cutoff.tv_sec -= get_options()->CircuitIdleTimeout; } - for (circ = global_circuitlist; circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; /* If the circuit has been dirty for too long, and there are no streams @@ -1113,7 +1297,7 @@ circuit_expire_old_circuits_clientside(void) if (circ->purpose != CIRCUIT_PURPOSE_PATH_BIAS_TESTING) circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); } else if (!circ->timestamp_dirty && circ->state == CIRCUIT_STATE_OPEN) { - if (timercmp(&circ->timestamp_began, &cutoff, <)) { + if (timercmp(&circ->timestamp_began, &cutoff, OP_LT)) { if (circ->purpose == CIRCUIT_PURPOSE_C_GENERAL || circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT || circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || @@ -1145,7 +1329,7 @@ circuit_expire_old_circuits_clientside(void) } } } - } + } SMARTLIST_FOREACH_END(circ); } /** How long do we wait before killing circuits with the properties @@ -1172,11 +1356,10 @@ circuit_expire_old_circuits_clientside(void) void circuit_expire_old_circuits_serverside(time_t now) { - circuit_t *circ; or_circuit_t *or_circ; time_t cutoff = now - IDLE_ONE_HOP_CIRC_TIMEOUT; - for (circ = global_circuitlist; circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (circ->marked_for_close || CIRCUIT_IS_ORIGIN(circ)) continue; or_circ = TO_OR_CIRCUIT(circ); @@ -1193,6 +1376,7 @@ circuit_expire_old_circuits_serverside(time_t now) circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); } } + SMARTLIST_FOREACH_END(circ); } /** Number of testing circuits we want open before testing our bandwidth. */ @@ -1217,18 +1401,18 @@ reset_bandwidth_test(void) int circuit_enough_testing_circs(void) { - circuit_t *circ; int num = 0; if (have_performed_bandwidth_test) return 1; - for (circ = global_circuitlist; circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->marked_for_close && CIRCUIT_IS_ORIGIN(circ) && circ->purpose == CIRCUIT_PURPOSE_TESTING && circ->state == CIRCUIT_STATE_OPEN) num++; } + SMARTLIST_FOREACH_END(circ); return num >= NUM_PARALLEL_TESTING_CIRCS; } @@ -1242,7 +1426,7 @@ static void circuit_testing_opened(origin_circuit_t *circ) { if (have_performed_bandwidth_test || - !check_whether_orport_reachable()) { + !check_whether_orport_reachable(get_options())) { /* either we've already done everything we want with testing circuits, * or this testing circuit became open due to a fluke, e.g. we picked * a last hop where we already had the connection open due to an @@ -1259,7 +1443,8 @@ circuit_testing_opened(origin_circuit_t *circ) static void circuit_testing_failed(origin_circuit_t *circ, int at_last_hop) { - if (server_mode(get_options()) && check_whether_orport_reachable()) + const or_options_t *options = get_options(); + if (server_mode(options) && check_whether_orport_reachable(options)) return; log_info(LD_GENERAL, @@ -1291,7 +1476,7 @@ circuit_has_opened(origin_circuit_t *circ) case CIRCUIT_PURPOSE_C_ESTABLISH_REND: rend_client_rendcirc_has_opened(circ); /* Start building an intro circ if we don't have one yet. */ - connection_ap_attach_pending(); + connection_ap_attach_pending(1); /* This isn't a call to circuit_try_attaching_streams because a * circuit in _C_ESTABLISH_REND state isn't connected to its * hidden service yet, thus we can't attach streams to it yet, @@ -1309,11 +1494,11 @@ circuit_has_opened(origin_circuit_t *circ) circuit_try_attaching_streams(circ); break; case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: - /* at Bob, waiting for introductions */ + /* at the service, waiting for introductions */ rend_service_intro_has_opened(circ); break; case CIRCUIT_PURPOSE_S_CONNECT_REND: - /* at Bob, connecting to rend point */ + /* at the service, connecting to rend point */ rend_service_rendezvous_has_opened(circ); break; case CIRCUIT_PURPOSE_TESTING: @@ -1353,14 +1538,14 @@ void circuit_try_attaching_streams(origin_circuit_t *circ) { /* Attach streams to this circuit if we can. */ - connection_ap_attach_pending(); + connection_ap_attach_pending(1); /* The call to circuit_try_clearing_isolation_state here will do * nothing and return 0 if we didn't attach any streams to circ * above. */ if (circuit_try_clearing_isolation_state(circ)) { /* Maybe *now* we can attach some streams to this circuit. */ - connection_ap_attach_pending(); + connection_ap_attach_pending(1); } } @@ -1433,32 +1618,32 @@ circuit_build_failed(origin_circuit_t *circ) circuit_testing_failed(circ, failed_at_last_hop); break; case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: - /* at Bob, waiting for introductions */ + /* at the service, waiting for introductions */ if (circ->base_.state != CIRCUIT_STATE_OPEN) { circuit_increment_failure_count(); } - /* no need to care here, because bob will rebuild intro + /* no need to care here, because the service will rebuild intro * points periodically. */ break; case CIRCUIT_PURPOSE_C_INTRODUCING: - /* at Alice, connecting to intro point */ - /* Don't increment failure count, since Bob may have picked + /* at the client, connecting to intro point */ + /* Don't increment failure count, since the service may have picked * the introduction point maliciously */ - /* Alice will pick a new intro point when this one dies, if + /* The client will pick a new intro point when this one dies, if * the stream in question still cares. No need to act here. */ break; case CIRCUIT_PURPOSE_C_ESTABLISH_REND: - /* at Alice, waiting for Bob */ + /* at the client, waiting for the service */ circuit_increment_failure_count(); - /* Alice will pick a new rend point when this one dies, if + /* the client will pick a new rend point when this one dies, if * the stream in question still cares. No need to act here. */ break; case CIRCUIT_PURPOSE_S_CONNECT_REND: - /* at Bob, connecting to rend point */ - /* Don't increment failure count, since Alice may have picked + /* at the service, connecting to rend point */ + /* Don't increment failure count, since the client may have picked * the rendezvous point maliciously */ log_info(LD_REND, - "Couldn't connect to Alice's chosen rend point %s " + "Couldn't connect to the client's chosen rend point %s " "(%s hop failed).", escaped(build_state_get_exit_nickname(circ->build_state)), failed_at_last_hop?"last":"non-last"); @@ -1490,6 +1675,20 @@ circuit_launch(uint8_t purpose, int flags) return circuit_launch_by_extend_info(purpose, NULL, flags); } +/* Do we have enough descriptors to build paths? + * If need_exit is true, return 1 if we can build exit paths. + * (We need at least one Exit in the consensus to build exit paths.) + * If need_exit is false, return 1 if we can build internal paths. + */ +static int +have_enough_path_info(int need_exit) +{ + if (need_exit) + return router_have_consensus_path() == CONSENSUS_PATH_EXIT; + else + return router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN; +} + /** Launch a new circuit with purpose <b>purpose</b> and exit node * <b>extend_info</b> (or NULL to select a random exit node). If flags * contains CIRCLAUNCH_NEED_UPTIME, choose among routers with high uptime. If @@ -1504,15 +1703,29 @@ circuit_launch_by_extend_info(uint8_t purpose, { origin_circuit_t *circ; int onehop_tunnel = (flags & CIRCLAUNCH_ONEHOP_TUNNEL) != 0; - - if (!onehop_tunnel && !router_have_minimum_dir_info()) { - log_debug(LD_CIRC,"Haven't fetched enough directory info yet; canceling " - "circuit launch."); + int have_path = have_enough_path_info(! (flags & CIRCLAUNCH_IS_INTERNAL) ); + int need_specific_rp = 0; + + if (!onehop_tunnel && (!router_have_minimum_dir_info() || !have_path)) { + log_debug(LD_CIRC,"Haven't %s yet; canceling " + "circuit launch.", + !router_have_minimum_dir_info() ? + "fetched enough directory info" : + "received a consensus with exits"); return NULL; } + /* If Tor2webRendezvousPoints is enabled and we are dealing with an + RP circuit, we want a specific RP node so we shouldn't canibalize + an already existing circuit. */ + if (get_options()->Tor2webRendezvousPoints && + purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND) { + need_specific_rp = 1; + } + if ((extend_info || purpose != CIRCUIT_PURPOSE_C_GENERAL) && - purpose != CIRCUIT_PURPOSE_TESTING && !onehop_tunnel) { + purpose != CIRCUIT_PURPOSE_TESTING && + !onehop_tunnel && !need_specific_rp) { /* see if there are appropriate circs available to cannibalize. */ /* XXX if we're planning to add a hop, perhaps we want to look for * internal circs rather than exit circs? -RD */ @@ -1561,12 +1774,12 @@ circuit_launch_by_extend_info(uint8_t purpose, switch (purpose) { case CIRCUIT_PURPOSE_C_ESTABLISH_REND: - case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: /* it's ready right now */ break; case CIRCUIT_PURPOSE_C_INTRODUCING: case CIRCUIT_PURPOSE_S_CONNECT_REND: case CIRCUIT_PURPOSE_C_GENERAL: + case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: /* need to add a new hop */ tor_assert(extend_info); if (circuit_extend_to_new_exit(circ, extend_info) < 0) @@ -1638,6 +1851,12 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, tor_assert(conn); tor_assert(circp); + if (ENTRY_TO_CONN(conn)->state != AP_CONN_STATE_CIRCUIT_WAIT) { + connection_t *c = ENTRY_TO_CONN(conn); + log_err(LD_BUG, "Connection state mismatch: wanted " + "AP_CONN_STATE_CIRCUIT_WAIT, but got %d (%s)", + c->state, conn_state_to_string(c->type, c->state)); + } tor_assert(ENTRY_TO_CONN(conn)->state == AP_CONN_STATE_CIRCUIT_WAIT); check_exit_policy = conn->socks_request->command == SOCKS_COMMAND_CONNECT && @@ -1664,7 +1883,9 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, return 1; /* we're happy */ } - if (!want_onehop && !router_have_minimum_dir_info()) { + int have_path = have_enough_path_info(!need_internal); + + if (!want_onehop && (!router_have_minimum_dir_info() || !have_path)) { if (!connection_get_by_type(CONN_TYPE_DIR)) { int severity = LOG_NOTICE; /* FFFF if this is a tunneled directory fetch, don't yell @@ -1672,14 +1893,20 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, if (entry_list_is_constrained(options) && entries_known_but_down(options)) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we haven't used client functionality " - "lately. Optimistically trying known %s again.", + "Application request when we haven't %s. " + "Optimistically trying known %s again.", + !router_have_minimum_dir_info() ? + "used client functionality lately" : + "received a consensus with exits", options->UseBridges ? "bridges" : "entrynodes"); entries_retry_all(options); } else if (!options->UseBridges || any_bridge_descriptors_known()) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we haven't used client functionality " - "lately. Optimistically trying directory fetches again."); + "Application request when we haven't %s. " + "Optimistically trying directory fetches again.", + !router_have_minimum_dir_info() ? + "used client functionality lately" : + "received a consensus with exits"); routerlist_retry_directory_downloads(time(NULL)); } } @@ -1764,6 +1991,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, "No intro points for '%s': re-fetching service descriptor.", safe_str_client(rend_data->onion_address)); rend_client_refetch_v2_renddesc(rend_data); + connection_ap_mark_as_non_pending_circuit(conn); ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_RENDDESC_WAIT; return 0; } @@ -1783,8 +2011,13 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, if (r && node_has_descriptor(r)) { /* We might want to connect to an IPv6 bridge for loading descriptors so we use the preferred address rather than - the primary. */ + the primary. */ extend_info = extend_info_from_node(r, conn->want_onehop ? 1 : 0); + if (!extend_info) { + log_warn(LD_CIRC,"Could not make a one-hop connection to %s. " + "Discarding this circuit.", conn->chosen_exit_name); + return -1; + } } else { log_debug(LD_DIR, "considering %d, %s", want_onehop, conn->chosen_exit_name); @@ -1834,11 +2067,13 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, else new_circ_purpose = desired_circuit_purpose; +#ifdef ENABLE_TOR2WEB_MODE if (options->Tor2webMode && (new_circ_purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND || new_circ_purpose == CIRCUIT_PURPOSE_C_INTRODUCING)) { want_onehop = 1; } +#endif { int flags = CIRCLAUNCH_NEED_CAPACITY; @@ -1870,7 +2105,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, circ->rend_data = rend_data_dup(ENTRY_TO_EDGE_CONN(conn)->rend_data); if (circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND && circ->base_.state == CIRCUIT_STATE_OPEN) - rend_client_rendcirc_has_opened(circ); + circuit_has_opened(circ); } } } /* endif (!circ) */ @@ -1928,7 +2163,7 @@ static void link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ, crypt_path_t *cpath) { - const node_t *exitnode; + const node_t *exitnode = NULL; /* add it into the linked list of streams on this circuit */ log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.", @@ -1945,7 +2180,7 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ, * that an attempt to connect to a hidden service just * succeeded. Tell rendclient.c. */ rend_client_note_connection_attempt_ended( - ENTRY_TO_EDGE_CONN(apconn)->rend_data->onion_address); + ENTRY_TO_EDGE_CONN(apconn)->rend_data); } if (cpath) { /* we were given one; use it */ @@ -1962,23 +2197,25 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ, circ->isolation_any_streams_attached = 1; connection_edge_update_circuit_isolation(apconn, circ, 0); + /* Compute the exitnode if possible, for logging below */ + if (cpath->extend_info) + exitnode = node_get_by_id(cpath->extend_info->identity_digest); + /* See if we can use optimistic data on this circuit */ - if (cpath->extend_info && - (exitnode = node_get_by_id(cpath->extend_info->identity_digest)) && - exitnode->rs) { - /* Okay; we know what exit node this is. */ - if (optimistic_data_enabled() && - circ->base_.purpose == CIRCUIT_PURPOSE_C_GENERAL && - exitnode->rs->version_supports_optimistic_data) - apconn->may_use_optimistic_data = 1; - else - apconn->may_use_optimistic_data = 0; - log_info(LD_APP, "Looks like completed circuit to %s %s allow " - "optimistic data for connection to %s", - safe_str_client(node_describe(exitnode)), - apconn->may_use_optimistic_data ? "does" : "doesn't", - safe_str_client(apconn->socks_request->address)); - } + if (optimistic_data_enabled() && + (circ->base_.purpose == CIRCUIT_PURPOSE_C_GENERAL || + circ->base_.purpose == CIRCUIT_PURPOSE_C_REND_JOINED)) + apconn->may_use_optimistic_data = 1; + else + apconn->may_use_optimistic_data = 0; + log_info(LD_APP, "Looks like completed circuit to %s %s allow " + "optimistic data for connection to %s", + circ->base_.purpose == CIRCUIT_PURPOSE_C_GENERAL ? + /* node_describe() does the right thing if exitnode is NULL */ + safe_str_client(node_describe(exitnode)) : + "hidden service", + apconn->may_use_optimistic_data ? "does" : "doesn't", + safe_str_client(apconn->socks_request->address)); } /** Return true iff <b>address</b> is matched by one of the entries in @@ -2014,7 +2251,7 @@ consider_recording_trackhost(const entry_connection_t *conn, char fp[HEX_DIGEST_LEN+1]; /* Search the addressmap for this conn's destination. */ - /* If he's not in the address map.. */ + /* If they're not in the address map.. */ if (!options->TrackHostExits || addressmap_have_mapping(conn->socks_request->address, options->TrackHostExitsExpire)) @@ -2058,8 +2295,15 @@ connection_ap_handshake_attach_chosen_circuit(entry_connection_t *conn, base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT; - if (!circ->base_.timestamp_dirty) - circ->base_.timestamp_dirty = time(NULL); + if (!circ->base_.timestamp_dirty || + ((conn->entry_cfg.isolation_flags & ISO_SOCKSAUTH) && + (conn->entry_cfg.socks_iso_keep_alive) && + (conn->socks_request->usernamelen || + conn->socks_request->passwordlen))) { + /* When stream isolation is in use and controlled by an application + * we are willing to keep using the stream. */ + circ->base_.timestamp_dirty = approx_time(); + } pathbias_count_use_attempt(circ); @@ -2116,6 +2360,25 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn) /* we're a general conn */ origin_circuit_t *circ=NULL; + /* Are we linked to a dir conn that aims to fetch a consensus? + * We check here because this conn might no longer be needed. */ + if (base_conn->linked_conn && + base_conn->linked_conn->type == CONN_TYPE_DIR && + base_conn->linked_conn->purpose == DIR_PURPOSE_FETCH_CONSENSUS) { + + /* Yes we are. Is there a consensus fetch farther along than us? */ + if (networkstatus_consensus_is_already_downloading( + TO_DIR_CONN(base_conn->linked_conn)->requested_resource)) { + /* We're doing the "multiple consensus fetch attempts" game from + * proposal 210, and we're late to the party. Just close this conn. + * The circuit and TLS conn that we made will time out after a while + * if nothing else wants to use them. */ + log_info(LD_DIR, "Closing extra consensus fetch (to %s) since one " + "is already downloading.", base_conn->linked_conn->address); + return -1; + } + } + if (conn->chosen_exit_name) { const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 1); int opt = conn->chosen_exit_optional; @@ -2180,7 +2443,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn) tor_assert(rendcirc); /* one is already established, attach */ log_info(LD_REND, - "rend joined circ %d already here. attaching. " + "rend joined circ %u already here. attaching. " "(stream %d sec old)", (unsigned)rendcirc->base_.n_circ_id, conn_age); /* Mark rendezvous circuits as 'newly dirty' every time you use @@ -2200,6 +2463,18 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn) return 1; } + /* At this point we need to re-check the state, since it's possible that + * our call to circuit_get_open_circ_or_launch() changed the connection's + * state from "CIRCUIT_WAIT" to "RENDDESC_WAIT" because we decided to + * re-fetch the descriptor. + */ + if (ENTRY_TO_CONN(conn)->state != AP_CONN_STATE_CIRCUIT_WAIT) { + log_info(LD_REND, "This connection is no longer ready to attach; its " + "state changed." + "(We probably have to re-fetch its descriptor.)"); + return 0; + } + if (rendcirc && (rendcirc->base_.purpose == CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED)) { log_info(LD_REND, diff --git a/src/or/circuituse.h b/src/or/circuituse.h index 11e5a64163..5973978c45 100644 --- a/src/or/circuituse.h +++ b/src/or/circuituse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,11 +16,13 @@ void circuit_expire_building(void); void circuit_remove_handled_ports(smartlist_t *needed_ports); int circuit_stream_is_being_handled(entry_connection_t *conn, uint16_t port, int min); +void circuit_log_ancient_one_hop_circuits(int age); #if 0 int circuit_conforms_to_options(const origin_circuit_t *circ, const or_options_t *options); #endif void circuit_build_needed_circs(time_t now); +void circuit_expire_old_circs_as_needed(time_t now); void circuit_detach_stream(circuit_t *circ, edge_connection_t *conn); void circuit_expire_old_circuits_serverside(time_t now); diff --git a/src/or/command.c b/src/or/command.c index 78fd4fad33..5ad92bed1e 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -53,6 +53,33 @@ static void command_process_created_cell(cell_t *cell, channel_t *chan); static void command_process_relay_cell(cell_t *cell, channel_t *chan); static void command_process_destroy_cell(cell_t *cell, channel_t *chan); +/** Convert the cell <b>command</b> into a lower-case, human-readable + * string. */ +const char * +cell_command_to_string(uint8_t command) +{ + switch (command) { + case CELL_PADDING: return "padding"; + case CELL_CREATE: return "create"; + case CELL_CREATED: return "created"; + case CELL_RELAY: return "relay"; + case CELL_DESTROY: return "destroy"; + case CELL_CREATE_FAST: return "create_fast"; + case CELL_CREATED_FAST: return "created_fast"; + case CELL_VERSIONS: return "versions"; + case CELL_NETINFO: return "netinfo"; + case CELL_RELAY_EARLY: return "relay_early"; + case CELL_CREATE2: return "create2"; + case CELL_CREATED2: return "created2"; + case CELL_VPADDING: return "vpadding"; + case CELL_CERTS: return "certs"; + case CELL_AUTH_CHALLENGE: return "auth_challenge"; + case CELL_AUTHENTICATE: return "authenticate"; + case CELL_AUTHORIZE: return "authorize"; + default: return "unrecognized"; + } +} + #ifdef KEEP_TIMING_STATS /** This is a wrapper function around the actual function that processes the * <b>cell</b> that just arrived on <b>conn</b>. Increment <b>*time</b> @@ -200,6 +227,34 @@ command_process_create_cell(cell_t *cell, channel_t *chan) (unsigned)cell->circ_id, U64_PRINTF_ARG(chan->global_identifier), chan); + /* We check for the conditions that would make us drop the cell before + * we check for the conditions that would make us send a DESTROY back, + * since those conditions would make a DESTROY nonsensical. */ + if (cell->circ_id == 0) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received a create cell (type %d) from %s with zero circID; " + " ignoring.", (int)cell->command, + channel_get_actual_remote_descr(chan)); + return; + } + + if (circuit_id_in_use_on_channel(cell->circ_id, chan)) { + const node_t *node = node_get_by_id(chan->identity_digest); + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Received CREATE cell (circID %u) for known circ. " + "Dropping (age %d).", + (unsigned)cell->circ_id, + (int)(time(NULL) - channel_when_created(chan))); + if (node) { + char *p = esc_for_log(node_get_platform(node)); + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Details: router %s, platform %s.", + node_describe(node), p); + tor_free(p); + } + return; + } + if (we_are_hibernating()) { log_info(LD_OR, "Received create cell but we're shutting down. Sending back " @@ -221,14 +276,6 @@ command_process_create_cell(cell_t *cell, channel_t *chan) return; } - if (cell->circ_id == 0) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Received a create cell (type %d) from %s with zero circID; " - " ignoring.", (int)cell->command, - channel_get_actual_remote_descr(chan)); - return; - } - /* If the high bit of the circuit ID is not as expected, close the * circ. */ if (chan->wide_circ_ids) @@ -247,23 +294,6 @@ command_process_create_cell(cell_t *cell, channel_t *chan) return; } - if (circuit_id_in_use_on_channel(cell->circ_id, chan)) { - const node_t *node = node_get_by_id(chan->identity_digest); - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Received CREATE cell (circID %u) for known circ. " - "Dropping (age %d).", - (unsigned)cell->circ_id, - (int)(time(NULL) - channel_when_created(chan))); - if (node) { - char *p = esc_for_log(node_get_platform(node)); - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Details: router %s, platform %s.", - node_describe(node), p); - tor_free(p); - } - return; - } - circ = or_circuit_new(cell->circ_id, chan); circ->base_.purpose = CIRCUIT_PURPOSE_OR; circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_ONIONSKIN_PENDING); @@ -280,7 +310,7 @@ command_process_create_cell(cell_t *cell, channel_t *chan) /* hand it off to the cpuworkers, and then return. */ if (connection_or_digest_is_known_relay(chan->identity_digest)) rep_hist_note_circuit_handshake_requested(create_cell->handshake_type); - if (assign_onionskin_to_cpuworker(NULL, circ, create_cell) < 0) { + if (assign_onionskin_to_cpuworker(circ, create_cell) < 0) { log_debug(LD_GENERAL,"Failed to hand off onionskin. Closing."); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); return; @@ -310,7 +340,6 @@ command_process_create_cell(cell_t *cell, channel_t *chan) if (len < 0) { log_warn(LD_OR,"Failed to generate key material. Closing."); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); - tor_free(create_cell); return; } created_cell.cell_type = CELL_CREATED_FAST; @@ -349,7 +378,7 @@ command_process_created_cell(cell_t *cell, channel_t *chan) return; } - if (circ->n_circ_id != cell->circ_id) { + if (circ->n_circ_id != cell->circ_id || circ->n_chan != chan) { log_fn(LOG_PROTOCOL_WARN,LD_PROTOCOL, "got created cell from Tor client? Closing."); circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); @@ -368,7 +397,6 @@ command_process_created_cell(cell_t *cell, channel_t *chan) log_debug(LD_OR,"at OP. Finishing handshake."); if ((err_reason = circuit_finish_handshake(origin_circ, &extended_cell.created_cell)) < 0) { - log_warn(LD_OR,"circuit_finish_handshake failed."); circuit_mark_for_close(circ, -err_reason); return; } @@ -408,6 +436,7 @@ command_process_created_cell(cell_t *cell, channel_t *chan) static void command_process_relay_cell(cell_t *cell, channel_t *chan) { + const or_options_t *options = get_options(); circuit_t *circ; int reason, direction; @@ -434,6 +463,7 @@ command_process_relay_cell(cell_t *cell, channel_t *chan) } if (!CIRCUIT_IS_ORIGIN(circ) && + chan == TO_OR_CIRCUIT(circ)->p_chan && cell->circ_id == TO_OR_CIRCUIT(circ)->p_circ_id) direction = CELL_DIRECTION_OUT; else @@ -480,6 +510,14 @@ command_process_relay_cell(cell_t *cell, channel_t *chan) direction==CELL_DIRECTION_OUT?"forward":"backward"); circuit_mark_for_close(circ, -reason); } + + /* If this is a cell in an RP circuit, count it as part of the + hidden service stats */ + if (options->HiddenServiceStatistics && + !CIRCUIT_IS_ORIGIN(circ) && + TO_OR_CIRCUIT(circ)->circuit_carries_hs_traffic_stats) { + rep_hist_seen_new_rp_cell(); + } } /** Process a 'destroy' <b>cell</b> that just arrived from @@ -514,6 +552,7 @@ command_process_destroy_cell(cell_t *cell, channel_t *chan) circ->received_destroy = 1; if (!CIRCUIT_IS_ORIGIN(circ) && + chan == TO_OR_CIRCUIT(circ)->p_chan && cell->circ_id == TO_OR_CIRCUIT(circ)->p_circ_id) { /* the destroy came from behind */ circuit_set_p_circid_chan(TO_OR_CIRCUIT(circ), 0, NULL); diff --git a/src/or/command.h b/src/or/command.h index 913f46a5cd..12cda6a463 100644 --- a/src/or/command.h +++ b/src/or/command.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,6 +19,8 @@ void command_process_var_cell(channel_t *chan, var_cell_t *cell); void command_setup_channel(channel_t *chan); void command_setup_listener(channel_listener_t *chan_l); +const char *cell_command_to_string(uint8_t command); + extern uint64_t stats_n_padding_cells_processed; extern uint64_t stats_n_create_cells_processed; extern uint64_t stats_n_created_cells_processed; diff --git a/src/or/config.c b/src/or/config.c index 709c5cc687..4b065a0053 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1,7 +1,7 @@ - /* Copyright (c) 2001 Matej Pfajfar. +/* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -10,8 +10,8 @@ **/ #define CONFIG_PRIVATE - #include "or.h" +#include "compat.h" #include "addressmap.h" #include "channel.h" #include "circuitbuild.h" @@ -40,20 +40,37 @@ #include "rendservice.h" #include "rephist.h" #include "router.h" +#include "sandbox.h" #include "util.h" #include "routerlist.h" #include "routerset.h" +#include "scheduler.h" #include "statefile.h" #include "transports.h" +#include "ext_orport.h" +#include "torgzip.h" #ifdef _WIN32 #include <shlobj.h> #endif #include "procmon.h" +#ifdef HAVE_SYSTEMD +# if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) +/* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse + * Coverity. Here's a kludge to unconfuse it. + */ +# define __INCLUDE_LEVEL__ 2 +# endif +#include <systemd/sd-daemon.h> +#endif + /* From main.c */ extern int quiet_level; +/* Prefix used to indicate a Unix socket in a FooPort configuration. */ +static const char unix_socket_prefix[] = "unix:"; + /** A list of abbreviations and aliases to map command-line options, obsolete * option names, or alternative option names, to their current values. */ static config_abbrev_t option_abbrevs_[] = { @@ -61,15 +78,16 @@ static config_abbrev_t option_abbrevs_[] = { PLURAL(AuthDirBadExitCC), PLURAL(AuthDirInvalidCC), PLURAL(AuthDirRejectCC), - PLURAL(ExitNode), PLURAL(EntryNode), PLURAL(ExcludeNode), + PLURAL(Tor2webRendezvousPoint), PLURAL(FirewallPort), PLURAL(LongLivedPort), PLURAL(HiddenServiceNode), PLURAL(HiddenServiceExcludeNode), PLURAL(NumCPU), PLURAL(RendNode), + PLURAL(RecommendedPackage), PLURAL(RendExcludeNode), PLURAL(StrictEntryNode), PLURAL(StrictExitNode), @@ -83,6 +101,7 @@ static config_abbrev_t option_abbrevs_[] = { { "DirFetchPostPeriod", "StatusFetchPeriod", 0, 0}, { "DirServer", "DirAuthority", 0, 0}, /* XXXX024 later, make this warn? */ { "MaxConn", "ConnLimit", 0, 1}, + { "MaxMemInCellQueues", "MaxMemInQueues", 0, 0}, { "ORBindAddress", "ORListenAddress", 0, 0}, { "DirBindAddress", "DirListenAddress", 0, 0}, { "SocksBindAddress", "SocksListenAddress", 0, 0}, @@ -96,8 +115,6 @@ static config_abbrev_t option_abbrevs_[] = { { "PreferTunnelledDirConns", "PreferTunneledDirConns", 0, 0}, { "BridgeAuthoritativeDirectory", "BridgeAuthoritativeDir", 0, 0}, { "HashedControlPassword", "__HashedControlSessionPassword", 1, 0}, - { "StrictEntryNodes", "StrictNodes", 0, 1}, - { "StrictExitNodes", "StrictNodes", 0, 1}, { "VirtualAddrNetwork", "VirtualAddrNetworkIPv4", 0, 0}, { "_UseFilteringSSLBufferevents", "UseFilteringSSLBufferevents", 0, 1}, { NULL, NULL, 0, 0}, @@ -124,8 +141,8 @@ static config_abbrev_t option_abbrevs_[] = { * be chosen first. */ static config_var_t option_vars_[] = { - OBSOLETE("AccountingMaxKB"), V(AccountingMax, MEMUNIT, "0 bytes"), + VAR("AccountingRule", STRING, AccountingRule_option, "max"), V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), V(AllowDotExit, BOOL, "0"), @@ -135,20 +152,21 @@ static config_var_t option_vars_[] = { V(AllowSingleHopExits, BOOL, "0"), V(AlternateBridgeAuthority, LINELIST, NULL), V(AlternateDirAuthority, LINELIST, NULL), - V(AlternateHSAuthority, LINELIST, NULL), + OBSOLETE("AlternateHSAuthority"), V(AssumeReachable, BOOL, "0"), - V(AuthDirBadDir, LINELIST, NULL), - V(AuthDirBadDirCCs, CSV, ""), + OBSOLETE("AuthDirBadDir"), + OBSOLETE("AuthDirBadDirCCs"), V(AuthDirBadExit, LINELIST, NULL), V(AuthDirBadExitCCs, CSV, ""), V(AuthDirInvalid, LINELIST, NULL), V(AuthDirInvalidCCs, CSV, ""), V(AuthDirFastGuarantee, MEMUNIT, "100 KB"), - V(AuthDirGuardBWGuarantee, MEMUNIT, "250 KB"), + V(AuthDirGuardBWGuarantee, MEMUNIT, "2 MB"), + V(AuthDirPinKeys, BOOL, "0"), V(AuthDirReject, LINELIST, NULL), V(AuthDirRejectCCs, CSV, ""), - V(AuthDirRejectUnlisted, BOOL, "0"), - V(AuthDirListBadDirs, BOOL, "0"), + OBSOLETE("AuthDirRejectUnlisted"), + OBSOLETE("AuthDirListBadDirs"), V(AuthDirListBadExits, BOOL, "0"), V(AuthDirMaxServersPerAddr, UINT, "2"), V(AuthDirMaxServersPerAuthAddr,UINT, "5"), @@ -172,10 +190,12 @@ static config_var_t option_vars_[] = { V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), V(ClientOnly, BOOL, "0"), - V(ClientPreferIPv6ORPort, BOOL, "0"), + V(ClientPreferIPv6ORPort, AUTOBOOL, "auto"), + V(ClientPreferIPv6DirPort, AUTOBOOL, "auto"), V(ClientRejectInternalAddresses, BOOL, "1"), V(ClientTransportPlugin, LINELIST, NULL), V(ClientUseIPv6, BOOL, "0"), + V(ClientUseIPv4, BOOL, "1"), V(ConsensusParams, STRING, NULL), V(ConnLimit, UINT, "1000"), V(ConnDirectionStatistics, BOOL, "0"), @@ -188,36 +208,35 @@ static config_var_t option_vars_[] = { V(ControlPortWriteToFile, FILENAME, NULL), V(ControlSocket, LINELIST, NULL), V(ControlSocketsGroupWritable, BOOL, "0"), + V(SocksSocketsGroupWritable, BOOL, "0"), V(CookieAuthentication, BOOL, "0"), V(CookieAuthFileGroupReadable, BOOL, "0"), V(CookieAuthFile, STRING, NULL), V(CountPrivateBandwidth, BOOL, "0"), V(DataDirectory, FILENAME, NULL), - OBSOLETE("DebugLogFile"), + V(DataDirectoryGroupReadable, BOOL, "0"), V(DisableNetwork, BOOL, "0"), V(DirAllowPrivateAddresses, BOOL, "0"), V(TestingAuthDirTimeToLearnReachability, INTERVAL, "30 minutes"), V(DirListenAddress, LINELIST, NULL), - OBSOLETE("DirFetchPeriod"), V(DirPolicy, LINELIST, NULL), VPORT(DirPort, LINELIST, NULL), V(DirPortFrontPage, FILENAME, NULL), - OBSOLETE("DirPostPeriod"), - OBSOLETE("DirRecordUsageByCountry"), - OBSOLETE("DirRecordUsageGranularity"), - OBSOLETE("DirRecordUsageRetainIPs"), - OBSOLETE("DirRecordUsageSaveInterval"), - V(DirReqStatistics, BOOL, "1"), + VAR("DirReqStatistics", BOOL, DirReqStatistics_option, "1"), VAR("DirAuthority", LINELIST, DirAuthorities, NULL), + V(DirCache, BOOL, "1"), V(DirAuthorityFallbackRate, DOUBLE, "1.0"), V(DisableAllSwap, BOOL, "0"), V(DisableDebuggerAttachment, BOOL, "1"), V(DisableIOCP, BOOL, "1"), - V(DisableV2DirectoryInfo_, BOOL, "0"), - V(DynamicDHGroups, BOOL, "0"), + OBSOLETE("DisableV2DirectoryInfo_"), + OBSOLETE("DynamicDHGroups"), VPORT(DNSPort, LINELIST, NULL), V(DNSListenAddress, LINELIST, NULL), V(DownloadExtraInfo, BOOL, "0"), + V(TestingEnableConnBwEvent, BOOL, "0"), + V(TestingEnableCellStatsEvent, BOOL, "0"), + V(TestingEnableTbEmptyEvent, BOOL, "0"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), V(EntryStatistics, BOOL, "0"), @@ -230,8 +249,13 @@ static config_var_t option_vars_[] = { V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPortStatistics, BOOL, "0"), V(ExtendAllowPrivateAddresses, BOOL, "0"), + V(ExitRelay, AUTOBOOL, "auto"), + VPORT(ExtORPort, LINELIST, NULL), + V(ExtORPortCookieAuthFile, STRING, NULL), + V(ExtORPortCookieAuthFileGroupReadable, BOOL, "0"), V(ExtraInfoStatistics, BOOL, "1"), V(FallbackDir, LINELIST, NULL), + V(UseDefaultFallbackDirs, BOOL, "1"), OBSOLETE("FallbackNetworkstatusFile"), V(FascistFirewall, BOOL, "0"), @@ -242,7 +266,7 @@ static config_var_t option_vars_[] = { V(FetchServerDescriptors, BOOL, "1"), V(FetchHidServDescriptors, BOOL, "1"), V(FetchUselessDescriptors, BOOL, "0"), - V(FetchV2Networkstatus, BOOL, "0"), + OBSOLETE("FetchV2Networkstatus"), V(GeoIPExcludeUnknown, AUTOBOOL, "auto"), #ifdef _WIN32 V(GeoIPFile, FILENAME, "<default>"), @@ -253,7 +277,6 @@ static config_var_t option_vars_[] = { V(GeoIPv6File, FILENAME, SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "geoip6"), #endif - OBSOLETE("GiveGuardFlagTo_CVE_2011_2768_VulnerableRelays"), OBSOLETE("Group"), V(GuardLifetime, INTERVAL, "0 minutes"), V(HardwareAccel, BOOL, "0"), @@ -261,17 +284,19 @@ static config_var_t option_vars_[] = { V(AccelName, STRING, NULL), V(AccelDir, FILENAME, NULL), V(HashedControlPassword, LINELIST, NULL), - V(HidServDirectoryV2, BOOL, "1"), + OBSOLETE("HidServDirectoryV2"), VAR("HiddenServiceDir", LINELIST_S, RendConfigLines, NULL), - OBSOLETE("HiddenServiceExcludeNodes"), - OBSOLETE("HiddenServiceNodes"), + VAR("HiddenServiceDirGroupReadable", LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceOptions",LINELIST_V, RendConfigLines, NULL), VAR("HiddenServicePort", LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceVersion",LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL), + VAR("HiddenServiceAllowUnknownPorts",LINELIST_S, RendConfigLines, NULL), + VAR("HiddenServiceMaxStreams",LINELIST_S, RendConfigLines, NULL), + VAR("HiddenServiceMaxStreamsCloseCircuit",LINELIST_S, RendConfigLines, NULL), + VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL), + V(HiddenServiceStatistics, BOOL, "1"), V(HidServAuth, LINELIST, NULL), - V(HSAuthoritativeDir, BOOL, "0"), - OBSOLETE("HSAuthorityRecordStats"), V(CloseHSClientCircuitsImmediatelyOnTimeout, BOOL, "0"), V(CloseHSServiceRendCircuitsImmediatelyOnTimeout, BOOL, "0"), V(HTTPProxy, STRING, NULL), @@ -281,41 +306,42 @@ static config_var_t option_vars_[] = { V(IPv6Exit, BOOL, "0"), VAR("ServerTransportPlugin", LINELIST, ServerTransportPlugin, NULL), V(ServerTransportListenAddr, LINELIST, NULL), + V(ServerTransportOptions, LINELIST, NULL), + V(SigningKeyLifetime, INTERVAL, "30 days"), V(Socks4Proxy, STRING, NULL), V(Socks5Proxy, STRING, NULL), V(Socks5ProxyUsername, STRING, NULL), V(Socks5ProxyPassword, STRING, NULL), - OBSOLETE("IgnoreVersion"), V(KeepalivePeriod, INTERVAL, "5 minutes"), + V(KeepBindCapabilities, AUTOBOOL, "auto"), VAR("Log", LINELIST, Logs, NULL), V(LogMessageDomains, BOOL, "0"), - OBSOLETE("LinkPadding"), - OBSOLETE("LogLevel"), - OBSOLETE("LogFile"), V(LogTimeGranularity, MSEC_INTERVAL, "1 second"), + V(TruncateLogFile, BOOL, "0"), + V(SyslogIdentityTag, STRING, NULL), V(LongLivedPorts, CSV, "21,22,706,1863,5050,5190,5222,5223,6523,6667,6697,8300"), VAR("MapAddress", LINELIST, AddressMap, NULL), V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), V(MaxClientCircuitsPending, UINT, "32"), - V(MaxMemInCellQueues, MEMUNIT, "8 GB"), + VAR("MaxMemInQueues", MEMUNIT, MaxMemInQueues_raw, "0"), OBSOLETE("MaxOnionsPending"), V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), - OBSOLETE("MonthlyAccountingStart"), V(MyFamily, STRING, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), - VAR("NamingAuthoritativeDirectory",BOOL, NamingAuthoritativeDir, "0"), + OBSOLETE("NamingAuthoritativeDirectory"), V(NATDListenAddress, LINELIST, NULL), VPORT(NATDPort, LINELIST, NULL), V(Nickname, STRING, NULL), + V(PredictedPortsRelevanceTime, INTERVAL, "1 hour"), V(WarnUnsafeSocks, BOOL, "1"), - OBSOLETE("NoPublish"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), V(NumCPUs, UINT, "0"), V(NumDirectoryGuards, UINT, "0"), V(NumEntryGuards, UINT, "0"), + V(OfflineMasterKey, BOOL, "0"), V(ORListenAddress, LINELIST, NULL), VPORT(ORPort, LINELIST, NULL), V(OutboundBindAddress, LINELIST, NULL), @@ -337,17 +363,23 @@ static config_var_t option_vars_[] = { V(PathBiasScaleUseThreshold, INT, "-1"), V(PathsNeededToBuildCircuits, DOUBLE, "-1"), - OBSOLETE("PathlenCoinWeight"), V(PerConnBWBurst, MEMUNIT, "0"), V(PerConnBWRate, MEMUNIT, "0"), V(PidFile, STRING, NULL), V(TestingTorNetwork, BOOL, "0"), V(TestingMinExitFlagThreshold, MEMUNIT, "0"), V(TestingMinFastFlagThreshold, MEMUNIT, "0"), + + V(TestingLinkCertLifetime, INTERVAL, "2 days"), + V(TestingAuthKeyLifetime, INTERVAL, "2 days"), + V(TestingLinkKeySlop, INTERVAL, "3 hours"), + V(TestingAuthKeySlop, INTERVAL, "3 hours"), + V(TestingSigningKeySlop, INTERVAL, "1 day"), + V(OptimisticData, AUTOBOOL, "auto"), V(PortForwarding, BOOL, "0"), V(PortForwardingHelper, FILENAME, "tor-fw-helper"), - V(PreferTunneledDirConns, BOOL, "1"), + OBSOLETE("PreferTunneledDirConns"), V(ProtocolWarnings, BOOL, "0"), V(PublishServerDescriptor, CSV, "1"), V(PublishHidServDescriptors, BOOL, "1"), @@ -357,19 +389,16 @@ static config_var_t option_vars_[] = { V(RecommendedVersions, LINELIST, NULL), V(RecommendedClientVersions, LINELIST, NULL), V(RecommendedServerVersions, LINELIST, NULL), - OBSOLETE("RedirectExit"), + V(RecommendedPackages, LINELIST, NULL), V(RefuseUnknownExits, AUTOBOOL, "auto"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), V(RelayBandwidthRate, MEMUNIT, "0"), - OBSOLETE("RendExcludeNodes"), - OBSOLETE("RendNodes"), V(RendPostPeriod, INTERVAL, "1 hour"), V(RephistTrackTime, INTERVAL, "24 hours"), - OBSOLETE("RouterFile"), V(RunAsDaemon, BOOL, "0"), -// V(RunTesting, BOOL, "0"), OBSOLETE("RunTesting"), // currently unused + V(Sandbox, BOOL, "0"), V(SafeLogging, STRING, "1"), V(SafeSocks, BOOL, "0"), V(ServerDNSAllowBrokenConfig, BOOL, "1"), @@ -380,48 +409,55 @@ static config_var_t option_vars_[] = { V(ServerDNSSearchDomains, BOOL, "0"), V(ServerDNSTestAddresses, CSV, "www.google.com,www.mit.edu,www.yahoo.com,www.slashdot.org"), + V(SchedulerLowWaterMark__, MEMUNIT, "100 MB"), + V(SchedulerHighWaterMark__, MEMUNIT, "101 MB"), + V(SchedulerMaxFlushCells__, UINT, "1000"), V(ShutdownWaitLength, INTERVAL, "30 seconds"), V(SocksListenAddress, LINELIST, NULL), V(SocksPolicy, LINELIST, NULL), VPORT(SocksPort, LINELIST, NULL), V(SocksTimeout, INTERVAL, "2 minutes"), V(SSLKeyLifetime, INTERVAL, "0"), - OBSOLETE("StatusFetchPeriod"), + OBSOLETE("StrictEntryNodes"), + OBSOLETE("StrictExitNodes"), V(StrictNodes, BOOL, "0"), - V(Support022HiddenServices, AUTOBOOL, "auto"), - OBSOLETE("SysLog"), + OBSOLETE("Support022HiddenServices"), V(TestSocks, BOOL, "0"), - OBSOLETE("TestVia"), V(TokenBucketRefillInterval, MSEC_INTERVAL, "100 msec"), V(Tor2webMode, BOOL, "0"), + V(Tor2webRendezvousPoints, ROUTERSET, NULL), V(TLSECGroup, STRING, NULL), V(TrackHostExits, CSV, NULL), V(TrackHostExitsExpire, INTERVAL, "30 minutes"), - OBSOLETE("TrafficShaping"), V(TransListenAddress, LINELIST, NULL), VPORT(TransPort, LINELIST, NULL), - V(TunnelDirConns, BOOL, "1"), + V(TransProxyType, STRING, "default"), + OBSOLETE("TunnelDirConns"), V(UpdateBridgesFromAuthority, BOOL, "0"), V(UseBridges, BOOL, "0"), V(UseEntryGuards, BOOL, "1"), V(UseEntryGuardsAsDirGuards, BOOL, "1"), + V(UseGuardFraction, AUTOBOOL, "auto"), V(UseMicrodescriptors, AUTOBOOL, "auto"), - V(UseNTorHandshake, AUTOBOOL, "auto"), + V(UseNTorHandshake, AUTOBOOL, "1"), V(User, STRING, NULL), V(UserspaceIOCPBuffers, BOOL, "0"), - VAR("V1AuthoritativeDirectory",BOOL, V1AuthoritativeDir, "0"), - VAR("V2AuthoritativeDirectory",BOOL, V2AuthoritativeDir, "0"), + OBSOLETE("V1AuthoritativeDirectory"), + OBSOLETE("V2AuthoritativeDirectory"), VAR("V3AuthoritativeDirectory",BOOL, V3AuthoritativeDir, "0"), V(TestingV3AuthInitialVotingInterval, INTERVAL, "30 minutes"), V(TestingV3AuthInitialVoteDelay, INTERVAL, "5 minutes"), V(TestingV3AuthInitialDistDelay, INTERVAL, "5 minutes"), + V(TestingV3AuthVotingStartOffset, INTERVAL, "0"), V(V3AuthVotingInterval, INTERVAL, "1 hour"), V(V3AuthVoteDelay, INTERVAL, "5 minutes"), V(V3AuthDistDelay, INTERVAL, "5 minutes"), V(V3AuthNIntervalsValid, UINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), V(V3BandwidthsFile, FILENAME, NULL), + V(GuardfractionFile, FILENAME, NULL), VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"), + OBSOLETE("VoteOnHidServDirectoriesV2"), V(VirtualAddrNetworkIPv4, STRING, "127.192.0.0/10"), V(VirtualAddrNetworkIPv6, STRING, "[FE80::]/10"), V(WarnPlaintextPorts, CSV, "23,109,110,143"), @@ -433,8 +469,60 @@ static config_var_t option_vars_[] = { VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword, NULL), VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), - V(MinUptimeHidServDirectoryV2, INTERVAL, "25 hours"), - V(VoteOnHidServDirectoriesV2, BOOL, "1"), + V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"), + V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 60, 60, 120, " + "300, 900, 2147483647"), + V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 60, 300, 600, " + "2147483647"), + V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, " + "300, 600, 1800, 1800, 1800, 1800, " + "1800, 3600, 7200"), + V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, " + "300, 600, 1800, 3600, 3600, 3600, " + "10800, 21600, 43200"), + /* With the ClientBootstrapConsensus*Download* below: + * Clients with only authorities will try: + * - 3 authorities over 10 seconds, then wait 60 minutes. + * Clients with authorities and fallbacks will try: + * - 2 authorities and 4 fallbacks over 21 seconds, then wait 60 minutes. + * Clients will also retry when an application request arrives. + * After a number of failed reqests, clients retry every 3 days + 1 hour. + * + * Clients used to try 2 authorities over 10 seconds, then wait for + * 60 minutes or an application request. + * + * When clients have authorities and fallbacks available, they use these + * schedules: (we stagger the times to avoid thundering herds) */ + V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL, + "10, 11, 3600, 10800, 25200, 54000, 111600, 262800" /* 3 days + 1 hour */), + V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL, + "0, 1, 4, 11, 3600, 10800, 25200, 54000, 111600, 262800"), + /* When clients only have authorities available, they use this schedule: */ + V(ClientBootstrapConsensusAuthorityOnlyDownloadSchedule, CSV_INTERVAL, + "0, 3, 7, 3600, 10800, 25200, 54000, 111600, 262800"), + /* We don't want to overwhelm slow networks (or mirrors whose replies are + * blocked), but we also don't want to fail if only some mirrors are + * blackholed. Clients will try 3 directories simultaneously. + * (Relays never use simultaneous connections.) */ + V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"), + V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "3600, 900, 900, 3600"), + V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "10 minutes"), + V(TestingDirConnectionMaxStall, INTERVAL, "5 minutes"), + V(TestingConsensusMaxDownloadTries, UINT, "8"), + /* Since we try connections rapidly and simultaneously, we can afford + * to give up earlier. (This protects against overloading directories.) */ + V(ClientBootstrapConsensusMaxDownloadTries, UINT, "7"), + /* We want to give up much earlier if we're only using authorities. */ + V(ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries, UINT, "4"), + V(TestingDescriptorMaxDownloadTries, UINT, "8"), + V(TestingMicrodescMaxDownloadTries, UINT, "8"), + V(TestingCertMaxDownloadTries, UINT, "8"), + V(TestingDirAuthVoteExit, ROUTERSET, NULL), + V(TestingDirAuthVoteExitIsStrict, BOOL, "0"), + V(TestingDirAuthVoteGuard, ROUTERSET, NULL), + V(TestingDirAuthVoteGuardIsStrict, BOOL, "0"), + V(TestingDirAuthVoteHSDir, ROUTERSET, NULL), + V(TestingDirAuthVoteHSDirIsStrict, BOOL, "0"), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } @@ -449,6 +537,14 @@ static const config_var_t testing_tor_network_defaults[] = { V(AssumeReachable, BOOL, "1"), V(AuthDirMaxServersPerAddr, UINT, "0"), V(AuthDirMaxServersPerAuthAddr,UINT, "0"), + V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL, + "0, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"), + V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL, + "0, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"), + V(ClientBootstrapConsensusAuthorityOnlyDownloadSchedule, CSV_INTERVAL, + "0, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"), + V(ClientBootstrapConsensusMaxDownloadTries, UINT, "80"), + V(ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries, UINT, "80"), V(ClientDNSRejectInternalAddresses, BOOL,"0"), V(ClientRejectInternalAddresses, BOOL, "0"), V(CountPrivateBandwidth, BOOL, "1"), @@ -457,13 +553,33 @@ static const config_var_t testing_tor_network_defaults[] = { V(V3AuthVotingInterval, INTERVAL, "5 minutes"), V(V3AuthVoteDelay, INTERVAL, "20 seconds"), V(V3AuthDistDelay, INTERVAL, "20 seconds"), - V(TestingV3AuthInitialVotingInterval, INTERVAL, "5 minutes"), + V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"), V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"), V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"), + V(TestingV3AuthVotingStartOffset, INTERVAL, "0"), V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"), V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"), + V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 5, 10, 15, " + "20, 30, 60"), + V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, 15, 20, " + "30, 60"), + V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " + "15, 20, 30, 60"), + V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " + "15, 20, 30, 60"), + V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "60, 30, 30, 60"), + V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"), + V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"), + V(TestingConsensusMaxDownloadTries, UINT, "80"), + V(TestingDescriptorMaxDownloadTries, UINT, "80"), + V(TestingMicrodescMaxDownloadTries, UINT, "80"), + V(TestingCertMaxDownloadTries, UINT, "80"), + V(TestingEnableConnBwEvent, BOOL, "1"), + V(TestingEnableCellStatsEvent, BOOL, "1"), + V(TestingEnableTbEmptyEvent, BOOL, "1"), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"), + V(RendPostPeriod, INTERVAL, "2 minutes"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -475,11 +591,7 @@ static const config_var_t testing_tor_network_defaults[] = { #ifdef _WIN32 static char *get_windows_conf_root(void); #endif -static int options_validate(or_options_t *old_options, - or_options_t *options, - int from_setconf, char **msg); static int options_act_reversible(const or_options_t *old_options, char **msg); -static int options_act(const or_options_t *old_options); static int options_transition_allowed(const or_options_t *old, const or_options_t *new, char **msg); @@ -487,29 +599,21 @@ static int options_transition_affects_workers( const or_options_t *old_options, const or_options_t *new_options); static int options_transition_affects_descriptor( const or_options_t *old_options, const or_options_t *new_options); -static int check_nickname_list(const char *lst, const char *name, char **msg); - -static int parse_bridge_line(const char *line, int validate_only); -static int parse_client_transport_line(const char *line, int validate_only); - -static int parse_server_transport_line(const char *line, int validate_only); +static int check_nickname_list(char **lst, const char *name, char **msg); static char *get_bindaddr_from_transport_listen_line(const char *line, const char *transport); -static int parse_dir_authority_line(const char *line, - dirinfo_type_t required_type, - int validate_only); -static int parse_dir_fallback_line(const char *line, - int validate_only); -static void port_cfg_free(port_cfg_t *port); static int parse_ports(or_options_t *options, int validate_only, - char **msg_out, int *n_ports_out); + char **msg_out, int *n_ports_out, + int *world_writable_control_socket); static int check_server_ports(const smartlist_t *ports, - const or_options_t *options); + const or_options_t *options, + int *num_low_ports_out); static int validate_data_directory(or_options_t *options); static int write_configuration_file(const char *fname, const or_options_t *options); -static int options_init_logs(or_options_t *options, int validate_only); +static int options_init_logs(const or_options_t *old_options, + or_options_t *options, int validate_only); static void init_libevent(const or_options_t *options); static int opt_streq(const char *s1, const char *s2); @@ -517,18 +621,23 @@ static int parse_outbound_addresses(or_options_t *options, int validate_only, char **msg); static void config_maybe_load_geoip_files_(const or_options_t *options, const or_options_t *old_options); +static int options_validate_cb(void *old_options, void *options, + void *default_options, + int from_setconf, char **msg); +static uint64_t compute_real_max_mem_in_queues(const uint64_t val, + int log_guess); /** Magic value for or_options_t. */ #define OR_OPTIONS_MAGIC 9090909 /** Configuration format for or_options_t. */ -static config_format_t options_format = { +STATIC config_format_t options_format = { sizeof(or_options_t), OR_OPTIONS_MAGIC, STRUCT_OFFSET(or_options_t, magic_), option_abbrevs_, option_vars_, - (validate_fn_t)options_validate, + options_validate_cb, NULL }; @@ -545,31 +654,35 @@ static or_options_t *global_default_options = NULL; static char *torrc_fname = NULL; /** Name of the most recently read torrc-defaults file.*/ static char *torrc_defaults_fname; -/** Configuration Options set by command line. */ +/** Configuration options set by command line. */ static config_line_t *global_cmdline_options = NULL; +/** Non-configuration options set by the command line */ +static config_line_t *global_cmdline_only_options = NULL; +/** Boolean: Have we parsed the command line? */ +static int have_parsed_cmdline = 0; /** Contents of most recently read DirPortFrontPage file. */ static char *global_dirfrontpagecontents = NULL; /** List of port_cfg_t for all configured ports. */ static smartlist_t *configured_ports = NULL; /** Return the contents of our frontpage string, or NULL if not configured. */ -const char * -get_dirportfrontpage(void) +MOCK_IMPL(const char*, +get_dirportfrontpage, (void)) { return global_dirfrontpagecontents; } -/** Return the currently configured options. */ -or_options_t * -get_options_mutable(void) +/** Returns the currently configured options. */ +MOCK_IMPL(or_options_t *, +get_options_mutable, (void)) { tor_assert(global_options); return global_options; } /** Returns the currently configured options */ -const or_options_t * -get_options(void) +MOCK_IMPL(const or_options_t *, +get_options,(void)) { return get_options_mutable(); } @@ -678,7 +791,7 @@ get_short_version(void) /** Release additional memory allocated in options */ -static void +STATIC void or_options_free(or_options_t *options) { if (!options) @@ -691,6 +804,8 @@ or_options_free(or_options_t *options) smartlist_free(options->NodeFamilySets); } tor_free(options->BridgePassword_AuthDigest_); + tor_free(options->command_arg); + tor_free(options->master_key_fname); config_free(&options_format, options); } @@ -707,6 +822,9 @@ config_free_all(void) config_free_lines(global_cmdline_options); global_cmdline_options = NULL; + config_free_lines(global_cmdline_only_options); + global_cmdline_only_options = NULL; + if (configured_ports) { SMARTLIST_FOREACH(configured_ports, port_cfg_t *, p, port_cfg_free(p)); @@ -716,7 +834,6 @@ config_free_all(void) tor_free(torrc_fname); tor_free(torrc_defaults_fname); - tor_free(the_tor_version); tor_free(global_dirfrontpagecontents); tor_free(the_short_tor_version); @@ -780,60 +897,76 @@ escaped_safe_str(const char *address) return escaped(address); } +/** List of default directory authorities */ + +static const char *default_authorities[] = { + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + "tor26 orport=443 " + "v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 " + "ipv6=[2001:858:2:2:aabb:0:563b:1526]:443 " + "86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D", + "dizum orport=443 " + "v3ident=E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 " + "194.109.206.212:80 7EA6 EAD6 FD83 083C 538F 4403 8BBF A077 587D D755", + "Bifroest orport=443 bridge " + "37.218.247.217:80 1D8F 3A91 C37C 5D1C 4C19 B1AD 1D0C FBE8 BF72 D8E1", + "gabelmoo orport=443 " + "v3ident=ED03BB616EB2F60BEC80151114BB25CEF515B226 " + "ipv6=[2001:638:a000:4140::ffff:189]:443 " + "131.188.40.189:80 F204 4413 DAC2 E02E 3D6B CF47 35A1 9BCA 1DE9 7281", + "dannenberg orport=443 " + "v3ident=0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "193.23.244.244:80 7BE6 83E6 5D48 1413 21C5 ED92 F075 C553 64AC 7123", + "maatuska orport=80 " + "v3ident=49015F787433103580E3B66A1707A00E60F2D15B " + "ipv6=[2001:67c:289c::9]:80 " + "171.25.193.9:443 BD6A 8292 55CB 08E6 6FBE 7D37 4836 3586 E46B 3810", + "Faravahar orport=443 " + "v3ident=EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 " + "154.35.175.225:80 CF6D 0AAF B385 BE71 B8E1 11FC 5CFF 4B47 9237 33BC", + "longclaw orport=443 " + "v3ident=23D15D965BC35114467363C165C4F724B64B4F66 " + "ipv6=[2620:13:4000:8000:60:f3ff:fea1:7cff]:443 " + "199.254.238.52:80 74A9 1064 6BCE EFBC D2E8 74FC 1DC9 9743 0F96 8145", + NULL +}; + +/** List of fallback directory authorities. The list is generated by opt-in of + * relays that meet certain stability criteria. + */ +static const char *default_fallbacks[] = { +#include "fallback_dirs.inc" + NULL +}; + /** Add the default directory authorities directly into the trusted dir list, - * but only add them insofar as they share bits with <b>type</b>. */ -static void + * but only add them insofar as they share bits with <b>type</b>. + * Each authority's bits are restricted to the bits shared with <b>type</b>. + * If <b>type</b> is ALL_DIRINFO or NO_DIRINFO (zero), add all authorities. */ +STATIC void add_default_trusted_dir_authorities(dirinfo_type_t type) { int i; - const char *authorities[] = { - "moria1 orport=9101 no-v2 " - "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " - "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", - "tor26 v1 orport=443 v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 " - "86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D", - "dizum orport=443 v3ident=E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 " - "194.109.206.212:80 7EA6 EAD6 FD83 083C 538F 4403 8BBF A077 587D D755", - "Tonga orport=443 bridge no-v2 82.94.251.203:80 " - "4A0C CD2D DC79 9508 3D73 F5D6 6710 0C8A 5831 F16D", - "gabelmoo orport=443 no-v2 " - "v3ident=ED03BB616EB2F60BEC80151114BB25CEF515B226 " - "131.188.40.189:80 F204 4413 DAC2 E02E 3D6B CF47 35A1 9BCA 1DE9 7281", - "dannenberg orport=443 no-v2 " - "v3ident=0232AF901C31A04EE9848595AF9BB7620D4C5B2E " - "193.23.244.244:80 7BE6 83E6 5D48 1413 21C5 ED92 F075 C553 64AC 7123", - "maatuska orport=80 no-v2 " - "v3ident=49015F787433103580E3B66A1707A00E60F2D15B " - "171.25.193.9:443 BD6A 8292 55CB 08E6 6FBE 7D37 4836 3586 E46B 3810", - "Faravahar orport=443 no-v2 " - "v3ident=EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97 " - "154.35.175.225:80 CF6D 0AAF B385 BE71 B8E1 11FC 5CFF 4B47 9237 33BC", - "longclaw orport=443 no-v2 " - "v3ident=23D15D965BC35114467363C165C4F724B64B4F66 " - "199.254.238.52:80 74A9 1064 6BCE EFBC D2E8 74FC 1DC9 9743 0F96 8145", - NULL - }; - for (i=0; authorities[i]; i++) { - if (parse_dir_authority_line(authorities[i], type, 0)<0) { + for (i=0; default_authorities[i]; i++) { + if (parse_dir_authority_line(default_authorities[i], type, 0)<0) { log_err(LD_BUG, "Couldn't parse internal DirAuthority line %s", - authorities[i]); + default_authorities[i]); } } } /** Add the default fallback directory servers into the fallback directory * server list. */ -static void -add_default_fallback_dir_servers(void) +MOCK_IMPL(void, +add_default_fallback_dir_servers,(void)) { int i; - const char *fallback[] = { - NULL - }; - for (i=0; fallback[i]; i++) { - if (parse_dir_fallback_line(fallback[i], 0)<0) { + for (i=0; default_fallbacks[i]; i++) { + if (parse_dir_fallback_line(default_fallbacks[i], 0)<0) { log_err(LD_BUG, "Couldn't parse internal FallbackDir line %s", - fallback[i]); + default_fallbacks[i]); } } } @@ -848,8 +981,7 @@ validate_dir_servers(or_options_t *options, or_options_t *old_options) config_line_t *cl; if (options->DirAuthorities && - (options->AlternateDirAuthority || options->AlternateBridgeAuthority || - options->AlternateHSAuthority)) { + (options->AlternateDirAuthority || options->AlternateBridgeAuthority)) { log_warn(LD_CONFIG, "You cannot set both DirAuthority and Alternate*Authority."); return -1; @@ -885,9 +1017,6 @@ validate_dir_servers(or_options_t *options, or_options_t *old_options) for (cl = options->AlternateDirAuthority; cl; cl = cl->next) if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0) return -1; - for (cl = options->AlternateHSAuthority; cl; cl = cl->next) - if (parse_dir_authority_line(cl->value, NO_DIRINFO, 1)<0) - return -1; for (cl = options->FallbackDir; cl; cl = cl->next) if (parse_dir_fallback_line(cl->value, 1)<0) return -1; @@ -897,7 +1026,7 @@ validate_dir_servers(or_options_t *options, or_options_t *old_options) /** Look at all the config options and assign new dir authorities * as appropriate. */ -static int +int consider_adding_dir_servers(const or_options_t *options, const or_options_t *old_options) { @@ -907,33 +1036,45 @@ consider_adding_dir_servers(const or_options_t *options, !smartlist_len(router_get_fallback_dir_servers()) || !old_options || !config_lines_eq(options->DirAuthorities, old_options->DirAuthorities) || !config_lines_eq(options->FallbackDir, old_options->FallbackDir) || + (options->UseDefaultFallbackDirs != old_options->UseDefaultFallbackDirs) || !config_lines_eq(options->AlternateBridgeAuthority, old_options->AlternateBridgeAuthority) || !config_lines_eq(options->AlternateDirAuthority, - old_options->AlternateDirAuthority) || - !config_lines_eq(options->AlternateHSAuthority, - old_options->AlternateHSAuthority); + old_options->AlternateDirAuthority); if (!need_to_update) return 0; /* all done */ + /* "You cannot set both DirAuthority and Alternate*Authority." + * Checking that this restriction holds allows us to simplify + * the unit tests. */ + tor_assert(!(options->DirAuthorities && + (options->AlternateDirAuthority + || options->AlternateBridgeAuthority))); + /* Start from a clean slate. */ clear_dir_servers(); if (!options->DirAuthorities) { /* then we may want some of the defaults */ dirinfo_type_t type = NO_DIRINFO; - if (!options->AlternateBridgeAuthority) + if (!options->AlternateBridgeAuthority) { type |= BRIDGE_DIRINFO; - if (!options->AlternateDirAuthority) - type |= V1_DIRINFO | V2_DIRINFO | V3_DIRINFO | EXTRAINFO_DIRINFO | - MICRODESC_DIRINFO; - if (!options->AlternateHSAuthority) - type |= HIDSERV_DIRINFO; - add_default_trusted_dir_authorities(type); + } + if (!options->AlternateDirAuthority) { + type |= V3_DIRINFO | EXTRAINFO_DIRINFO | MICRODESC_DIRINFO; + /* Only add the default fallback directories when the DirAuthorities, + * AlternateDirAuthority, and FallbackDir directory config options + * are set to their defaults, and when UseDefaultFallbackDirs is 1. */ + if (!options->FallbackDir && options->UseDefaultFallbackDirs) { + add_default_fallback_dir_servers(); + } + } + /* if type == NO_DIRINFO, we don't want to add any of the + * default authorities, because we've replaced them all */ + if (type != NO_DIRINFO) + add_default_trusted_dir_authorities(type); } - if (!options->FallbackDir) - add_default_fallback_dir_servers(); for (cl = options->DirAuthorities; cl; cl = cl->next) if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0) @@ -944,15 +1085,15 @@ consider_adding_dir_servers(const or_options_t *options, for (cl = options->AlternateDirAuthority; cl; cl = cl->next) if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0) return -1; - for (cl = options->AlternateHSAuthority; cl; cl = cl->next) - if (parse_dir_authority_line(cl->value, NO_DIRINFO, 0)<0) - return -1; for (cl = options->FallbackDir; cl; cl = cl->next) if (parse_dir_fallback_line(cl->value, 0)<0) return -1; return 0; } +/* Helps determine flags to pass to switch_id. */ +static int have_low_ports = -1; + /** Fetch the active option list, and take actions based on it. All of the * things we do should survive being done repeatedly. If present, * <b>old_options</b> contains the previous value of the options. @@ -969,7 +1110,8 @@ options_act_reversible(const or_options_t *old_options, char **msg) int running_tor = options->command == CMD_RUN_TOR; int set_conn_limit = 0; int r = -1; - int logs_marked = 0; + int logs_marked = 0, logs_initialized = 0; + int old_min_log_level = get_min_log_level(); /* Daemonize _first_, since we only want to open most of this stuff in * the subprocess. Libevent bases can't be reliably inherited across @@ -979,6 +1121,11 @@ options_act_reversible(const or_options_t *old_options, char **msg) start_daemon(); } +#ifdef HAVE_SYSTEMD + /* Our PID may have changed, inform supervisor */ + sd_notifyf(0, "MAINPID=%ld\n", (long int)getpid()); +#endif + #ifndef HAVE_SYS_UN_H if (options->ControlSocket || options->ControlSocketsGroupWritable) { *msg = tor_strdup("Unix domain sockets (ControlSocket) not supported " @@ -996,22 +1143,39 @@ options_act_reversible(const or_options_t *old_options, char **msg) if (running_tor) { int n_ports=0; /* We need to set the connection limit before we can open the listeners. */ - if (set_max_file_descriptors((unsigned)options->ConnLimit, - &options->ConnLimit_) < 0) { - *msg = tor_strdup("Problem with ConnLimit value. See logs for details."); - goto rollback; + if (! sandbox_is_active()) { + if (set_max_file_descriptors((unsigned)options->ConnLimit, + &options->ConnLimit_) < 0) { + *msg = tor_strdup("Problem with ConnLimit value. " + "See logs for details."); + goto rollback; + } + set_conn_limit = 1; + } else { + tor_assert(old_options); + options->ConnLimit_ = old_options->ConnLimit_; } - set_conn_limit = 1; /* Set up libevent. (We need to do this before we can register the * listeners as listeners.) */ if (running_tor && !libevent_initialized) { init_libevent(options); libevent_initialized = 1; + + /* This has to come up after libevent is initialized. */ + control_initialize_event_queue(); + + /* + * Initialize the scheduler - this has to come after + * options_init_from_torrc() sets up libevent - why yes, that seems + * completely sensible to hide the libevent setup in the option parsing + * code! It also needs to happen before init_keys(), so it needs to + * happen here too. How yucky. */ + scheduler_init(); } /* Adjust the port configuration so we can launch listeners. */ - if (parse_ports(options, 0, msg, &n_ports)) { + if (parse_ports(options, 0, msg, &n_ports, NULL)) { if (!*msg) *msg = tor_strdup("Unexpected problem parsing port config"); goto rollback; @@ -1037,12 +1201,15 @@ options_act_reversible(const or_options_t *old_options, char **msg) "non-control network connections. Shutting down all existing " "connections."); connection_mark_all_noncontrol_connections(); + /* We can't complete circuits until the network is re-enabled. */ + note_that_we_maybe_cant_complete_circuits(); } } #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) /* Open /dev/pf before dropping privileges. */ - if (options->TransPort_set) { + if (options->TransPort_set && + options->TransProxyType_parsed == TPT_DEFAULT) { if (get_pf_socket() < 0) { *msg = tor_strdup("Unable to open /dev/pf for transparent proxy."); goto rollback; @@ -1061,7 +1228,16 @@ options_act_reversible(const or_options_t *old_options, char **msg) /* Setuid/setgid as appropriate */ if (options->User) { - if (switch_id(options->User) != 0) { + tor_assert(have_low_ports != -1); + unsigned switch_id_flags = 0; + if (options->KeepBindCapabilities == 1) { + switch_id_flags |= SWITCH_ID_KEEP_BINDLOW; + switch_id_flags |= SWITCH_ID_WARN_IF_NO_CAPS; + } + if (options->KeepBindCapabilities == -1 && have_low_ports) { + switch_id_flags |= SWITCH_ID_KEEP_BINDLOW; + } + if (switch_id(options->User, switch_id_flags) != 0) { /* No need to roll back, since you can't change the value. */ *msg = tor_strdup("Problem with User value. See logs for details."); goto done; @@ -1069,32 +1245,29 @@ options_act_reversible(const or_options_t *old_options, char **msg) } /* Ensure data directory is private; create if possible. */ + cpd_check_t cpd_opts = running_tor ? CPD_CREATE : CPD_CHECK; + if (options->DataDirectoryGroupReadable) + cpd_opts |= CPD_GROUP_READ; if (check_private_dir(options->DataDirectory, - running_tor ? CPD_CREATE : CPD_CHECK, + cpd_opts, options->User)<0) { tor_asprintf(msg, "Couldn't access/create private data directory \"%s\"", options->DataDirectory); + goto done; /* No need to roll back, since you can't change the value. */ } - /* Write control ports to disk as appropriate */ - control_ports_write_to_file(); - - if (directory_caches_v2_dir_info(options)) { - char *fn = NULL; - tor_asprintf(&fn, "%s"PATH_SEPARATOR"cached-status", - options->DataDirectory); - if (check_private_dir(fn, running_tor ? CPD_CREATE : CPD_CHECK, - options->User) < 0) { - tor_asprintf(msg, - "Couldn't access/create private data directory \"%s\"", fn); - tor_free(fn); - goto done; +#ifndef _WIN32 + if (options->DataDirectoryGroupReadable) { + /* Only new dirs created get new opts, also enforce group read. */ + if (chmod(options->DataDirectory, 0750)) { + log_warn(LD_FS,"Unable to make %s group-readable: %s", + options->DataDirectory, strerror(errno)); } - tor_free(fn); } +#endif /* Bail out at this point if we're not going to be a client or server: * we don't run Tor itself. */ @@ -1103,10 +1276,12 @@ options_act_reversible(const or_options_t *old_options, char **msg) mark_logs_temp(); /* Close current logs once new logs are open. */ logs_marked = 1; - if (options_init_logs(options, 0)<0) { /* Configure the tor_log(s) */ + /* Configure the tor_log(s) */ + if (options_init_logs(old_options, options, 0)<0) { *msg = tor_strdup("Failed to init Log options. See logs for details."); goto rollback; } + logs_initialized = 1; commit: r = 0; @@ -1117,13 +1292,47 @@ options_act_reversible(const or_options_t *old_options, char **msg) add_callback_log(severity, control_event_logmsg); control_adjust_event_log_severity(); tor_free(severity); + tor_log_update_sigsafe_err_fds(); + } + if (logs_initialized) { + flush_log_messages_from_startup(); + } + + { + const char *badness = NULL; + int bad_safelog = 0, bad_severity = 0, new_badness = 0; + if (options->SafeLogging_ != SAFELOG_SCRUB_ALL) { + bad_safelog = 1; + if (!old_options || old_options->SafeLogging_ != options->SafeLogging_) + new_badness = 1; + } + if (get_min_log_level() >= LOG_INFO) { + bad_severity = 1; + if (get_min_log_level() != old_min_log_level) + new_badness = 1; + } + if (bad_safelog && bad_severity) + badness = "you disabled SafeLogging, and " + "you're logging more than \"notice\""; + else if (bad_safelog) + badness = "you disabled SafeLogging"; + else + badness = "you're logging more than \"notice\""; + if (new_badness) + log_warn(LD_GENERAL, "Your log may contain sensitive information - %s. " + "Don't log unless it serves an important reason. " + "Overwrite the log afterwards.", badness); } + SMARTLIST_FOREACH(replaced_listeners, connection_t *, conn, { + int marked = conn->marked_for_close; log_notice(LD_NET, "Closing old %s on %s:%d", conn_type_to_string(conn->type), conn->address, conn->port); connection_close_immediate(conn); - connection_mark_for_close(conn); + if (!marked) { + connection_mark_for_close(conn); + } }); goto done; @@ -1165,7 +1374,8 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out) routerset_needs_geoip(options->EntryNodes) || routerset_needs_geoip(options->ExitNodes) || routerset_needs_geoip(options->ExcludeExitNodes) || - routerset_needs_geoip(options->ExcludeNodes); + routerset_needs_geoip(options->ExcludeNodes) || + routerset_needs_geoip(options->Tor2webRendezvousPoints); if (routerset_usage && reason_out) { *reason_out = "We've been configured to use (or avoid) nodes in certain " @@ -1216,10 +1426,6 @@ options_transition_requires_fresh_tls_context(const or_options_t *old_options, if (!old_options) return 0; - if ((old_options->DynamicDHGroups != new_options->DynamicDHGroups)) { - return 1; - } - if (!opt_streq(old_options->TLSECGroup, new_options->TLSECGroup)) return 1; @@ -1235,7 +1441,7 @@ options_transition_requires_fresh_tls_context(const or_options_t *old_options, * Note: We haven't moved all the "act on new configuration" logic * here yet. Some is still in do_hup() and other places. */ -static int +STATIC int options_act(const or_options_t *old_options) { config_line_t *cl; @@ -1257,10 +1463,12 @@ options_act(const or_options_t *old_options) if (options->DisableDebuggerAttachment && !disabled_debugger_attach && running_tor) { int ok = tor_disable_debugger_attach(); + /* LCOV_EXCL_START the warned_debugger_attach is 0 can't reach inside. */ if (warned_debugger_attach && ok == 1) { log_notice(LD_CONFIG, "Disabled attaching debuggers for unprivileged " "users."); } + /* LCOV_EXCL_STOP */ disabled_debugger_attach = (ok == 1); } else if (!options->DisableDebuggerAttachment && !warned_debugger_attach) { @@ -1270,6 +1478,9 @@ options_act(const or_options_t *old_options) } } + /* Write control ports to disk as appropriate */ + control_ports_write_to_file(); + if (running_tor && !have_lockfile()) { if (try_locking(options, 1) < 0) return -1; @@ -1284,30 +1495,47 @@ options_act(const or_options_t *old_options) #endif #ifdef ENABLE_TOR2WEB_MODE +/* LCOV_EXCL_START */ if (!options->Tor2webMode) { log_err(LD_CONFIG, "This copy of Tor was compiled to run in " "'tor2web mode'. It can only be run with the Tor2webMode torrc " "option enabled."); return -1; } +/* LCOV_EXCL_STOP */ #else if (options->Tor2webMode) { log_err(LD_CONFIG, "This copy of Tor was not compiled to run in " "'tor2web mode'. It cannot be run with the Tor2webMode torrc " "option enabled. To enable Tor2webMode recompile with the " - "--enable-tor2webmode option."); + "--enable-tor2web-mode option."); return -1; } #endif + /* If we are a bridge with a pluggable transport proxy but no + Extended ORPort, inform the user that they are missing out. */ + if (server_mode(options) && options->ServerTransportPlugin && + !options->ExtORPort_lines) { + log_notice(LD_CONFIG, "We use pluggable transports but the Extended " + "ORPort is disabled. Tor and your pluggable transports proxy " + "communicate with each other via the Extended ORPort so it " + "is suggested you enable it: it will also allow your Bridge " + "to collect statistics about its clients that use pluggable " + "transports. Please enable it using the ExtORPort torrc option " + "(e.g. set 'ExtORPort auto')."); + } + if (options->Bridges) { mark_bridge_list(); for (cl = options->Bridges; cl; cl = cl->next) { - if (parse_bridge_line(cl->value, 0)<0) { + bridge_line_t *bridge_line = parse_bridge_line(cl->value); + if (!bridge_line) { log_warn(LD_BUG, "Previously validated Bridge line could not be added!"); return -1; } + bridge_add_from_config(bridge_line); } sweep_bridge_list(); } @@ -1331,32 +1559,47 @@ options_act(const or_options_t *old_options) rep_hist_load_mtbf_data(time(NULL)); } + /* If we have an ExtORPort, initialize its auth cookie. */ + if (running_tor && + init_ext_or_cookie_authentication(!!options->ExtORPort_lines) < 0) { + log_warn(LD_CONFIG,"Error creating Extended ORPort cookie file."); + return -1; + } + mark_transport_list(); pt_prepare_proxy_list_for_config_read(); - if (options->ClientTransportPlugin) { - for (cl = options->ClientTransportPlugin; cl; cl = cl->next) { - if (parse_client_transport_line(cl->value, 0)<0) { - log_warn(LD_BUG, - "Previously validated ClientTransportPlugin line " - "could not be added!"); - return -1; + if (!options->DisableNetwork) { + if (options->ClientTransportPlugin) { + for (cl = options->ClientTransportPlugin; cl; cl = cl->next) { + if (parse_transport_line(options, cl->value, 0, 0) < 0) { + log_warn(LD_BUG, + "Previously validated ClientTransportPlugin line " + "could not be added!"); + return -1; + } } } - } - if (options->ServerTransportPlugin && server_mode(options)) { - for (cl = options->ServerTransportPlugin; cl; cl = cl->next) { - if (parse_server_transport_line(cl->value, 0)<0) { - log_warn(LD_BUG, - "Previously validated ServerTransportPlugin line " - "could not be added!"); - return -1; + if (options->ServerTransportPlugin && server_mode(options)) { + for (cl = options->ServerTransportPlugin; cl; cl = cl->next) { + if (parse_transport_line(options, cl->value, 0, 1) < 0) { + log_warn(LD_BUG, + "Previously validated ServerTransportPlugin line " + "could not be added!"); + return -1; + } } } } sweep_transport_list(); sweep_proxy_list(); + /* Start the PT proxy configuration. By doing this configuration + here, we also figure out which proxies need to be restarted and + which not. */ + if (pt_proxies_configuration_pending() && !net_is_disabled()) + pt_configure_remaining_proxies(); + /* Bail out at this point if we're not going to be a client or server: * we want to not fork, and to log stuff to stderr. */ if (!running_tor) @@ -1368,24 +1611,6 @@ options_act(const or_options_t *old_options) finish_daemon(options->DataDirectory); } - /* If needed, generate a new TLS DH prime according to the current torrc. */ - if (server_mode(options) && options->DynamicDHGroups) { - char *keydir = get_datadir_fname("keys"); - if (check_private_dir(keydir, CPD_CREATE, options->User)) { - tor_free(keydir); - return -1; - } - tor_free(keydir); - - if (!old_options || !old_options->DynamicDHGroups) { - char *fname = get_datadir_fname2("keys", "dynamic_dh_params"); - crypto_set_tls_dh_prime(fname); - tor_free(fname); - } - } else { /* clients don't need a dynamic DH prime. */ - crypto_set_tls_dh_prime(NULL); - } - /* We want to reinit keys as needed before we do much of anything else: keys are important, and other things can depend on them. */ if (transition_affects_workers || @@ -1406,8 +1631,9 @@ options_act(const or_options_t *old_options) /* Write our PID to the PID file. If we do not have write permissions we * will log a warning */ - if (options->PidFile) + if (options->PidFile && !sandbox_is_active()) { write_pidfile(options->PidFile); + } /* Register addressmap directives */ config_register_addressmaps(options); @@ -1421,8 +1647,8 @@ options_act(const or_options_t *old_options) return -1; } - if (init_cookie_authentication(options->CookieAuthentication) < 0) { - log_warn(LD_CONFIG,"Error creating cookie authentication file."); + if (init_control_cookie_authentication(options->CookieAuthentication) < 0) { + log_warn(LD_CONFIG,"Error creating control cookie authentication file."); return -1; } @@ -1434,6 +1660,12 @@ options_act(const or_options_t *old_options) return -1; } + /* Set up scheduler thresholds */ + scheduler_set_watermarks((uint32_t)options->SchedulerLowWaterMark__, + (uint32_t)options->SchedulerHighWaterMark__, + (options->SchedulerMaxFlushCells__ > 0) ? + options->SchedulerMaxFlushCells__ : 1000); + /* Set up accounting */ if (accounting_parse_options(options, 0)<0) { log_warn(LD_CONFIG,"Error in accounting options"); @@ -1483,11 +1715,25 @@ options_act(const or_options_t *old_options) } if (parse_outbound_addresses(options, 0, &msg) < 0) { - log_warn(LD_BUG, "Failed parsing oubound bind addresses: %s", msg); + log_warn(LD_BUG, "Failed parsing outbound bind addresses: %s", msg); tor_free(msg); return -1; } + config_maybe_load_geoip_files_(options, old_options); + + if (geoip_is_loaded(AF_INET) && options->GeoIPExcludeUnknown) { + /* ExcludeUnknown is true or "auto" */ + const int is_auto = options->GeoIPExcludeUnknown == -1; + int changed; + + changed = routerset_add_unknown_ccs(&options->ExcludeNodes, is_auto); + changed += routerset_add_unknown_ccs(&options->ExcludeExitNodes, is_auto); + + if (changed) + routerset_add_unknown_ccs(&options->ExcludeExitNodesUnion_, is_auto); + } + /* Check for transitions that need action. */ if (old_options) { int revise_trackexithosts = 0; @@ -1501,6 +1747,8 @@ options_act(const or_options_t *old_options) options->ExcludeExitNodes) || !routerset_equal(old_options->EntryNodes, options->EntryNodes) || !routerset_equal(old_options->ExitNodes, options->ExitNodes) || + !routerset_equal(old_options->Tor2webRendezvousPoints, + options->Tor2webRendezvousPoints) || options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, "Changed to using entry guards or bridges, or changed " @@ -1518,8 +1766,8 @@ options_act(const or_options_t *old_options) if (revise_trackexithosts) addressmap_clear_excluded_trackexithosts(options); - if (!options->AutomapHostsOnResolve) { - if (old_options->AutomapHostsOnResolve) + if (!options->AutomapHostsOnResolve && + old_options->AutomapHostsOnResolve) { revise_automap_entries = 1; } else { if (!smartlist_strings_eq(old_options->AutomapHostsSuffixes, @@ -1566,11 +1814,12 @@ options_act(const or_options_t *old_options) "Worker-related options changed. Rotating workers."); if (server_mode(options) && !server_mode(old_options)) { + cpu_init(); ip_address_changed(0); - if (can_complete_circuit || !any_predicted_circuits(time(NULL))) + if (have_completed_a_circuit() || !any_predicted_circuits(time(NULL))) inform_testing_reachability(); } - cpuworkers_rotate(); + cpuworkers_rotate_keyinfo(); if (dns_reset()) return -1; } else { @@ -1583,32 +1832,24 @@ options_act(const or_options_t *old_options) connection_or_update_token_buckets(get_connection_array(), options); } - config_maybe_load_geoip_files_(options, old_options); - - if (geoip_is_loaded(AF_INET) && options->GeoIPExcludeUnknown) { - /* ExcludeUnknown is true or "auto" */ - const int is_auto = options->GeoIPExcludeUnknown == -1; - int changed; - - changed = routerset_add_unknown_ccs(&options->ExcludeNodes, is_auto); - changed += routerset_add_unknown_ccs(&options->ExcludeExitNodes, is_auto); - - if (changed) - routerset_add_unknown_ccs(&options->ExcludeExitNodesUnion_, is_auto); - } + /* Only collect directory-request statistics on relays and bridges. */ + options->DirReqStatistics = options->DirReqStatistics_option && + server_mode(options); if (options->CellStatistics || options->DirReqStatistics || options->EntryStatistics || options->ExitPortStatistics || options->ConnDirectionStatistics || + options->HiddenServiceStatistics || options->BridgeAuthoritativeDir) { time_t now = time(NULL); int print_notice = 0; - /* If we aren't acting as a server, we can't collect stats anyway. */ - if (!server_mode(options)) { + /* Only collect other relay-only statistics on relays. */ + if (!public_server_mode(options)) { options->CellStatistics = 0; - options->DirReqStatistics = 0; options->EntryStatistics = 0; + options->ConnDirectionStatistics = 0; + options->HiddenServiceStatistics = 0; options->ExitPortStatistics = 0; } @@ -1623,8 +1864,8 @@ options_act(const or_options_t *old_options) geoip_dirreq_stats_init(now); print_notice = 1; } else { + /* disable statistics collection since we have no geoip file */ options->DirReqStatistics = 0; - /* Don't warn Tor clients, they don't use statistics */ if (options->ORPort_set) log_notice(LD_CONFIG, "Configured to measure directory request " "statistics, but no GeoIP database found. " @@ -1654,17 +1895,24 @@ options_act(const or_options_t *old_options) options->ConnDirectionStatistics) { rep_hist_conn_stats_init(now); } + if ((!old_options || !old_options->HiddenServiceStatistics) && + options->HiddenServiceStatistics) { + log_info(LD_CONFIG, "Configured to measure hidden service statistics."); + rep_hist_hs_stats_init(now); + } if ((!old_options || !old_options->BridgeAuthoritativeDir) && options->BridgeAuthoritativeDir) { rep_hist_desc_stats_init(now); print_notice = 1; } if (print_notice) - log_notice(LD_CONFIG, "Configured to measure statistics. Look for " - "the *-stats files that will first be written to the " + log_notice(LD_CONFIG, "Configured to measure statistics. Look for " + "the *-stats files that will first be written to the " "data directory in 24 hours from now."); } + /* If we used to have statistics enabled but we just disabled them, + stop gathering them. */ if (old_options && old_options->CellStatistics && !options->CellStatistics) rep_hist_buffer_stats_term(); @@ -1674,6 +1922,9 @@ options_act(const or_options_t *old_options) if (old_options && old_options->EntryStatistics && !options->EntryStatistics) geoip_entry_stats_term(); + if (old_options && old_options->HiddenServiceStatistics && + !options->HiddenServiceStatistics) + rep_hist_hs_stats_term(); if (old_options && old_options->ExitPortStatistics && !options->ExitPortStatistics) rep_hist_exit_stats_term(); @@ -1706,7 +1957,7 @@ options_act(const or_options_t *old_options) directory_fetches_dir_info_early(old_options)) || !bool_eq(directory_fetches_dir_info_later(options), directory_fetches_dir_info_later(old_options))) { - /* Make sure update_router_have_min_dir_info gets called. */ + /* Make sure update_router_have_minimum_dir_info() gets called. */ router_dir_info_changed(); /* We might need to download a new consensus status later or sooner than * we had expected. */ @@ -1730,40 +1981,80 @@ options_act(const or_options_t *old_options) return 0; } -/** Helper: Read a list of configuration options from the command line. - * If successful, put them in *<b>result</b> and return 0, and return - * -1 and leave *<b>result</b> alone. */ -static int -config_get_commandlines(int argc, char **argv, config_line_t **result) +typedef enum { + TAKES_NO_ARGUMENT = 0, + ARGUMENT_NECESSARY = 1, + ARGUMENT_OPTIONAL = 2 +} takes_argument_t; + +static const struct { + const char *name; + takes_argument_t takes_argument; +} CMDLINE_ONLY_OPTIONS[] = { + { "-f", ARGUMENT_NECESSARY }, + { "--allow-missing-torrc", TAKES_NO_ARGUMENT }, + { "--defaults-torrc", ARGUMENT_NECESSARY }, + { "--hash-password", ARGUMENT_NECESSARY }, + { "--dump-config", ARGUMENT_OPTIONAL }, + { "--list-fingerprint", TAKES_NO_ARGUMENT }, + { "--keygen", TAKES_NO_ARGUMENT }, + { "--newpass", TAKES_NO_ARGUMENT }, +#if 0 +/* XXXX028: This is not working yet in 0.2.7, so disabling with the + * minimal code modification. */ + { "--master-key", ARGUMENT_NECESSARY }, +#endif + { "--no-passphrase", TAKES_NO_ARGUMENT }, + { "--passphrase-fd", ARGUMENT_NECESSARY }, + { "--verify-config", TAKES_NO_ARGUMENT }, + { "--ignore-missing-torrc", TAKES_NO_ARGUMENT }, + { "--quiet", TAKES_NO_ARGUMENT }, + { "--hush", TAKES_NO_ARGUMENT }, + { "--version", TAKES_NO_ARGUMENT }, + { "--library-versions", TAKES_NO_ARGUMENT }, + { "-h", TAKES_NO_ARGUMENT }, + { "--help", TAKES_NO_ARGUMENT }, + { "--list-torrc-options", TAKES_NO_ARGUMENT }, + { "--nt-service", TAKES_NO_ARGUMENT }, + { "-nt-service", TAKES_NO_ARGUMENT }, + { NULL, 0 }, +}; + +/** Helper: Read a list of configuration options from the command line. If + * successful, or if ignore_errors is set, put them in *<b>result</b>, put the + * commandline-only options in *<b>cmdline_result</b>, and return 0; + * otherwise, return -1 and leave *<b>result</b> and <b>cmdline_result</b> + * alone. */ +int +config_parse_commandline(int argc, char **argv, int ignore_errors, + config_line_t **result, + config_line_t **cmdline_result) { + config_line_t *param = NULL; + config_line_t *front = NULL; config_line_t **new = &front; - char *s; + + config_line_t *front_cmdline = NULL; + config_line_t **new_cmdline = &front_cmdline; + + char *s, *arg; int i = 1; while (i < argc) { unsigned command = CONFIG_LINE_NORMAL; - int want_arg = 1; - - if (!strcmp(argv[i],"-f") || - !strcmp(argv[i],"--defaults-torrc") || - !strcmp(argv[i],"--hash-password")) { - i += 2; /* command-line option with argument. ignore them. */ - continue; - } else if (!strcmp(argv[i],"--list-fingerprint") || - !strcmp(argv[i],"--verify-config") || - !strcmp(argv[i],"--ignore-missing-torrc") || - !strcmp(argv[i],"--quiet") || - !strcmp(argv[i],"--hush")) { - i += 1; /* command-line option. ignore it. */ - continue; - } else if (!strcmp(argv[i],"--nt-service") || - !strcmp(argv[i],"-nt-service")) { - i += 1; - continue; + takes_argument_t want_arg = ARGUMENT_NECESSARY; + int is_cmdline = 0; + int j; + + for (j = 0; CMDLINE_ONLY_OPTIONS[j].name != NULL; ++j) { + if (!strcmp(argv[i], CMDLINE_ONLY_OPTIONS[j].name)) { + is_cmdline = 1; + want_arg = CMDLINE_ONLY_OPTIONS[j].takes_argument; + break; + } } - *new = tor_malloc_zero(sizeof(config_line_t)); s = argv[i]; /* Each keyword may be prefixed with one or two dashes. */ @@ -1782,23 +2073,45 @@ config_get_commandlines(int argc, char **argv, config_line_t **result) want_arg = 0; } - if (want_arg && i == argc-1) { - log_warn(LD_CONFIG,"Command-line option '%s' with no value. Failing.", - argv[i]); - config_free_lines(front); - return -1; + const int is_last = (i == argc-1); + + if (want_arg == ARGUMENT_NECESSARY && is_last) { + if (ignore_errors) { + arg = strdup(""); + } else { + log_warn(LD_CONFIG,"Command-line option '%s' with no value. Failing.", + argv[i]); + config_free_lines(front); + config_free_lines(front_cmdline); + return -1; + } + } else if (want_arg == ARGUMENT_OPTIONAL && is_last) { + arg = tor_strdup(""); + } else { + arg = (want_arg != TAKES_NO_ARGUMENT) ? tor_strdup(argv[i+1]) : + tor_strdup(""); } - (*new)->key = tor_strdup(config_expand_abbrev(&options_format, s, 1, 1)); - (*new)->value = want_arg ? tor_strdup(argv[i+1]) : tor_strdup(""); - (*new)->command = command; - (*new)->next = NULL; + param = tor_malloc_zero(sizeof(config_line_t)); + param->key = is_cmdline ? tor_strdup(argv[i]) : + tor_strdup(config_expand_abbrev(&options_format, s, 1, 1)); + param->value = arg; + param->command = command; + param->next = NULL; log_debug(LD_CONFIG, "command line: parsed keyword '%s', value '%s'", - (*new)->key, (*new)->value); + param->key, param->value); + + if (is_cmdline) { + *new_cmdline = param; + new_cmdline = &((*new_cmdline)->next); + } else { + *new = param; + new = &((*new)->next); + } - new = &((*new)->next); i += want_arg ? 2 : 1; } + *cmdline_result = front_cmdline; *result = front; return 0; } @@ -1850,7 +2163,8 @@ options_trial_assign(config_line_t *list, int use_defaults, return r; } - if (options_validate(get_options_mutable(), trial_options, 1, msg) < 0) { + if (options_validate(get_options_mutable(), trial_options, + global_default_options, 1, msg) < 0) { config_free(&options_format, trial_options); return SETOPT_ERR_PARSE; /*XXX make this a separate return value. */ } @@ -1876,7 +2190,7 @@ print_usage(void) printf( "Copyright (c) 2001-2004, Roger Dingledine\n" "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n" -"Copyright (c) 2007-2013, The Tor Project, Inc.\n\n" +"Copyright (c) 2007-2016, The Tor Project, Inc.\n\n" "tor -f <torrc> [args]\n" "See man page for options, or https://www.torproject.org/ for " "documentation.\n"); @@ -1908,8 +2222,41 @@ get_last_resolved_addr(void) return last_resolved_addr; } +/** Reset last_resolved_addr from outside this file. */ +void +reset_last_resolved_addr(void) +{ + last_resolved_addr = 0; +} + /** - * Use <b>options-\>Address</b> to guess our public IP address. + * Attempt getting our non-local (as judged by tor_addr_is_internal() + * function) IP address using following techniques, listed in + * order from best (most desirable, try first) to worst (least + * desirable, try if everything else fails). + * + * First, attempt using <b>options-\>Address</b> to get our + * non-local IP address. + * + * If <b>options-\>Address</b> represents a non-local IP address, + * consider it ours. + * + * If <b>options-\>Address</b> is a DNS name that resolves to + * a non-local IP address, consider this IP address ours. + * + * If <b>options-\>Address</b> is NULL, fall back to getting local + * hostname and using it in above-described ways to try and + * get our IP address. + * + * In case local hostname cannot be resolved to a non-local IP + * address, try getting an IP address of network interface + * in hopes it will be non-local one. + * + * Fail if one or more of the following is true: + * - DNS name in <b>options-\>Address</b> cannot be resolved. + * - <b>options-\>Address</b> is a local host address. + * - Attempt to getting local hostname fails. + * - Attempt to getting network interface address fails. * * Return 0 if all is well, or -1 if we can't find a suitable * public IP address. @@ -1918,6 +2265,11 @@ get_last_resolved_addr(void) * - Put our public IP address (in host order) into *<b>addr_out</b>. * - If <b>method_out</b> is non-NULL, set *<b>method_out</b> to a static * string describing how we arrived at our answer. + * - "CONFIGURED" - parsed from IP address string in + * <b>options-\>Address</b> + * - "RESOLVED" - resolved from DNS name in <b>options-\>Address</b> + * - "GETHOSTNAME" - resolved from a local hostname. + * - "INTERFACE" - retrieved from a network interface. * - If <b>hostname_out</b> is non-NULL, and we resolved a hostname to * get our address, set *<b>hostname_out</b> to a newly allocated string * holding that hostname. (If we didn't get our address by resolving a @@ -1943,6 +2295,7 @@ resolve_my_address(int warn_severity, const or_options_t *options, int notice_severity = warn_severity <= LOG_NOTICE ? LOG_NOTICE : warn_severity; + tor_addr_t myaddr; tor_assert(addr_out); /* @@ -1955,7 +2308,7 @@ resolve_my_address(int warn_severity, const or_options_t *options, explicit_ip = 0; /* it's implicit */ explicit_hostname = 0; /* it's implicit */ - if (gethostname(hostname, sizeof(hostname)) < 0) { + if (tor_gethostname(hostname, sizeof(hostname)) < 0) { log_fn(warn_severity, LD_NET,"Error obtaining local hostname"); return -1; } @@ -1993,24 +2346,26 @@ resolve_my_address(int warn_severity, const or_options_t *options, "local interface. Using that.", fmt_addr32(addr)); strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname)); } else { /* resolved hostname into addr */ + tor_addr_from_ipv4h(&myaddr, addr); + if (!explicit_hostname && - is_internal_IP(addr, 0)) { - uint32_t interface_ip; + tor_addr_is_internal(&myaddr, 0)) { + tor_addr_t interface_ip; log_fn(notice_severity, LD_CONFIG, "Guessed local hostname '%s' " "resolves to a private IP address (%s). Trying something " "else.", hostname, fmt_addr32(addr)); - if (get_interface_address(warn_severity, &interface_ip)) { + if (get_interface_address6(warn_severity, AF_INET, &interface_ip)<0) { log_fn(warn_severity, LD_CONFIG, "Could not get local interface IP address. Too bad."); - } else if (is_internal_IP(interface_ip, 0)) { + } else if (tor_addr_is_internal(&interface_ip, 0)) { log_fn(notice_severity, LD_CONFIG, "Interface IP address '%s' is a private address too. " - "Ignoring.", fmt_addr32(interface_ip)); + "Ignoring.", fmt_addr(&interface_ip)); } else { from_interface = 1; - addr = interface_ip; + addr = tor_addr_to_ipv4h(&interface_ip); log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for local interface." " Using that.", fmt_addr32(addr)); @@ -2028,8 +2383,10 @@ resolve_my_address(int warn_severity, const or_options_t *options, * out if it is and we don't want that. */ + tor_addr_from_ipv4h(&myaddr,addr); + addr_string = tor_dup_ip(addr); - if (is_internal_IP(addr, 0)) { + if (tor_addr_is_internal(&myaddr, 0)) { /* make sure we're ok with publishing an internal IP */ if (!options->DirAuthorities && !options->AlternateDirAuthority) { /* if they are using the default authorities, disallow internal IPs @@ -2118,8 +2475,8 @@ resolve_my_address(int warn_severity, const or_options_t *options, /** Return true iff <b>addr</b> is judged to be on the same network as us, or * on a private network. */ -int -is_local_addr(const tor_addr_t *addr) +MOCK_IMPL(int, +is_local_addr, (const tor_addr_t *addr)) { if (tor_addr_is_internal(addr, 0)) return 1; @@ -2135,7 +2492,7 @@ is_local_addr(const tor_addr_t *addr) * resolve_my_address will never be called at all). In those cases, * last_resolved_addr will be 0, and so checking to see whether ip is on * the same /24 as last_resolved_addr will be the same as checking whether - * it was on net 0, which is already done by is_internal_IP. + * it was on net 0, which is already done by tor_addr_is_internal. */ if ((last_resolved_addr & (uint32_t)0xffffff00ul) == (ip & (uint32_t)0xffffff00ul)) @@ -2164,10 +2521,29 @@ options_init(or_options_t *options) * include options that are the same as Tor's defaults. */ char * -options_dump(const or_options_t *options, int minimal) +options_dump(const or_options_t *options, int how_to_dump) { - return config_dump(&options_format, global_default_options, - options, minimal, 0); + const or_options_t *use_defaults; + int minimal; + switch (how_to_dump) { + case OPTIONS_DUMP_MINIMAL: + use_defaults = global_default_options; + minimal = 1; + break; + case OPTIONS_DUMP_DEFAULTS: + use_defaults = NULL; + minimal = 1; + break; + case OPTIONS_DUMP_ALL: + use_defaults = NULL; + minimal = 0; + break; + default: + log_warn(LD_BUG, "Bogus value for how_to_dump==%d", how_to_dump); + return NULL; + } + + return config_dump(&options_format, use_defaults, options, minimal, 0); } /** Return 0 if every element of sl is a string holding a decimal @@ -2216,7 +2592,7 @@ ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg) /** Parse an authority type from <b>options</b>-\>PublishServerDescriptor * and write it to <b>options</b>-\>PublishServerDescriptor_. Treat "1" - * as "v2,v3" unless BridgeRelay is 1, in which case treat it as "bridge". + * as "v3" unless BridgeRelay is 1, in which case treat it as "bridge". * Treat "0" as "". * Return 0 on success or -1 if not a recognized authority type (in which * case the value of PublishServerDescriptor_ is undefined). */ @@ -2230,14 +2606,16 @@ compute_publishserverdescriptor(or_options_t *options) return 0; SMARTLIST_FOREACH_BEGIN(list, const char *, string) { if (!strcasecmp(string, "v1")) - *auth |= V1_DIRINFO; + log_warn(LD_CONFIG, "PublishServerDescriptor v1 has no effect, because " + "there are no v1 directory authorities anymore."); else if (!strcmp(string, "1")) if (options->BridgeRelay) *auth |= BRIDGE_DIRINFO; else - *auth |= V2_DIRINFO | V3_DIRINFO; + *auth |= V3_DIRINFO; else if (!strcasecmp(string, "v2")) - *auth |= V2_DIRINFO; + log_warn(LD_CONFIG, "PublishServerDescriptor v2 has no effect, because " + "there are no v2 directory authorities anymore."); else if (!strcasecmp(string, "v3")) *auth |= V3_DIRINFO; else if (!strcasecmp(string, "bridge")) @@ -2257,6 +2635,12 @@ compute_publishserverdescriptor(or_options_t *options) /** Lowest allowable value for RendPostPeriod; if this is too low, hidden * services can overload the directory system. */ #define MIN_REND_POST_PERIOD (10*60) +#define MIN_REND_POST_PERIOD_TESTING (5) + +/** Higest allowable value for PredictedPortsRelevanceTime; if this is + * too high, our selection of exits will decrease for an extended + * period of time to an uncomfortable level .*/ +#define MAX_PREDICTED_CIRCS_RELEVANCE (60*60) /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) @@ -2284,10 +2668,79 @@ compute_publishserverdescriptor(or_options_t *options) * */ #define RECOMMENDED_MIN_CIRCUIT_BUILD_TIMEOUT (10) -/** Return 0 if every setting in <b>options</b> is reasonable, and a - * permissible transition from <b>old_options</b>. Else return -1. - * Should have no side effects, except for normalizing the contents of - * <b>options</b>. +static int +options_validate_cb(void *old_options, void *options, void *default_options, + int from_setconf, char **msg) +{ + return options_validate(old_options, options, default_options, + from_setconf, msg); +} + +#define REJECT(arg) \ + STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END +#ifdef __GNUC__ +#define COMPLAIN(args...) \ + STMT_BEGIN log_warn(LD_CONFIG, args); STMT_END +#else +#define COMPLAIN(args, ...) \ + STMT_BEGIN log_warn(LD_CONFIG, args, ##__VA_ARGS__); STMT_END +#endif + +/** Log a warning message iff <b>filepath</b> is not absolute. + * Warning message must contain option name <b>option</b> and + * an absolute path that <b>filepath</b> will resolve to. + * + * In case <b>filepath</b> is absolute, do nothing. + */ +static void +warn_if_option_path_is_relative(const char *option, + char *filepath) +{ + if (filepath && path_is_relative(filepath)) { + char *abs_path = make_path_absolute(filepath); + COMPLAIN("Path for %s (%s) is relative and will resolve to %s." + " Is this what you wanted?", option, filepath, abs_path); + tor_free(abs_path); + } +} + +/** Scan <b>options</b> for occurances of relative file/directory + * path and log a warning whenever it is found. + */ +static void +warn_about_relative_paths(or_options_t *options) +{ + tor_assert(options); + + warn_if_option_path_is_relative("CookieAuthFile", + options->CookieAuthFile); + warn_if_option_path_is_relative("ExtORPortCookieAuthFile", + options->ExtORPortCookieAuthFile); + warn_if_option_path_is_relative("DirPortFrontPage", + options->DirPortFrontPage); + warn_if_option_path_is_relative("V3BandwidthsFile", + options->V3BandwidthsFile); + warn_if_option_path_is_relative("ControlPortWriteToFile", + options->ControlPortWriteToFile); + warn_if_option_path_is_relative("GeoIPFile",options->GeoIPFile); + warn_if_option_path_is_relative("GeoIPv6File",options->GeoIPv6File); + warn_if_option_path_is_relative("Log",options->DebugLogFile); + warn_if_option_path_is_relative("AccelDir",options->AccelDir); + warn_if_option_path_is_relative("DataDirectory",options->DataDirectory); + warn_if_option_path_is_relative("PidFile",options->PidFile); + + for (config_line_t *hs_line = options->RendConfigLines; hs_line; + hs_line = hs_line->next) { + if (!strcasecmp(hs_line->key, "HiddenServiceDir")) + warn_if_option_path_is_relative("HiddenServiceDir",hs_line->value); + } +} + +/** Return 0 if every setting in <b>options</b> is reasonable, is a + * permissible transition from <b>old_options</b>, and none of the + * testing-only settings differ from <b>default_options</b> unless in + * testing mode. Else return -1. Should have no side effects, except for + * normalizing the contents of <b>options</b>. * * On error, tor_strdup an error explanation into *<b>msg</b>. * @@ -2296,21 +2749,21 @@ compute_publishserverdescriptor(or_options_t *options) * Log line should stay empty. If it's 0, then give us a default log * if there are no logs defined. */ -static int +STATIC int options_validate(or_options_t *old_options, or_options_t *options, - int from_setconf, char **msg) + or_options_t *default_options, int from_setconf, char **msg) { int i; config_line_t *cl; const char *uname = get_uname(); int n_ports=0; -#define REJECT(arg) \ - STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END -#define COMPLAIN(arg) STMT_BEGIN log_warn(LD_CONFIG, arg); STMT_END + int world_writable_control_socket=0; tor_assert(msg); *msg = NULL; + warn_about_relative_paths(options); + if (server_mode(options) && (!strcmpstart(uname, "Windows 95") || !strcmpstart(uname, "Windows 98") || @@ -2321,7 +2774,8 @@ options_validate(or_options_t *old_options, or_options_t *options, "for details.", uname); } - if (parse_ports(options, 1, msg, &n_ports) < 0) + if (parse_ports(options, 1, msg, &n_ports, + &world_writable_control_socket) < 0) return -1; if (parse_outbound_addresses(options, 1, msg) < 0) @@ -2356,7 +2810,8 @@ options_validate(or_options_t *old_options, or_options_t *options, config_line_append(&options->Logs, "Log", "warn stdout"); } - if (options_init_logs(options, 1)<0) /* Validate the tor_log(s) */ + /* Validate the tor_log(s) */ + if (options_init_logs(old_options, options, 1)<0) REJECT("Failed to validate Log options. See logs for details."); if (authdir_mode(options)) { @@ -2366,10 +2821,12 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Failed to resolve/guess local address. See logs for details."); } -#ifndef _WIN32 - if (options->RunAsDaemon && torrc_fname && path_is_relative(torrc_fname)) - REJECT("Can't use a relative path to torrc when RunAsDaemon is set."); -#endif + if (server_mode(options) && options->RendConfigLines) + log_warn(LD_CONFIG, + "Tor is currently configured as a relay and a hidden service. " + "That's not very secure: you should probably run your hidden service " + "in a separate Tor process, at least -- see " + "https://trac.torproject.org/8742"); /* XXXX require that the only port not be DirPort? */ /* XXXX require that at least one port be listened-upon. */ @@ -2379,10 +2836,47 @@ options_validate(or_options_t *old_options, or_options_t *options, "undefined, and there aren't any hidden services configured. " "Tor will still run, but probably won't do anything."); -#ifndef USE_TRANSPARENT - /* XXXX024 I think we can remove this TransListenAddress */ - if (options->TransPort_set || options->TransListenAddress) - REJECT("TransPort and TransListenAddress are disabled in this build."); + options->TransProxyType_parsed = TPT_DEFAULT; +#ifdef USE_TRANSPARENT + if (options->TransProxyType) { + if (!strcasecmp(options->TransProxyType, "default")) { + options->TransProxyType_parsed = TPT_DEFAULT; + } else if (!strcasecmp(options->TransProxyType, "pf-divert")) { +#if !defined(__OpenBSD__) && !defined( DARWIN ) + /* Later versions of OS X have pf */ + REJECT("pf-divert is a OpenBSD-specific " + "and OS X/Darwin-specific feature."); +#else + options->TransProxyType_parsed = TPT_PF_DIVERT; +#endif + } else if (!strcasecmp(options->TransProxyType, "tproxy")) { +#if !defined(__linux__) + REJECT("TPROXY is a Linux-specific feature."); +#else + options->TransProxyType_parsed = TPT_TPROXY; +#endif + } else if (!strcasecmp(options->TransProxyType, "ipfw")) { +#ifndef KERNEL_MAY_SUPPORT_IPFW + /* Earlier versions of OS X have ipfw */ + REJECT("ipfw is a FreeBSD-specific" + "and OS X/Darwin-specific feature."); +#else + options->TransProxyType_parsed = TPT_IPFW; +#endif + } else { + REJECT("Unrecognized value for TransProxyType"); + } + + if (strcasecmp(options->TransProxyType, "default") && + !options->TransPort_set) { + REJECT("Cannot use TransProxyType without any valid TransPort or " + "TransListenAddress."); + } + } +#else + if (options->TransPort_set) + REJECT("TransPort and TransListenAddress are disabled " + "in this build."); #endif if (options->TokenBucketRefillInterval <= 0 @@ -2390,16 +2884,23 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("TokenBucketRefillInterval must be between 1 and 1000 inclusive."); } - if (options->DisableV2DirectoryInfo_ && ! authdir_mode(options)) { - REJECT("DisableV2DirectoryInfo_ set, but we aren't an authority."); - } - if (options->ExcludeExitNodes || options->ExcludeNodes) { options->ExcludeExitNodesUnion_ = routerset_new(); routerset_union(options->ExcludeExitNodesUnion_,options->ExcludeExitNodes); routerset_union(options->ExcludeExitNodesUnion_,options->ExcludeNodes); } + if (options->SchedulerLowWaterMark__ == 0 || + options->SchedulerLowWaterMark__ > UINT32_MAX) { + log_warn(LD_GENERAL, "Bad SchedulerLowWaterMark__ option"); + return -1; + } else if (options->SchedulerHighWaterMark__ <= + options->SchedulerLowWaterMark__ || + options->SchedulerHighWaterMark__ > UINT32_MAX) { + log_warn(LD_GENERAL, "Bad SchedulerHighWaterMark option"); + return -1; + } + if (options->NodeFamilies) { options->NodeFamilySets = smartlist_new(); for (cl = options->NodeFamilies; cl; cl = cl->next) { @@ -2417,6 +2918,9 @@ options_validate(or_options_t *old_options, or_options_t *options, COMPLAIN("Unrecognized TLSECGroup: Falling back to the default."); tor_free(options->TLSECGroup); } + if (!evaluate_ecgroup_for_tls(options->TLSECGroup)) { + REJECT("Unsupported TLSECGroup."); + } if (options->ExcludeNodes && options->StrictNodes) { COMPLAIN("You have asked to exclude certain relays from all positions " @@ -2424,11 +2928,16 @@ options_validate(or_options_t *old_options, or_options_t *options, "features to be broken in unpredictable ways."); } + for (cl = options->RecommendedPackages; cl; cl = cl->next) { + if (! validate_recommended_package_line(cl->value)) { + log_warn(LD_CONFIG, "Invalid RecommendedPackage line %s will be ignored", + escaped(cl->value)); + } + } + if (options->AuthoritativeDir) { if (!options->ContactInfo && !options->TestingTorNetwork) REJECT("Authoritative directory servers must set ContactInfo"); - if (options->V1AuthoritativeDir && !options->RecommendedVersions) - REJECT("V1 authoritative dir servers must set RecommendedVersions."); if (!options->RecommendedClientVersions) options->RecommendedClientVersions = config_lines_dup(options->RecommendedVersions); @@ -2450,15 +2959,18 @@ options_validate(or_options_t *old_options, or_options_t *options, "extra-info documents. Setting DownloadExtraInfo."); options->DownloadExtraInfo = 1; } - if (!(options->BridgeAuthoritativeDir || options->HSAuthoritativeDir || - options->V1AuthoritativeDir || options->V2AuthoritativeDir || + if (!(options->BridgeAuthoritativeDir || options->V3AuthoritativeDir)) REJECT("AuthoritativeDir is set, but none of " - "(Bridge/HS/V1/V2/V3)AuthoritativeDir is set."); + "(Bridge/V3)AuthoritativeDir is set."); /* If we have a v3bandwidthsfile and it's broken, complain on startup */ if (options->V3BandwidthsFile && !old_options) { dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); } + /* same for guardfraction file */ + if (options->GuardfractionFile && !old_options) { + dirserv_read_guardfraction_file(options->GuardfractionFile, NULL); + } } if (options->AuthoritativeDir && !options->DirPort_set) @@ -2474,10 +2986,6 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("FetchDirInfoExtraEarly requires that you also set " "FetchDirInfoEarly"); - if (options->HSAuthoritativeDir && proxy_mode(options)) - REJECT("Running as authoritative v0 HS directory, but also configured " - "as a client."); - if (options->ConnLimit <= 0) { tor_asprintf(msg, "ConnLimit must be greater than 0, but was set to %d", @@ -2567,6 +3075,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } } + /* Terminate Reachable*Addresses with reject * + */ for (i=0; i<3; i++) { config_line_t **linep = (i==0) ? &options->ReachableAddresses : @@ -2576,8 +3086,6 @@ options_validate(or_options_t *old_options, or_options_t *options, continue; /* We need to end with a reject *:*, not an implicit accept *:* */ for (;;) { - if (!strcmp((*linep)->value, "reject *:*")) /* already there */ - break; linep = &((*linep)->next); if (!*linep) { *linep = tor_malloc_zero(sizeof(config_line_t)); @@ -2593,11 +3101,29 @@ options_validate(or_options_t *old_options, or_options_t *options, if ((options->ReachableAddresses || options->ReachableORAddresses || - options->ReachableDirAddresses) && + options->ReachableDirAddresses || + options->ClientUseIPv4 == 0) && server_mode(options)) REJECT("Servers must be able to freely connect to the rest " "of the Internet, so they must not set Reachable*Addresses " - "or FascistFirewall."); + "or FascistFirewall or FirewallPorts or ClientUseIPv4 0."); + + /* We check if Reachable*Addresses blocks all addresses in + * parse_reachable_addresses(). */ + +#define WARN_PLEASE_USE_IPV6_LOG_MSG \ + "ClientPreferIPv6%sPort 1 is ignored unless tor is using IPv6. " \ + "Please set ClientUseIPv6 1, ClientUseIPv4 0, or configure bridges." + + if (!fascist_firewall_use_ipv6(options) + && options->ClientPreferIPv6ORPort == 1) + log_warn(LD_CONFIG, WARN_PLEASE_USE_IPV6_LOG_MSG, "OR"); + + if (!fascist_firewall_use_ipv6(options) + && options->ClientPreferIPv6DirPort == 1) + log_warn(LD_CONFIG, WARN_PLEASE_USE_IPV6_LOG_MSG, "Dir"); + +#undef WARN_PLEASE_USE_IPV6_LOG_MSG if (options->UseBridges && server_mode(options)) @@ -2614,11 +3140,10 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("If EntryNodes is set, UseEntryGuards must be enabled."); } - if (options->MaxMemInCellQueues < (256 << 20)) { - log_warn(LD_CONFIG, "MaxMemInCellQueues must be at least 256 MB for now. " - "Ideally, have it as large as you can afford."); - options->MaxMemInCellQueues = (256 << 20); - } + options->MaxMemInQueues = + compute_real_max_mem_in_queues(options->MaxMemInQueues_raw, + server_mode(options)); + options->MaxMemInQueues_low_threshold = (options->MaxMemInQueues / 4) * 3; options->AllowInvalid_ = 0; @@ -2663,8 +3188,7 @@ options_validate(or_options_t *old_options, or_options_t *options, if ((options->BridgeRelay || options->PublishServerDescriptor_ & BRIDGE_DIRINFO) - && (options->PublishServerDescriptor_ - & (V1_DIRINFO|V2_DIRINFO|V3_DIRINFO))) { + && (options->PublishServerDescriptor_ & V3_DIRINFO)) { REJECT("Bridges are not supposed to publish router descriptors to the " "directory authorities. Please correct your " "PublishServerDescriptor line."); @@ -2684,10 +3208,13 @@ options_validate(or_options_t *old_options, or_options_t *options, options->MinUptimeHidServDirectoryV2 = 0; } - if (options->RendPostPeriod < MIN_REND_POST_PERIOD) { + const int min_rendpostperiod = + options->TestingTorNetwork ? + MIN_REND_POST_PERIOD_TESTING : MIN_REND_POST_PERIOD; + if (options->RendPostPeriod < min_rendpostperiod) { log_warn(LD_CONFIG, "RendPostPeriod option is too short; " - "raising to %d seconds.", MIN_REND_POST_PERIOD); - options->RendPostPeriod = MIN_REND_POST_PERIOD; + "raising to %d seconds.", min_rendpostperiod); + options->RendPostPeriod = min_rendpostperiod;; } if (options->RendPostPeriod > MAX_DIR_PERIOD) { @@ -2696,6 +3223,14 @@ options_validate(or_options_t *old_options, or_options_t *options, options->RendPostPeriod = MAX_DIR_PERIOD; } + if (options->PredictedPortsRelevanceTime > + MAX_PREDICTED_CIRCS_RELEVANCE) { + log_warn(LD_CONFIG, "PredictedPortsRelevanceTime is too large; " + "clipping to %ds.", MAX_PREDICTED_CIRCS_RELEVANCE); + options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE; + } + +#ifdef ENABLE_TOR2WEB_MODE if (options->Tor2webMode && options->LearnCircuitBuildTimeout) { /* LearnCircuitBuildTimeout and Tor2webMode are incompatible in * two ways: @@ -2727,6 +3262,11 @@ options_validate(or_options_t *old_options, or_options_t *options, "Tor2WebMode is enabled; disabling UseEntryGuards."); options->UseEntryGuards = 0; } +#endif + + if (options->Tor2webRendezvousPoints && !options->Tor2webMode) { + REJECT("Tor2webRendezvousPoints cannot be set without Tor2webMode."); + } if (!(options->UseEntryGuards) && (options->RendConfigLines != NULL)) { @@ -2737,6 +3277,21 @@ options_validate(or_options_t *old_options, or_options_t *options, "http://freehaven.net/anonbib/#hs-attack06 for details."); } + if (options->EntryNodes && + routerset_is_list(options->EntryNodes) && + (routerset_len(options->EntryNodes) == 1) && + (options->RendConfigLines != NULL)) { + tor_asprintf(msg, + "You have one single EntryNodes and at least one hidden service " + "configured. This is bad because it's very easy to locate your " + "entry guard which can then lead to the deanonymization of your " + "hidden service -- for more details, see " + "https://trac.torproject.org/projects/tor/ticket/14917. " + "For this reason, the use of one EntryNodes with an hidden " + "service is prohibited until a better solution is found."); + return -1; + } + if (!options->LearnCircuitBuildTimeout && options->CircuitBuildTimeout && options->CircuitBuildTimeout < RECOMMENDED_MIN_CIRCUIT_BUILD_TIMEOUT) { log_warn(LD_CONFIG, @@ -2812,6 +3367,11 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); + if (options->PortForwarding && options->Sandbox) { + REJECT("PortForwarding is not compatible with Sandbox; at most one can " + "be set"); + } + if (ensure_bandwidth_cap(&options->BandwidthRate, "BandwidthRate", msg) < 0) return -1; @@ -2846,29 +3406,34 @@ options_validate(or_options_t *old_options, or_options_t *options, options->RelayBandwidthRate = options->RelayBandwidthBurst; if (server_mode(options)) { - if (options->BandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { + const unsigned required_min_bw = + public_server_mode(options) ? + RELAY_REQUIRED_MIN_BANDWIDTH : BRIDGE_REQUIRED_MIN_BANDWIDTH; + const char * const optbridge = + public_server_mode(options) ? "" : "bridge "; + if (options->BandwidthRate < required_min_bw) { tor_asprintf(msg, "BandwidthRate is set to %d bytes/second. " - "For servers, it must be at least %d.", - (int)options->BandwidthRate, - ROUTER_REQUIRED_MIN_BANDWIDTH); + "For %sservers, it must be at least %u.", + (int)options->BandwidthRate, optbridge, + required_min_bw); return -1; } else if (options->MaxAdvertisedBandwidth < - ROUTER_REQUIRED_MIN_BANDWIDTH/2) { + required_min_bw/2) { tor_asprintf(msg, "MaxAdvertisedBandwidth is set to %d bytes/second. " - "For servers, it must be at least %d.", - (int)options->MaxAdvertisedBandwidth, - ROUTER_REQUIRED_MIN_BANDWIDTH/2); + "For %sservers, it must be at least %u.", + (int)options->MaxAdvertisedBandwidth, optbridge, + required_min_bw/2); return -1; } if (options->RelayBandwidthRate && - options->RelayBandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { + options->RelayBandwidthRate < required_min_bw) { tor_asprintf(msg, "RelayBandwidthRate is set to %d bytes/second. " - "For servers, it must be at least %d.", - (int)options->RelayBandwidthRate, - ROUTER_REQUIRED_MIN_BANDWIDTH); + "For %sservers, it must be at least %u.", + (int)options->RelayBandwidthRate, optbridge, + required_min_bw); return -1; } } @@ -2904,6 +3469,38 @@ options_validate(or_options_t *old_options, or_options_t *options, } } + options->AccountingRule = ACCT_MAX; + if (options->AccountingRule_option) { + if (!strcmp(options->AccountingRule_option, "sum")) + options->AccountingRule = ACCT_SUM; + else if (!strcmp(options->AccountingRule_option, "max")) + options->AccountingRule = ACCT_MAX; + else if (!strcmp(options->AccountingRule_option, "in")) + options->AccountingRule = ACCT_IN; + else if (!strcmp(options->AccountingRule_option, "out")) + options->AccountingRule = ACCT_OUT; + else + REJECT("AccountingRule must be 'sum', 'max', 'in', or 'out'"); + } + + if (options->DirPort_set && !options->DirCache) { + REJECT("DirPort configured but DirCache disabled. DirPort requires " + "DirCache."); + } + + if (options->BridgeRelay && !options->DirCache) { + REJECT("We're a bridge but DirCache is disabled. BridgeRelay requires " + "DirCache."); + } + + if (server_mode(options)) { + char *msg = NULL; + if (have_enough_mem_for_dircache(options, 0, &msg)) { + log_warn(LD_CONFIG, "%s", msg); + tor_free(msg); + } + } + if (options->HTTPProxy) { /* parse it now */ if (tor_addr_port_lookup(options->HTTPProxy, &options->HTTPProxyAddr, &options->HTTPProxyPort) < 0) @@ -2952,11 +3549,11 @@ options_validate(or_options_t *old_options, or_options_t *options, } } - /* Check if more than one proxy type has been enabled. */ + /* Check if more than one exclusive proxy type has been enabled. */ if (!!options->Socks4Proxy + !!options->Socks5Proxy + - !!options->HTTPSProxy + !!options->ClientTransportPlugin > 1) + !!options->HTTPSProxy > 1) REJECT("You have configured more than one proxy type. " - "(Socks4Proxy|Socks5Proxy|HTTPSProxy|ClientTransportPlugin)"); + "(Socks4Proxy|Socks5Proxy|HTTPSProxy)"); /* Check if the proxies will give surprising behavior. */ if (options->HTTPProxy && !(options->Socks4Proxy || @@ -2971,14 +3568,14 @@ options_validate(or_options_t *old_options, or_options_t *options, size_t len; len = strlen(options->Socks5ProxyUsername); - if (len < 1 || len > 255) + if (len < 1 || len > MAX_SOCKS5_AUTH_FIELD_SIZE) REJECT("Socks5ProxyUsername must be between 1 and 255 characters."); if (!options->Socks5ProxyPassword) REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); len = strlen(options->Socks5ProxyPassword); - if (len < 1 || len > 255) + if (len < 1 || len > MAX_SOCKS5_AUTH_FIELD_SIZE) REJECT("Socks5ProxyPassword must be between 1 and 255 characters."); } else if (options->Socks5ProxyPassword) REJECT("Socks5ProxyPassword must be included with Socks5ProxyUsername."); @@ -3014,13 +3611,16 @@ options_validate(or_options_t *old_options, or_options_t *options, } } - if (options->ControlPort_set && !options->HashedControlPassword && + if ((options->ControlPort_set || world_writable_control_socket) && + !options->HashedControlPassword && !options->HashedControlSessionPassword && !options->CookieAuthentication) { - log_warn(LD_CONFIG, "ControlPort is open, but no authentication method " + log_warn(LD_CONFIG, "Control%s is %s, but no authentication method " "has been configured. This means that any program on your " "computer can reconfigure your Tor. That's bad! You should " - "upgrade your Tor controller as soon as possible."); + "upgrade your Tor controller as soon as possible.", + options->ControlPort_set ? "Port" : "Socket", + options->ControlPort_set ? "open" : "world writable"); } if (options->CookieAuthFileGroupReadable && !options->CookieAuthFile) { @@ -3035,7 +3635,7 @@ options_validate(or_options_t *old_options, or_options_t *options, "You should also make sure you aren't listing this bridge's " "fingerprint in any other MyFamily."); } - if (check_nickname_list(options->MyFamily, "MyFamily", msg)) + if (check_nickname_list(&options->MyFamily, "MyFamily", msg)) return -1; for (cl = options->NodeFamilies; cl; cl = cl->next) { routerset_t *rs = routerset_new(); @@ -3049,32 +3649,35 @@ options_validate(or_options_t *old_options, or_options_t *options, if (validate_addr_policies(options, msg) < 0) return -1; + /* If FallbackDir is set, we don't UseDefaultFallbackDirs */ + if (options->UseDefaultFallbackDirs && options->FallbackDir) { + log_info(LD_CONFIG, "You have set UseDefaultFallbackDirs 1 and " + "FallbackDir(s). Ignoring UseDefaultFallbackDirs, and " + "using the FallbackDir(s) you have set."); + } + if (validate_dir_servers(options, old_options) < 0) REJECT("Directory authority/fallback line did not parse. See logs " "for details."); if (options->UseBridges && !options->Bridges) REJECT("If you set UseBridges, you must specify at least one bridge."); - if (options->UseBridges && !options->TunnelDirConns) - REJECT("If you set UseBridges, you must set TunnelDirConns."); - if (options->RendConfigLines && - (!options->TunnelDirConns || !options->PreferTunneledDirConns)) - REJECT("If you are running a hidden service, you must set TunnelDirConns " - "and PreferTunneledDirConns"); for (cl = options->Bridges; cl; cl = cl->next) { - if (parse_bridge_line(cl->value, 1)<0) - REJECT("Bridge line did not parse. See logs for details."); + bridge_line_t *bridge_line = parse_bridge_line(cl->value); + if (!bridge_line) + REJECT("Bridge line did not parse. See logs for details."); + bridge_line_free(bridge_line); } for (cl = options->ClientTransportPlugin; cl; cl = cl->next) { - if (parse_client_transport_line(cl->value, 1)<0) - REJECT("Transport line did not parse. See logs for details."); + if (parse_transport_line(options, cl->value, 1, 0) < 0) + REJECT("Invalid client transport line. See logs for details."); } for (cl = options->ServerTransportPlugin; cl; cl = cl->next) { - if (parse_server_transport_line(cl->value, 1)<0) - REJECT("Server transport line did not parse. See logs for details."); + if (parse_transport_line(options, cl->value, 1, 1) < 0) + REJECT("Invalid server transport line. See logs for details."); } if (options->ServerTransportPlugin && !server_mode(options)) { @@ -3100,6 +3703,19 @@ options_validate(or_options_t *old_options, or_options_t *options, "ServerTransportListenAddr line will be ignored."); } + for (cl = options->ServerTransportOptions; cl; cl = cl->next) { + /** If get_options_from_transport_options_line() fails with + 'transport' being NULL, it means that something went wrong + while parsing the ServerTransportOptions line. */ + smartlist_t *options_sl = + get_options_from_transport_options_line(cl->value, NULL); + if (!options_sl) + REJECT("ServerTransportOptions did not parse. See logs for details."); + + SMARTLIST_FOREACH(options_sl, char *, cp, tor_free(cp)); + smartlist_free(options_sl); + } + if (options->ConstrainedSockets) { /* If the user wants to constrain socket buffer use, make sure the desired * limit is between MIN|MAX_TCPSOCK_BUFFER in k increments. */ @@ -3124,19 +3740,68 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->V3AuthVoteDelay + options->V3AuthDistDelay >= options->V3AuthVotingInterval/2) { - REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half " - "V3AuthVotingInterval"); + /* + This doesn't work, but it seems like it should: + what code is preventing the interval being less than twice the lead-up? + if (options->TestingTorNetwork) { + if (options->V3AuthVoteDelay + options->V3AuthDistDelay >= + options->V3AuthVotingInterval) { + REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than " + "V3AuthVotingInterval"); + } else { + COMPLAIN("V3AuthVoteDelay plus V3AuthDistDelay is more than half " + "V3AuthVotingInterval. This may lead to " + "consensus instability, particularly if clocks drift."); + } + } else { + */ + REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half " + "V3AuthVotingInterval"); + /* + } + */ + } + + if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) { + if (options->TestingTorNetwork) { + if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS_TESTING) { + REJECT("V3AuthVoteDelay is way too low."); + } else { + COMPLAIN("V3AuthVoteDelay is very low. " + "This may lead to failure to vote for a consensus."); + } + } else { + REJECT("V3AuthVoteDelay is way too low."); + } + } + + if (options->V3AuthDistDelay < MIN_DIST_SECONDS) { + if (options->TestingTorNetwork) { + if (options->V3AuthDistDelay < MIN_DIST_SECONDS_TESTING) { + REJECT("V3AuthDistDelay is way too low."); + } else { + COMPLAIN("V3AuthDistDelay is very low. " + "This may lead to missing votes in a consensus."); + } + } else { + REJECT("V3AuthDistDelay is way too low."); + } } - if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) - REJECT("V3AuthVoteDelay is way too low."); - if (options->V3AuthDistDelay < MIN_DIST_SECONDS) - REJECT("V3AuthDistDelay is way too low."); if (options->V3AuthNIntervalsValid < 2) REJECT("V3AuthNIntervalsValid must be at least 2."); if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL) { - REJECT("V3AuthVotingInterval is insanely low."); + if (options->TestingTorNetwork) { + if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL_TESTING) { + REJECT("V3AuthVotingInterval is insanely low."); + } else { + COMPLAIN("V3AuthVotingInterval is very low. " + "This may lead to failure to synchronise for a consensus."); + } + } else { + REJECT("V3AuthVotingInterval is insanely low."); + } } else if (options->V3AuthVotingInterval > 24*60*60) { REJECT("V3AuthVotingInterval is insanely high."); } else if (((24*60*60) % options->V3AuthVotingInterval) != 0) { @@ -3158,24 +3823,6 @@ options_validate(or_options_t *old_options, or_options_t *options, AF_INET6, 1, msg)<0) return -1; - if (options->PreferTunneledDirConns && !options->TunnelDirConns) - REJECT("Must set TunnelDirConns if PreferTunneledDirConns is set."); - - if ((options->Socks4Proxy || options->Socks5Proxy) && - !options->HTTPProxy && !options->PreferTunneledDirConns) - REJECT("When Socks4Proxy or Socks5Proxy is configured, " - "PreferTunneledDirConns and TunnelDirConns must both be " - "set to 1, or HTTPProxy must be configured."); - - if (options->AutomapHostsSuffixes) { - SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, - { - size_t len = strlen(suf); - if (len && suf[len-1] == '.') - suf[len-1] = '\0'; - }); - } - if (options->TestingTorNetwork && !(options->DirAuthorities || (options->AlternateDirAuthority && @@ -3192,65 +3839,174 @@ options_validate(or_options_t *old_options, or_options_t *options, "ignore you."); } - /*XXXX checking for defaults manually like this is a bit fragile.*/ - - /* Keep changes to hard-coded values synchronous to man page and default - * values table. */ - if (options->TestingV3AuthInitialVotingInterval != 30*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingV3AuthInitialVotingInterval may only be changed in testing " - "Tor networks!"); - } else if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) { +#define CHECK_DEFAULT(arg) \ + STMT_BEGIN \ + if (!options->TestingTorNetwork && \ + !options->UsingTestNetworkDefaults_ && \ + !config_is_same(&options_format,options, \ + default_options,#arg)) { \ + REJECT(#arg " may only be changed in testing Tor " \ + "networks!"); \ + } STMT_END + CHECK_DEFAULT(TestingV3AuthInitialVotingInterval); + CHECK_DEFAULT(TestingV3AuthInitialVoteDelay); + CHECK_DEFAULT(TestingV3AuthInitialDistDelay); + CHECK_DEFAULT(TestingV3AuthVotingStartOffset); + CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability); + CHECK_DEFAULT(TestingEstimatedDescriptorPropagationTime); + CHECK_DEFAULT(TestingServerDownloadSchedule); + CHECK_DEFAULT(TestingClientDownloadSchedule); + CHECK_DEFAULT(TestingServerConsensusDownloadSchedule); + CHECK_DEFAULT(TestingClientConsensusDownloadSchedule); + CHECK_DEFAULT(TestingBridgeDownloadSchedule); + CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest); + CHECK_DEFAULT(TestingDirConnectionMaxStall); + CHECK_DEFAULT(TestingConsensusMaxDownloadTries); + CHECK_DEFAULT(TestingDescriptorMaxDownloadTries); + CHECK_DEFAULT(TestingMicrodescMaxDownloadTries); + CHECK_DEFAULT(TestingCertMaxDownloadTries); + CHECK_DEFAULT(TestingAuthKeyLifetime); + CHECK_DEFAULT(TestingLinkCertLifetime); + CHECK_DEFAULT(TestingSigningKeySlop); + CHECK_DEFAULT(TestingAuthKeySlop); + CHECK_DEFAULT(TestingLinkKeySlop); +#undef CHECK_DEFAULT + + if (options->SigningKeyLifetime < options->TestingSigningKeySlop*2) + REJECT("SigningKeyLifetime is too short."); + if (options->TestingLinkCertLifetime < options->TestingAuthKeySlop*2) + REJECT("LinkCertLifetime is too short."); + if (options->TestingAuthKeyLifetime < options->TestingLinkKeySlop*2) + REJECT("TestingAuthKeyLifetime is too short."); + + if (options->TestingV3AuthInitialVotingInterval + < MIN_VOTE_INTERVAL_TESTING_INITIAL) { REJECT("TestingV3AuthInitialVotingInterval is insanely low."); } else if (((30*60) % options->TestingV3AuthInitialVotingInterval) != 0) { REJECT("TestingV3AuthInitialVotingInterval does not divide evenly into " "30 minutes."); } - if (options->TestingV3AuthInitialVoteDelay != 5*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - - REJECT("TestingV3AuthInitialVoteDelay may only be changed in testing " - "Tor networks!"); - } else if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) { + if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS_TESTING) { REJECT("TestingV3AuthInitialVoteDelay is way too low."); } - if (options->TestingV3AuthInitialDistDelay != 5*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingV3AuthInitialDistDelay may only be changed in testing " - "Tor networks!"); - } else if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) { + if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS_TESTING) { REJECT("TestingV3AuthInitialDistDelay is way too low."); } if (options->TestingV3AuthInitialVoteDelay + options->TestingV3AuthInitialDistDelay >= - options->TestingV3AuthInitialVotingInterval/2) { + options->TestingV3AuthInitialVotingInterval) { REJECT("TestingV3AuthInitialVoteDelay plus TestingV3AuthInitialDistDelay " - "must be less than half TestingV3AuthInitialVotingInterval"); + "must be less than TestingV3AuthInitialVotingInterval"); } - if (options->TestingAuthDirTimeToLearnReachability != 30*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingAuthDirTimeToLearnReachability may only be changed in " - "testing Tor networks!"); - } else if (options->TestingAuthDirTimeToLearnReachability < 0) { + if (options->TestingV3AuthVotingStartOffset > + MIN(options->TestingV3AuthInitialVotingInterval, + options->V3AuthVotingInterval)) { + REJECT("TestingV3AuthVotingStartOffset is higher than the voting " + "interval."); + } else if (options->TestingV3AuthVotingStartOffset < 0) { + REJECT("TestingV3AuthVotingStartOffset must be non-negative."); + } + + if (options->TestingAuthDirTimeToLearnReachability < 0) { REJECT("TestingAuthDirTimeToLearnReachability must be non-negative."); } else if (options->TestingAuthDirTimeToLearnReachability > 2*60*60) { COMPLAIN("TestingAuthDirTimeToLearnReachability is insanely high."); } - if (options->TestingEstimatedDescriptorPropagationTime != 10*60 && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingEstimatedDescriptorPropagationTime may only be changed in " - "testing Tor networks!"); - } else if (options->TestingEstimatedDescriptorPropagationTime < 0) { + if (options->TestingEstimatedDescriptorPropagationTime < 0) { REJECT("TestingEstimatedDescriptorPropagationTime must be non-negative."); } else if (options->TestingEstimatedDescriptorPropagationTime > 60*60) { COMPLAIN("TestingEstimatedDescriptorPropagationTime is insanely high."); } + if (options->TestingClientMaxIntervalWithoutRequest < 1) { + REJECT("TestingClientMaxIntervalWithoutRequest is way too low."); + } else if (options->TestingClientMaxIntervalWithoutRequest > 3600) { + COMPLAIN("TestingClientMaxIntervalWithoutRequest is insanely high."); + } + + if (options->TestingDirConnectionMaxStall < 5) { + REJECT("TestingDirConnectionMaxStall is way too low."); + } else if (options->TestingDirConnectionMaxStall > 3600) { + COMPLAIN("TestingDirConnectionMaxStall is insanely high."); + } + + if (options->TestingConsensusMaxDownloadTries < 2) { + REJECT("TestingConsensusMaxDownloadTries must be greater than 2."); + } else if (options->TestingConsensusMaxDownloadTries > 800) { + COMPLAIN("TestingConsensusMaxDownloadTries is insanely high."); + } + + if (options->ClientBootstrapConsensusMaxDownloadTries < 2) { + REJECT("ClientBootstrapConsensusMaxDownloadTries must be greater " + "than 2." + ); + } else if (options->ClientBootstrapConsensusMaxDownloadTries > 800) { + COMPLAIN("ClientBootstrapConsensusMaxDownloadTries is insanely " + "high."); + } + + if (options->ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries + < 2) { + REJECT("ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries must " + "be greater than 2." + ); + } else if ( + options->ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries + > 800) { + COMPLAIN("ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries is " + "insanely high."); + } + + if (options->ClientBootstrapConsensusMaxInProgressTries < 1) { + REJECT("ClientBootstrapConsensusMaxInProgressTries must be greater " + "than 0."); + } else if (options->ClientBootstrapConsensusMaxInProgressTries + > 100) { + COMPLAIN("ClientBootstrapConsensusMaxInProgressTries is insanely " + "high."); + } + + if (options->TestingDescriptorMaxDownloadTries < 2) { + REJECT("TestingDescriptorMaxDownloadTries must be greater than 1."); + } else if (options->TestingDescriptorMaxDownloadTries > 800) { + COMPLAIN("TestingDescriptorMaxDownloadTries is insanely high."); + } + + if (options->TestingMicrodescMaxDownloadTries < 2) { + REJECT("TestingMicrodescMaxDownloadTries must be greater than 1."); + } else if (options->TestingMicrodescMaxDownloadTries > 800) { + COMPLAIN("TestingMicrodescMaxDownloadTries is insanely high."); + } + + if (options->TestingCertMaxDownloadTries < 2) { + REJECT("TestingCertMaxDownloadTries must be greater than 1."); + } else if (options->TestingCertMaxDownloadTries > 800) { + COMPLAIN("TestingCertMaxDownloadTries is insanely high."); + } + + if (options->TestingEnableConnBwEvent && + !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { + REJECT("TestingEnableConnBwEvent may only be changed in testing " + "Tor networks!"); + } + + if (options->TestingEnableCellStatsEvent && + !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { + REJECT("TestingEnableCellStatsEvent may only be changed in testing " + "Tor networks!"); + } + + if (options->TestingEnableTbEmptyEvent && + !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { + REJECT("TestingEnableTbEmptyEvent may only be changed in testing " + "Tor networks!"); + } + if (options->TestingTorNetwork) { log_warn(LD_CONFIG, "TestingTorNetwork is set. This will make your node " "almost unusable in the public Tor network, and is " @@ -3280,9 +4036,118 @@ options_validate(or_options_t *old_options, or_options_t *options, "combination."); return 0; +} + #undef REJECT #undef COMPLAIN + +/* Given the value that the user has set for MaxMemInQueues, compute the + * actual maximum value. We clip this value if it's too low, and autodetect + * it if it's set to 0. */ +static uint64_t +compute_real_max_mem_in_queues(const uint64_t val, int log_guess) +{ + uint64_t result; + + if (val == 0) { +#define ONE_GIGABYTE (U64_LITERAL(1) << 30) +#define ONE_MEGABYTE (U64_LITERAL(1) << 20) +#if SIZEOF_VOID_P >= 8 +#define MAX_DEFAULT_MAXMEM (8*ONE_GIGABYTE) +#else +#define MAX_DEFAULT_MAXMEM (2*ONE_GIGABYTE) +#endif + /* The user didn't pick a memory limit. Choose a very large one + * that is still smaller than the system memory */ + static int notice_sent = 0; + size_t ram = 0; + if (get_total_system_memory(&ram) < 0) { + /* We couldn't determine our total system memory! */ +#if SIZEOF_VOID_P >= 8 + /* 64-bit system. Let's hope for 8 GB. */ + result = 8 * ONE_GIGABYTE; +#else + /* (presumably) 32-bit system. Let's hope for 1 GB. */ + result = ONE_GIGABYTE; +#endif + } else { + /* We detected it, so let's pick 3/4 of the total RAM as our limit. */ + const uint64_t avail = (ram / 4) * 3; + + /* Make sure it's in range from 0.25 GB to 8 GB. */ + if (avail > MAX_DEFAULT_MAXMEM) { + /* If you want to use more than this much RAM, you need to configure + it yourself */ + result = MAX_DEFAULT_MAXMEM; + } else if (avail < ONE_GIGABYTE / 4) { + result = ONE_GIGABYTE / 4; + } else { + result = avail; + } + } + if (log_guess && ! notice_sent) { + log_notice(LD_CONFIG, "%sMaxMemInQueues is set to "U64_FORMAT" MB. " + "You can override this by setting MaxMemInQueues by hand.", + ram ? "Based on detected system memory, " : "", + U64_PRINTF_ARG(result / ONE_MEGABYTE)); + notice_sent = 1; + } + return result; + } else if (val < ONE_GIGABYTE / 4) { + log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " + "Ideally, have it as large as you can afford."); + return ONE_GIGABYTE / 4; + } else { + /* The value was fine all along */ + return val; + } +} + +/* If we have less than 300 MB suggest disabling dircache */ +#define DIRCACHE_MIN_MB_BANDWIDTH 300 +#define DIRCACHE_MIN_BANDWIDTH (DIRCACHE_MIN_MB_BANDWIDTH*ONE_MEGABYTE) +#define STRINGIFY(val) #val + +/** Create a warning message for emitting if we are a dircache but may not have + * enough system memory, or if we are not a dircache but probably should be. + * Return -1 when a message is returned in *msg*, else return 0. */ +STATIC int +have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem, + char **msg) +{ + *msg = NULL; + /* XXX We should possibly be looking at MaxMemInQueues here + * unconditionally. Or we should believe total_mem unconditionally. */ + if (total_mem == 0) { + if (get_total_system_memory(&total_mem) < 0) { + total_mem = options->MaxMemInQueues >= SIZE_MAX ? + SIZE_MAX : (size_t)options->MaxMemInQueues; + } + } + if (options->DirCache) { + if (total_mem < DIRCACHE_MIN_BANDWIDTH) { + if (options->BridgeRelay) { + *msg = strdup("Running a Bridge with less than " + STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is " + "not recommended."); + } else { + *msg = strdup("Being a directory cache (default) with less than " + STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is " + "not recommended and may consume most of the available " + "resources, consider disabling this functionality by " + "setting the DirCache option to 0."); + } + } + } else { + if (total_mem >= DIRCACHE_MIN_BANDWIDTH) { + *msg = strdup("DirCache is disabled and we are configured as a " + "relay. This may disqualify us from becoming a guard in the " + "future."); + } + } + return *msg == NULL ? 0 : -1; } +#undef STRINGIFY /** Helper: return true iff s1 and s2 are both NULL, or both non-NULL * equal strings. */ @@ -3312,6 +4177,12 @@ options_transition_allowed(const or_options_t *old, return -1; } + if (old->Sandbox != new_val->Sandbox) { + *msg = tor_strdup("While Tor is running, changing Sandbox " + "is not allowed."); + return -1; + } + if (strcmp(old->DataDirectory,new_val->DataDirectory)!=0) { tor_asprintf(msg, "While Tor is running, changing DataDirectory " @@ -3325,6 +4196,18 @@ options_transition_allowed(const or_options_t *old, return -1; } + if (old->KeepBindCapabilities != new_val->KeepBindCapabilities) { + *msg = tor_strdup("While Tor is running, changing KeepBindCapabilities is " + "not allowed."); + return -1; + } + + if (!opt_streq(old->SyslogIdentityTag, new_val->SyslogIdentityTag)) { + *msg = tor_strdup("While Tor is running, changing " + "SyslogIdentityTag is not allowed."); + return -1; + } + if ((old->HardwareAccel != new_val->HardwareAccel) || !opt_streq(old->AccelName, new_val->AccelName) || !opt_streq(old->AccelDir, new_val->AccelDir)) { @@ -3364,6 +4247,39 @@ options_transition_allowed(const or_options_t *old, return -1; } + if (sandbox_is_active()) { +#define SB_NOCHANGE_STR(opt) \ + do { \ + if (! opt_streq(old->opt, new_val->opt)) { \ + *msg = tor_strdup("Can't change " #opt " while Sandbox is active"); \ + return -1; \ + } \ + } while (0) + + SB_NOCHANGE_STR(Address); + SB_NOCHANGE_STR(PidFile); + SB_NOCHANGE_STR(ServerDNSResolvConfFile); + SB_NOCHANGE_STR(DirPortFrontPage); + SB_NOCHANGE_STR(CookieAuthFile); + SB_NOCHANGE_STR(ExtORPortCookieAuthFile); + +#undef SB_NOCHANGE_STR + + if (! config_lines_eq(old->Logs, new_val->Logs)) { + *msg = tor_strdup("Can't change Logs while Sandbox is active"); + return -1; + } + if (old->ConnLimit != new_val->ConnLimit) { + *msg = tor_strdup("Can't change ConnLimit while Sandbox is active"); + return -1; + } + if (server_mode(old) != server_mode(new_val)) { + *msg = tor_strdup("Can't start/stop being a server while " + "Sandbox is active"); + return -1; + } + } + return 0; } @@ -3403,6 +4319,7 @@ options_transition_affects_descriptor(const or_options_t *old_options, !opt_streq(old_options->Nickname,new_options->Nickname) || !opt_streq(old_options->Address,new_options->Address) || !config_lines_eq(old_options->ExitPolicy,new_options->ExitPolicy) || + old_options->ExitRelay != new_options->ExitRelay || old_options->ExitPolicyRejectPrivate != new_options->ExitPolicyRejectPrivate || old_options->IPv6Exit != new_options->IPv6Exit || @@ -3421,7 +4338,10 @@ options_transition_affects_descriptor(const or_options_t *old_options, !opt_streq(old_options->MyFamily, new_options->MyFamily) || !opt_streq(old_options->AccountingStart, new_options->AccountingStart) || old_options->AccountingMax != new_options->AccountingMax || - public_server_mode(old_options) != public_server_mode(new_options)) + old_options->AccountingRule != new_options->AccountingRule || + public_server_mode(old_options) != public_server_mode(new_options) || + old_options->DirCache != new_options->DirCache || + old_options->AssumeReachable != new_options->AssumeReachable) return 1; return 0; @@ -3491,7 +4411,10 @@ get_windows_conf_root(void) static const char * get_default_conf_file(int defaults_file) { -#ifdef _WIN32 +#ifdef DISABLE_SYSTEM_TORRC + (void) defaults_file; + return NULL; +#elif defined(_WIN32) if (defaults_file) { static char defaults_path[MAX_PATH+1]; tor_snprintf(defaults_path, MAX_PATH, "%s\\torrc-defaults", @@ -3509,31 +4432,63 @@ get_default_conf_file(int defaults_file) } /** Verify whether lst is a string containing valid-looking comma-separated - * nicknames, or NULL. Return 0 on success. Warn and return -1 on failure. + * nicknames, or NULL. Will normalise <b>lst</b> to prefix '$' to any nickname + * or fingerprint that needs it. Return 0 on success. + * Warn and return -1 on failure. */ static int -check_nickname_list(const char *lst, const char *name, char **msg) +check_nickname_list(char **lst, const char *name, char **msg) { int r = 0; smartlist_t *sl; + int changes = 0; - if (!lst) + if (!*lst) return 0; sl = smartlist_new(); - smartlist_split_string(sl, lst, ",", + smartlist_split_string(sl, *lst, ",", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0); - SMARTLIST_FOREACH(sl, const char *, s, + SMARTLIST_FOREACH_BEGIN(sl, char *, s) { if (!is_legal_nickname_or_hexdigest(s)) { + // check if first char is dollar + if (s[0] != '$') { + // Try again but with a dollar symbol prepended + char *prepended; + tor_asprintf(&prepended, "$%s", s); + + if (is_legal_nickname_or_hexdigest(prepended)) { + // The nickname is valid when it's prepended, swap the current + // version with a prepended one + tor_free(s); + SMARTLIST_REPLACE_CURRENT(sl, s, prepended); + changes = 1; + continue; + } + + // Still not valid, free and fallback to error message + tor_free(prepended); + } + tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name); r = -1; break; } - }); + } + SMARTLIST_FOREACH_END(s); + + // Replace the caller's nickname list with a fixed one + if (changes && r == 0) { + char *newNicknames = smartlist_join_strings(sl, ", ", 0, NULL); + tor_free(*lst); + *lst = newNicknames; + } + SMARTLIST_FOREACH(sl, char *, s, tor_free(s)); smartlist_free(sl); + return r; } @@ -3549,26 +4504,26 @@ check_nickname_list(const char *lst, const char *name, char **msg) * filename if it doesn't exist. */ static char * -find_torrc_filename(int argc, char **argv, +find_torrc_filename(config_line_t *cmd_arg, int defaults_file, int *using_default_fname, int *ignore_missing_torrc) { char *fname=NULL; - int i; + config_line_t *p_index; const char *fname_opt = defaults_file ? "--defaults-torrc" : "-f"; const char *ignore_opt = defaults_file ? NULL : "--ignore-missing-torrc"; if (defaults_file) *ignore_missing_torrc = 1; - for (i = 1; i < argc; ++i) { - if (i < argc-1 && !strcmp(argv[i],fname_opt)) { + for (p_index = cmd_arg; p_index; p_index = p_index->next) { + if (!strcmp(p_index->key, fname_opt)) { if (fname) { log_warn(LD_CONFIG, "Duplicate %s options on command line.", fname_opt); tor_free(fname); } - fname = expand_filename(argv[i+1]); + fname = expand_filename(p_index->value); { char *absfname; @@ -3578,8 +4533,7 @@ find_torrc_filename(int argc, char **argv, } *using_default_fname = 0; - ++i; - } else if (ignore_opt && !strcmp(argv[i],ignore_opt)) { + } else if (ignore_opt && !strcmp(p_index->key,ignore_opt)) { *ignore_missing_torrc = 1; } } @@ -3587,27 +4541,45 @@ find_torrc_filename(int argc, char **argv, if (*using_default_fname) { /* didn't find one, try CONFDIR */ const char *dflt = get_default_conf_file(defaults_file); - if (dflt && file_status(dflt) == FN_FILE) { + file_status_t st = file_status(dflt); + if (dflt && (st == FN_FILE || st == FN_EMPTY)) { fname = tor_strdup(dflt); } else { #ifndef _WIN32 char *fn = NULL; - if (!defaults_file) + if (!defaults_file) { fn = expand_filename("~/.torrc"); - if (fn && file_status(fn) == FN_FILE) { - fname = fn; + } + if (fn) { + file_status_t hmst = file_status(fn); + if (hmst == FN_FILE || hmst == FN_EMPTY || dflt == NULL) { + fname = fn; + } else { + tor_free(fn); + fname = tor_strdup(dflt); + } } else { - tor_free(fn); - fname = tor_strdup(dflt); + fname = dflt ? tor_strdup(dflt) : NULL; } #else - fname = tor_strdup(dflt); + fname = dflt ? tor_strdup(dflt) : NULL; #endif } } return fname; } +/** Read the torrc from standard input and return it as a string. + * Upon failure, return NULL. + */ +static char * +load_torrc_from_stdin(void) +{ + size_t sz_out; + + return read_file_to_str_until_eof(STDIN_FILENO,SIZE_MAX,&sz_out); +} + /** Load a configuration file from disk, setting torrc_fname or * torrc_defaults_fname if successful. * @@ -3616,7 +4588,7 @@ find_torrc_filename(int argc, char **argv, * Return the contents of the file on success, and NULL on failure. */ static char * -load_torrc_from_disk(int argc, char **argv, int defaults_file) +load_torrc_from_disk(config_line_t *cmd_arg, int defaults_file) { char *fname=NULL; char *cf = NULL; @@ -3624,16 +4596,20 @@ load_torrc_from_disk(int argc, char **argv, int defaults_file) int ignore_missing_torrc = 0; char **fname_var = defaults_file ? &torrc_defaults_fname : &torrc_fname; - fname = find_torrc_filename(argc, argv, defaults_file, - &using_default_torrc, &ignore_missing_torrc); - tor_assert(fname); - log_debug(LD_CONFIG, "Opening config file \"%s\"", fname); - - tor_free(*fname_var); - *fname_var = fname; + if (*fname_var == NULL) { + fname = find_torrc_filename(cmd_arg, defaults_file, + &using_default_torrc, &ignore_missing_torrc); + tor_free(*fname_var); + *fname_var = fname; + } else { + fname = *fname_var; + } + log_debug(LD_CONFIG, "Opening config file \"%s\"", fname?fname:"<NULL>"); /* Open config file */ - if (file_status(fname) != FN_FILE || + file_status_t st = fname ? file_status(fname) : FN_EMPTY; + if (fname == NULL || + !(st == FN_FILE || st == FN_EMPTY) || !(cf = read_file_to_str(fname,0,NULL))) { if (using_default_torrc == 1 || ignore_missing_torrc) { if (!defaults_file) @@ -3666,59 +4642,70 @@ int options_init_from_torrc(int argc, char **argv) { char *cf=NULL, *cf_defaults=NULL; - int i, command; + int command; int retval = -1; - static char **backup_argv; - static int backup_argc; char *command_arg = NULL; char *errmsg=NULL; + config_line_t *p_index = NULL; + config_line_t *cmdline_only_options = NULL; - if (argv) { /* first time we're called. save command line args */ - backup_argv = argv; - backup_argc = argc; - } else { /* we're reloading. need to clean up old options first. */ - argv = backup_argv; - argc = backup_argc; + /* Go through command-line variables */ + if (! have_parsed_cmdline) { + /* Or we could redo the list every time we pass this place. + * It does not really matter */ + if (config_parse_commandline(argc, argv, 0, &global_cmdline_options, + &global_cmdline_only_options) < 0) { + goto err; + } + have_parsed_cmdline = 1; } - if (argc > 1 && (!strcmp(argv[1], "-h") || !strcmp(argv[1],"--help"))) { + cmdline_only_options = global_cmdline_only_options; + + if (config_line_find(cmdline_only_options, "-h") || + config_line_find(cmdline_only_options, "--help")) { print_usage(); exit(0); } - if (argc > 1 && !strcmp(argv[1], "--list-torrc-options")) { + if (config_line_find(cmdline_only_options, "--list-torrc-options")) { /* For documenting validating whether we've documented everything. */ list_torrc_options(); exit(0); } - if (argc > 1 && (!strcmp(argv[1],"--version"))) { + if (config_line_find(cmdline_only_options, "--version")) { printf("Tor version %s.\n",get_version()); exit(0); } - if (argc > 1 && (!strcmp(argv[1],"--digests"))) { - printf("Tor version %s.\n",get_version()); - printf("%s", libor_get_digests()); - printf("%s", tor_get_digests()); - exit(0); - } - /* Go through command-line variables */ - if (!global_cmdline_options) { - /* Or we could redo the list every time we pass this place. - * It does not really matter */ - if (config_get_commandlines(argc, argv, &global_cmdline_options) < 0) { - goto err; - } + if (config_line_find(cmdline_only_options, "--library-versions")) { + printf("Tor version %s. \n", get_version()); + printf("Library versions\tCompiled\t\tRuntime\n"); + printf("Libevent\t\t%-15s\t\t%s\n", + tor_libevent_get_header_version_str(), + tor_libevent_get_version_str()); + printf("OpenSSL \t\t%-15s\t\t%s\n", + crypto_openssl_get_header_version_str(), + crypto_openssl_get_version_str()); + printf("Zlib \t\t%-15s\t\t%s\n", + tor_zlib_get_header_version_str(), + tor_zlib_get_version_str()); + //TODO: Hex versions? + exit(0); } command = CMD_RUN_TOR; - for (i = 1; i < argc; ++i) { - if (!strcmp(argv[i],"--list-fingerprint")) { + for (p_index = cmdline_only_options; p_index; p_index = p_index->next) { + if (!strcmp(p_index->key,"--keygen")) { + command = CMD_KEYGEN; + } else if (!strcmp(p_index->key,"--list-fingerprint")) { command = CMD_LIST_FINGERPRINT; - } else if (!strcmp(argv[i],"--hash-password")) { + } else if (!strcmp(p_index->key, "--hash-password")) { command = CMD_HASH_PASSWORD; - command_arg = tor_strdup( (i < argc-1) ? argv[i+1] : ""); - ++i; - } else if (!strcmp(argv[i],"--verify-config")) { + command_arg = p_index->value; + } else if (!strcmp(p_index->key, "--dump-config")) { + command = CMD_DUMP_CONFIG; + command_arg = p_index->value; + } else if (!strcmp(p_index->key, "--verify-config")) { command = CMD_VERIFY_CONFIG; } } @@ -3727,15 +4714,91 @@ options_init_from_torrc(int argc, char **argv) cf_defaults = tor_strdup(""); cf = tor_strdup(""); } else { - cf_defaults = load_torrc_from_disk(argc, argv, 1); - cf = load_torrc_from_disk(argc, argv, 0); - if (!cf) - goto err; + cf_defaults = load_torrc_from_disk(cmdline_only_options, 1); + + const config_line_t *f_line = config_line_find(cmdline_only_options, + "-f"); + + const int read_torrc_from_stdin = + (f_line != NULL && strcmp(f_line->value, "-") == 0); + + if (read_torrc_from_stdin) { + cf = load_torrc_from_stdin(); + } else { + cf = load_torrc_from_disk(cmdline_only_options, 0); + } + + if (!cf) { + if (config_line_find(cmdline_only_options, "--allow-missing-torrc")) { + cf = tor_strdup(""); + } else { + goto err; + } + } } retval = options_init_from_string(cf_defaults, cf, command, command_arg, &errmsg); + if (retval < 0) + goto err; + + if (config_line_find(cmdline_only_options, "--no-passphrase")) { + if (command == CMD_KEYGEN) { + get_options_mutable()->keygen_force_passphrase = FORCE_PASSPHRASE_OFF; + } else { + log_err(LD_CONFIG, "--no-passphrase specified without --keygen!"); + exit(1); + } + } + + if (config_line_find(cmdline_only_options, "--newpass")) { + if (command == CMD_KEYGEN) { + get_options_mutable()->change_key_passphrase = 1; + } else { + log_err(LD_CONFIG, "--newpass specified without --keygen!"); + exit(1); + } + } + + { + const config_line_t *fd_line = config_line_find(cmdline_only_options, + "--passphrase-fd"); + if (fd_line) { + if (get_options()->keygen_force_passphrase == FORCE_PASSPHRASE_OFF) { + log_err(LD_CONFIG, "--no-passphrase specified with --passphrase-fd!"); + exit(1); + } else if (command != CMD_KEYGEN) { + log_err(LD_CONFIG, "--passphrase-fd specified without --keygen!"); + exit(1); + } else { + const char *v = fd_line->value; + int ok = 1; + long fd = tor_parse_long(v, 10, 0, INT_MAX, &ok, NULL); + if (fd < 0 || ok == 0) { + log_err(LD_CONFIG, "Invalid --passphrase-fd value %s", escaped(v)); + exit(1); + } + get_options_mutable()->keygen_passphrase_fd = (int)fd; + get_options_mutable()->use_keygen_passphrase_fd = 1; + get_options_mutable()->keygen_force_passphrase = FORCE_PASSPHRASE_ON; + } + } + } + + { + const config_line_t *key_line = config_line_find(cmdline_only_options, + "--master-key"); + if (key_line) { + if (command != CMD_KEYGEN) { + log_err(LD_CONFIG, "--master-key without --keygen!"); + exit(1); + } else { + get_options_mutable()->master_key_fname = tor_strdup(key_line->value); + } + } + } + err: tor_free(cf); @@ -3774,7 +4837,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, newoptions->magic_ = OR_OPTIONS_MAGIC; options_init(newoptions); newoptions->command = command; - newoptions->command_arg = command_arg; + newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL; for (i = 0; i < 2; ++i) { const char *body = i==0 ? cf_defaults : cf; @@ -3838,7 +4901,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, newoptions->magic_ = OR_OPTIONS_MAGIC; options_init(newoptions); newoptions->command = command; - newoptions->command_arg = command_arg; + newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL; /* Assign all options a second time. */ for (i = 0; i < 2; ++i) { @@ -3870,7 +4933,8 @@ options_init_from_string(const char *cf_defaults, const char *cf, } /* Validate newoptions */ - if (options_validate(oldoptions, newoptions, 0, msg) < 0) { + if (options_validate(oldoptions, newoptions, newdefaultoptions, + 0, msg) < 0) { err = SETOPT_ERR_PARSE; /*XXX make this a separate return value.*/ goto err; } @@ -3900,7 +4964,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, return err; } -/** Return the location for our configuration file. +/** Return the location for our configuration file. May return NULL. */ const char * get_torrc_fname(int defaults_fname) @@ -4007,7 +5071,8 @@ addressmap_register_auto(const char *from, const char *to, * Initialize the logs based on the configuration file. */ static int -options_init_logs(or_options_t *options, int validate_only) +options_init_logs(const or_options_t *old_options, or_options_t *options, + int validate_only) { config_line_t *opt; int ok; @@ -4088,7 +5153,7 @@ options_init_logs(or_options_t *options, int validate_only) !strcasecmp(smartlist_get(elts,0), "syslog")) { #ifdef HAVE_SYSLOG_H if (!validate_only) { - add_syslog_log(severity); + add_syslog_log(severity, options->SyslogIdentityTag); } #else log_warn(LD_CONFIG, "Syslog is not supported on this system. Sorry."); @@ -4100,7 +5165,21 @@ options_init_logs(or_options_t *options, int validate_only) !strcasecmp(smartlist_get(elts,0), "file")) { if (!validate_only) { char *fname = expand_filename(smartlist_get(elts, 1)); - if (add_file_log(severity, fname) < 0) { + /* Truncate if TruncateLogFile is set and we haven't seen this option + line before. */ + int truncate = 0; + if (options->TruncateLogFile) { + truncate = 1; + if (old_options) { + config_line_t *opt2; + for (opt2 = old_options->Logs; opt2; opt2 = opt2->next) + if (!strcmp(opt->value, opt2->value)) { + truncate = 0; + break; + } + } + } + if (add_file_log(severity, fname, truncate) < 0) { log_warn(LD_CONFIG, "Couldn't open file for 'Log %s': %s", opt->value, strerror(errno)); ok = 0; @@ -4127,21 +5206,72 @@ options_init_logs(or_options_t *options, int validate_only) return ok?0:-1; } +/** Given a smartlist of SOCKS arguments to be passed to a transport + * proxy in <b>args</b>, validate them and return -1 if they are + * corrupted. Return 0 if they seem OK. */ +static int +validate_transport_socks_arguments(const smartlist_t *args) +{ + char *socks_string = NULL; + size_t socks_string_len; + + tor_assert(args); + tor_assert(smartlist_len(args) > 0); + + SMARTLIST_FOREACH_BEGIN(args, const char *, s) { + if (!string_is_key_value(LOG_WARN, s)) { /* items should be k=v items */ + log_warn(LD_CONFIG, "'%s' is not a k=v item.", s); + return -1; + } + } SMARTLIST_FOREACH_END(s); + + socks_string = pt_stringify_socks_args(args); + if (!socks_string) + return -1; + + socks_string_len = strlen(socks_string); + tor_free(socks_string); + + if (socks_string_len > MAX_SOCKS5_AUTH_SIZE_TOTAL) { + log_warn(LD_CONFIG, "SOCKS arguments can't be more than %u bytes (%lu).", + MAX_SOCKS5_AUTH_SIZE_TOTAL, + (unsigned long) socks_string_len); + return -1; + } + + return 0; +} + +/** Deallocate a bridge_line_t structure. */ +/* private */ void +bridge_line_free(bridge_line_t *bridge_line) +{ + if (!bridge_line) + return; + + if (bridge_line->socks_args) { + SMARTLIST_FOREACH(bridge_line->socks_args, char*, s, tor_free(s)); + smartlist_free(bridge_line->socks_args); + } + tor_free(bridge_line->transport_name); + tor_free(bridge_line); +} + /** Read the contents of a Bridge line from <b>line</b>. Return 0 * if the line is well-formed, and -1 if it isn't. If * <b>validate_only</b> is 0, and the line is well-formed, then add - * the bridge described in the line to our internal bridge list. */ -static int -parse_bridge_line(const char *line, int validate_only) + * the bridge described in the line to our internal bridge list. + * + * Bridge line format: + * Bridge [transport] IP:PORT [id-fingerprint] [k=v] [k=v] ... + */ +/* private */ bridge_line_t * +parse_bridge_line(const char *line) { smartlist_t *items = NULL; - int r; char *addrport=NULL, *fingerprint=NULL; - char *transport_name=NULL; - char *field1=NULL; - tor_addr_t addr; - uint16_t port = 0; - char digest[DIGEST_LEN]; + char *field=NULL; + bridge_line_t *bridge_line = tor_malloc_zero(sizeof(bridge_line_t)); items = smartlist_new(); smartlist_split_string(items, line, NULL, @@ -4151,107 +5281,143 @@ parse_bridge_line(const char *line, int validate_only) goto err; } - /* field1 is either a transport name or addrport */ - field1 = smartlist_get(items, 0); + /* first field is either a transport name or addrport */ + field = smartlist_get(items, 0); smartlist_del_keeporder(items, 0); - if (!(strstr(field1, ".") || strstr(field1, ":"))) { - /* new-style bridge line */ - transport_name = field1; + if (string_is_C_identifier(field)) { + /* It's a transport name. */ + bridge_line->transport_name = field; if (smartlist_len(items) < 1) { log_warn(LD_CONFIG, "Too few items to Bridge line."); goto err; } - addrport = smartlist_get(items, 0); + addrport = smartlist_get(items, 0); /* Next field is addrport then. */ smartlist_del_keeporder(items, 0); } else { - addrport = field1; + addrport = field; } - if (tor_addr_port_lookup(addrport, &addr, &port)<0) { + if (tor_addr_port_parse(LOG_INFO, addrport, + &bridge_line->addr, &bridge_line->port, 443)<0) { log_warn(LD_CONFIG, "Error parsing Bridge address '%s'", addrport); goto err; } - if (!port) { - log_info(LD_CONFIG, - "Bridge address '%s' has no port; using default port 443.", - addrport); - port = 443; - } + /* If transports are enabled, next field could be a fingerprint or a + socks argument. If transports are disabled, next field must be + a fingerprint. */ if (smartlist_len(items)) { - fingerprint = smartlist_join_strings(items, "", 0, NULL); + if (bridge_line->transport_name) { /* transports enabled: */ + field = smartlist_get(items, 0); + smartlist_del_keeporder(items, 0); + + /* If it's a key=value pair, then it's a SOCKS argument for the + transport proxy... */ + if (string_is_key_value(LOG_DEBUG, field)) { + bridge_line->socks_args = smartlist_new(); + smartlist_add(bridge_line->socks_args, field); + } else { /* ...otherwise, it's the bridge fingerprint. */ + fingerprint = field; + } + + } else { /* transports disabled: */ + fingerprint = smartlist_join_strings(items, "", 0, NULL); + } + } + + /* Handle fingerprint, if it was provided. */ + if (fingerprint) { if (strlen(fingerprint) != HEX_DIGEST_LEN) { log_warn(LD_CONFIG, "Key digest for Bridge is wrong length."); goto err; } - if (base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN)<0) { + if (base16_decode(bridge_line->digest, DIGEST_LEN, + fingerprint, HEX_DIGEST_LEN)<0) { log_warn(LD_CONFIG, "Unable to decode Bridge key digest."); goto err; } } - if (!validate_only) { - log_debug(LD_DIR, "Bridge at %s (transport: %s) (%s)", - fmt_addrport(&addr, port), - transport_name ? transport_name : "no transport", - fingerprint ? fingerprint : "no key listed"); - bridge_add_from_config(&addr, port, - fingerprint ? digest : NULL, transport_name); + /* If we are using transports, any remaining items in the smartlist + should be k=v values. */ + if (bridge_line->transport_name && smartlist_len(items)) { + if (!bridge_line->socks_args) + bridge_line->socks_args = smartlist_new(); + + /* append remaining items of 'items' to 'socks_args' */ + smartlist_add_all(bridge_line->socks_args, items); + smartlist_clear(items); + + tor_assert(smartlist_len(bridge_line->socks_args) > 0); + } + + if (bridge_line->socks_args) { + if (validate_transport_socks_arguments(bridge_line->socks_args) < 0) + goto err; } - r = 0; goto done; err: - r = -1; + bridge_line_free(bridge_line); + bridge_line = NULL; done: SMARTLIST_FOREACH(items, char*, s, tor_free(s)); smartlist_free(items); tor_free(addrport); - tor_free(transport_name); tor_free(fingerprint); - return r; + + return bridge_line; } -/** Read the contents of a ClientTransportPlugin line from - * <b>line</b>. Return 0 if the line is well-formed, and -1 if it - * isn't. +/** Read the contents of a ClientTransportPlugin or ServerTransportPlugin + * line from <b>line</b>, depending on the value of <b>server</b>. Return 0 + * if the line is well-formed, and -1 if it isn't. * - * If <b>validate_only</b> is 0, and the line is well-formed: + * If <b>validate_only</b> is 0, the line is well-formed, and the transport is + * needed by some bridge: * - If it's an external proxy line, add the transport described in the line to * our internal transport list. - * - If it's a managed proxy line, launch the managed proxy. */ -static int -parse_client_transport_line(const char *line, int validate_only) + * - If it's a managed proxy line, launch the managed proxy. + */ + +STATIC int +parse_transport_line(const or_options_t *options, + const char *line, int validate_only, + int server) { + smartlist_t *items = NULL; int r; - char *field2=NULL; - - const char *transports=NULL; - smartlist_t *transport_list=NULL; - char *addrport=NULL; + const char *transports = NULL; + smartlist_t *transport_list = NULL; + char *type = NULL; + char *addrport = NULL; tor_addr_t addr; uint16_t port = 0; - int socks_ver=PROXY_NONE; + int socks_ver = PROXY_NONE; /* managed proxy options */ - int is_managed=0; - char **proxy_argv=NULL; - char **tmp=NULL; - int proxy_argc,i; + int is_managed = 0; + char **proxy_argv = NULL; + char **tmp = NULL; + int proxy_argc, i; + int is_useless_proxy = 1; int line_length; + /* Split the line into space-separated tokens */ items = smartlist_new(); smartlist_split_string(items, line, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); + line_length = smartlist_len(items); - line_length = smartlist_len(items); if (line_length < 3) { - log_warn(LD_CONFIG, "Too few arguments on ClientTransportPlugin line."); + log_warn(LD_CONFIG, + "Too few arguments on %sTransportPlugin line.", + server ? "Server" : "Client"); goto err; } @@ -4263,59 +5429,109 @@ parse_client_transport_line(const char *line, int validate_only) smartlist_split_string(transport_list, transports, ",", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH_BEGIN(transport_list, const char *, transport_name) { + /* validate transport names */ if (!string_is_C_identifier(transport_name)) { log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).", transport_name); goto err; } - } SMARTLIST_FOREACH_END(transport_name); - /* field2 is either a SOCKS version or "exec" */ - field2 = smartlist_get(items, 1); + /* see if we actually need the transports provided by this proxy */ + if (!validate_only && transport_is_needed(transport_name)) + is_useless_proxy = 0; + } SMARTLIST_FOREACH_END(transport_name); - if (!strcmp(field2,"socks4")) { + type = smartlist_get(items, 1); + if (!strcmp(type, "exec")) { + is_managed = 1; + } else if (server && !strcmp(type, "proxy")) { + /* 'proxy' syntax only with ServerTransportPlugin */ + is_managed = 0; + } else if (!server && !strcmp(type, "socks4")) { + /* 'socks4' syntax only with ClientTransportPlugin */ + is_managed = 0; socks_ver = PROXY_SOCKS4; - } else if (!strcmp(field2,"socks5")) { + } else if (!server && !strcmp(type, "socks5")) { + /* 'socks5' syntax only with ClientTransportPlugin */ + is_managed = 0; socks_ver = PROXY_SOCKS5; - } else if (!strcmp(field2,"exec")) { - is_managed=1; } else { - log_warn(LD_CONFIG, "Strange ClientTransportPlugin field '%s'.", - field2); + log_warn(LD_CONFIG, + "Strange %sTransportPlugin type '%s'", + server ? "Server" : "Client", type); + goto err; + } + + if (is_managed && options->Sandbox) { + log_warn(LD_CONFIG, + "Managed proxies are not compatible with Sandbox mode." + "(%sTransportPlugin line was %s)", + server ? "Server" : "Client", escaped(line)); goto err; } - if (is_managed) { /* managed */ - if (!validate_only) { /* if we are not just validating, use the - rest of the line as the argv of the proxy - to be launched */ - proxy_argc = line_length-2; + if (is_managed) { + /* managed */ + + if (!server && !validate_only && is_useless_proxy) { + log_info(LD_GENERAL, + "Pluggable transport proxy (%s) does not provide " + "any needed transports and will not be launched.", + line); + } + + /* + * If we are not just validating, use the rest of the line as the + * argv of the proxy to be launched. Also, make sure that we are + * only launching proxies that contribute useful transports. + */ + + if (!validate_only && (server || !is_useless_proxy)) { + proxy_argc = line_length - 2; tor_assert(proxy_argc > 0); - proxy_argv = tor_malloc_zero(sizeof(char*)*(proxy_argc+1)); + proxy_argv = tor_calloc((proxy_argc + 1), sizeof(char *)); tmp = proxy_argv; - for (i=0;i<proxy_argc;i++) { /* store arguments */ + + for (i = 0; i < proxy_argc; i++) { + /* store arguments */ *tmp++ = smartlist_get(items, 2); smartlist_del_keeporder(items, 2); } - *tmp = NULL; /*terminated with NULL, just like execve() likes it*/ + *tmp = NULL; /* terminated with NULL, just like execve() likes it */ /* kickstart the thing */ - pt_kickstart_client_proxy(transport_list, proxy_argv); + if (server) { + pt_kickstart_server_proxy(transport_list, proxy_argv); + } else { + pt_kickstart_client_proxy(transport_list, proxy_argv); + } + } + } else { + /* external */ + + /* ClientTransportPlugins connecting through a proxy is managed only. */ + if (!server && (options->Socks4Proxy || options->Socks5Proxy || + options->HTTPSProxy)) { + log_warn(LD_CONFIG, "You have configured an external proxy with another " + "proxy type. (Socks4Proxy|Socks5Proxy|HTTPSProxy)"); + goto err; } - } else { /* external */ + if (smartlist_len(transport_list) != 1) { - log_warn(LD_CONFIG, "You can't have an external proxy with " - "more than one transports."); + log_warn(LD_CONFIG, + "You can't have an external proxy with more than " + "one transport."); goto err; } addrport = smartlist_get(items, 2); - if (tor_addr_port_lookup(addrport, &addr, &port)<0) { - log_warn(LD_CONFIG, "Error parsing transport " - "address '%s'", addrport); + if (tor_addr_port_lookup(addrport, &addr, &port) < 0) { + log_warn(LD_CONFIG, + "Error parsing transport address '%s'", addrport); goto err; } + if (!port) { log_warn(LD_CONFIG, "Transport address '%s' has no port.", addrport); @@ -4323,11 +5539,15 @@ parse_client_transport_line(const char *line, int validate_only) } if (!validate_only) { - transport_add_from_config(&addr, port, smartlist_get(transport_list, 0), - socks_ver); - - log_info(LD_DIR, "Transport '%s' found at %s", + log_info(LD_DIR, "%s '%s' at %s.", + server ? "Server transport" : "Transport", transports, fmt_addrport(&addr, port)); + + if (!server) { + transport_add_from_config(&addr, port, + smartlist_get(transport_list, 0), + socks_ver); + } } } @@ -4383,7 +5603,7 @@ get_bindaddr_from_transport_listen_line(const char *line,const char *transport) goto err; /* Validate addrport */ - if (tor_addr_port_parse(LOG_WARN, addrport, &addr, &port)<0) { + if (tor_addr_port_parse(LOG_WARN, addrport, &addr, &port, -1)<0) { log_warn(LD_CONFIG, "Error parsing ServerTransportListenAddr " "address '%s'", addrport); goto err; @@ -4402,164 +5622,122 @@ get_bindaddr_from_transport_listen_line(const char *line,const char *transport) return addrport; } -/** Given the name of a pluggable transport in <b>transport</b>, check - * the configuration file to see if the user has explicitly asked for - * it to listen on a specific port. Return a <address:port> string if - * so, otherwise NULL. */ -char * -get_transport_bindaddr_from_config(const char *transport) -{ - config_line_t *cl; - const or_options_t *options = get_options(); - - for (cl = options->ServerTransportListenAddr; cl; cl = cl->next) { - char *bindaddr = - get_bindaddr_from_transport_listen_line(cl->value, transport); - if (bindaddr) - return bindaddr; - } - - return NULL; -} - -/** Read the contents of a ServerTransportPlugin line from - * <b>line</b>. Return 0 if the line is well-formed, and -1 if it - * isn't. - * If <b>validate_only</b> is 0, the line is well-formed, and it's a - * managed proxy line, launch the managed proxy. */ -static int -parse_server_transport_line(const char *line, int validate_only) +/** Given a ServerTransportOptions <b>line</b>, return a smartlist + * with the options. Return NULL if the line was not well-formed. + * + * If <b>transport</b> is set, return NULL if the line is not + * referring to <b>transport</b>. + * + * The returned smartlist and its strings are allocated on the heap + * and it's the responsibility of the caller to free it. */ +smartlist_t * +get_options_from_transport_options_line(const char *line,const char *transport) { - smartlist_t *items = NULL; - int r; - const char *transports=NULL; - smartlist_t *transport_list=NULL; - char *type=NULL; - char *addrport=NULL; - tor_addr_t addr; - uint16_t port = 0; - - /* managed proxy options */ - int is_managed=0; - char **proxy_argv=NULL; - char **tmp=NULL; - int proxy_argc,i; - - int line_length; + smartlist_t *items = smartlist_new(); + smartlist_t *options = smartlist_new(); + const char *parsed_transport = NULL; - items = smartlist_new(); smartlist_split_string(items, line, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - line_length = smartlist_len(items); - if (line_length < 3) { - log_warn(LD_CONFIG, "Too few arguments on ServerTransportPlugin line."); + if (smartlist_len(items) < 2) { + log_warn(LD_CONFIG,"Too few arguments on ServerTransportOptions line."); goto err; } - /* Get the first line element, split it to commas into - transport_list (in case it's multiple transports) and validate - the transport names. */ - transports = smartlist_get(items, 0); - transport_list = smartlist_new(); - smartlist_split_string(transport_list, transports, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(transport_list, const char *, transport_name) { - if (!string_is_C_identifier(transport_name)) { - log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).", - transport_name); - goto err; - } - } SMARTLIST_FOREACH_END(transport_name); - - type = smartlist_get(items, 1); - - if (!strcmp(type, "exec")) { - is_managed=1; - } else if (!strcmp(type, "proxy")) { - is_managed=0; - } else { - log_warn(LD_CONFIG, "Strange ServerTransportPlugin type '%s'", type); + parsed_transport = smartlist_get(items, 0); + /* If 'transport' is given, check if it matches the one on the line */ + if (transport && strcmp(transport, parsed_transport)) goto err; - } - - if (is_managed) { /* managed */ - if (!validate_only) { - proxy_argc = line_length-2; - tor_assert(proxy_argc > 0); - proxy_argv = tor_malloc_zero(sizeof(char*)*(proxy_argc+1)); - tmp = proxy_argv; - - for (i=0;i<proxy_argc;i++) { /* store arguments */ - *tmp++ = smartlist_get(items, 2); - smartlist_del_keeporder(items, 2); - } - *tmp = NULL; /*terminated with NULL, just like execve() likes it*/ - - /* kickstart the thing */ - pt_kickstart_server_proxy(transport_list, proxy_argv); - } - } else { /* external */ - if (smartlist_len(transport_list) != 1) { - log_warn(LD_CONFIG, "You can't have an external proxy with " - "more than one transports."); - goto err; - } - addrport = smartlist_get(items, 2); + SMARTLIST_FOREACH_BEGIN(items, const char *, option) { + if (option_sl_idx == 0) /* skip the transport field (first field)*/ + continue; - if (tor_addr_port_lookup(addrport, &addr, &port)<0) { - log_warn(LD_CONFIG, "Error parsing transport " - "address '%s'", addrport); - goto err; - } - if (!port) { - log_warn(LD_CONFIG, - "Transport address '%s' has no port.", addrport); + /* validate that it's a k=v value */ + if (!string_is_key_value(LOG_WARN, option)) { + log_warn(LD_CONFIG, "%s is not a k=v value.", escaped(option)); goto err; } - if (!validate_only) { - log_info(LD_DIR, "Server transport '%s' at %s.", - transports, fmt_addrport(&addr, port)); - } - } + /* add it to the options smartlist */ + smartlist_add(options, tor_strdup(option)); + log_debug(LD_CONFIG, "Added %s to the list of options", escaped(option)); + } SMARTLIST_FOREACH_END(option); - r = 0; goto done; err: - r = -1; + SMARTLIST_FOREACH(options, char*, s, tor_free(s)); + smartlist_free(options); + options = NULL; done: SMARTLIST_FOREACH(items, char*, s, tor_free(s)); smartlist_free(items); - if (transport_list) { - SMARTLIST_FOREACH(transport_list, char*, s, tor_free(s)); - smartlist_free(transport_list); + + return options; +} + +/** Given the name of a pluggable transport in <b>transport</b>, check + * the configuration file to see if the user has explicitly asked for + * it to listen on a specific port. Return a <address:port> string if + * so, otherwise NULL. */ +char * +get_transport_bindaddr_from_config(const char *transport) +{ + config_line_t *cl; + const or_options_t *options = get_options(); + + for (cl = options->ServerTransportListenAddr; cl; cl = cl->next) { + char *bindaddr = + get_bindaddr_from_transport_listen_line(cl->value, transport); + if (bindaddr) + return bindaddr; } - return r; + return NULL; +} + +/** Given the name of a pluggable transport in <b>transport</b>, check + * the configuration file to see if the user has asked us to pass any + * parameters to the pluggable transport. Return a smartlist + * containing the parameters, otherwise NULL. */ +smartlist_t * +get_options_for_server_transport(const char *transport) +{ + config_line_t *cl; + const or_options_t *options = get_options(); + + for (cl = options->ServerTransportOptions; cl; cl = cl->next) { + smartlist_t *options_sl = + get_options_from_transport_options_line(cl->value, transport); + if (options_sl) + return options_sl; + } + + return NULL; } /** Read the contents of a DirAuthority line from <b>line</b>. If * <b>validate_only</b> is 0, and the line is well-formed, and it * shares any bits with <b>required_type</b> or <b>required_type</b> - * is 0, then add the dirserver described in the line (minus whatever - * bits it's missing) as a valid authority. Return 0 on success, + * is NO_DIRINFO (zero), then add the dirserver described in the line + * (minus whatever bits it's missing) as a valid authority. + * Return 0 on success or filtering out by type, * or -1 if the line isn't well-formed or if we can't add it. */ -static int +STATIC int parse_dir_authority_line(const char *line, dirinfo_type_t required_type, int validate_only) { smartlist_t *items = NULL; int r; char *addrport=NULL, *address=NULL, *nickname=NULL, *fingerprint=NULL; + tor_addr_port_t ipv6_addrport, *ipv6_addrport_ptr = NULL; uint16_t dir_port = 0, or_port = 0; char digest[DIGEST_LEN]; char v3_digest[DIGEST_LEN]; - dirinfo_type_t type = V2_DIRINFO; - int is_not_hidserv_authority = 0, is_not_v2_authority = 0; + dirinfo_type_t type = 0; double weight = 1.0; items = smartlist_new(); @@ -4579,16 +5757,15 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, char *flag = smartlist_get(items, 0); if (TOR_ISDIGIT(flag[0])) break; - if (!strcasecmp(flag, "v1")) { - type |= (V1_DIRINFO | HIDSERV_DIRINFO); - } else if (!strcasecmp(flag, "hs")) { - type |= HIDSERV_DIRINFO; - } else if (!strcasecmp(flag, "no-hs")) { - is_not_hidserv_authority = 1; + if (!strcasecmp(flag, "hs") || + !strcasecmp(flag, "no-hs")) { + log_warn(LD_CONFIG, "The DirAuthority options 'hs' and 'no-hs' are " + "obsolete; you don't need them any more."); } else if (!strcasecmp(flag, "bridge")) { type |= BRIDGE_DIRINFO; } else if (!strcasecmp(flag, "no-v2")) { - is_not_v2_authority = 1; + /* obsolete, but may still be contained in DirAuthority lines generated + by various tools */; } else if (!strcasecmpstart(flag, "orport=")) { int ok; char *portstring = flag + strlen("orport="); @@ -4613,6 +5790,20 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, } else { type |= V3_DIRINFO|EXTRAINFO_DIRINFO|MICRODESC_DIRINFO; } + } else if (!strcasecmpstart(flag, "ipv6=")) { + if (ipv6_addrport_ptr) { + log_warn(LD_CONFIG, "Redundant ipv6 addr/port on DirAuthority line"); + } else { + if (tor_addr_port_parse(LOG_WARN, flag+strlen("ipv6="), + &ipv6_addrport.addr, &ipv6_addrport.port, + -1) < 0 + || tor_addr_family(&ipv6_addrport.addr) != AF_INET6) { + log_warn(LD_CONFIG, "Bad ipv6 addr/port %s on DirAuthority line", + escaped(flag)); + goto err; + } + ipv6_addrport_ptr = &ipv6_addrport; + } } else { log_warn(LD_CONFIG, "Unrecognized flag '%s' on DirAuthority line", flag); @@ -4620,10 +5811,6 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, tor_free(flag); smartlist_del_keeporder(items, 0); } - if (is_not_hidserv_authority) - type &= ~HIDSERV_DIRINFO; - if (is_not_v2_authority) - type &= ~V2_DIRINFO; if (smartlist_len(items) < 2) { log_warn(LD_CONFIG, "Too few arguments to DirAuthority line."); @@ -4646,14 +5833,6 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, fingerprint, (int)strlen(fingerprint)); goto err; } - if (!strcmp(fingerprint, "E623F7625FBE0C87820F11EC5F6D5377ED816294")) { - /* a known bad fingerprint. refuse to use it. We can remove this - * clause once Tor 0.1.2.17 is obsolete. */ - log_warn(LD_CONFIG, "Dangerous dirserver line. To correct, erase your " - "torrc file (%s), or reinstall Tor and use the default torrc.", - get_torrc_fname(0)); - goto err; - } if (base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN)<0) { log_warn(LD_CONFIG, "Unable to decode DirAuthority key digest."); goto err; @@ -4667,6 +5846,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, log_debug(LD_DIR, "Trusted %d dirserver at %s:%d (%s)", (int)type, address, (int)dir_port, (char*)smartlist_get(items,0)); if (!(ds = trusted_dir_server_new(nickname, address, dir_port, or_port, + ipv6_addrport_ptr, digest, v3_digest, type, weight))) goto err; dir_server_add(ds); @@ -4692,7 +5872,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, * <b>validate_only</b> is 0, and the line is well-formed, then add the * dirserver described in the line as a fallback directory. Return 0 on * success, or -1 if the line isn't well-formed or if we can't add it. */ -static int +int parse_dir_fallback_line(const char *line, int validate_only) { @@ -4704,6 +5884,7 @@ parse_dir_fallback_line(const char *line, int ok; char id[DIGEST_LEN]; char *address=NULL; + tor_addr_port_t ipv6_addrport, *ipv6_addrport_ptr = NULL; double weight=1.0; memset(id, 0, sizeof(id)); @@ -4722,6 +5903,20 @@ parse_dir_fallback_line(const char *line, } else if (!strcmpstart(cp, "id=")) { ok = !base16_decode(id, DIGEST_LEN, cp+strlen("id="), strlen(cp)-strlen("id=")); + } else if (!strcasecmpstart(cp, "ipv6=")) { + if (ipv6_addrport_ptr) { + log_warn(LD_CONFIG, "Redundant ipv6 addr/port on FallbackDir line"); + } else { + if (tor_addr_port_parse(LOG_WARN, cp+strlen("ipv6="), + &ipv6_addrport.addr, &ipv6_addrport.port, + -1) < 0 + || tor_addr_family(&ipv6_addrport.addr) != AF_INET6) { + log_warn(LD_CONFIG, "Bad ipv6 addr/port %s on FallbackDir line", + escaped(cp)); + goto end; + } + ipv6_addrport_ptr = &ipv6_addrport; + } } else if (!strcmpstart(cp, "weight=")) { int ok; const char *wstring = cp + strlen("weight="); @@ -4763,7 +5958,8 @@ parse_dir_fallback_line(const char *line, if (!validate_only) { dir_server_t *ds; - ds = fallback_dir_server_new(&addr, dirport, orport, id, weight); + ds = fallback_dir_server_new(&addr, dirport, orport, ipv6_addrport_ptr, + id, weight); if (!ds) { log_warn(LD_CONFIG, "Couldn't create FallbackDir %s", escaped(line)); goto end; @@ -4782,18 +5978,19 @@ parse_dir_fallback_line(const char *line, } /** Allocate and return a new port_cfg_t with reasonable defaults. */ -static port_cfg_t * -port_cfg_new(void) +STATIC port_cfg_t * +port_cfg_new(size_t namelen) { - port_cfg_t *cfg = tor_malloc_zero(sizeof(port_cfg_t)); - cfg->ipv4_traffic = 1; - cfg->cache_ipv4_answers = 1; - cfg->prefer_ipv6_virtaddr = 1; + tor_assert(namelen <= SIZE_T_CEILING - sizeof(port_cfg_t) - 1); + port_cfg_t *cfg = tor_malloc_zero(sizeof(port_cfg_t) + namelen + 1); + cfg->entry_cfg.ipv4_traffic = 1; + cfg->entry_cfg.cache_ipv4_answers = 1; + cfg->entry_cfg.prefer_ipv6_virtaddr = 1; return cfg; } /** Free all storage held in <b>port</b> */ -static void +STATIC void port_cfg_free(port_cfg_t *port) { tor_free(port); @@ -4826,12 +6023,33 @@ warn_nonlocal_client_ports(const smartlist_t *ports, const char *portname, } SMARTLIST_FOREACH_END(port); } -/** Given a list of port_cfg_t in <b>ports</b>, warn any controller port there - * is listening on any non-loopback address. If <b>forbid</b> is true, - * then emit a stronger warning and remove the port from the list. +/** Warn for every Extended ORPort port in <b>ports</b> that is on a + * publicly routable address. */ +static void +warn_nonlocal_ext_orports(const smartlist_t *ports, const char *portname) +{ + SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) { + if (port->type != CONN_TYPE_EXT_OR_LISTENER) + continue; + if (port->is_unix_addr) + continue; + /* XXX maybe warn even if address is RFC1918? */ + if (!tor_addr_is_internal(&port->addr, 1)) { + log_warn(LD_CONFIG, "You specified a public address '%s' for %sPort. " + "This is not advised; this address is supposed to only be " + "exposed on localhost so that your pluggable transport " + "proxies can connect to it.", + fmt_addrport(&port->addr, port->port), portname); + } + } SMARTLIST_FOREACH_END(port); +} + +/** Given a list of port_cfg_t in <b>ports</b>, warn if any controller port + * there is listening on any non-loopback address. If <b>forbid_nonlocal</b> + * is true, then emit a stronger warning and remove the port from the list. */ static void -warn_nonlocal_controller_ports(smartlist_t *ports, unsigned forbid) +warn_nonlocal_controller_ports(smartlist_t *ports, unsigned forbid_nonlocal) { int warned = 0; SMARTLIST_FOREACH_BEGIN(ports, port_cfg_t *, port) { @@ -4840,7 +6058,7 @@ warn_nonlocal_controller_ports(smartlist_t *ports, unsigned forbid) if (port->is_unix_addr) continue; if (!tor_addr_is_loopback(&port->addr)) { - if (forbid) { + if (forbid_nonlocal) { if (!warned) log_warn(LD_CONFIG, "You have a ControlPort set to accept " @@ -4868,12 +6086,54 @@ warn_nonlocal_controller_ports(smartlist_t *ports, unsigned forbid) } SMARTLIST_FOREACH_END(port); } -#define CL_PORT_NO_OPTIONS (1u<<0) -#define CL_PORT_WARN_NONLOCAL (1u<<1) -#define CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2) -#define CL_PORT_SERVER_OPTIONS (1u<<3) -#define CL_PORT_FORBID_NONLOCAL (1u<<4) -#define CL_PORT_TAKES_HOSTNAMES (1u<<5) +#ifdef HAVE_SYS_UN_H + +/** Parse the given <b>addrport</b> and set <b>path_out</b> if a Unix socket + * path is found. Return 0 on success. On error, a negative value is + * returned, -ENOENT if no Unix statement found, -EINVAL if the socket path + * is empty and -ENOSYS if AF_UNIX is not supported (see function in the + * #else statement below). */ + +int +config_parse_unix_port(const char *addrport, char **path_out) +{ + tor_assert(path_out); + tor_assert(addrport); + + if (strcmpstart(addrport, unix_socket_prefix)) { + /* Not a Unix socket path. */ + return -ENOENT; + } + + if (strlen(addrport + strlen(unix_socket_prefix)) == 0) { + /* Empty socket path, not very usable. */ + return -EINVAL; + } + + *path_out = tor_strdup(addrport + strlen(unix_socket_prefix)); + return 0; +} + +#else /* defined(HAVE_SYS_UN_H) */ + +int +config_parse_unix_port(const char *addrport, char **path_out) +{ + tor_assert(path_out); + tor_assert(addrport); + + if (strcmpstart(addrport, unix_socket_prefix)) { + /* Not a Unix socket path. */ + return -ENOENT; + } + + log_warn(LD_CONFIG, + "Port configuration %s is for an AF_UNIX socket, but we have no" + "support available on this platform", + escaped(addrport)); + return -ENOSYS; +} +#endif /* defined(HAVE_SYS_UN_H) */ /** * Parse port configuration for a single port type. @@ -4892,12 +6152,12 @@ warn_nonlocal_controller_ports(smartlist_t *ports, unsigned forbid) * If no address is specified, default to <b>defaultaddr</b>. If no * FooPort is given, default to defaultport (if 0, there is no default). * - * If CL_PORT_NO_OPTIONS is set in <b>flags</b>, do not allow stream + * If CL_PORT_NO_STREAM_OPTIONS is set in <b>flags</b>, do not allow stream * isolation options in the FooPort entries. * * If CL_PORT_WARN_NONLOCAL is set in <b>flags</b>, warn if any of the * ports are not on a local address. If CL_PORT_FORBID_NONLOCAL is set, - * this is a contrl port with no password set: don't even allow it. + * this is a control port with no password set: don't even allow it. * * Unless CL_PORT_ALLOW_EXTRA_LISTENADDR is set in <b>flags</b>, warn * if FooListenAddress is set but FooPort is 0. @@ -4913,7 +6173,7 @@ warn_nonlocal_controller_ports(smartlist_t *ports, unsigned forbid) * <b>out</b> for every port that the client should listen on. Return 0 * on success, -1 on failure. */ -static int +STATIC int parse_port_config(smartlist_t *out, const config_line_t *ports, const config_line_t *listenaddrs, @@ -4921,19 +6181,24 @@ parse_port_config(smartlist_t *out, int listener_type, const char *defaultaddr, int defaultport, - unsigned flags) + const unsigned flags) { smartlist_t *elts; int retval = -1; const unsigned is_control = (listener_type == CONN_TYPE_CONTROL_LISTENER); - const unsigned allow_no_options = flags & CL_PORT_NO_OPTIONS; + const unsigned is_ext_orport = (listener_type == CONN_TYPE_EXT_OR_LISTENER); + const unsigned allow_no_stream_options = flags & CL_PORT_NO_STREAM_OPTIONS; const unsigned use_server_options = flags & CL_PORT_SERVER_OPTIONS; const unsigned warn_nonlocal = flags & CL_PORT_WARN_NONLOCAL; const unsigned forbid_nonlocal = flags & CL_PORT_FORBID_NONLOCAL; + const unsigned default_to_group_writable = + flags & CL_PORT_DFLT_GROUP_WRITABLE; const unsigned allow_spurious_listenaddr = flags & CL_PORT_ALLOW_EXTRA_LISTENADDR; const unsigned takes_hostnames = flags & CL_PORT_TAKES_HOSTNAMES; + const unsigned is_unix_socket = flags & CL_PORT_IS_UNIXSOCKET; int got_zero_port=0, got_nonzero_port=0; + char *unix_socket_path = NULL; /* FooListenAddress is deprecated; let's make it work like it used to work, * though. */ @@ -4969,14 +6234,14 @@ parse_port_config(smartlist_t *out, if (use_server_options && out) { /* Add a no_listen port. */ - port_cfg_t *cfg = port_cfg_new(); + port_cfg_t *cfg = port_cfg_new(0); cfg->type = listener_type; cfg->port = mainport; tor_addr_make_unspec(&cfg->addr); /* Server ports default to 0.0.0.0 */ - cfg->no_listen = 1; - cfg->bind_ipv4_only = 1; - cfg->ipv4_traffic = 1; - cfg->prefer_ipv6_virtaddr = 1; + cfg->server_cfg.no_listen = 1; + cfg->server_cfg.bind_ipv4_only = 1; + cfg->entry_cfg.ipv4_traffic = 1; + cfg->entry_cfg.prefer_ipv6_virtaddr = 1; smartlist_add(out, cfg); } @@ -4989,13 +6254,13 @@ parse_port_config(smartlist_t *out, return -1; } if (out) { - port_cfg_t *cfg = port_cfg_new(); + port_cfg_t *cfg = port_cfg_new(0); cfg->type = listener_type; cfg->port = port ? port : mainport; tor_addr_copy(&cfg->addr, &addr); - cfg->session_group = SESSION_GROUP_UNSET; - cfg->isolation_flags = ISO_DEFAULT; - cfg->no_advertise = 1; + cfg->entry_cfg.session_group = SESSION_GROUP_UNSET; + cfg->entry_cfg.isolation_flags = ISO_DEFAULT; + cfg->server_cfg.no_advertise = 1; smartlist_add(out, cfg); } } @@ -5003,6 +6268,8 @@ parse_port_config(smartlist_t *out, if (warn_nonlocal && out) { if (is_control) warn_nonlocal_controller_ports(out, forbid_nonlocal); + else if (is_ext_orport) + warn_nonlocal_ext_orports(out, portname); else warn_nonlocal_client_ports(out, portname, listener_type); } @@ -5012,13 +6279,19 @@ parse_port_config(smartlist_t *out, /* No ListenAddress lines. If there's no FooPort, then maybe make a default * one. */ if (! ports) { - if (defaultport && out) { - port_cfg_t *cfg = port_cfg_new(); + if (defaultport && defaultaddr && out) { + port_cfg_t *cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0); cfg->type = listener_type; - cfg->port = defaultport; - tor_addr_parse(&cfg->addr, defaultaddr); - cfg->session_group = SESSION_GROUP_UNSET; - cfg->isolation_flags = ISO_DEFAULT; + if (is_unix_socket) { + tor_addr_make_unspec(&cfg->addr); + memcpy(cfg->unix_addr, defaultaddr, strlen(defaultaddr) + 1); + cfg->is_unix_addr = 1; + } else { + cfg->port = defaultport; + tor_addr_parse(&cfg->addr, defaultaddr); + } + cfg->entry_cfg.session_group = SESSION_GROUP_UNSET; + cfg->entry_cfg.isolation_flags = ISO_DEFAULT; smartlist_add(out, cfg); } return 0; @@ -5030,10 +6303,11 @@ parse_port_config(smartlist_t *out, for (; ports; ports = ports->next) { tor_addr_t addr; - int port; + int port, ret; int sessiongroup = SESSION_GROUP_UNSET; unsigned isolation = ISO_DEFAULT; int prefer_no_auth = 0; + int socks_iso_keep_alive = 0; char *addrport; uint16_t ptmp=0; @@ -5043,7 +6317,9 @@ parse_port_config(smartlist_t *out, ipv4_traffic = 1, ipv6_traffic = 0, prefer_ipv6 = 0, cache_ipv4 = 1, use_cached_ipv4 = 0, cache_ipv6 = 0, use_cached_ipv6 = 0, - prefer_ipv6_automap = 1; + prefer_ipv6_automap = 1, world_writable = 0, group_writable = 0, + relax_dirmode_check = 0, + has_used_unix_socket_only_option = 0; smartlist_split_string(elts, ports->value, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); @@ -5052,16 +6328,36 @@ parse_port_config(smartlist_t *out, goto err; } - if (allow_no_options && smartlist_len(elts) > 1) { - log_warn(LD_CONFIG, "Too many options on %sPort line", portname); + /* Now parse the addr/port value */ + addrport = smartlist_get(elts, 0); + + /* Let's start to check if it's a Unix socket path. */ + ret = config_parse_unix_port(addrport, &unix_socket_path); + if (ret < 0 && ret != -ENOENT) { + if (ret == -EINVAL) { + log_warn(LD_CONFIG, "Empty Unix socket path."); + } goto err; } - /* Now parse the addr/port value */ - addrport = smartlist_get(elts, 0); - if (!strcmp(addrport, "auto")) { + if (unix_socket_path && + ! conn_listener_type_supports_af_unix(listener_type)) { + log_warn(LD_CONFIG, "%sPort does not support unix sockets", portname); + goto err; + } + + if (unix_socket_path) { + port = 1; + } else if (is_unix_socket) { + unix_socket_path = tor_strdup(addrport); + if (!strcmp(addrport, "0")) + port = 0; + else + port = 1; + } else if (!strcmp(addrport, "auto")) { port = CFG_AUTO_PORT; - tor_addr_parse(&addr, defaultaddr); + int af = tor_addr_parse(&addr, defaultaddr); + tor_assert(af >= 0); } else if (!strcasecmpend(addrport, ":auto")) { char *addrtmp = tor_strndup(addrport, strlen(addrport)-5); port = CFG_AUTO_PORT; @@ -5071,12 +6367,14 @@ parse_port_config(smartlist_t *out, tor_free(addrtmp); goto err; } + tor_free(addrtmp); } else { /* Try parsing integer port before address, because, who knows? "9050" might be a valid address. */ port = (int) tor_parse_long(addrport, 10, 0, 65535, &ok, NULL); if (ok) { - tor_addr_parse(&addr, defaultaddr); + int af = tor_addr_parse(&addr, defaultaddr); + tor_assert(af >= 0); } else if (tor_addr_port_lookup(addrport, &addr, &ptmp) == 0) { if (ptmp == 0) { log_warn(LD_CONFIG, "%sPort line has address but no port", portname); @@ -5084,12 +6382,15 @@ parse_port_config(smartlist_t *out, } port = ptmp; } else { - log_warn(LD_CONFIG, "Couldn't parse address '%s' for %sPort", + log_warn(LD_CONFIG, "Couldn't parse address %s for %sPort", escaped(addrport), portname); goto err; } } + if (unix_socket_path && default_to_group_writable) + group_writable = 1; + /* Now parse the rest of the options, if any. */ if (use_server_options) { /* This is a server port; parse advertising options */ @@ -5146,10 +6447,11 @@ parse_port_config(smartlist_t *out, const char *elt_orig = elt; if (elt_sl_idx == 0) continue; /* Skip addr:port */ + if (!strcasecmpstart(elt, "SessionGroup=")) { int group = (int)tor_parse_long(elt+strlen("SessionGroup="), 10, 0, INT_MAX, &ok, NULL); - if (!ok) { + if (!ok || !allow_no_stream_options) { log_warn(LD_CONFIG, "Invalid %sPort option '%s'", portname, escaped(elt)); goto err; @@ -5168,6 +6470,26 @@ parse_port_config(smartlist_t *out, elt += 2; } + if (!strcasecmp(elt, "GroupWritable")) { + group_writable = !no; + has_used_unix_socket_only_option = 1; + continue; + } else if (!strcasecmp(elt, "WorldWritable")) { + world_writable = !no; + has_used_unix_socket_only_option = 1; + continue; + } else if (!strcasecmp(elt, "RelaxDirModeCheck")) { + relax_dirmode_check = !no; + has_used_unix_socket_only_option = 1; + continue; + } + + if (allow_no_stream_options) { + log_warn(LD_CONFIG, "Unrecognized %sPort option '%s'", + portname, escaped(elt)); + continue; + } + if (takes_hostnames) { if (!strcasecmp(elt, "IPv4Traffic")) { ipv4_traffic = ! no; @@ -5204,6 +6526,9 @@ parse_port_config(smartlist_t *out, } else if (!strcasecmp(elt, "PreferSOCKSNoAuth")) { prefer_no_auth = ! no; continue; + } else if (!strcasecmp(elt, "KeepAliveIsolateSOCKSAuth")) { + socks_iso_keep_alive = ! no; + continue; } if (!strcasecmpend(elt, "s")) @@ -5243,29 +6568,55 @@ parse_port_config(smartlist_t *out, goto err; } + if ( has_used_unix_socket_only_option && ! unix_socket_path) { + log_warn(LD_CONFIG, "You have a %sPort entry with GroupWritable, " + "WorldWritable, or RelaxDirModeCheck, but it is not a " + "unix socket.", portname); + goto err; + } + + if (!(isolation & ISO_SOCKSAUTH) && socks_iso_keep_alive) { + log_warn(LD_CONFIG, "You have a %sPort entry with both " + "NoIsolateSOCKSAuth and KeepAliveIsolateSOCKSAuth set.", + portname); + goto err; + } + if (out && port) { - port_cfg_t *cfg = port_cfg_new(); - tor_addr_copy(&cfg->addr, &addr); - cfg->port = port; + size_t namelen = unix_socket_path ? strlen(unix_socket_path) : 0; + port_cfg_t *cfg = port_cfg_new(namelen); + if (unix_socket_path) { + tor_addr_make_unspec(&cfg->addr); + memcpy(cfg->unix_addr, unix_socket_path, namelen + 1); + cfg->is_unix_addr = 1; + tor_free(unix_socket_path); + } else { + tor_addr_copy(&cfg->addr, &addr); + cfg->port = port; + } cfg->type = listener_type; - cfg->isolation_flags = isolation; - cfg->session_group = sessiongroup; - cfg->no_advertise = no_advertise; - cfg->no_listen = no_listen; - cfg->all_addrs = all_addrs; - cfg->bind_ipv4_only = bind_ipv4_only; - cfg->bind_ipv6_only = bind_ipv6_only; - cfg->ipv4_traffic = ipv4_traffic; - cfg->ipv6_traffic = ipv6_traffic; - cfg->prefer_ipv6 = prefer_ipv6; - cfg->cache_ipv4_answers = cache_ipv4; - cfg->cache_ipv6_answers = cache_ipv6; - cfg->use_cached_ipv4_answers = use_cached_ipv4; - cfg->use_cached_ipv6_answers = use_cached_ipv6; - cfg->prefer_ipv6_virtaddr = prefer_ipv6_automap; - cfg->socks_prefer_no_auth = prefer_no_auth; + cfg->is_world_writable = world_writable; + cfg->is_group_writable = group_writable; + cfg->relax_dirmode_check = relax_dirmode_check; + cfg->entry_cfg.isolation_flags = isolation; + cfg->entry_cfg.session_group = sessiongroup; + cfg->server_cfg.no_advertise = no_advertise; + cfg->server_cfg.no_listen = no_listen; + cfg->server_cfg.all_addrs = all_addrs; + cfg->server_cfg.bind_ipv4_only = bind_ipv4_only; + cfg->server_cfg.bind_ipv6_only = bind_ipv6_only; + cfg->entry_cfg.ipv4_traffic = ipv4_traffic; + cfg->entry_cfg.ipv6_traffic = ipv6_traffic; + cfg->entry_cfg.prefer_ipv6 = prefer_ipv6; + cfg->entry_cfg.cache_ipv4_answers = cache_ipv4; + cfg->entry_cfg.cache_ipv6_answers = cache_ipv6; + cfg->entry_cfg.use_cached_ipv4_answers = use_cached_ipv4; + cfg->entry_cfg.use_cached_ipv6_answers = use_cached_ipv6; + cfg->entry_cfg.prefer_ipv6_virtaddr = prefer_ipv6_automap; + cfg->entry_cfg.socks_prefer_no_auth = prefer_no_auth; if (! (isolation & ISO_SOCKSAUTH)) - cfg->socks_prefer_no_auth = 1; + cfg->entry_cfg.socks_prefer_no_auth = 1; + cfg->entry_cfg.socks_iso_keep_alive = socks_iso_keep_alive; smartlist_add(out, cfg); } @@ -5276,6 +6627,8 @@ parse_port_config(smartlist_t *out, if (warn_nonlocal && out) { if (is_control) warn_nonlocal_controller_ports(out, forbid_nonlocal); + else if (is_ext_orport) + warn_nonlocal_ext_orports(out, portname); else warn_nonlocal_client_ports(out, portname, listener_type); } @@ -5291,32 +6644,10 @@ parse_port_config(smartlist_t *out, err: SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); smartlist_free(elts); + tor_free(unix_socket_path); return retval; } -/** Parse a list of config_line_t for an AF_UNIX unix socket listener option - * from <b>cfg</b> and add them to <b>out</b>. No fancy options are - * supported: the line contains nothing but the path to the AF_UNIX socket. */ -static int -parse_unix_socket_config(smartlist_t *out, const config_line_t *cfg, - int listener_type) -{ - - if (!out) - return 0; - - for ( ; cfg; cfg = cfg->next) { - size_t len = strlen(cfg->value); - port_cfg_t *port = tor_malloc_zero(sizeof(port_cfg_t) + len + 1); - port->is_unix_addr = 1; - memcpy(port->unix_addr, cfg->value, len+1); - port->type = listener_type; - smartlist_add(out, port); - } - - return 0; -} - /** Return the number of ports which are actually going to listen with type * <b>listenertype</b>. Do not count no_listen ports. Do not count unix * sockets. */ @@ -5325,7 +6656,7 @@ count_real_listeners(const smartlist_t *ports, int listenertype) { int n = 0; SMARTLIST_FOREACH_BEGIN(ports, port_cfg_t *, port) { - if (port->no_listen || port->is_unix_addr) + if (port->server_cfg.no_listen || port->is_unix_addr) continue; if (port->type != listenertype) continue; @@ -5345,7 +6676,8 @@ count_real_listeners(const smartlist_t *ports, int listenertype) **/ static int parse_ports(or_options_t *options, int validate_only, - char **msg, int *n_ports_out) + char **msg, int *n_ports_out, + int *world_writable_control_socket) { smartlist_t *ports; int retval = -1; @@ -5354,12 +6686,14 @@ parse_ports(or_options_t *options, int validate_only, *n_ports_out = 0; + const unsigned gw_flag = options->SocksSocketsGroupWritable ? + CL_PORT_DFLT_GROUP_WRITABLE : 0; if (parse_port_config(ports, options->SocksPort_lines, options->SocksListenAddress, "Socks", CONN_TYPE_AP_LISTENER, "127.0.0.1", 9050, CL_PORT_WARN_NONLOCAL|CL_PORT_ALLOW_EXTRA_LISTENADDR| - CL_PORT_TAKES_HOSTNAMES) < 0) { + CL_PORT_TAKES_HOSTNAMES|gw_flag) < 0) { *msg = tor_strdup("Invalid SocksPort/SocksListenAddress configuration"); goto err; } @@ -5388,12 +6722,15 @@ parse_ports(or_options_t *options, int validate_only, goto err; } { - unsigned control_port_flags = CL_PORT_NO_OPTIONS | CL_PORT_WARN_NONLOCAL; + unsigned control_port_flags = CL_PORT_NO_STREAM_OPTIONS | + CL_PORT_WARN_NONLOCAL; const int any_passwords = (options->HashedControlPassword || options->HashedControlSessionPassword || options->CookieAuthentication); if (! any_passwords) control_port_flags |= CL_PORT_FORBID_NONLOCAL; + if (options->ControlSocketsGroupWritable) + control_port_flags |= CL_PORT_DFLT_GROUP_WRITABLE; if (parse_port_config(ports, options->ControlPort_lines, @@ -5405,9 +6742,11 @@ parse_ports(or_options_t *options, int validate_only, "configuration"); goto err; } - if (parse_unix_socket_config(ports, - options->ControlSocket, - CONN_TYPE_CONTROL_LISTENER) < 0) { + + if (parse_port_config(ports, options->ControlSocket, NULL, + "ControlSocket", + CONN_TYPE_CONTROL_LISTENER, NULL, 0, + control_port_flags | CL_PORT_IS_UNIXSOCKET) < 0) { *msg = tor_strdup("Invalid ControlSocket configuration"); goto err; } @@ -5422,6 +6761,14 @@ parse_ports(or_options_t *options, int validate_only, goto err; } if (parse_port_config(ports, + options->ExtORPort_lines, NULL, + "ExtOR", CONN_TYPE_EXT_OR_LISTENER, + "127.0.0.1", 0, + CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) { + *msg = tor_strdup("Invalid ExtORPort configuration"); + goto err; + } + if (parse_port_config(ports, options->DirPort_lines, options->DirListenAddress, "Dir", CONN_TYPE_DIR_LISTENER, "0.0.0.0", 0, @@ -5431,10 +6778,13 @@ parse_ports(or_options_t *options, int validate_only, } } - if (check_server_ports(ports, options) < 0) { + int n_low_ports = 0; + if (check_server_ports(ports, options, &n_low_ports) < 0) { *msg = tor_strdup("Misconfigured server ports"); goto err; } + if (have_low_ports < 0) + have_low_ports = (n_low_ports > 0); *n_ports_out = smartlist_len(ports); @@ -5456,6 +6806,18 @@ parse_ports(or_options_t *options, int validate_only, !! count_real_listeners(ports, CONN_TYPE_DIR_LISTENER); options->DNSPort_set = !! count_real_listeners(ports, CONN_TYPE_AP_DNS_LISTENER); + options->ExtORPort_set = + !! count_real_listeners(ports, CONN_TYPE_EXT_OR_LISTENER); + + if (world_writable_control_socket) { + SMARTLIST_FOREACH(ports, port_cfg_t *, p, + if (p->type == CONN_TYPE_CONTROL_LISTENER && + p->is_unix_addr && + p->is_world_writable) { + *world_writable_control_socket = 1; + break; + }); + } if (!validate_only) { if (configured_ports) { @@ -5476,10 +6838,12 @@ parse_ports(or_options_t *options, int validate_only, } /** Given a list of <b>port_cfg_t</b> in <b>ports</b>, check them for internal - * consistency and warn as appropriate. */ + * consistency and warn as appropriate. Set *<b>n_low_ports_out</b> to the + * number of sub-1024 ports we will be binding. */ static int check_server_ports(const smartlist_t *ports, - const or_options_t *options) + const or_options_t *options, + int *n_low_ports_out) { int n_orport_advertised = 0; int n_orport_advertised_ipv4 = 0; @@ -5491,25 +6855,25 @@ check_server_ports(const smartlist_t *ports, SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) { if (port->type == CONN_TYPE_DIR_LISTENER) { - if (! port->no_advertise) + if (! port->server_cfg.no_advertise) ++n_dirport_advertised; - if (! port->no_listen) + if (! port->server_cfg.no_listen) ++n_dirport_listeners; } else if (port->type == CONN_TYPE_OR_LISTENER) { - if (! port->no_advertise) { + if (! port->server_cfg.no_advertise) { ++n_orport_advertised; if (tor_addr_family(&port->addr) == AF_INET || (tor_addr_family(&port->addr) == AF_UNSPEC && - !port->bind_ipv6_only)) + !port->server_cfg.bind_ipv6_only)) ++n_orport_advertised_ipv4; } - if (! port->no_listen) + if (! port->server_cfg.no_listen) ++n_orport_listeners; } else { continue; } #ifndef _WIN32 - if (!port->no_listen && port->port < 1024) + if (!port->server_cfg.no_listen && port->port < 1024) ++n_low_port; #endif } SMARTLIST_FOREACH_END(port); @@ -5542,23 +6906,31 @@ check_server_ports(const smartlist_t *ports, r = -1; } - if (n_low_port && options->AccountingMax) { + if (n_low_port && options->AccountingMax && + (!have_capability_support() || options->KeepBindCapabilities == 0)) { + const char *extra = ""; + if (options->KeepBindCapabilities == 0 && have_capability_support()) + extra = ", and you have disabled KeepBindCapabilities."; log_warn(LD_CONFIG, "You have set AccountingMax to use hibernation. You have also " - "chosen a low DirPort or OrPort. This combination can make Tor stop " + "chosen a low DirPort or OrPort%s." + "This combination can make Tor stop " "working when it tries to re-attach the port after a period of " "hibernation. Please choose a different port or turn off " "hibernation unless you know this combination will work on your " - "platform."); + "platform.", extra); } + if (n_low_ports_out) + *n_low_ports_out = n_low_port; + return r; } /** Return a list of port_cfg_t for client ports parsed from the * options. */ -const smartlist_t * -get_configured_ports(void) +MOCK_IMPL(const smartlist_t *, +get_configured_ports,(void)) { if (!configured_ports) configured_ports = smartlist_new(); @@ -5587,7 +6959,7 @@ get_first_listener_addrport_string(int listener_type) return NULL; SMARTLIST_FOREACH_BEGIN(configured_ports, const port_cfg_t *, cfg) { - if (cfg->no_listen) + if (cfg->server_cfg.no_listen) continue; if (cfg->type == listener_type && @@ -5630,16 +7002,15 @@ get_first_listener_addrport_string(int listener_type) int get_first_advertised_port_by_type_af(int listener_type, int address_family) { - if (!configured_ports) - return 0; - SMARTLIST_FOREACH_BEGIN(configured_ports, const port_cfg_t *, cfg) { + const smartlist_t *conf_ports = get_configured_ports(); + SMARTLIST_FOREACH_BEGIN(conf_ports, const port_cfg_t *, cfg) { if (cfg->type == listener_type && - !cfg->no_advertise && + !cfg->server_cfg.no_advertise && (tor_addr_family(&cfg->addr) == address_family || tor_addr_family(&cfg->addr) == AF_UNSPEC)) { if (tor_addr_family(&cfg->addr) != AF_UNSPEC || - (address_family == AF_INET && !cfg->bind_ipv6_only) || - (address_family == AF_INET6 && !cfg->bind_ipv4_only)) { + (address_family == AF_INET && !cfg->server_cfg.bind_ipv6_only) || + (address_family == AF_INET6 && !cfg->server_cfg.bind_ipv4_only)) { return cfg->port; } } @@ -5723,10 +7094,13 @@ write_configuration_file(const char *fname, const or_options_t *options) char *old_val=NULL, *new_val=NULL, *new_conf=NULL; int rename_old = 0, r; - tor_assert(fname); + if (!fname) + return -1; switch (file_status(fname)) { + /* create backups of old config files, even if they're empty */ case FN_FILE: + case FN_EMPTY: old_val = read_file_to_str(fname, 0, NULL); if (!old_val || strcmpstart(old_val, GENERATED_FILE_PREFIX)) { rename_old = 1; @@ -5743,7 +7117,7 @@ write_configuration_file(const char *fname, const or_options_t *options) return -1; } - if (!(new_conf = options_dump(options, 1))) { + if (!(new_conf = options_dump(options, OPTIONS_DUMP_MINIMAL))) { log_warn(LD_BUG, "Couldn't get configuration string"); goto err; } @@ -5762,7 +7136,7 @@ write_configuration_file(const char *fname, const or_options_t *options) ++i; } log_notice(LD_CONFIG, "Renaming old configuration file to \"%s\"", fn_tmp); - if (rename(fname, fn_tmp) < 0) { + if (tor_rename(fname, fn_tmp) < 0) {//XXXX sandbox doesn't allow log_warn(LD_FS, "Couldn't rename configuration file \"%s\" to \"%s\": %s", fname, fn_tmp, strerror(errno)); @@ -5819,7 +7193,6 @@ get_num_cpus(const or_options_t *options) static void init_libevent(const or_options_t *options) { - const char *badness=NULL; tor_libevent_cfg cfg; tor_assert(options); @@ -5840,17 +7213,6 @@ init_libevent(const or_options_t *options) tor_libevent_initialize(&cfg); suppress_libevent_log_msg(NULL); - - tor_check_libevent_version(tor_libevent_get_method(), - server_mode(get_options()), - &badness); - if (badness) { - const char *v = tor_libevent_get_version_str(); - const char *m = tor_libevent_get_method(); - control_event_general_status(LOG_WARN, - "BAD_LIBEVENT VERSION=%s METHOD=%s BADNESS=%s RECOVERED=NO", - v, m, badness); - } } /** Return a newly allocated string holding a filename relative to the data @@ -5903,6 +7265,43 @@ options_get_datadir_fname2_suffix(const or_options_t *options, return fname; } +/** Check wether the data directory has a private subdirectory + * <b>subdir</b>. If not, try to create it. Return 0 on success, + * -1 otherwise. */ +int +check_or_create_data_subdir(const char *subdir) +{ + char *statsdir = get_datadir_fname(subdir); + int return_val = 0; + + if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { + log_warn(LD_HIST, "Unable to create %s/ directory!", subdir); + return_val = -1; + } + tor_free(statsdir); + return return_val; +} + +/** Create a file named <b>fname</b> with contents <b>str</b> in the + * subdirectory <b>subdir</b> of the data directory. <b>descr</b> + * should be a short description of the file's content and will be + * used for the warning message, if it's present and the write process + * fails. Return 0 on success, -1 otherwise.*/ +int +write_to_data_subdir(const char* subdir, const char* fname, + const char* str, const char* descr) +{ + char *filename = get_datadir_fname2(subdir, fname); + int return_val = 0; + + if (write_str_to_file(filename, str, 0) < 0) { + log_warn(LD_HIST, "Unable to write %s to disk!", descr ? descr : fname); + return_val = -1; + } + tor_free(filename); + return return_val; +} + /** Given a file name check to see whether the file exists but has not been * modified for a very long time. If so, remove it. */ void @@ -5911,12 +7310,17 @@ remove_file_if_very_old(const char *fname, time_t now) #define VERY_OLD_FILE_AGE (28*24*60*60) struct stat st; - if (stat(fname, &st)==0 && st.st_mtime < now-VERY_OLD_FILE_AGE) { + log_debug(LD_FS, "stat()ing %s", fname); + if (stat(sandbox_intern_string(fname), &st)==0 && + st.st_mtime < now-VERY_OLD_FILE_AGE) { char buf[ISO_TIME_LEN+1]; format_local_iso_time(buf, st.st_mtime); log_notice(LD_GENERAL, "Obsolete file %s hasn't been modified since %s. " "Removing it.", fname, buf); - unlink(fname); + if (unlink(fname) != 0) { + log_warn(LD_FS, "Failed to unlink %s: %s", + fname, strerror(errno)); + } } } @@ -5992,6 +7396,7 @@ getinfo_helper_config(control_connection_t *conn, case CONFIG_TYPE_ISOTIME: type = "Time"; break; case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break; case CONFIG_TYPE_CSV: type = "CommaList"; break; + case CONFIG_TYPE_CSV_INTERVAL: type = "TimeIntervalCommaList"; break; case CONFIG_TYPE_LINELIST: type = "LineList"; break; case CONFIG_TYPE_LINELIST_S: type = "Dependant"; break; case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break; @@ -6008,15 +7413,67 @@ getinfo_helper_config(control_connection_t *conn, smartlist_free(sl); } else if (!strcmp(question, "config/defaults")) { smartlist_t *sl = smartlist_new(); - int i; + int i, dirauth_lines_seen = 0, fallback_lines_seen = 0; for (i = 0; option_vars_[i].name; ++i) { const config_var_t *var = &option_vars_[i]; if (var->initvalue != NULL) { - char *val = esc_for_log(var->initvalue); - smartlist_add_asprintf(sl, "%s %s\n",var->name,val); - tor_free(val); + if (strcmp(option_vars_[i].name, "DirAuthority") == 0) { + /* + * Count dirauth lines we have a default for; we'll use the + * count later to decide whether to add the defaults manually + */ + ++dirauth_lines_seen; + } + if (strcmp(option_vars_[i].name, "FallbackDir") == 0) { + /* + * Similarly count fallback lines, so that we can decided later + * to add the defaults manually. + */ + ++fallback_lines_seen; + } + char *val = esc_for_log(var->initvalue); + smartlist_add_asprintf(sl, "%s %s\n",var->name,val); + tor_free(val); + } + } + + if (dirauth_lines_seen == 0) { + /* + * We didn't see any directory authorities with default values, + * so add the list of default authorities manually. + */ + const char **i; + + /* + * default_authorities is defined earlier in this file and + * is a const char ** NULL-terminated array of dirauth config + * lines. + */ + for (i = default_authorities; *i != NULL; ++i) { + char *val = esc_for_log(*i); + smartlist_add_asprintf(sl, "DirAuthority %s\n", val); + tor_free(val); } } + + if (fallback_lines_seen == 0 && + get_options()->UseDefaultFallbackDirs == 1) { + /* + * We didn't see any explicitly configured fallback mirrors, + * so add the defaults to the list manually. + * + * default_fallbacks is included earlier in this file and + * is a const char ** NULL-terminated array of fallback config lines. + */ + const char **i; + + for (i = default_fallbacks; *i != NULL; ++i) { + char *val = esc_for_log(*i); + smartlist_add_asprintf(sl, "FallbackDir %s\n", val); + tor_free(val); + } + } + *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); @@ -6123,3 +7580,70 @@ config_maybe_load_geoip_files_(const or_options_t *options, config_load_geoip_file_(AF_INET6, options->GeoIPv6File, "geoip6"); } +/** Initialize cookie authentication (used so far by the ControlPort + * and Extended ORPort). + * + * Allocate memory and create a cookie (of length <b>cookie_len</b>) + * in <b>cookie_out</b>. + * Then write it down to <b>fname</b> and prepend it with <b>header</b>. + * + * If <b>group_readable</b> is set, set <b>fname</b> to be readable + * by the default GID. + * + * If the whole procedure was successful, set + * <b>cookie_is_set_out</b> to True. */ +int +init_cookie_authentication(const char *fname, const char *header, + int cookie_len, int group_readable, + uint8_t **cookie_out, int *cookie_is_set_out) +{ + char cookie_file_str_len = strlen(header) + cookie_len; + char *cookie_file_str = tor_malloc(cookie_file_str_len); + int retval = -1; + + /* We don't want to generate a new cookie every time we call + * options_act(). One should be enough. */ + if (*cookie_is_set_out) { + retval = 0; /* we are all set */ + goto done; + } + + /* If we've already set the cookie, free it before re-setting + it. This can happen if we previously generated a cookie, but + couldn't write it to a disk. */ + if (*cookie_out) + tor_free(*cookie_out); + + /* Generate the cookie */ + *cookie_out = tor_malloc(cookie_len); + crypto_rand((char *)*cookie_out, cookie_len); + + /* Create the string that should be written on the file. */ + memcpy(cookie_file_str, header, strlen(header)); + memcpy(cookie_file_str+strlen(header), *cookie_out, cookie_len); + if (write_bytes_to_file(fname, cookie_file_str, cookie_file_str_len, 1)) { + log_warn(LD_FS,"Error writing auth cookie to %s.", escaped(fname)); + goto done; + } + +#ifndef _WIN32 + if (group_readable) { + if (chmod(fname, 0640)) { + log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname)); + } + } +#else + (void) group_readable; +#endif + + /* Success! */ + log_info(LD_GENERAL, "Generated auth cookie file in '%s'.", escaped(fname)); + *cookie_is_set_out = 1; + retval = 0; + + done: + memwipe(cookie_file_str, 0, cookie_file_str_len); + tor_free(cookie_file_str); + return retval; +} + diff --git a/src/or/config.h b/src/or/config.h index ef4acac514..02121cf95c 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,9 +12,15 @@ #ifndef TOR_CONFIG_H #define TOR_CONFIG_H -const char *get_dirportfrontpage(void); -const or_options_t *get_options(void); -or_options_t *get_options_mutable(void); +#include "testsupport.h" + +#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(DARWIN) +#define KERNEL_MAY_SUPPORT_IPFW +#endif + +MOCK_DECL(const char*, get_dirportfrontpage, (void)); +MOCK_DECL(const or_options_t *, get_options, (void)); +MOCK_DECL(or_options_t *, get_options_mutable, (void)); int set_options(or_options_t *new_val, char **msg); void config_free_all(void); const char *safe_str_client(const char *address); @@ -27,12 +33,17 @@ setopt_err_t options_trial_assign(config_line_t *list, int use_defaults, int clear_first, char **msg); uint32_t get_last_resolved_addr(void); +void reset_last_resolved_addr(void); int resolve_my_address(int warn_severity, const or_options_t *options, uint32_t *addr_out, const char **method_out, char **hostname_out); -int is_local_addr(const tor_addr_t *addr); +MOCK_DECL(int, is_local_addr, (const tor_addr_t *addr)); void options_init(or_options_t *options); -char *options_dump(const or_options_t *options, int minimal); + +#define OPTIONS_DUMP_MINIMAL 1 +#define OPTIONS_DUMP_DEFAULTS 2 +#define OPTIONS_DUMP_ALL 3 +char *options_dump(const or_options_t *options, int how_to_dump); int options_init_from_torrc(int argc, char **argv); setopt_err_t options_init_from_string(const char *cf_defaults, const char *cf, int command, const char *command_arg, char **msg); @@ -54,14 +65,22 @@ char *options_get_datadir_fname2_suffix(const or_options_t *options, * get_datadir_fname2_suffix. */ #define get_datadir_fname2(sub1,sub2) \ get_datadir_fname2_suffix((sub1), (sub2), NULL) +/** Return a newly allocated string containing datadir/sub1/sub2 relative to + * opts. See get_datadir_fname2_suffix. */ +#define options_get_datadir_fname2(opts,sub1,sub2) \ + options_get_datadir_fname2_suffix((opts),(sub1), (sub2), NULL) /** Return a newly allocated string containing datadir/sub1suffix. See * get_datadir_fname2_suffix. */ #define get_datadir_fname_suffix(sub1, suffix) \ get_datadir_fname2_suffix((sub1), NULL, (suffix)) +int check_or_create_data_subdir(const char *subdir); +int write_to_data_subdir(const char* subdir, const char* fname, + const char* str, const char* descr); + int get_num_cpus(const or_options_t *options); -const smartlist_t *get_configured_ports(void); +MOCK_DECL(const smartlist_t *,get_configured_ports,(void)); int get_first_advertised_port_by_type_af(int listener_type, int address_family); #define get_primary_or_port() \ @@ -80,16 +99,20 @@ int getinfo_helper_config(control_connection_t *conn, const char *question, char **answer, const char **errmsg); -const char *tor_get_digests(void); uint32_t get_effective_bwrate(const or_options_t *options); uint32_t get_effective_bwburst(const or_options_t *options); char *get_transport_bindaddr_from_config(const char *transport); -#ifdef CONFIG_PRIVATE -/* Used only by config.c and test.c */ +int init_cookie_authentication(const char *fname, const char *header, + int cookie_len, int group_readable, + uint8_t **cookie_out, int *cookie_is_set_out); + or_options_t *options_new(void); -#endif + +int config_parse_commandline(int argc, char **argv, int ignore_errors, + config_line_t **result, + config_line_t **cmdline_result); void config_register_addressmaps(const or_options_t *options); /* XXXX024 move to connection_edge.h */ @@ -97,6 +120,70 @@ int addressmap_register_auto(const char *from, const char *to, time_t expires, addressmap_entry_source_t addrmap_source, const char **msg); +int config_parse_unix_port(const char *addrport, char **path_out); + +/** Represents the information stored in a torrc Bridge line. */ +typedef struct bridge_line_t { + tor_addr_t addr; /* The IP address of the bridge. */ + uint16_t port; /* The TCP port of the bridge. */ + char *transport_name; /* The name of the pluggable transport that + should be used to connect to the bridge. */ + char digest[DIGEST_LEN]; /* The bridge's identity key digest. */ + smartlist_t *socks_args; /* SOCKS arguments for the pluggable + transport proxy. */ +} bridge_line_t; + +void bridge_line_free(bridge_line_t *bridge_line); +bridge_line_t *parse_bridge_line(const char *line); +smartlist_t *get_options_from_transport_options_line(const char *line, + const char *transport); +smartlist_t *get_options_for_server_transport(const char *transport); + +#ifdef CONFIG_PRIVATE + +#define CL_PORT_NO_STREAM_OPTIONS (1u<<0) +#define CL_PORT_WARN_NONLOCAL (1u<<1) +#define CL_PORT_ALLOW_EXTRA_LISTENADDR (1u<<2) +#define CL_PORT_SERVER_OPTIONS (1u<<3) +#define CL_PORT_FORBID_NONLOCAL (1u<<4) +#define CL_PORT_TAKES_HOSTNAMES (1u<<5) +#define CL_PORT_IS_UNIXSOCKET (1u<<6) +#define CL_PORT_DFLT_GROUP_WRITABLE (1u<<7) + +STATIC int options_act(const or_options_t *old_options); +#ifdef TOR_UNIT_TESTS +extern struct config_format_t options_format; +#endif + +STATIC port_cfg_t *port_cfg_new(size_t namelen); +STATIC void port_cfg_free(port_cfg_t *port); +STATIC void or_options_free(or_options_t *options); +STATIC int options_validate(or_options_t *old_options, + or_options_t *options, + or_options_t *default_options, + int from_setconf, char **msg); +STATIC int parse_transport_line(const or_options_t *options, + const char *line, int validate_only, + int server); +STATIC int consider_adding_dir_servers(const or_options_t *options, + const or_options_t *old_options); +STATIC void add_default_trusted_dir_authorities(dirinfo_type_t type); +MOCK_DECL(STATIC void, add_default_fallback_dir_servers, (void)); +STATIC int parse_dir_authority_line(const char *line, + dirinfo_type_t required_type, + int validate_only); +STATIC int parse_dir_fallback_line(const char *line, int validate_only); +STATIC int have_enough_mem_for_dircache(const or_options_t *options, + size_t total_mem, char **msg); +STATIC int parse_port_config(smartlist_t *out, + const config_line_t *ports, + const config_line_t *listenaddrs, + const char *portname, + int listener_type, + const char *defaultaddr, + int defaultport, + const unsigned flags); +#endif #endif diff --git a/src/or/config_codedigest.c b/src/or/config_codedigest.c deleted file mode 100644 index 86d14bacef..0000000000 --- a/src/or/config_codedigest.c +++ /dev/null @@ -1,13 +0,0 @@ - -const char *tor_get_digests(void); - -/** Return a string describing the digest of the source files in src/or/ - */ -const char * -tor_get_digests(void) -{ - return "" -#include "or_sha1.i" - ; -} - diff --git a/src/or/confparse.c b/src/or/confparse.c index 8863d92409..4f446d07c3 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -1,9 +1,16 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * \file confparse.c + * + * \brief Back-end for parsing and generating key-value files, used to + * implement the torrc file format and the state file. + */ + #include "or.h" #include "confparse.h" #include "routerset.h" @@ -79,6 +86,21 @@ config_line_append(config_line_t **lst, (*lst) = newline; } +/** Return the line in <b>lines</b> whose key is exactly <b>key</b>, or NULL + * if no such key exists. For handling commandline-only options only; other + * options should be looked up in the appropriate data structure. */ +const config_line_t * +config_line_find(const config_line_t *lines, + const char *key) +{ + const config_line_t *cl; + for (cl = lines; cl; cl = cl->next) { + if (!strcmp(cl->key, key)) + return cl; + } + return NULL; +} + /** Helper: parse the config string and strdup into key/value * strings. Set *result to the list, or NULL if parsing the string * failed. Return 0 on success, -1 on failure. Warn and ignore any @@ -223,6 +245,8 @@ config_assign_value(const config_format_t *fmt, void *options, int i, ok; const config_var_t *var; void *lvalue; + int *csv_int; + smartlist_t *csv_str; CONFIG_CHECK(fmt, options); @@ -357,6 +381,36 @@ config_assign_value(const config_format_t *fmt, void *options, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); break; + case CONFIG_TYPE_CSV_INTERVAL: + if (*(smartlist_t**)lvalue) { + SMARTLIST_FOREACH(*(smartlist_t**)lvalue, int *, cp, tor_free(cp)); + smartlist_clear(*(smartlist_t**)lvalue); + } else { + *(smartlist_t**)lvalue = smartlist_new(); + } + csv_str = smartlist_new(); + smartlist_split_string(csv_str, c->value, ",", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(csv_str, char *, str) + { + i = config_parse_interval(str, &ok); + if (!ok) { + tor_asprintf(msg, + "Interval in '%s %s' is malformed or out of bounds.", + c->key, c->value); + SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); + smartlist_free(csv_str); + return -1; + } + csv_int = tor_malloc_zero(sizeof(int)); + *csv_int = i; + smartlist_add(*(smartlist_t**)lvalue, csv_int); + } + SMARTLIST_FOREACH_END(str); + SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); + smartlist_free(csv_str); + break; + case CONFIG_TYPE_LINELIST: case CONFIG_TYPE_LINELIST_S: { @@ -555,6 +609,7 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, const config_var_t *var; const void *value; config_line_t *result; + smartlist_t *csv_str; tor_assert(options && key); CONFIG_CHECK(fmt, options); @@ -637,6 +692,20 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, else result->value = tor_strdup(""); break; + case CONFIG_TYPE_CSV_INTERVAL: + if (*(smartlist_t**)value) { + csv_str = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(*(smartlist_t**)value, int *, i) + { + smartlist_add_asprintf(csv_str, "%d", *i); + } + SMARTLIST_FOREACH_END(i); + result->value = smartlist_join_strings(csv_str, ",", 0, NULL); + SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); + smartlist_free(csv_str); + } else + result->value = tor_strdup(""); + break; case CONFIG_TYPE_OBSOLETE: log_fn(LOG_INFO, LD_CONFIG, "You asked me for the value of an obsolete config option '%s'.", @@ -826,6 +895,13 @@ config_clear(const config_format_t *fmt, void *options, *(smartlist_t **)lvalue = NULL; } break; + case CONFIG_TYPE_CSV_INTERVAL: + if (*(smartlist_t**)lvalue) { + SMARTLIST_FOREACH(*(smartlist_t **)lvalue, int *, cp, tor_free(cp)); + smartlist_free(*(smartlist_t **)lvalue); + *(smartlist_t **)lvalue = NULL; + } + break; case CONFIG_TYPE_LINELIST: case CONFIG_TYPE_LINELIST_S: config_free_lines(*(config_line_t **)lvalue); @@ -1005,8 +1081,8 @@ config_dump(const config_format_t *fmt, const void *default_options, /* XXX use a 1 here so we don't add a new log line while dumping */ if (default_options == NULL) { - if (fmt->validate_fn(NULL, defaults_tmp, 1, &msg) < 0) { - log_err(LD_BUG, "Failed to validate default config."); + if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) { + log_err(LD_BUG, "Failed to validate default config: %s", msg); tor_free(msg); tor_assert(0); } @@ -1072,20 +1148,36 @@ static struct unit_table_t memory_units[] = { { "kbytes", 1<<10 }, { "kilobyte", 1<<10 }, { "kilobytes", 1<<10 }, + { "kilobits", 1<<7 }, + { "kilobit", 1<<7 }, + { "kbits", 1<<7 }, + { "kbit", 1<<7 }, { "m", 1<<20 }, { "mb", 1<<20 }, { "mbyte", 1<<20 }, { "mbytes", 1<<20 }, { "megabyte", 1<<20 }, { "megabytes", 1<<20 }, + { "megabits", 1<<17 }, + { "megabit", 1<<17 }, + { "mbits", 1<<17 }, + { "mbit", 1<<17 }, { "gb", 1<<30 }, { "gbyte", 1<<30 }, { "gbytes", 1<<30 }, { "gigabyte", 1<<30 }, { "gigabytes", 1<<30 }, + { "gigabits", 1<<27 }, + { "gigabit", 1<<27 }, + { "gbits", 1<<27 }, + { "gbit", 1<<27 }, { "tb", U64_LITERAL(1)<<40 }, { "terabyte", U64_LITERAL(1)<<40 }, { "terabytes", U64_LITERAL(1)<<40 }, + { "terabits", U64_LITERAL(1)<<37 }, + { "terabit", U64_LITERAL(1)<<37 }, + { "tbits", U64_LITERAL(1)<<37 }, + { "tbit", U64_LITERAL(1)<<37 }, { NULL, 0 }, }; diff --git a/src/or/confparse.h b/src/or/confparse.h index 1b987f3bf9..885c615202 100644 --- a/src/or/confparse.h +++ b/src/or/confparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONFPARSE_H @@ -26,6 +26,9 @@ typedef enum config_type_t { CONFIG_TYPE_ISOTIME, /**< An ISO-formatted time relative to UTC. */ CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and * optional whitespace. */ + CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and + * optional whitespace, representing intervals in + * seconds, with optional units */ CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, * mixed with other keywords. */ @@ -68,12 +71,12 @@ typedef struct config_var_description_t { /** Type of a callback to validate whether a given configuration is * well-formed and consistent. See options_trial_assign() for documentation * of arguments. */ -typedef int (*validate_fn_t)(void*,void*,int,char**); +typedef int (*validate_fn_t)(void*,void*,void*,int,char**); /** Information on the keys, value types, key-to-struct-member mappings, * variable descriptions, validation functions, and abbreviations for a * configuration or storage format. */ -typedef struct { +typedef struct config_format_t { size_t size; /**< Size of the struct that everything gets parsed into. */ uint32_t magic; /**< Required 'magic value' to make sure we have a struct * of the right type. */ @@ -100,6 +103,8 @@ void *config_new(const config_format_t *fmt); void config_line_append(config_line_t **lst, const char *key, const char *val); config_line_t *config_lines_dup(const config_line_t *inp); +const config_line_t *config_line_find(const config_line_t *lines, + const char *key); void config_free(const config_format_t *fmt, void *options); int config_lines_eq(config_line_t *a, config_line_t *b); int config_count_key(const config_line_t *a, const char *key); diff --git a/src/or/connection.c b/src/or/connection.c index 4f74a1d04b..4fbbaf1abd 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -10,6 +10,7 @@ * on connections. **/ +#define CONNECTION_PRIVATE #include "or.h" #include "buffers.h" /* @@ -17,6 +18,8 @@ * part of a subclass (channel_tls_t). */ #define TOR_CHANNEL_INTERNAL_ +#define CONNECTION_PRIVATE +#include "backtrace.h" #include "channel.h" #include "channeltls.h" #include "circuitbuild.h" @@ -27,14 +30,15 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" -#include "cpuworker.h" #include "directory.h" #include "dirserv.h" #include "dns.h" #include "dnsserv.h" #include "entrynodes.h" +#include "ext_orport.h" #include "geoip.h" #include "main.h" +#include "nodelist.h" #include "policies.h" #include "reasons.h" #include "relay.h" @@ -42,8 +46,11 @@ #include "rendcommon.h" #include "rephist.h" #include "router.h" +#include "routerlist.h" #include "transports.h" #include "routerparse.h" +#include "sandbox.h" +#include "transports.h" #ifdef USE_BUFFEREVENTS #include <event2/event.h> @@ -53,6 +60,11 @@ #include <pwd.h> #endif +#ifdef HAVE_SYS_UN_H +#include <sys/socket.h> +#include <sys/un.h> +#endif + static connection_t *connection_listener_new( const struct sockaddr *listensockaddr, socklen_t listensocklen, int type, @@ -97,6 +109,7 @@ static smartlist_t *outgoing_addrs = NULL; #define CASE_ANY_LISTENER_TYPE \ case CONN_TYPE_OR_LISTENER: \ + case CONN_TYPE_EXT_OR_LISTENER: \ case CONN_TYPE_AP_LISTENER: \ case CONN_TYPE_DIR_LISTENER: \ case CONN_TYPE_CONTROL_LISTENER: \ @@ -125,9 +138,10 @@ conn_type_to_string(int type) case CONN_TYPE_AP: return "Socks"; case CONN_TYPE_DIR_LISTENER: return "Directory listener"; case CONN_TYPE_DIR: return "Directory"; - case CONN_TYPE_CPUWORKER: return "CPU worker"; case CONN_TYPE_CONTROL_LISTENER: return "Control listener"; case CONN_TYPE_CONTROL: return "Control"; + case CONN_TYPE_EXT_OR: return "Extended OR"; + case CONN_TYPE_EXT_OR_LISTENER: return "Extended OR listener"; default: log_warn(LD_BUG, "unknown connection type %d", type); tor_snprintf(buf, sizeof(buf), "unknown [%d]", type); @@ -164,6 +178,18 @@ conn_state_to_string(int type, int state) case OR_CONN_STATE_OPEN: return "open"; } break; + case CONN_TYPE_EXT_OR: + switch (state) { + case EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE: + return "waiting for authentication type"; + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE: + return "waiting for client nonce"; + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH: + return "waiting for client hash"; + case EXT_OR_CONN_STATE_OPEN: return "open"; + case EXT_OR_CONN_STATE_FLUSHING: return "flushing final OKAY"; + } + break; case CONN_TYPE_EXIT: switch (state) { case EXIT_CONN_STATE_RESOLVING: return "waiting for dest info"; @@ -194,12 +220,6 @@ conn_state_to_string(int type, int state) case DIR_CONN_STATE_SERVER_WRITING: return "writing"; } break; - case CONN_TYPE_CPUWORKER: - switch (state) { - case CPUWORKER_STATE_IDLE: return "idle"; - case CPUWORKER_STATE_BUSY_ONION: return "busy with onion"; - } - break; case CONN_TYPE_CONTROL: switch (state) { case CONTROL_CONN_STATE_OPEN: return "open (protocol v1)"; @@ -228,7 +248,7 @@ connection_type_uses_bufferevent(connection_t *conn) case CONN_TYPE_DIR: case CONN_TYPE_CONTROL: case CONN_TYPE_OR: - case CONN_TYPE_CPUWORKER: + case CONN_TYPE_EXT_OR: return 1; default: return 0; @@ -249,22 +269,22 @@ dir_connection_new(int socket_family) /** Allocate and return a new or_connection_t, initialized as by * connection_init(). * - * Set timestamp_last_added_nonpadding to now. - * - * Assign a pseudorandom next_circ_id between 0 and 2**15. - * * Initialize active_circuit_pqueue. * * Set active_circuit_pqueue_last_recalibrated to current cell_ewma tick. */ or_connection_t * -or_connection_new(int socket_family) +or_connection_new(int type, int socket_family) { or_connection_t *or_conn = tor_malloc_zero(sizeof(or_connection_t)); time_t now = time(NULL); - connection_init(now, TO_CONN(or_conn), CONN_TYPE_OR, socket_family); + tor_assert(type == CONN_TYPE_OR || type == CONN_TYPE_EXT_OR); + connection_init(now, TO_CONN(or_conn), type, socket_family); - or_conn->timestamp_last_added_nonpadding = time(NULL); + connection_or_set_canonical(or_conn, 0); + + if (type == CONN_TYPE_EXT_OR) + connection_or_set_ext_or_identifier(or_conn); return or_conn; } @@ -285,9 +305,11 @@ entry_connection_new(int type, int socket_family) * in a little while. Otherwise, we're doing this as a linked connection * of some kind, and we should set it up here based on the socket family */ if (socket_family == AF_INET) - entry_conn->ipv4_traffic_ok = 1; + entry_conn->entry_cfg.ipv4_traffic = 1; else if (socket_family == AF_INET6) - entry_conn->ipv6_traffic_ok = 1; + entry_conn->entry_cfg.ipv6_traffic = 1; + else if (socket_family == AF_UNIX) + entry_conn->is_socks_socket = 1; return entry_conn; } @@ -311,7 +333,6 @@ control_connection_new(int socket_family) tor_malloc_zero(sizeof(control_connection_t)); connection_init(time(NULL), TO_CONN(control_conn), CONN_TYPE_CONTROL, socket_family); - log_notice(LD_CONTROL, "New control connection opened."); return control_conn; } @@ -334,7 +355,8 @@ connection_new(int type, int socket_family) { switch (type) { case CONN_TYPE_OR: - return TO_CONN(or_connection_new(socket_family)); + case CONN_TYPE_EXT_OR: + return TO_CONN(or_connection_new(type, socket_family)); case CONN_TYPE_EXIT: return TO_CONN(edge_connection_new(type, socket_family)); @@ -376,6 +398,7 @@ connection_init(time_t now, connection_t *conn, int type, int socket_family) switch (type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: conn->magic = OR_CONNECTION_MAGIC; break; case CONN_TYPE_EXIT: @@ -430,11 +453,27 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) conn_b->linked_conn = conn_a; } +/** Return true iff the provided connection listener type supports AF_UNIX + * sockets. */ +int +conn_listener_type_supports_af_unix(int type) +{ + /* For now only control ports or SOCKS ports can be Unix domain sockets + * and listeners at the same time */ + switch (type) { + case CONN_TYPE_CONTROL_LISTENER: + case CONN_TYPE_AP_LISTENER: + return 1; + default: + return 0; + } +} + /** Deallocate memory used by <b>conn</b>. Deallocate its buffers if * necessary, close its socket if necessary, and mark the directory as dirty * if <b>conn</b> is an OR or OP connection. */ -static void +STATIC void connection_free_(connection_t *conn) { void *mem; @@ -444,6 +483,7 @@ connection_free_(connection_t *conn) switch (conn->type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); mem = TO_OR_CONN(conn); memlen = sizeof(or_connection_t); @@ -494,9 +534,9 @@ connection_free_(connection_t *conn) buf_free(conn->outbuf); } else { if (conn->socket_family == AF_UNIX) { - /* For now only control ports can be Unix domain sockets + /* For now only control and SOCKS ports can be Unix domain sockets * and listeners at the same time */ - tor_assert(conn->type == CONN_TYPE_CONTROL_LISTENER); + tor_assert(conn_listener_type_supports_af_unix(conn->type)); if (unlink(conn->address) < 0 && errno != ENOENT) { log_warn(LD_NET, "Could not unlink %s: %s", conn->address, @@ -522,8 +562,7 @@ connection_free_(connection_t *conn) or_conn, TLS_CHAN_TO_BASE(or_conn->chan), U64_PRINTF_ARG( TLS_CHAN_TO_BASE(or_conn->chan)->global_identifier)); - if (!(TLS_CHAN_TO_BASE(or_conn->chan)->state == CHANNEL_STATE_CLOSED || - TLS_CHAN_TO_BASE(or_conn->chan)->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_FINISHED(TLS_CHAN_TO_BASE(or_conn->chan))) { channel_close_for_error(TLS_CHAN_TO_BASE(or_conn->chan)); } @@ -551,10 +590,19 @@ connection_free_(connection_t *conn) control_connection_t *control_conn = TO_CONTROL_CONN(conn); tor_free(control_conn->safecookie_client_hash); tor_free(control_conn->incoming_cmd); + if (control_conn->ephemeral_onion_services) { + SMARTLIST_FOREACH(control_conn->ephemeral_onion_services, char *, cp, { + memwipe(cp, 0, strlen(cp)); + tor_free(cp); + }); + smartlist_free(control_conn->ephemeral_onion_services); + } } - tor_free(conn->read_event); /* Probably already freed by connection_free. */ - tor_free(conn->write_event); /* Probably already freed by connection_free. */ + /* Probably already freed by connection_free. */ + tor_event_free(conn->read_event); + tor_event_free(conn->write_event); + conn->read_event = conn->write_event = NULL; IF_HAS_BUFFEREVENT(conn, { /* This was a workaround to handle bugs in some old versions of libevent * where callbacks can occur after calling bufferevent_free(). Setting @@ -590,6 +638,13 @@ connection_free_(connection_t *conn) log_warn(LD_BUG, "called on OR conn with non-zeroed identity_digest"); connection_or_remove_from_identity_map(TO_OR_CONN(conn)); } + if (conn->type == CONN_TYPE_OR || conn->type == CONN_TYPE_EXT_OR) { + connection_or_remove_from_ext_or_id_map(TO_OR_CONN(conn)); + tor_free(TO_OR_CONN(conn)->ext_or_conn_id); + tor_free(TO_OR_CONN(conn)->ext_or_auth_correct_client_hash); + tor_free(TO_OR_CONN(conn)->ext_or_transport); + } + #ifdef USE_BUFFEREVENTS if (conn->type == CONN_TYPE_OR && TO_OR_CONN(conn)->bucket_cfg) { ev_token_bucket_cfg_free(TO_OR_CONN(conn)->bucket_cfg); @@ -603,8 +658,8 @@ connection_free_(connection_t *conn) /** Make sure <b>conn</b> isn't in any of the global conn lists; then free it. */ -void -connection_free(connection_t *conn) +MOCK_IMPL(void, +connection_free,(connection_t *conn)) { if (!conn) return; @@ -627,6 +682,13 @@ connection_free(connection_t *conn) if (conn->type == CONN_TYPE_CONTROL) { connection_control_closed(TO_CONTROL_CONN(conn)); } +#if 1 + /* DEBUGGING */ + if (conn->type == CONN_TYPE_AP) { + connection_ap_warn_and_unmark_if_pending_circ(TO_ENTRY_CONN(conn), + "connection_free"); + } +#endif connection_unregister_events(conn); connection_free_(conn); } @@ -653,6 +715,7 @@ connection_about_to_close_connection(connection_t *conn) connection_dir_about_to_close(TO_DIR_CONN(conn)); break; case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: connection_or_about_to_close(TO_OR_CONN(conn)); break; case CONN_TYPE_AP: @@ -864,9 +927,9 @@ create_unix_sockaddr(const char *listenaddress, char **readable_address, } #endif /* HAVE_SYS_UN_H */ -/** Warn that an accept or a connect has failed because we're running up - * against our ulimit. Rate-limit these warnings so that we don't spam - * the log. */ +/** Warn that an accept or a connect has failed because we're running out of + * TCP sockets we can use on current system. Rate-limit these warnings so + * that we don't spam the log. */ static void warn_too_many_conns(void) { @@ -876,7 +939,7 @@ warn_too_many_conns(void) if ((m = rate_limit_log(&last_warned, approx_time()))) { int n_conns = get_n_open_sockets(); log_warn(LD_NET,"Failing because we have %d connections already. Please " - "raise your ulimit -n.%s", n_conns, m); + "read doc/TUNING for guidance.%s", n_conns, m); tor_free(m); control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", n_conns); @@ -884,30 +947,89 @@ warn_too_many_conns(void) } #ifdef HAVE_SYS_UN_H + +#define UNIX_SOCKET_PURPOSE_CONTROL_SOCKET 0 +#define UNIX_SOCKET_PURPOSE_SOCKS_SOCKET 1 + +/** Check if the purpose isn't one of the ones we know what to do with */ + +static int +is_valid_unix_socket_purpose(int purpose) +{ + int valid = 0; + + switch (purpose) { + case UNIX_SOCKET_PURPOSE_CONTROL_SOCKET: + case UNIX_SOCKET_PURPOSE_SOCKS_SOCKET: + valid = 1; + break; + } + + return valid; +} + +/** Return a string description of a unix socket purpose */ +static const char * +unix_socket_purpose_to_string(int purpose) +{ + const char *s = "unknown-purpose socket"; + + switch (purpose) { + case UNIX_SOCKET_PURPOSE_CONTROL_SOCKET: + s = "control socket"; + break; + case UNIX_SOCKET_PURPOSE_SOCKS_SOCKET: + s = "SOCKS socket"; + break; + } + + return s; +} + /** Check whether we should be willing to open an AF_UNIX socket in * <b>path</b>. Return 0 if we should go ahead and -1 if we shouldn't. */ static int -check_location_for_unix_socket(const or_options_t *options, const char *path) +check_location_for_unix_socket(const or_options_t *options, const char *path, + int purpose, const port_cfg_t *port) { int r = -1; - char *p = tor_strdup(path); + char *p = NULL; + + tor_assert(is_valid_unix_socket_purpose(purpose)); + + p = tor_strdup(path); cpd_check_t flags = CPD_CHECK_MODE_ONLY; - if (get_parent_directory(p)<0) + if (get_parent_directory(p)<0 || p[0] != '/') { + log_warn(LD_GENERAL, "Bad unix socket address '%s'. Tor does not support " + "relative paths for unix sockets.", path); goto done; + } - if (options->ControlSocketsGroupWritable) + if (port->is_world_writable) { + /* World-writable sockets can go anywhere. */ + r = 0; + goto done; + } + + if (port->is_group_writable) { flags |= CPD_GROUP_OK; + } + + if (port->relax_dirmode_check) { + flags |= CPD_RELAX_DIRMODE_CHECK; + } if (check_private_dir(p, flags, options->User) < 0) { char *escpath, *escdir; escpath = esc_for_log(path); escdir = esc_for_log(p); - log_warn(LD_GENERAL, "Before Tor can create a control socket in %s, the " - "directory %s needs to exist, and to be accessible only by the " - "user%s account that is running Tor. (On some Unix systems, " - "anybody who can list a socket can connect to it, so Tor is " - "being careful.)", escpath, escdir, - options->ControlSocketsGroupWritable ? " and group" : ""); + log_warn(LD_GENERAL, "Before Tor can create a %s in %s, the directory " + "%s needs to exist, and to be accessible only by the user%s " + "account that is running Tor. (On some Unix systems, anybody " + "who can list a socket can connect to it, so Tor is being " + "careful.)", + unix_socket_purpose_to_string(purpose), escpath, escdir, + port->is_group_writable ? " and group" : ""); tor_free(escpath); tor_free(escdir); goto done; @@ -921,12 +1043,14 @@ check_location_for_unix_socket(const or_options_t *options, const char *path) #endif /** Tell the TCP stack that it shouldn't wait for a long time after - * <b>sock</b> has closed before reusing its port. */ -static void + * <b>sock</b> has closed before reusing its port. Return 0 on success, + * -1 on failure. */ +static int make_socket_reuseable(tor_socket_t sock) { #ifdef _WIN32 (void) sock; + return 0; #else int one=1; @@ -936,12 +1060,37 @@ make_socket_reuseable(tor_socket_t sock) * already has it bound_. So, don't do that on Win32. */ if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*) &one, (socklen_t)sizeof(one)) == -1) { - log_warn(LD_NET, "Error setting SO_REUSEADDR flag: %s", - tor_socket_strerror(errno)); + return -1; } + return 0; #endif } +#ifdef _WIN32 +/** Tell the Windows TCP stack to prevent other applications from receiving + * traffic from tor's open ports. Return 0 on success, -1 on failure. */ +static int +make_win32_socket_exclusive(tor_socket_t sock) +{ +#ifdef SO_EXCLUSIVEADDRUSE + int one=1; + + /* Any socket that sets REUSEADDR on win32 can bind to a port _even when + * somebody else already has it bound_, and _even if the original socket + * didn't set REUSEADDR_. Use EXCLUSIVEADDRUSE to prevent this port-stealing + * on win32. */ + if (setsockopt(sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, (void*) &one, + (socklen_t)sizeof(one))) { + return -1; + } + return 0; +#else + (void) sock; + return 0; +#endif +} +#endif + /** Max backlog to pass to listen. We start at */ static int listen_limit = INT_MAX; @@ -971,61 +1120,85 @@ tor_listen(tor_socket_t fd) */ static connection_t * connection_listener_new(const struct sockaddr *listensockaddr, - socklen_t socklen, - int type, const char *address, - const port_cfg_t *port_cfg) + socklen_t socklen, + int type, const char *address, + const port_cfg_t *port_cfg) { listener_connection_t *lis_conn; - connection_t *conn; - tor_socket_t s; /* the socket we're going to make */ + connection_t *conn = NULL; + tor_socket_t s = TOR_INVALID_SOCKET; /* the socket we're going to make */ or_options_t const *options = get_options(); + (void) options; /* Windows doesn't use this. */ #if defined(HAVE_PWD_H) && defined(HAVE_SYS_UN_H) - struct passwd *pw = NULL; + const struct passwd *pw = NULL; #endif uint16_t usePort = 0, gotPort = 0; int start_reading = 0; static int global_next_session_group = SESSION_GROUP_FIRST_AUTO; tor_addr_t addr; - if (get_n_open_sockets() >= get_options()->ConnLimit_-1) { - warn_too_many_conns(); - return NULL; - } - if (listensockaddr->sa_family == AF_INET || listensockaddr->sa_family == AF_INET6) { - int is_tcp = (type != CONN_TYPE_AP_DNS_LISTENER); - if (is_tcp) + int is_stream = (type != CONN_TYPE_AP_DNS_LISTENER); + if (is_stream) start_reading = 1; tor_addr_from_sockaddr(&addr, listensockaddr, &usePort); - log_notice(LD_NET, "Opening %s on %s", conn_type_to_string(type), fmt_addrport(&addr, usePort)); - s = tor_open_socket(tor_addr_family(&addr), - is_tcp ? SOCK_STREAM : SOCK_DGRAM, - is_tcp ? IPPROTO_TCP: IPPROTO_UDP); + s = tor_open_socket_nonblocking(tor_addr_family(&addr), + is_stream ? SOCK_STREAM : SOCK_DGRAM, + is_stream ? IPPROTO_TCP: IPPROTO_UDP); if (!SOCKET_OK(s)) { - log_warn(LD_NET,"Socket creation failed: %s", - tor_socket_strerror(tor_socket_errno(-1))); + int e = tor_socket_errno(s); + if (ERRNO_IS_RESOURCE_LIMIT(e)) { + warn_too_many_conns(); + } else { + log_warn(LD_NET, "Socket creation failed: %s", + tor_socket_strerror(e)); + } goto err; } - make_socket_reuseable(s); + if (make_socket_reuseable(s) < 0) { + log_warn(LD_NET, "Error setting SO_REUSEADDR flag on %s: %s", + conn_type_to_string(type), + tor_socket_strerror(errno)); + } -#ifdef IPV6_V6ONLY - if (listensockaddr->sa_family == AF_INET6) { #ifdef _WIN32 - /* In Redmond, this kind of thing passes for standards-conformance. */ - DWORD one = 1; -#else + if (make_win32_socket_exclusive(s) < 0) { + log_warn(LD_NET, "Error setting SO_EXCLUSIVEADDRUSE flag on %s: %s", + conn_type_to_string(type), + tor_socket_strerror(errno)); + } +#endif + +#if defined(USE_TRANSPARENT) && defined(IP_TRANSPARENT) + if (options->TransProxyType_parsed == TPT_TPROXY && + type == CONN_TYPE_AP_TRANS_LISTENER) { int one = 1; + if (setsockopt(s, SOL_IP, IP_TRANSPARENT, (void*)&one, + (socklen_t)sizeof(one)) < 0) { + const char *extra = ""; + int e = tor_socket_errno(s); + if (e == EPERM) + extra = "TransTPROXY requires root privileges or similar" + " capabilities."; + log_warn(LD_NET, "Error setting IP_TRANSPARENT flag: %s.%s", + tor_socket_strerror(e), extra); + } + } #endif + +#ifdef IPV6_V6ONLY + if (listensockaddr->sa_family == AF_INET6) { + int one = 1; /* We need to set IPV6_V6ONLY so that this socket can't get used for * IPv4 connections. */ if (setsockopt(s,IPPROTO_IPV6, IPV6_V6ONLY, - (void*)&one, sizeof(one))<0) { + (void*)&one, (socklen_t)sizeof(one)) < 0) { int e = tor_socket_errno(s); log_warn(LD_NET, "Error setting IPV6_V6ONLY flag: %s", tor_socket_strerror(e)); @@ -1041,15 +1214,13 @@ connection_listener_new(const struct sockaddr *listensockaddr, helpfulhint = ". Is Tor already running?"; log_warn(LD_NET, "Could not bind to %s:%u: %s%s", address, usePort, tor_socket_strerror(e), helpfulhint); - tor_close_socket(s); goto err; } - if (is_tcp) { + if (is_stream) { if (tor_listen(s) < 0) { log_warn(LD_NET, "Could not listen on %s:%u: %s", address, usePort, tor_socket_strerror(tor_socket_errno(s))); - tor_close_socket(s); goto err; } } @@ -1069,15 +1240,21 @@ connection_listener_new(const struct sockaddr *listensockaddr, tor_addr_from_sockaddr(&addr2, (struct sockaddr*)&ss, &gotPort); } #ifdef HAVE_SYS_UN_H + /* + * AF_UNIX generic setup stuff + */ } else if (listensockaddr->sa_family == AF_UNIX) { + /* We want to start reading for both AF_UNIX cases */ start_reading = 1; - /* For now only control ports can be Unix domain sockets - * and listeners at the same time */ - tor_assert(type == CONN_TYPE_CONTROL_LISTENER); + tor_assert(conn_listener_type_supports_af_unix(type)); - if (check_location_for_unix_socket(options, address) < 0) - goto err; + if (check_location_for_unix_socket(options, address, + (type == CONN_TYPE_CONTROL_LISTENER) ? + UNIX_SOCKET_PURPOSE_CONTROL_SOCKET : + UNIX_SOCKET_PURPOSE_SOCKS_SOCKET, port_cfg) < 0) { + goto err; + } log_notice(LD_NET, "Opening %s on %s", conn_type_to_string(type), address); @@ -1089,76 +1266,96 @@ connection_listener_new(const struct sockaddr *listensockaddr, strerror(errno)); goto err; } - s = tor_open_socket(AF_UNIX, SOCK_STREAM, 0); + + s = tor_open_socket_nonblocking(AF_UNIX, SOCK_STREAM, 0); if (! SOCKET_OK(s)) { - log_warn(LD_NET,"Socket creation failed: %s.", strerror(errno)); + int e = tor_socket_errno(s); + if (ERRNO_IS_RESOURCE_LIMIT(e)) { + warn_too_many_conns(); + } else { + log_warn(LD_NET,"Socket creation failed: %s.", strerror(e)); + } goto err; } - if (bind(s, listensockaddr, (socklen_t)sizeof(struct sockaddr_un)) == -1) { + if (bind(s, listensockaddr, + (socklen_t)sizeof(struct sockaddr_un)) == -1) { log_warn(LD_NET,"Bind to %s failed: %s.", address, tor_socket_strerror(tor_socket_errno(s))); - tor_close_socket(s); goto err; } + #ifdef HAVE_PWD_H if (options->User) { - pw = getpwnam(options->User); + pw = tor_getpwnam(options->User); + struct stat st; if (pw == NULL) { log_warn(LD_NET,"Unable to chown() %s socket: user %s not found.", address, options->User); - tor_close_socket(s); goto err; - } else if (chown(address, pw->pw_uid, pw->pw_gid) < 0) { + } else if (fstat(s, &st) == 0 && + st.st_uid == pw->pw_uid && st.st_gid == pw->pw_gid) { + /* No change needed */ + } else if (chown(sandbox_intern_string(address), + pw->pw_uid, pw->pw_gid) < 0) { log_warn(LD_NET,"Unable to chown() %s socket: %s.", address, strerror(errno)); - tor_close_socket(s); goto err; } } #endif - if (options->ControlSocketsGroupWritable) { + + { + unsigned mode; + const char *status; + struct stat st; + if (port_cfg->is_world_writable) { + mode = 0666; + status = "world-writable"; + } else if (port_cfg->is_group_writable) { + mode = 0660; + status = "group-writable"; + } else { + mode = 0600; + status = "private"; + } /* We need to use chmod; fchmod doesn't work on sockets on all * platforms. */ - if (chmod(address, 0660) < 0) { - log_warn(LD_FS,"Unable to make %s group-writable.", address); - tor_close_socket(s); + if (fstat(s, &st) == 0 && (st.st_mode & 0777) == mode) { + /* no change needed */ + } else if (chmod(sandbox_intern_string(address), mode) < 0) { + log_warn(LD_FS,"Unable to make %s %s.", address, status); goto err; } } - if (listen(s,SOMAXCONN) < 0) { + if (listen(s, SOMAXCONN) < 0) { log_warn(LD_NET, "Could not listen on %s: %s", address, tor_socket_strerror(tor_socket_errno(s))); - tor_close_socket(s); goto err; } -#else - (void)options; #endif /* HAVE_SYS_UN_H */ } else { - log_err(LD_BUG,"Got unexpected address family %d.", - listensockaddr->sa_family); - tor_assert(0); - } - - if (set_socket_nonblocking(s) == -1) { - tor_close_socket(s); - goto err; + log_err(LD_BUG, "Got unexpected address family %d.", + listensockaddr->sa_family); + tor_assert(0); } lis_conn = listener_connection_new(type, listensockaddr->sa_family); conn = TO_CONN(lis_conn); conn->socket_family = listensockaddr->sa_family; conn->s = s; + s = TOR_INVALID_SOCKET; /* Prevent double-close */ conn->address = tor_strdup(address); conn->port = gotPort; tor_addr_copy(&conn->addr, &addr); - if (port_cfg->isolation_flags) { - lis_conn->isolation_flags = port_cfg->isolation_flags; - if (port_cfg->session_group >= 0) { - lis_conn->session_group = port_cfg->session_group; + memcpy(&lis_conn->entry_cfg, &port_cfg->entry_cfg, sizeof(entry_port_cfg_t)); + + if (port_cfg->entry_cfg.isolation_flags) { + lis_conn->entry_cfg.isolation_flags = port_cfg->entry_cfg.isolation_flags; + if (port_cfg->entry_cfg.session_group >= 0) { + lis_conn->entry_cfg.session_group = port_cfg->entry_cfg.session_group; } else { /* This can wrap after around INT_MAX listeners are opened. But I don't * believe that matters, since you would need to open a ridiculous @@ -1166,27 +1363,18 @@ connection_listener_new(const struct sockaddr *listensockaddr, * hit this. An OR with a dozen ports open, for example, would have to * close and re-open its listeners every second for 4 years nonstop. */ - lis_conn->session_group = global_next_session_group--; + lis_conn->entry_cfg.session_group = global_next_session_group--; } } - if (type == CONN_TYPE_AP_LISTENER) { - lis_conn->socks_ipv4_traffic = port_cfg->ipv4_traffic; - lis_conn->socks_ipv6_traffic = port_cfg->ipv6_traffic; - lis_conn->socks_prefer_ipv6 = port_cfg->prefer_ipv6; - } else { - lis_conn->socks_ipv4_traffic = 1; - lis_conn->socks_ipv6_traffic = 1; + + if (type != CONN_TYPE_AP_LISTENER) { + lis_conn->entry_cfg.ipv4_traffic = 1; + lis_conn->entry_cfg.ipv6_traffic = 1; + lis_conn->entry_cfg.prefer_ipv6 = 0; } - lis_conn->cache_ipv4_answers = port_cfg->cache_ipv4_answers; - lis_conn->cache_ipv6_answers = port_cfg->cache_ipv6_answers; - lis_conn->use_cached_ipv4_answers = port_cfg->use_cached_ipv4_answers; - lis_conn->use_cached_ipv6_answers = port_cfg->use_cached_ipv6_answers; - lis_conn->prefer_ipv6_virtaddr = port_cfg->prefer_ipv6_virtaddr; - lis_conn->socks_prefer_no_auth = port_cfg->socks_prefer_no_auth; if (connection_add(conn) < 0) { /* no space, forget it */ log_warn(LD_NET,"connection_add for listener failed. Giving up."); - connection_free(conn); goto err; } @@ -1205,6 +1393,11 @@ connection_listener_new(const struct sockaddr *listensockaddr, return conn; err: + if (SOCKET_OK(s)) + tor_close_socket(s); + if (conn) + connection_free(conn); + return NULL; } @@ -1245,6 +1438,8 @@ check_sockaddr(const struct sockaddr *sa, int len, int level) "Address for new connection has address/port equal to zero."); ok = 0; } + } else if (sa->sa_family == AF_UNIX) { + ok = 1; } else { ok = 0; } @@ -1278,7 +1473,7 @@ static int connection_handle_listener_read(connection_t *conn, int new_type) { tor_socket_t news; /* the new socket */ - connection_t *newconn; + connection_t *newconn = 0; /* information about the remote peer when connecting to other routers */ struct sockaddr_storage addrbuf; struct sockaddr *remote = (struct sockaddr*)&addrbuf; @@ -1289,12 +1484,12 @@ connection_handle_listener_read(connection_t *conn, int new_type) tor_assert((size_t)remotelen >= sizeof(struct sockaddr_in)); memset(&addrbuf, 0, sizeof(addrbuf)); - news = tor_accept_socket(conn->s,remote,&remotelen); + news = tor_accept_socket_nonblocking(conn->s,remote,&remotelen); if (!SOCKET_OK(news)) { /* accept() error */ int e = tor_socket_errno(conn->s); if (ERRNO_IS_ACCEPT_EAGAIN(e)) { - return 0; /* he hung up before we could accept(). that's fine. */ - } else if (ERRNO_IS_ACCEPT_RESOURCE_LIMIT(e)) { + return 0; /* they hung up before we could accept(). that's fine. */ + } else if (ERRNO_IS_RESOURCE_LIMIT(e)) { warn_too_many_conns(); return 0; } @@ -1308,8 +1503,15 @@ connection_handle_listener_read(connection_t *conn, int new_type) "Connection accepted on socket %d (child of fd %d).", (int)news,(int)conn->s); - make_socket_reuseable(news); - if (set_socket_nonblocking(news) == -1) { + if (make_socket_reuseable(news) < 0) { + if (tor_socket_errno(news) == EINVAL) { + /* This can happen on OSX if we get a badly timed shutdown. */ + log_debug(LD_NET, "make_socket_reuseable returned EINVAL"); + } else { + log_warn(LD_NET, "Error setting SO_REUSEADDR flag on %s: %s", + conn_type_to_string(new_type), + tor_socket_strerror(errno)); + } tor_close_socket(news); return 0; } @@ -1322,7 +1524,8 @@ connection_handle_listener_read(connection_t *conn, int new_type) return 0; } - if (conn->socket_family == AF_INET || conn->socket_family == AF_INET6) { + if (conn->socket_family == AF_INET || conn->socket_family == AF_INET6 || + (conn->socket_family == AF_UNIX && new_type == CONN_TYPE_AP)) { tor_addr_t addr; uint16_t port; if (check_sockaddr(remote, remotelen, LOG_INFO)<0) { @@ -1363,15 +1566,24 @@ connection_handle_listener_read(connection_t *conn, int new_type) newconn->port = port; newconn->address = tor_dup_addr(&addr); - if (new_type == CONN_TYPE_AP) { - TO_ENTRY_CONN(newconn)->socks_request->socks_prefer_no_auth = - TO_LISTENER_CONN(conn)->socks_prefer_no_auth; + if (new_type == CONN_TYPE_AP && conn->socket_family != AF_UNIX) { + log_info(LD_NET, "New SOCKS connection opened from %s.", + fmt_and_decorate_addr(&addr)); + } + if (new_type == CONN_TYPE_AP && conn->socket_family == AF_UNIX) { + newconn->port = 0; + newconn->address = tor_strdup(conn->address); + log_info(LD_NET, "New SOCKS AF_UNIX connection opened"); + } + if (new_type == CONN_TYPE_CONTROL) { + log_notice(LD_CONTROL, "New control connection opened from %s.", + fmt_and_decorate_addr(&addr)); } - } else if (conn->socket_family == AF_UNIX) { - /* For now only control ports can be Unix domain sockets - * and listeners at the same time */ + } else if (conn->socket_family == AF_UNIX && conn->type != CONN_TYPE_AP) { tor_assert(conn->type == CONN_TYPE_CONTROL_LISTENER); + tor_assert(new_type == CONN_TYPE_CONTROL); + log_notice(LD_CONTROL, "New control connection opened."); newconn = connection_new(new_type, conn->socket_family); newconn->s = news; @@ -1411,6 +1623,9 @@ connection_init_accepted_conn(connection_t *conn, connection_start_reading(conn); switch (conn->type) { + case CONN_TYPE_EXT_OR: + /* Initiate Extended ORPort authentication. */ + return connection_ext_or_start_auth(TO_OR_CONN(conn)); case CONN_TYPE_OR: control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1); @@ -1420,28 +1635,21 @@ connection_init_accepted_conn(connection_t *conn, return rv; break; case CONN_TYPE_AP: - TO_ENTRY_CONN(conn)->isolation_flags = listener->isolation_flags; - TO_ENTRY_CONN(conn)->session_group = listener->session_group; + memcpy(&TO_ENTRY_CONN(conn)->entry_cfg, &listener->entry_cfg, + sizeof(entry_port_cfg_t)); TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch(); TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type; - TO_ENTRY_CONN(conn)->ipv4_traffic_ok = listener->socks_ipv4_traffic; - TO_ENTRY_CONN(conn)->ipv6_traffic_ok = listener->socks_ipv6_traffic; - TO_ENTRY_CONN(conn)->prefer_ipv6_traffic = listener->socks_prefer_ipv6; - TO_ENTRY_CONN(conn)->cache_ipv4_answers = listener->cache_ipv4_answers; - TO_ENTRY_CONN(conn)->cache_ipv6_answers = listener->cache_ipv6_answers; - TO_ENTRY_CONN(conn)->use_cached_ipv4_answers = - listener->use_cached_ipv4_answers; - TO_ENTRY_CONN(conn)->use_cached_ipv6_answers = - listener->use_cached_ipv6_answers; - TO_ENTRY_CONN(conn)->prefer_ipv6_virtaddr = - listener->prefer_ipv6_virtaddr; switch (TO_CONN(listener)->type) { case CONN_TYPE_AP_LISTENER: conn->state = AP_CONN_STATE_SOCKS_WAIT; + TO_ENTRY_CONN(conn)->socks_request->socks_prefer_no_auth = + listener->entry_cfg.socks_prefer_no_auth; break; case CONN_TYPE_AP_TRANS_LISTENER: TO_ENTRY_CONN(conn)->is_transparent_ap = 1; + /* XXXX028 -- is this correct still, with the addition of + * pending_entry_connections ? */ conn->state = AP_CONN_STATE_CIRCUIT_WAIT; return connection_ap_process_transparent(TO_ENTRY_CONN(conn)); case CONN_TYPE_AP_NATD_LISTENER: @@ -1462,36 +1670,25 @@ connection_init_accepted_conn(connection_t *conn, } /** Take conn, make a nonblocking socket; try to connect to - * addr:port (they arrive in *host order*). If fail, return -1 and if + * sa, binding to bindaddr if sa is not localhost. If fail, return -1 and if * applicable put your best guess about errno into *<b>socket_error</b>. - * Else assign s to conn-\>s: if connected return 1, if EAGAIN return 0. - * - * address is used to make the logs useful. - * - * On success, add conn to the list of polled connections. + * If connected return 1, if EAGAIN return 0. */ -int -connection_connect(connection_t *conn, const char *address, - const tor_addr_t *addr, uint16_t port, int *socket_error) +MOCK_IMPL(STATIC int, +connection_connect_sockaddr,(connection_t *conn, + const struct sockaddr *sa, + socklen_t sa_len, + const struct sockaddr *bindaddr, + socklen_t bindaddr_len, + int *socket_error)) { tor_socket_t s; int inprogress = 0; - struct sockaddr_storage addrbuf; - struct sockaddr *dest_addr; - int dest_addr_len; const or_options_t *options = get_options(); - int protocol_family; - if (get_n_open_sockets() >= get_options()->ConnLimit_-1) { - warn_too_many_conns(); - *socket_error = SOCK_ERRNO(ENOBUFS); - return -1; - } - - if (tor_addr_family(addr) == AF_INET6) - protocol_family = PF_INET6; - else - protocol_family = PF_INET; + tor_assert(conn); + tor_assert(sa); + tor_assert(socket_error); if (get_options()->DisableNetwork) { /* We should never even try to connect anyplace if DisableNetwork is set. @@ -1504,74 +1701,47 @@ connection_connect(connection_t *conn, const char *address, return -1; } - s = tor_open_socket(protocol_family,SOCK_STREAM,IPPROTO_TCP); + const int protocol_family = sa->sa_family; + const int proto = (sa->sa_family == AF_INET6 || + sa->sa_family == AF_INET) ? IPPROTO_TCP : 0; + + s = tor_open_socket_nonblocking(protocol_family, SOCK_STREAM, proto); if (! SOCKET_OK(s)) { - *socket_error = tor_socket_errno(-1); - log_warn(LD_NET,"Error creating network socket: %s", - tor_socket_strerror(*socket_error)); + *socket_error = tor_socket_errno(s); + if (ERRNO_IS_RESOURCE_LIMIT(*socket_error)) { + warn_too_many_conns(); + } else { + log_warn(LD_NET,"Error creating network socket: %s", + tor_socket_strerror(*socket_error)); + } return -1; } - make_socket_reuseable(s); - - if (!tor_addr_is_loopback(addr)) { - const tor_addr_t *ext_addr = NULL; - if (protocol_family == AF_INET && - !tor_addr_is_null(&options->OutboundBindAddressIPv4_)) - ext_addr = &options->OutboundBindAddressIPv4_; - else if (protocol_family == AF_INET6 && - !tor_addr_is_null(&options->OutboundBindAddressIPv6_)) - ext_addr = &options->OutboundBindAddressIPv6_; - if (ext_addr) { - struct sockaddr_storage ext_addr_sa; - socklen_t ext_addr_len = 0; - memset(&ext_addr_sa, 0, sizeof(ext_addr_sa)); - ext_addr_len = tor_addr_to_sockaddr(ext_addr, 0, - (struct sockaddr *) &ext_addr_sa, - sizeof(ext_addr_sa)); - if (ext_addr_len == 0) { - log_warn(LD_NET, - "Error converting OutboundBindAddress %s into sockaddr. " - "Ignoring.", fmt_and_decorate_addr(ext_addr)); - } else { - if (bind(s, (struct sockaddr *) &ext_addr_sa, ext_addr_len) < 0) { - *socket_error = tor_socket_errno(s); - log_warn(LD_NET,"Error binding network socket to %s: %s", - fmt_and_decorate_addr(ext_addr), - tor_socket_strerror(*socket_error)); - tor_close_socket(s); - return -1; - } - } - } + if (make_socket_reuseable(s) < 0) { + log_warn(LD_NET, "Error setting SO_REUSEADDR flag on new connection: %s", + tor_socket_strerror(errno)); } - if (set_socket_nonblocking(s) == -1) { + if (bindaddr && bind(s, bindaddr, bindaddr_len) < 0) { *socket_error = tor_socket_errno(s); + log_warn(LD_NET,"Error binding network socket: %s", + tor_socket_strerror(*socket_error)); tor_close_socket(s); return -1; } + tor_assert(options); if (options->ConstrainedSockets) set_constrained_socket_buffers(s, (int)options->ConstrainedSockSize); - memset(&addrbuf,0,sizeof(addrbuf)); - dest_addr = (struct sockaddr*) &addrbuf; - dest_addr_len = tor_addr_to_sockaddr(addr, port, dest_addr, sizeof(addrbuf)); - tor_assert(dest_addr_len > 0); - - log_debug(LD_NET, "Connecting to %s:%u.", - escaped_safe_str_client(address), port); - - if (connect(s, dest_addr, (socklen_t)dest_addr_len) < 0) { + if (connect(s, sa, sa_len) < 0) { int e = tor_socket_errno(s); if (!ERRNO_IS_CONN_EINPROGRESS(e)) { /* yuck. kill it. */ *socket_error = e; log_info(LD_NET, - "connect() to %s:%u failed: %s", - escaped_safe_str_client(address), - port, tor_socket_strerror(e)); + "connect() to socket failed: %s", + tor_socket_strerror(e)); tor_close_socket(s); return -1; } else { @@ -1580,19 +1750,201 @@ connection_connect(connection_t *conn, const char *address, } /* it succeeded. we're connected. */ - log_fn(inprogress?LOG_DEBUG:LOG_INFO, LD_NET, - "Connection to %s:%u %s (sock "TOR_SOCKET_T_FORMAT").", - escaped_safe_str_client(address), - port, inprogress?"in progress":"established", s); + log_fn(inprogress ? LOG_DEBUG : LOG_INFO, LD_NET, + "Connection to socket %s (sock "TOR_SOCKET_T_FORMAT").", + inprogress ? "in progress" : "established", s); conn->s = s; if (connection_add_connecting(conn) < 0) { /* no space, forget it */ *socket_error = SOCK_ERRNO(ENOBUFS); return -1; } + return inprogress ? 0 : 1; } +/* Log a message if connection attempt is made when IPv4 or IPv6 is disabled. + * Log a less severe message if we couldn't conform to ClientPreferIPv6ORPort + * or ClientPreferIPv6ORPort. */ +static void +connection_connect_log_client_use_ip_version(const connection_t *conn) +{ + const or_options_t *options = get_options(); + + /* Only clients care about ClientUseIPv4/6, bail out early on servers, and + * on connections we don't care about */ + if (server_mode(options) || !conn || conn->type == CONN_TYPE_EXIT) { + return; + } + + /* We're only prepared to log OR and DIR connections here */ + if (conn->type != CONN_TYPE_OR && conn->type != CONN_TYPE_DIR) { + return; + } + + const int must_ipv4 = !fascist_firewall_use_ipv6(options); + const int must_ipv6 = (options->ClientUseIPv4 == 0); + const int pref_ipv6 = (conn->type == CONN_TYPE_OR + ? fascist_firewall_prefer_ipv6_orport(options) + : fascist_firewall_prefer_ipv6_dirport(options)); + tor_addr_t real_addr; + tor_addr_make_null(&real_addr, AF_UNSPEC); + + /* OR conns keep the original address in real_addr, as addr gets overwritten + * with the descriptor address */ + if (conn->type == CONN_TYPE_OR) { + const or_connection_t *or_conn = TO_OR_CONN((connection_t *)conn); + tor_addr_copy(&real_addr, &or_conn->real_addr); + } else if (conn->type == CONN_TYPE_DIR) { + tor_addr_copy(&real_addr, &conn->addr); + } + + /* Check if we broke a mandatory address family restriction */ + if ((must_ipv4 && tor_addr_family(&real_addr) == AF_INET6) + || (must_ipv6 && tor_addr_family(&real_addr) == AF_INET)) { + static int logged_backtrace = 0; + log_info(LD_BUG, "Outgoing %s connection to %s violated ClientUseIPv%s 0.", + conn->type == CONN_TYPE_OR ? "OR" : "Dir", + fmt_addr(&real_addr), + options->ClientUseIPv4 == 0 ? "4" : "6"); + if (!logged_backtrace) { + log_backtrace(LOG_INFO, LD_BUG, "Address came from"); + logged_backtrace = 1; + } + } + + /* Bridges are allowed to break IPv4/IPv6 ORPort preferences to connect to + * the node's configured address when ClientPreferIPv6ORPort is auto */ + if (options->UseBridges && conn->type == CONN_TYPE_OR + && options->ClientPreferIPv6ORPort == -1) { + return; + } + + /* Check if we couldn't satisfy an address family preference */ + if ((!pref_ipv6 && tor_addr_family(&real_addr) == AF_INET6) + || (pref_ipv6 && tor_addr_family(&real_addr) == AF_INET)) { + log_info(LD_NET, "Outgoing connection to %s doesn't satisfy " + "ClientPreferIPv6%sPort %d, with ClientUseIPv4 %d, and " + "fascist_firewall_use_ipv6 %d (ClientUseIPv6 %d and UseBridges " + "%d).", + fmt_addr(&real_addr), + conn->type == CONN_TYPE_OR ? "OR" : "Dir", + conn->type == CONN_TYPE_OR ? options->ClientPreferIPv6ORPort + : options->ClientPreferIPv6DirPort, + options->ClientUseIPv4, fascist_firewall_use_ipv6(options), + options->ClientUseIPv6, options->UseBridges); + } +} + +/** Take conn, make a nonblocking socket; try to connect to + * addr:port (port arrives in *host order*). If fail, return -1 and if + * applicable put your best guess about errno into *<b>socket_error</b>. + * Else assign s to conn-\>s: if connected return 1, if EAGAIN return 0. + * + * addr:port can be different to conn->addr:conn->port if connecting through + * a proxy. + * + * address is used to make the logs useful. + * + * On success, add conn to the list of polled connections. + */ +int +connection_connect(connection_t *conn, const char *address, + const tor_addr_t *addr, uint16_t port, int *socket_error) +{ + struct sockaddr_storage addrbuf; + struct sockaddr_storage bind_addr_ss; + struct sockaddr *bind_addr = NULL; + struct sockaddr *dest_addr; + int dest_addr_len, bind_addr_len = 0; + const or_options_t *options = get_options(); + int protocol_family; + + /* Log if we didn't stick to ClientUseIPv4/6 or ClientPreferIPv6OR/DirPort + */ + connection_connect_log_client_use_ip_version(conn); + + if (tor_addr_family(addr) == AF_INET6) + protocol_family = PF_INET6; + else + protocol_family = PF_INET; + + if (!tor_addr_is_loopback(addr)) { + const tor_addr_t *ext_addr = NULL; + if (protocol_family == AF_INET && + !tor_addr_is_null(&options->OutboundBindAddressIPv4_)) + ext_addr = &options->OutboundBindAddressIPv4_; + else if (protocol_family == AF_INET6 && + !tor_addr_is_null(&options->OutboundBindAddressIPv6_)) + ext_addr = &options->OutboundBindAddressIPv6_; + if (ext_addr) { + memset(&bind_addr_ss, 0, sizeof(bind_addr_ss)); + bind_addr_len = tor_addr_to_sockaddr(ext_addr, 0, + (struct sockaddr *) &bind_addr_ss, + sizeof(bind_addr_ss)); + if (bind_addr_len == 0) { + log_warn(LD_NET, + "Error converting OutboundBindAddress %s into sockaddr. " + "Ignoring.", fmt_and_decorate_addr(ext_addr)); + } else { + bind_addr = (struct sockaddr *)&bind_addr_ss; + } + } + } + + memset(&addrbuf,0,sizeof(addrbuf)); + dest_addr = (struct sockaddr*) &addrbuf; + dest_addr_len = tor_addr_to_sockaddr(addr, port, dest_addr, sizeof(addrbuf)); + tor_assert(dest_addr_len > 0); + + log_debug(LD_NET, "Connecting to %s:%u.", + escaped_safe_str_client(address), port); + + return connection_connect_sockaddr(conn, dest_addr, dest_addr_len, + bind_addr, bind_addr_len, socket_error); +} + +#ifdef HAVE_SYS_UN_H + +/** Take conn, make a nonblocking socket; try to connect to + * an AF_UNIX socket at socket_path. If fail, return -1 and if applicable + * put your best guess about errno into *<b>socket_error</b>. Else assign s + * to conn-\>s: if connected return 1, if EAGAIN return 0. + * + * On success, add conn to the list of polled connections. + */ +int +connection_connect_unix(connection_t *conn, const char *socket_path, + int *socket_error) +{ + struct sockaddr_un dest_addr; + + tor_assert(socket_path); + + /* Check that we'll be able to fit it into dest_addr later */ + if (strlen(socket_path) + 1 > sizeof(dest_addr.sun_path)) { + log_warn(LD_NET, + "Path %s is too long for an AF_UNIX socket\n", + escaped_safe_str_client(socket_path)); + *socket_error = SOCK_ERRNO(ENAMETOOLONG); + return -1; + } + + memset(&dest_addr, 0, sizeof(dest_addr)); + dest_addr.sun_family = AF_UNIX; + strlcpy(dest_addr.sun_path, socket_path, sizeof(dest_addr.sun_path)); + + log_debug(LD_NET, + "Connecting to AF_UNIX socket at %s.", + escaped_safe_str_client(socket_path)); + + return connection_connect_sockaddr(conn, + (struct sockaddr *)&dest_addr, sizeof(dest_addr), + NULL, 0, socket_error); +} + +#endif /* defined(HAVE_SYS_UN_H) */ + /** Convert state number to string representation for logging purposes. */ static const char * @@ -1617,6 +1969,32 @@ connection_proxy_state_to_string(int state) return states[state]; } +/** Returns the global proxy type used by tor. Use this function for + * logging or high-level purposes, don't use it to fill the + * <b>proxy_type</b> field of or_connection_t; use the actual proxy + * protocol instead.*/ +static int +get_proxy_type(void) +{ + const or_options_t *options = get_options(); + + if (options->ClientTransportPlugin) + return PROXY_PLUGGABLE; + else if (options->HTTPSProxy) + return PROXY_CONNECT; + else if (options->Socks4Proxy) + return PROXY_SOCKS4; + else if (options->Socks5Proxy) + return PROXY_SOCKS5; + else + return PROXY_NONE; +} + +/* One byte for the version, one for the command, two for the + port, and four for the addr... and, one more for the + username NUL: */ +#define SOCKS4_STANDARD_BUFFER_SIZE (1 + 1 + 2 + 4 + 1) + /** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn * for conn->addr:conn->port, authenticating with the auth details given * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies @@ -1671,17 +2049,45 @@ connection_proxy_connect(connection_t *conn, int type) } case PROXY_SOCKS4: { - unsigned char buf[9]; + unsigned char *buf; uint16_t portn; uint32_t ip4addr; + size_t buf_size = 0; + char *socks_args_string = NULL; - /* Send a SOCKS4 connect request with empty user id */ + /* Send a SOCKS4 connect request */ if (tor_addr_family(&conn->addr) != AF_INET) { log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6"); return -1; } + { /* If we are here because we are trying to connect to a + pluggable transport proxy, check if we have any SOCKS + arguments to transmit. If we do, compress all arguments to + a single string in 'socks_args_string': */ + + if (get_proxy_type() == PROXY_PLUGGABLE) { + socks_args_string = + pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port); + if (socks_args_string) + log_debug(LD_NET, "Sending out '%s' as our SOCKS argument string.", + socks_args_string); + } + } + + { /* Figure out the buffer size we need for the SOCKS message: */ + + buf_size = SOCKS4_STANDARD_BUFFER_SIZE; + + /* If we have a SOCKS argument string, consider its size when + calculating the buffer size: */ + if (socks_args_string) + buf_size += strlen(socks_args_string); + } + + buf = tor_malloc_zero(buf_size); + ip4addr = tor_addr_to_ipv4n(&conn->addr); portn = htons(conn->port); @@ -1689,9 +2095,23 @@ connection_proxy_connect(connection_t *conn, int type) buf[1] = SOCKS_COMMAND_CONNECT; /* command */ memcpy(buf + 2, &portn, 2); /* port */ memcpy(buf + 4, &ip4addr, 4); /* addr */ - buf[8] = 0; /* userid (empty) */ - connection_write_to_buf((char *)buf, sizeof(buf), conn); + /* Next packet field is the userid. If we have pluggable + transport SOCKS arguments, we have to embed them + there. Otherwise, we use an empty userid. */ + if (socks_args_string) { /* place the SOCKS args string: */ + tor_assert(strlen(socks_args_string) > 0); + tor_assert(buf_size >= + SOCKS4_STANDARD_BUFFER_SIZE + strlen(socks_args_string)); + strlcpy((char *)buf + 8, socks_args_string, buf_size - 8); + tor_free(socks_args_string); + } else { + buf[8] = 0; /* no userid */ + } + + connection_write_to_buf((char *)buf, buf_size, conn); + tor_free(buf); + conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK; break; } @@ -1703,8 +2123,13 @@ connection_proxy_connect(connection_t *conn, int type) buf[0] = 5; /* version */ + /* We have to use SOCKS5 authentication, if we have a + Socks5ProxyUsername or if we want to pass arguments to our + pluggable transport proxy: */ + if ((options->Socks5ProxyUsername) || + (get_proxy_type() == PROXY_PLUGGABLE && + (get_socks_args_by_bridge_addrport(&conn->addr, conn->port)))) { /* number of auth methods */ - if (options->Socks5ProxyUsername) { buf[1] = 2; buf[2] = 0x00; /* no authentication */ buf[3] = 0x02; /* rfc1929 Username/Passwd auth */ @@ -1898,15 +2323,49 @@ connection_read_proxy_handshake(connection_t *conn) unsigned char buf[1024]; size_t reqsize, usize, psize; const char *user, *pass; + char *socks_args_string = NULL; + + if (get_proxy_type() == PROXY_PLUGGABLE) { + socks_args_string = + pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port); + if (!socks_args_string) { + log_warn(LD_NET, "Could not create SOCKS args string."); + ret = -1; + break; + } + + log_debug(LD_NET, "SOCKS5 arguments: %s", socks_args_string); + tor_assert(strlen(socks_args_string) > 0); + tor_assert(strlen(socks_args_string) <= MAX_SOCKS5_AUTH_SIZE_TOTAL); + + if (strlen(socks_args_string) > MAX_SOCKS5_AUTH_FIELD_SIZE) { + user = socks_args_string; + usize = MAX_SOCKS5_AUTH_FIELD_SIZE; + pass = socks_args_string + MAX_SOCKS5_AUTH_FIELD_SIZE; + psize = strlen(socks_args_string) - MAX_SOCKS5_AUTH_FIELD_SIZE; + } else { + user = socks_args_string; + usize = strlen(socks_args_string); + pass = "\0"; + psize = 1; + } + } else if (get_options()->Socks5ProxyUsername) { + user = get_options()->Socks5ProxyUsername; + pass = get_options()->Socks5ProxyPassword; + tor_assert(user && pass); + usize = strlen(user); + psize = strlen(pass); + } else { + log_err(LD_BUG, "We entered %s for no reason!", __func__); + tor_fragile_assert(); + ret = -1; + break; + } - user = get_options()->Socks5ProxyUsername; - pass = get_options()->Socks5ProxyPassword; - tor_assert(user && pass); - - /* XXX len of user and pass must be <= 255 !!! */ - usize = strlen(user); - psize = strlen(pass); - tor_assert(usize <= 255 && psize <= 255); + /* Username and password lengths should have been checked + above and during torrc parsing. */ + tor_assert(usize <= MAX_SOCKS5_AUTH_FIELD_SIZE && + psize <= MAX_SOCKS5_AUTH_FIELD_SIZE); reqsize = 3 + usize + psize; buf[0] = 1; /* negotiation version */ @@ -1915,6 +2374,9 @@ connection_read_proxy_handshake(connection_t *conn) buf[2 + usize] = psize; memcpy(buf + 3 + usize, pass, psize); + if (socks_args_string) + tor_free(socks_args_string); + connection_write_to_buf((char *)buf, reqsize, conn); conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_RFC1929_OK; @@ -2013,7 +2475,7 @@ retry_listener_ports(smartlist_t *old_conns, (conn->socket_family == AF_UNIX && ! wanted->is_unix_addr)) continue; - if (wanted->no_listen) + if (wanted->server_cfg.no_listen) continue; /* We don't want to open a listener for this one */ if (wanted->is_unix_addr) { @@ -2054,8 +2516,17 @@ retry_listener_ports(smartlist_t *old_conns, connection_t *conn; int real_port = port->port == CFG_AUTO_PORT ? 0 : port->port; tor_assert(real_port <= UINT16_MAX); - if (port->no_listen) + if (port->server_cfg.no_listen) + continue; + +#ifndef _WIN32 + /* We don't need to be root to create a UNIX socket, so defer until after + * setuid. */ + const or_options_t *options = get_options(); + if (port->is_unix_addr && !geteuid() && (options->User) && + strcmp(options->User, "root")) continue; +#endif if (port->is_unix_addr) { listensockaddr = (struct sockaddr *) @@ -2072,7 +2543,7 @@ retry_listener_ports(smartlist_t *old_conns, if (listensockaddr) { conn = connection_listener_new(listensockaddr, listensocklen, - port->type, address, port); + port->type, address, port); tor_free(listensockaddr); tor_free(address); } else { @@ -2176,7 +2647,6 @@ connection_mark_all_noncontrol_connections(void) if (conn->marked_for_close) continue; switch (conn->type) { - case CONN_TYPE_CPUWORKER: case CONN_TYPE_CONTROL_LISTENER: case CONN_TYPE_CONTROL: break; @@ -2184,6 +2654,20 @@ connection_mark_all_noncontrol_connections(void) connection_mark_unattached_ap(TO_ENTRY_CONN(conn), END_STREAM_REASON_HIBERNATING); break; + case CONN_TYPE_OR: + { + or_connection_t *orconn = TO_OR_CONN(conn); + if (orconn->chan) { + connection_or_close_normally(orconn, 0); + } else { + /* + * There should have been one, but mark for close and hope + * for the best.. + */ + connection_mark_for_close(conn); + } + } + break; default: connection_mark_for_close(conn); break; @@ -2205,6 +2689,7 @@ connection_is_rate_limited(connection_t *conn) return 0; /* Internal connection */ else if (! options->CountPrivateBandwidth && (tor_addr_family(&conn->addr) == AF_UNSPEC || /* no address */ + tor_addr_family(&conn->addr) == AF_UNIX || /* no address */ tor_addr_is_internal(&conn->addr, 0))) return 0; /* Internal address */ else @@ -2358,9 +2843,8 @@ connection_bucket_write_limit(connection_t *conn, time_t now) * shouldn't send <b>attempt</b> bytes of low-priority directory stuff * out to <b>conn</b>. Else return 0. - * Priority is 1 for v1 requests (directories and running-routers), - * and 2 for v2 requests (statuses and descriptors). But see FFFF in - * directory_handle_command_get() for why we don't use priority 2 yet. + * Priority was 1 for v1 requests (directories and running-routers), + * and 2 for v2 requests and later (statuses and descriptors). * * There are a lot of parameters we could use here: * - global_relayed_write_bucket. Low is bad. @@ -2466,7 +2950,58 @@ record_num_bytes_transferred(connection_t *conn, } #endif +/** Helper: convert given <b>tvnow</b> time value to milliseconds since + * midnight. */ +static uint32_t +msec_since_midnight(const struct timeval *tvnow) +{ + return (uint32_t)(((tvnow->tv_sec % 86400L) * 1000L) + + ((uint32_t)tvnow->tv_usec / (uint32_t)1000L)); +} + +/** Helper: return the time in milliseconds since <b>last_empty_time</b> + * when a bucket ran empty that previously had <b>tokens_before</b> tokens + * now has <b>tokens_after</b> tokens after refilling at timestamp + * <b>tvnow</b>, capped at <b>milliseconds_elapsed</b> milliseconds since + * last refilling that bucket. Return 0 if the bucket has not been empty + * since the last refill or has not been refilled. */ +uint32_t +bucket_millis_empty(int tokens_before, uint32_t last_empty_time, + int tokens_after, int milliseconds_elapsed, + const struct timeval *tvnow) +{ + uint32_t result = 0, refilled; + if (tokens_before <= 0 && tokens_after > tokens_before) { + refilled = msec_since_midnight(tvnow); + result = (uint32_t)((refilled + 86400L * 1000L - last_empty_time) % + (86400L * 1000L)); + if (result > (uint32_t)milliseconds_elapsed) + result = (uint32_t)milliseconds_elapsed; + } + return result; +} + +/** Check if a bucket which had <b>tokens_before</b> tokens and which got + * <b>tokens_removed</b> tokens removed at timestamp <b>tvnow</b> has run + * out of tokens, and if so, note the milliseconds since midnight in + * <b>timestamp_var</b> for the next TB_EMPTY event. */ +void +connection_buckets_note_empty_ts(uint32_t *timestamp_var, + int tokens_before, size_t tokens_removed, + const struct timeval *tvnow) +{ + if (tokens_before > 0 && (uint32_t)tokens_before <= tokens_removed) + *timestamp_var = msec_since_midnight(tvnow); +} + #ifndef USE_BUFFEREVENTS +/** Last time at which the global or relay buckets were emptied in msec + * since midnight. */ +static uint32_t global_relayed_read_emptied = 0, + global_relayed_write_emptied = 0, + global_read_emptied = 0, + global_write_emptied = 0; + /** We just read <b>num_read</b> and wrote <b>num_written</b> bytes * onto <b>conn</b>. Decrement buckets appropriately. */ static void @@ -2489,6 +3024,30 @@ connection_buckets_decrement(connection_t *conn, time_t now, if (!connection_is_rate_limited(conn)) return; /* local IPs are free */ + /* If one or more of our token buckets ran dry just now, note the + * timestamp for TB_EMPTY events. */ + if (get_options()->TestingEnableTbEmptyEvent) { + struct timeval tvnow; + tor_gettimeofday_cached(&tvnow); + if (connection_counts_as_relayed_traffic(conn, now)) { + connection_buckets_note_empty_ts(&global_relayed_read_emptied, + global_relayed_read_bucket, num_read, &tvnow); + connection_buckets_note_empty_ts(&global_relayed_write_emptied, + global_relayed_write_bucket, num_written, &tvnow); + } + connection_buckets_note_empty_ts(&global_read_emptied, + global_read_bucket, num_read, &tvnow); + connection_buckets_note_empty_ts(&global_write_emptied, + global_write_bucket, num_written, &tvnow); + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { + or_connection_t *or_conn = TO_OR_CONN(conn); + connection_buckets_note_empty_ts(&or_conn->read_emptied_time, + or_conn->read_bucket, num_read, &tvnow); + connection_buckets_note_empty_ts(&or_conn->write_emptied_time, + or_conn->write_bucket, num_written, &tvnow); + } + } + if (connection_counts_as_relayed_traffic(conn, now)) { global_relayed_read_bucket -= (int)num_read; global_relayed_write_bucket -= (int)num_written; @@ -2508,6 +3067,9 @@ connection_consider_empty_read_buckets(connection_t *conn) { const char *reason; + if (!connection_is_rate_limited(conn)) + return; /* Always okay. */ + if (global_read_bucket <= 0) { reason = "global read bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && @@ -2520,9 +3082,6 @@ connection_consider_empty_read_buckets(connection_t *conn) } else return; /* all good, no need to stop it */ - if (conn->type == CONN_TYPE_CPUWORKER) - return; /* Always okay. */ - LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); conn->read_blocked_on_bw = 1; connection_stop_reading(conn); @@ -2535,6 +3094,9 @@ connection_consider_empty_write_buckets(connection_t *conn) { const char *reason; + if (!connection_is_rate_limited(conn)) + return; /* Always okay. */ + if (global_write_bucket <= 0) { reason = "global write bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && @@ -2547,9 +3109,6 @@ connection_consider_empty_write_buckets(connection_t *conn) } else return; /* all good, no need to stop it */ - if (conn->type == CONN_TYPE_CPUWORKER) - return; /* Always okay. */ - LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); conn->write_blocked_on_bw = 1; connection_stop_writing(conn); @@ -2609,6 +3168,12 @@ connection_bucket_refill(int milliseconds_elapsed, time_t now) smartlist_t *conns = get_connection_array(); int bandwidthrate, bandwidthburst, relayrate, relayburst; + int prev_global_read = global_read_bucket; + int prev_global_write = global_write_bucket; + int prev_relay_read = global_relayed_read_bucket; + int prev_relay_write = global_relayed_write_bucket; + struct timeval tvnow; /*< Only used if TB_EMPTY events are enabled. */ + bandwidthrate = (int)options->BandwidthRate; bandwidthburst = (int)options->BandwidthBurst; @@ -2643,12 +3208,42 @@ connection_bucket_refill(int milliseconds_elapsed, time_t now) milliseconds_elapsed, "global_relayed_write_bucket"); + /* If buckets were empty before and have now been refilled, tell any + * interested controllers. */ + if (get_options()->TestingEnableTbEmptyEvent) { + uint32_t global_read_empty_time, global_write_empty_time, + relay_read_empty_time, relay_write_empty_time; + tor_gettimeofday_cached(&tvnow); + global_read_empty_time = bucket_millis_empty(prev_global_read, + global_read_emptied, global_read_bucket, + milliseconds_elapsed, &tvnow); + global_write_empty_time = bucket_millis_empty(prev_global_write, + global_write_emptied, global_write_bucket, + milliseconds_elapsed, &tvnow); + control_event_tb_empty("GLOBAL", global_read_empty_time, + global_write_empty_time, milliseconds_elapsed); + relay_read_empty_time = bucket_millis_empty(prev_relay_read, + global_relayed_read_emptied, + global_relayed_read_bucket, + milliseconds_elapsed, &tvnow); + relay_write_empty_time = bucket_millis_empty(prev_relay_write, + global_relayed_write_emptied, + global_relayed_write_bucket, + milliseconds_elapsed, &tvnow); + control_event_tb_empty("RELAY", relay_read_empty_time, + relay_write_empty_time, milliseconds_elapsed); + } + /* refill the per-connection buckets */ SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); int orbandwidthrate = or_conn->bandwidthrate; int orbandwidthburst = or_conn->bandwidthburst; + + int prev_conn_read = or_conn->read_bucket; + int prev_conn_write = or_conn->write_bucket; + if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { connection_bucket_refill_helper(&or_conn->read_bucket, orbandwidthrate, @@ -2663,6 +3258,27 @@ connection_bucket_refill(int milliseconds_elapsed, time_t now) milliseconds_elapsed, "or_conn->write_bucket"); } + + /* If buckets were empty before and have now been refilled, tell any + * interested controllers. */ + if (get_options()->TestingEnableTbEmptyEvent) { + char *bucket; + uint32_t conn_read_empty_time, conn_write_empty_time; + tor_asprintf(&bucket, "ORCONN ID="U64_FORMAT, + U64_PRINTF_ARG(or_conn->base_.global_identifier)); + conn_read_empty_time = bucket_millis_empty(prev_conn_read, + or_conn->read_emptied_time, + or_conn->read_bucket, + milliseconds_elapsed, &tvnow); + conn_write_empty_time = bucket_millis_empty(prev_conn_write, + or_conn->write_emptied_time, + or_conn->write_bucket, + milliseconds_elapsed, &tvnow); + control_event_tb_empty(bucket, conn_read_empty_time, + conn_write_empty_time, + milliseconds_elapsed); + tor_free(bucket); + } } if (conn->read_blocked_on_bw == 1 /* marked to turn reading back on now */ @@ -2819,6 +3435,8 @@ connection_handle_read_impl(connection_t *conn) switch (conn->type) { case CONN_TYPE_OR_LISTENER: return connection_handle_listener_read(conn, CONN_TYPE_OR); + case CONN_TYPE_EXT_OR_LISTENER: + return connection_handle_listener_read(conn, CONN_TYPE_EXT_OR); case CONN_TYPE_AP_LISTENER: case CONN_TYPE_AP_TRANS_LISTENER: case CONN_TYPE_AP_NATD_LISTENER: @@ -3071,14 +3689,37 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, /* change *max_to_read */ *max_to_read = at_most - n_read; - /* Update edge_conn->n_read */ + /* Update edge_conn->n_read and ocirc->n_read_circ_bw */ if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + circuit_t *circ = circuit_get_by_edge_conn(edge_conn); + origin_circuit_t *ocirc; + /* Check for overflow: */ if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_read > n_read)) edge_conn->n_read += (int)n_read; else edge_conn->n_read = UINT32_MAX; + + if (circ && CIRCUIT_IS_ORIGIN(circ)) { + ocirc = TO_ORIGIN_CIRCUIT(circ); + if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_read_circ_bw > n_read)) + ocirc->n_read_circ_bw += (int)n_read; + else + ocirc->n_read_circ_bw = UINT32_MAX; + } + } + + /* If CONN_BW events are enabled, update conn->n_read_conn_bw for + * OR/DIR/EXIT connections, checking for overflow. */ + if (get_options()->TestingEnableConnBwEvent && + (conn->type == CONN_TYPE_OR || + conn->type == CONN_TYPE_DIR || + conn->type == CONN_TYPE_EXIT)) { + if (PREDICT_LIKELY(UINT32_MAX - conn->n_read_conn_bw > n_read)) + conn->n_read_conn_bw += (int)n_read; + else + conn->n_read_conn_bw = UINT32_MAX; } } @@ -3091,7 +3732,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, } /* Call even if result is 0, since the global read bucket may - * have reached 0 on a different conn, and this guy needs to + * have reached 0 on a different conn, and this connection needs to * know to stop reading. */ connection_consider_empty_read_buckets(conn); if (n_written > 0 && connection_is_writing(conn)) @@ -3293,7 +3934,7 @@ connection_fetch_from_buf_line(connection_t *conn, char *data, } } -/** As fetch_from_buf_http, but fetches from a conncetion's input buffer_t or +/** As fetch_from_buf_http, but fetches from a connection's input buffer_t or * its bufferevent as appropriate. */ int connection_fetch_from_buf_http(connection_t *conn, @@ -3331,8 +3972,8 @@ connection_outbuf_too_full(connection_t *conn) /** Try to flush more bytes onto <b>conn</b>-\>s. * - * This function gets called either from conn_write() in main.c - * when poll() has declared that conn wants to write, or below + * This function gets called either from conn_write_callback() in main.c + * when libevent tells us that conn wants to write, or below * from connection_write_to_buf() when an entire TLS record is ready. * * Update <b>conn</b>-\>timestamp_lastwritten to now, and call flush_buf @@ -3374,9 +4015,15 @@ connection_handle_write_impl(connection_t *conn, int force) if (connection_state_is_connecting(conn)) { if (getsockopt(conn->s, SOL_SOCKET, SO_ERROR, (void*)&e, &len) < 0) { log_warn(LD_BUG, "getsockopt() syscall failed"); - if (CONN_IS_EDGE(conn)) - connection_edge_end_errno(TO_EDGE_CONN(conn)); - connection_mark_for_close(conn); + if (conn->type == CONN_TYPE_OR) { + or_connection_t *orconn = TO_OR_CONN(conn); + connection_or_close_for_error(orconn, 0); + } else { + if (CONN_IS_EDGE(conn)) { + connection_edge_end_errno(TO_EDGE_CONN(conn)); + } + connection_mark_for_close(conn); + } return -1; } if (e) { @@ -3491,6 +4138,8 @@ connection_handle_write_impl(connection_t *conn, int force) tor_tls_get_n_raw_bytes(or_conn->tls, &n_read, &n_written); log_debug(LD_GENERAL, "After TLS write of %d: %ld read, %ld written", result, (long)n_read, (long)n_written); + or_conn->bytes_xmitted += result; + or_conn->bytes_xmitted_by_tls += n_written; /* So we notice bytes were written even on error */ /* XXXX024 This cast is safe since we can never write INT_MAX bytes in a * single set of TLS operations. But it looks kinda ugly. If we refactor @@ -3518,12 +4167,34 @@ connection_handle_write_impl(connection_t *conn, int force) if (n_written && conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + circuit_t *circ = circuit_get_by_edge_conn(edge_conn); + origin_circuit_t *ocirc; /* Check for overflow: */ if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_written > n_written)) edge_conn->n_written += (int)n_written; else edge_conn->n_written = UINT32_MAX; + + if (circ && CIRCUIT_IS_ORIGIN(circ)) { + ocirc = TO_ORIGIN_CIRCUIT(circ); + if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_written_circ_bw > n_written)) + ocirc->n_written_circ_bw += (int)n_written; + else + ocirc->n_written_circ_bw = UINT32_MAX; + } + } + + /* If CONN_BW events are enabled, update conn->n_written_conn_bw for + * OR/DIR/EXIT connections, checking for overflow. */ + if (n_written && get_options()->TestingEnableConnBwEvent && + (conn->type == CONN_TYPE_OR || + conn->type == CONN_TYPE_DIR || + conn->type == CONN_TYPE_EXIT)) { + if (PREDICT_LIKELY(UINT32_MAX - conn->n_written_conn_bw > n_written)) + conn->n_written_conn_bw += (int)n_written; + else + conn->n_written_conn_bw = UINT32_MAX; } connection_buckets_decrement(conn, approx_time(), n_read, n_written); @@ -3557,7 +4228,7 @@ connection_handle_write_impl(connection_t *conn, int force) } /* Call even if result is 0, since the global write bucket may - * have reached 0 on a different conn, and this guy needs to + * have reached 0 on a different conn, and this connection needs to * know to stop writing. */ connection_consider_empty_write_buckets(conn); if (n_read > 0 && connection_is_reading(conn)) @@ -3609,9 +4280,9 @@ connection_flush(connection_t *conn) * it all, so we don't end up with many megabytes of controller info queued at * once. */ -void -connection_write_to_buf_impl_(const char *string, size_t len, - connection_t *conn, int zlib) +MOCK_IMPL(void, +connection_write_to_buf_impl_,(const char *string, size_t len, + connection_t *conn, int zlib)) { /* XXXX This function really needs to return -1 on failure. */ int r; @@ -3656,6 +4327,12 @@ connection_write_to_buf_impl_(const char *string, size_t len, "write_to_buf failed. Closing circuit (fd %d).", (int)conn->s); circuit_mark_for_close(circuit_get_by_edge_conn(TO_EDGE_CONN(conn)), END_CIRC_REASON_INTERNAL); + } else if (conn->type == CONN_TYPE_OR) { + or_connection_t *orconn = TO_OR_CONN(conn); + log_warn(LD_NET, + "write_to_buf failed on an orconn; notifying of error " + "(fd %d)", (int)(conn->s)); + connection_or_close_for_error(orconn, 0); } else { log_warn(LD_NET, "write_to_buf failed. Closing connection (fd %d).", @@ -3676,55 +4353,35 @@ connection_write_to_buf_impl_(const char *string, size_t len, conn->outbuf_flushlen += buf_datalen(conn->outbuf) - old_datalen; } else { conn->outbuf_flushlen += len; - - /* Should we try flushing the outbuf now? */ - if (conn->in_flushed_some) { - /* Don't flush the outbuf when the reason we're writing more stuff is - * _because_ we flushed the outbuf. That's unfair. */ - return; - } - - if (conn->type == CONN_TYPE_CONTROL && - !connection_is_rate_limited(conn) && - conn->outbuf_flushlen-len < 1<<16 && - conn->outbuf_flushlen >= 1<<16) { - /* just try to flush all of it */ - } else - return; /* no need to try flushing */ - - if (connection_handle_write(conn, 0) < 0) { - if (!conn->marked_for_close) { - /* this connection is broken. remove it. */ - log_warn(LD_BUG, "unhandled error on write for " - "conn (type %d, fd %d); removing", - conn->type, (int)conn->s); - tor_fragile_assert(); - /* do a close-immediate here, so we don't try to flush */ - connection_close_immediate(conn); - } - return; - } } } +/** Return a connection_t * from get_connection_array() that satisfies test on + * var, and that is not marked for close. */ +#define CONN_GET_TEMPLATE(var, test) \ + STMT_BEGIN \ + smartlist_t *conns = get_connection_array(); \ + SMARTLIST_FOREACH(conns, connection_t *, var, \ + { \ + if (var && (test) && !var->marked_for_close) \ + return var; \ + }); \ + return NULL; \ + STMT_END + /** Return a connection with given type, address, port, and purpose; - * or NULL if no such connection exists. */ -connection_t * -connection_get_by_type_addr_port_purpose(int type, + * or NULL if no such connection exists (or if all such connections are marked + * for close). */ +MOCK_IMPL(connection_t *, +connection_get_by_type_addr_port_purpose,(int type, const tor_addr_t *addr, uint16_t port, - int purpose) + int purpose)) { - smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { - if (conn->type == type && + CONN_GET_TEMPLATE(conn, + (conn->type == type && tor_addr_eq(&conn->addr, addr) && conn->port == port && - conn->purpose == purpose && - !conn->marked_for_close) - return conn; - }); - return NULL; + conn->purpose == purpose)); } /** Return the stream with id <b>id</b> if it is not already marked for @@ -3733,13 +4390,7 @@ connection_get_by_type_addr_port_purpose(int type, connection_t * connection_get_by_global_id(uint64_t id) { - smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { - if (conn->global_identifier == id) - return conn; - }); - return NULL; + CONN_GET_TEMPLATE(conn, conn->global_identifier == id); } /** Return a connection of type <b>type</b> that is not marked for close. @@ -3747,13 +4398,7 @@ connection_get_by_global_id(uint64_t id) connection_t * connection_get_by_type(int type) { - smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { - if (conn->type == type && !conn->marked_for_close) - return conn; - }); - return NULL; + CONN_GET_TEMPLATE(conn, conn->type == type); } /** Return a connection of type <b>type</b> that is in state <b>state</b>, @@ -3762,13 +4407,7 @@ connection_get_by_type(int type) connection_t * connection_get_by_type_state(int type, int state) { - smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { - if (conn->type == type && conn->state == state && !conn->marked_for_close) - return conn; - }); - return NULL; + CONN_GET_TEMPLATE(conn, conn->type == type && conn->state == state); } /** Return a connection of type <b>type</b> that has rendquery equal @@ -3779,78 +4418,124 @@ connection_t * connection_get_by_type_state_rendquery(int type, int state, const char *rendquery) { - smartlist_t *conns = get_connection_array(); - tor_assert(type == CONN_TYPE_DIR || type == CONN_TYPE_AP || type == CONN_TYPE_EXIT); tor_assert(rendquery); - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { - if (conn->type == type && - !conn->marked_for_close && - (!state || state == conn->state)) { - if (type == CONN_TYPE_DIR && + CONN_GET_TEMPLATE(conn, + (conn->type == type && + (!state || state == conn->state)) && + ( + (type == CONN_TYPE_DIR && TO_DIR_CONN(conn)->rend_data && !rend_cmp_service_ids(rendquery, TO_DIR_CONN(conn)->rend_data->onion_address)) - return conn; - else if (CONN_IS_EDGE(conn) && + || + (CONN_IS_EDGE(conn) && TO_EDGE_CONN(conn)->rend_data && !rend_cmp_service_ids(rendquery, TO_EDGE_CONN(conn)->rend_data->onion_address)) - return conn; - } - } SMARTLIST_FOREACH_END(conn); - return NULL; + )); } -/** Return a directory connection (if any one exists) that is fetching - * the item described by <b>state</b>/<b>resource</b> */ -dir_connection_t * -connection_dir_get_by_purpose_and_resource(int purpose, - const char *resource) +/** Return a new smartlist of dir_connection_t * from get_connection_array() + * that satisfy conn_test on connection_t *conn_var, and dirconn_test on + * dir_connection_t *dirconn_var. conn_var must be of CONN_TYPE_DIR and not + * marked for close to be included in the list. */ +#define DIR_CONN_LIST_TEMPLATE(conn_var, conn_test, \ + dirconn_var, dirconn_test) \ + STMT_BEGIN \ + smartlist_t *conns = get_connection_array(); \ + smartlist_t *dir_conns = smartlist_new(); \ + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn_var) { \ + if (conn_var && (conn_test) \ + && conn_var->type == CONN_TYPE_DIR \ + && !conn_var->marked_for_close) { \ + dir_connection_t *dirconn_var = TO_DIR_CONN(conn_var); \ + if (dirconn_var && (dirconn_test)) { \ + smartlist_add(dir_conns, dirconn_var); \ + } \ + } \ + } SMARTLIST_FOREACH_END(conn_var); \ + return dir_conns; \ + STMT_END + +/** Return a list of directory connections that are fetching the item + * described by <b>purpose</b>/<b>resource</b>. If there are none, + * return an empty list. This list must be freed using smartlist_free, + * but the pointers in it must not be freed. + * Note that this list should not be cached, as the pointers in it can be + * freed if their connections close. */ +smartlist_t * +connection_dir_list_by_purpose_and_resource( + int purpose, + const char *resource) { - smartlist_t *conns = get_connection_array(); + DIR_CONN_LIST_TEMPLATE(conn, + conn->purpose == purpose, + dirconn, + 0 == strcmp_opt(resource, + dirconn->requested_resource)); +} - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { - dir_connection_t *dirconn; - if (conn->type != CONN_TYPE_DIR || conn->marked_for_close || - conn->purpose != purpose) - continue; - dirconn = TO_DIR_CONN(conn); - if (dirconn->requested_resource == NULL) { - if (resource == NULL) - return dirconn; - } else if (resource) { - if (0 == strcmp(resource, dirconn->requested_resource)) - return dirconn; - } - } SMARTLIST_FOREACH_END(conn); +/** Return a list of directory connections that are fetching the item + * described by <b>purpose</b>/<b>resource</b>/<b>state</b>. If there are + * none, return an empty list. This list must be freed using smartlist_free, + * but the pointers in it must not be freed. + * Note that this list should not be cached, as the pointers in it can be + * freed if their connections close. */ +smartlist_t * +connection_dir_list_by_purpose_resource_and_state( + int purpose, + const char *resource, + int state) +{ + DIR_CONN_LIST_TEMPLATE(conn, + conn->purpose == purpose && conn->state == state, + dirconn, + 0 == strcmp_opt(resource, + dirconn->requested_resource)); +} - return NULL; +#undef DIR_CONN_LIST_TEMPLATE + +/** Return an arbitrary active OR connection that isn't <b>this_conn</b>. + * + * We use this to guess if we should tell the controller that we + * didn't manage to connect to any of our bridges. */ +static connection_t * +connection_get_another_active_or_conn(const or_connection_t *this_conn) +{ + CONN_GET_TEMPLATE(conn, + conn != TO_CONN(this_conn) && conn->type == CONN_TYPE_OR); } -/** Return an open, non-marked connection of a given type and purpose, or NULL - * if no such connection exists. */ -connection_t * -connection_get_by_type_purpose(int type, int purpose) +/** Return 1 if there are any active OR connections apart from + * <b>this_conn</b>. + * + * We use this to guess if we should tell the controller that we + * didn't manage to connect to any of our bridges. */ +int +any_other_active_or_conns(const or_connection_t *this_conn) { - smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH(conns, connection_t *, conn, - { - if (conn->type == type && - !conn->marked_for_close && - (purpose == conn->purpose)) - return conn; - }); - return NULL; + connection_t *conn = connection_get_another_active_or_conn(this_conn); + if (conn != NULL) { + log_debug(LD_DIR, "%s: Found an OR connection: %s", + __func__, conn->address); + return 1; + } + + return 0; } +#undef CONN_GET_TEMPLATE + /** Return 1 if <b>conn</b> is a listener conn, else return 0. */ int connection_is_listener(connection_t *conn) { if (conn->type == CONN_TYPE_OR_LISTENER || + conn->type == CONN_TYPE_EXT_OR_LISTENER || conn->type == CONN_TYPE_AP_LISTENER || conn->type == CONN_TYPE_AP_TRANS_LISTENER || conn->type == CONN_TYPE_AP_DNS_LISTENER || @@ -3873,6 +4558,7 @@ connection_state_is_open(connection_t *conn) return 0; if ((conn->type == CONN_TYPE_OR && conn->state == OR_CONN_STATE_OPEN) || + (conn->type == CONN_TYPE_EXT_OR) || (conn->type == CONN_TYPE_AP && conn->state == AP_CONN_STATE_OPEN) || (conn->type == CONN_TYPE_EXIT && conn->state == EXIT_CONN_STATE_OPEN) || (conn->type == CONN_TYPE_CONTROL && @@ -3912,25 +4598,12 @@ alloc_http_authenticator(const char *authenticator) /* an authenticator in Basic authentication * is just the string "username:password" */ const size_t authenticator_length = strlen(authenticator); - /* The base64_encode function needs a minimum buffer length - * of 66 bytes. */ - const size_t base64_authenticator_length = (authenticator_length/48+1)*66; + const size_t base64_authenticator_length = + base64_encode_size(authenticator_length, 0) + 1; char *base64_authenticator = tor_malloc(base64_authenticator_length); if (base64_encode(base64_authenticator, base64_authenticator_length, - authenticator, authenticator_length) < 0) { + authenticator, authenticator_length, 0) < 0) { tor_free(base64_authenticator); /* free and set to null */ - } else { - int i = 0, j = 0; - ssize_t len = strlen(base64_authenticator); - - /* remove all newline occurrences within the string */ - for (i=0; i < len; ++i) { - if ('\n' != base64_authenticator[i]) { - base64_authenticator[j] = base64_authenticator[i]; - ++j; - } - } - base64_authenticator[j]='\0'; } return base64_authenticator; } @@ -3999,6 +4672,8 @@ client_check_address_changed(tor_socket_t sock) SMARTLIST_FOREACH(outgoing_addrs, tor_addr_t*, a_ptr, tor_free(a_ptr)); smartlist_clear(outgoing_addrs); smartlist_add(outgoing_addrs, tor_memdup(&out_addr, sizeof(tor_addr_t))); + /* We'll need to resolve ourselves again. */ + reset_last_resolved_addr(); /* Okay, now change our keys. */ ip_address_changed(1); } @@ -4042,14 +4717,14 @@ connection_process_inbuf(connection_t *conn, int package_partial) switch (conn->type) { case CONN_TYPE_OR: return connection_or_process_inbuf(TO_OR_CONN(conn)); + case CONN_TYPE_EXT_OR: + return connection_ext_or_process_inbuf(TO_OR_CONN(conn)); case CONN_TYPE_EXIT: case CONN_TYPE_AP: return connection_edge_process_inbuf(TO_EDGE_CONN(conn), package_partial); case CONN_TYPE_DIR: return connection_dir_process_inbuf(TO_DIR_CONN(conn)); - case CONN_TYPE_CPUWORKER: - return connection_cpu_process_inbuf(conn); case CONN_TYPE_CONTROL: return connection_control_process_inbuf(TO_CONTROL_CONN(conn)); default: @@ -4102,13 +4777,13 @@ connection_finished_flushing(connection_t *conn) switch (conn->type) { case CONN_TYPE_OR: return connection_or_finished_flushing(TO_OR_CONN(conn)); + case CONN_TYPE_EXT_OR: + return connection_ext_or_finished_flushing(TO_OR_CONN(conn)); case CONN_TYPE_AP: case CONN_TYPE_EXIT: return connection_edge_finished_flushing(TO_EDGE_CONN(conn)); case CONN_TYPE_DIR: return connection_dir_finished_flushing(TO_DIR_CONN(conn)); - case CONN_TYPE_CPUWORKER: - return connection_cpu_finished_flushing(conn); case CONN_TYPE_CONTROL: return connection_control_finished_flushing(TO_CONTROL_CONN(conn)); default: @@ -4157,14 +4832,13 @@ connection_reached_eof(connection_t *conn) { switch (conn->type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: return connection_or_reached_eof(TO_OR_CONN(conn)); case CONN_TYPE_AP: case CONN_TYPE_EXIT: return connection_edge_reached_eof(TO_EDGE_CONN(conn)); case CONN_TYPE_DIR: return connection_dir_reached_eof(TO_DIR_CONN(conn)); - case CONN_TYPE_CPUWORKER: - return connection_cpu_reached_eof(conn); case CONN_TYPE_CONTROL: return connection_control_reached_eof(TO_CONTROL_CONN(conn)); default: @@ -4243,6 +4917,7 @@ assert_connection_ok(connection_t *conn, time_t now) switch (conn->type) { case CONN_TYPE_OR: + case CONN_TYPE_EXT_OR: tor_assert(conn->magic == OR_CONNECTION_MAGIC); break; case CONN_TYPE_AP: @@ -4348,6 +5023,10 @@ assert_connection_ok(connection_t *conn, time_t now) tor_assert(conn->state >= OR_CONN_STATE_MIN_); tor_assert(conn->state <= OR_CONN_STATE_MAX_); break; + case CONN_TYPE_EXT_OR: + tor_assert(conn->state >= EXT_OR_CONN_STATE_MIN_); + tor_assert(conn->state <= EXT_OR_CONN_STATE_MAX_); + break; case CONN_TYPE_EXIT: tor_assert(conn->state >= EXIT_CONN_STATE_MIN_); tor_assert(conn->state <= EXIT_CONN_STATE_MAX_); @@ -4365,10 +5044,6 @@ assert_connection_ok(connection_t *conn, time_t now) tor_assert(conn->purpose >= DIR_PURPOSE_MIN_); tor_assert(conn->purpose <= DIR_PURPOSE_MAX_); break; - case CONN_TYPE_CPUWORKER: - tor_assert(conn->state >= CPUWORKER_STATE_MIN_); - tor_assert(conn->state <= CPUWORKER_STATE_MAX_); - break; case CONN_TYPE_CONTROL: tor_assert(conn->state >= CONTROL_CONN_STATE_MIN_); tor_assert(conn->state <= CONTROL_CONN_STATE_MAX_); @@ -4390,6 +5065,27 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, { const or_options_t *options = get_options(); + /* Client Transport Plugins can use another proxy, but that should be hidden + * from the rest of tor (as the plugin is responsible for dealing with the + * proxy), check it first, then check the rest of the proxy types to allow + * the config to have unused ClientTransportPlugin entries. + */ + if (options->ClientTransportPlugin) { + const transport_t *transport = NULL; + int r; + r = get_transport_by_bridge_addrport(&conn->addr, conn->port, &transport); + if (r<0) + return -1; + if (transport) { /* transport found */ + tor_addr_copy(addr, &transport->addr); + *port = transport->port; + *proxy_type = transport->socks_version; + return 0; + } + + /* Unused ClientTransportPlugin. */ + } + if (options->HTTPSProxy) { tor_addr_copy(addr, &options->HTTPSProxyAddr); *port = options->HTTPSProxyPort; @@ -4405,43 +5101,14 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, *port = options->Socks5ProxyPort; *proxy_type = PROXY_SOCKS5; return 0; - } else if (options->ClientTransportPlugin || - options->Bridges) { - const transport_t *transport = NULL; - int r; - r = find_transport_by_bridge_addrport(&conn->addr, conn->port, &transport); - if (r<0) - return -1; - if (transport) { /* transport found */ - tor_addr_copy(addr, &transport->addr); - *port = transport->port; - *proxy_type = transport->socks_version; - return 0; - } } + tor_addr_make_unspec(addr); + *port = 0; *proxy_type = PROXY_NONE; return 0; } -/** Returns the global proxy type used by tor. */ -static int -get_proxy_type(void) -{ - const or_options_t *options = get_options(); - - if (options->HTTPSProxy) - return PROXY_CONNECT; - else if (options->Socks4Proxy) - return PROXY_SOCKS4; - else if (options->Socks5Proxy) - return PROXY_SOCKS5; - else if (options->ClientTransportPlugin) - return PROXY_PLUGGABLE; - else - return PROXY_NONE; -} - /** Log a failed connection to a proxy server. * <b>conn</b> is the connection we use the proxy server for. */ void @@ -4457,7 +5124,7 @@ log_failed_proxy_connection(connection_t *conn) log_warn(LD_NET, "The connection to the %s proxy server at %s just failed. " "Make sure that the proxy server is up and running.", - proxy_type_to_string(get_proxy_type()), + proxy_type_to_string(proxy_type), fmt_addrport(&proxy_addr, proxy_port)); } @@ -4477,9 +5144,7 @@ proxy_type_to_string(int proxy_type) } /** Call connection_free_() on every connection in our array, and release all - * storage held by connection.c. This is used by cpuworkers and dnsworkers - * when they fork, so they don't keep resources held open (especially - * sockets). + * storage held by connection.c. * * Don't do the checks in connection_free(), because they will * fail. @@ -4498,6 +5163,7 @@ connection_free_all(void) /* Unlink everything from the identity map. */ connection_or_clear_identity_map(); + connection_or_clear_ext_or_id_map(); /* Clear out our list of broken connections */ clear_broken_connection_map(0); @@ -4519,3 +5185,34 @@ connection_free_all(void) #endif } +/** Log a warning, and possibly emit a control event, that <b>received</b> came + * at a skewed time. <b>trusted</b> indicates that the <b>source</b> was one + * that we had more faith in and therefore the warning level should have higher + * severity. + */ +void +clock_skew_warning(const connection_t *conn, long apparent_skew, int trusted, + log_domain_mask_t domain, const char *received, + const char *source) +{ + char dbuf[64]; + char *ext_source = NULL; + format_time_interval(dbuf, sizeof(dbuf), apparent_skew); + if (conn) + tor_asprintf(&ext_source, "%s:%s:%d", source, conn->address, conn->port); + else + ext_source = tor_strdup(source); + log_fn(trusted ? LOG_WARN : LOG_INFO, domain, + "Received %s with skewed time (%s): " + "It seems that our clock is %s by %s, or that theirs is %s%s. " + "Tor requires an accurate clock to work: please check your time, " + "timezone, and date settings.", received, ext_source, + apparent_skew > 0 ? "ahead" : "behind", dbuf, + apparent_skew > 0 ? "behind" : "ahead", + (!conn || trusted) ? "" : ", or they are sending us the wrong time"); + if (trusted) + control_event_general_status(LOG_WARN, "CLOCK_SKEW SKEW=%ld SOURCE=%s", + apparent_skew, ext_source); + tor_free(ext_source); +} + diff --git a/src/or/connection.h b/src/or/connection.h index c78fe6e652..4835235fba 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,9 +17,10 @@ const char *conn_type_to_string(int type); const char *conn_state_to_string(int type, int state); +int conn_listener_type_supports_af_unix(int type); dir_connection_t *dir_connection_new(int socket_family); -or_connection_t *or_connection_new(int socket_family); +or_connection_t *or_connection_new(int type, int socket_family); edge_connection_t *edge_connection_new(int type, int socket_family); entry_connection_t *entry_connection_new(int type, int socket_family); control_connection_t *control_connection_new(int socket_family); @@ -27,7 +28,7 @@ listener_connection_t *listener_connection_new(int type, int socket_family); connection_t *connection_new(int type, int socket_family); void connection_link_connections(connection_t *conn_a, connection_t *conn_b); -void connection_free(connection_t *conn); +MOCK_DECL(void,connection_free,(connection_t *conn)); void connection_free_all(void); void connection_about_to_close_connection(connection_t *conn); void connection_close_immediate(connection_t *conn); @@ -89,6 +90,21 @@ int connection_connect(connection_t *conn, const char *address, const tor_addr_t *addr, uint16_t port, int *socket_error); +#ifdef HAVE_SYS_UN_H + +int connection_connect_unix(connection_t *conn, const char *socket_path, + int *socket_error); + +#endif /* defined(HAVE_SYS_UN_H) */ + +/** Maximum size of information that we can fit into SOCKS5 username + or password fields. */ +#define MAX_SOCKS5_AUTH_FIELD_SIZE 255 + +/** Total maximum size of information that we can fit into SOCKS5 + username and password fields. */ +#define MAX_SOCKS5_AUTH_SIZE_TOTAL 2*MAX_SOCKS5_AUTH_FIELD_SIZE + int connection_proxy_connect(connection_t *conn, int type); int connection_read_proxy_handshake(connection_t *conn); void log_failed_proxy_connection(connection_t *conn); @@ -122,20 +138,20 @@ int connection_outbuf_too_full(connection_t *conn); int connection_handle_write(connection_t *conn, int force); int connection_flush(connection_t *conn); -void connection_write_to_buf_impl_(const char *string, size_t len, - connection_t *conn, int zlib); +MOCK_DECL(void, connection_write_to_buf_impl_, + (const char *string, size_t len, connection_t *conn, int zlib)); /* DOCDOC connection_write_to_buf */ static void connection_write_to_buf(const char *string, size_t len, connection_t *conn); /* DOCDOC connection_write_to_buf_zlib */ static void connection_write_to_buf_zlib(const char *string, size_t len, dir_connection_t *conn, int done); -static INLINE void +static inline void connection_write_to_buf(const char *string, size_t len, connection_t *conn) { connection_write_to_buf_impl_(string, len, conn, 0); } -static INLINE void +static inline void connection_write_to_buf_zlib(const char *string, size_t len, dir_connection_t *conn, int done) { @@ -147,7 +163,7 @@ static size_t connection_get_inbuf_len(connection_t *conn); /* DOCDOC connection_get_outbuf_len */ static size_t connection_get_outbuf_len(connection_t *conn); -static INLINE size_t +static inline size_t connection_get_inbuf_len(connection_t *conn) { IF_HAS_BUFFEREVENT(conn, { @@ -157,7 +173,7 @@ connection_get_inbuf_len(connection_t *conn) } } -static INLINE size_t +static inline size_t connection_get_outbuf_len(connection_t *conn) { IF_HAS_BUFFEREVENT(conn, { @@ -170,17 +186,62 @@ connection_get_outbuf_len(connection_t *conn) connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); -connection_t *connection_get_by_type_purpose(int type, int purpose); -connection_t *connection_get_by_type_addr_port_purpose(int type, - const tor_addr_t *addr, - uint16_t port, int purpose); +MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type, + const tor_addr_t *addr, + uint16_t port, int purpose)); connection_t *connection_get_by_type_state(int type, int state); connection_t *connection_get_by_type_state_rendquery(int type, int state, const char *rendquery); -dir_connection_t *connection_dir_get_by_purpose_and_resource( - int state, const char *resource); +smartlist_t *connection_dir_list_by_purpose_and_resource( + int purpose, + const char *resource); +smartlist_t *connection_dir_list_by_purpose_resource_and_state( + int purpose, + const char *resource, + int state); + +#define CONN_LEN_AND_FREE_TEMPLATE(sl) \ + STMT_BEGIN \ + int len = smartlist_len(sl); \ + smartlist_free(sl); \ + return len; \ + STMT_END + +/** Return a count of directory connections that are fetching the item + * described by <b>purpose</b>/<b>resource</b>. */ +static inline int +connection_dir_count_by_purpose_and_resource( + int purpose, + const char *resource) +{ + smartlist_t *conns = connection_dir_list_by_purpose_and_resource( + purpose, + resource); + CONN_LEN_AND_FREE_TEMPLATE(conns); +} -#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) +/** Return a count of directory connections that are fetching the item + * described by <b>purpose</b>/<b>resource</b>/<b>state</b>. */ +static inline int +connection_dir_count_by_purpose_resource_and_state( + int purpose, + const char *resource, + int state) +{ + smartlist_t *conns = + connection_dir_list_by_purpose_resource_and_state( + purpose, + resource, + state); + CONN_LEN_AND_FREE_TEMPLATE(conns); +} + +#undef CONN_LEN_AND_FREE_TEMPLATE + +int any_other_active_or_conns(const or_connection_t *this_conn); + +/* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */ +#define connection_speaks_cells(conn) (((conn)->type == CONN_TYPE_OR) || 0) int connection_is_listener(connection_t *conn); int connection_state_is_open(connection_t *conn); int connection_state_is_connecting(connection_t *conn); @@ -192,6 +253,10 @@ int connection_or_nonopen_was_started_here(or_connection_t *conn); void connection_dump_buffer_mem_stats(int severity); void remove_file_if_very_old(const char *fname, time_t now); +void clock_skew_warning(const connection_t *conn, long apparent_skew, + int trusted, log_domain_mask_t domain, + const char *received, const char *source); + #ifdef USE_BUFFEREVENTS int connection_type_uses_bufferevent(connection_t *conn); void connection_configure_bufferevent_callbacks(connection_t *conn); @@ -206,5 +271,25 @@ void connection_enable_rate_limiting(connection_t *conn); #define connection_type_uses_bufferevent(c) (0) #endif +#ifdef CONNECTION_PRIVATE +STATIC void connection_free_(connection_t *conn); + +/* Used only by connection.c and test*.c */ +uint32_t bucket_millis_empty(int tokens_before, uint32_t last_empty_time, + int tokens_after, int milliseconds_elapsed, + const struct timeval *tvnow); +void connection_buckets_note_empty_ts(uint32_t *timestamp_var, + int tokens_before, + size_t tokens_removed, + const struct timeval *tvnow); +MOCK_DECL(STATIC int,connection_connect_sockaddr, + (connection_t *conn, + const struct sockaddr *sa, + socklen_t sa_len, + const struct sockaddr *bindaddr, + socklen_t bindaddr_len, + int *socket_error)); +#endif + #endif diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 39f8af61f6..8098fb017b 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,9 +11,13 @@ #define CONNECTION_EDGE_PRIVATE #include "or.h" + +#include "backtrace.h" + #include "addressmap.h" #include "buffers.h" #include "channel.h" +#include "circpathbias.h" #include "circuitlist.h" #include "circuituse.h" #include "config.h" @@ -45,6 +49,19 @@ #ifdef HAVE_LINUX_NETFILTER_IPV4_H #include <linux/netfilter_ipv4.h> #define TRANS_NETFILTER +#define TRANS_NETFILTER_IPV4 +#endif + +#ifdef HAVE_LINUX_IF_H +#include <linux/if.h> +#endif + +#ifdef HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H +#include <linux/netfilter_ipv6/ip6_tables.h> +#if defined(IP6T_SO_ORIGINAL_DST) +#define TRANS_NETFILTER +#define TRANS_NETFILTER_IPV6 +#endif #endif #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) @@ -53,6 +70,10 @@ #define TRANS_PF #endif +#ifdef IP_TRANSPARENT +#define TRANS_TPROXY +#endif + #define SOCKS4_GRANTED 90 #define SOCKS4_REJECT 91 @@ -61,19 +82,14 @@ static int connection_ap_process_natd(entry_connection_t *conn); static int connection_exit_connect_dir(edge_connection_t *exitconn); static int consider_plaintext_ports(entry_connection_t *conn, uint16_t port); static int connection_ap_supports_optimistic_data(const entry_connection_t *); -static void connection_ap_handshake_socks_resolved_addr( - entry_connection_t *conn, - const tor_addr_t *answer, - int ttl, - time_t expires); /** An AP stream has failed/finished. If it hasn't already sent back * a socks reply, send one now (based on endreason). Also set * has_sent_end to 1, and mark the conn. */ -void -connection_mark_unattached_ap_(entry_connection_t *conn, int endreason, - int line, const char *file) +MOCK_IMPL(void, +connection_mark_unattached_ap_,(entry_connection_t *conn, int endreason, + int line, const char *file)) { connection_t *base_conn = ENTRY_TO_CONN(conn); edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(conn); @@ -89,8 +105,7 @@ connection_mark_unattached_ap_(entry_connection_t *conn, int endreason, * but we should fix it someday anyway. */ if ((edge_conn->on_circuit != NULL || edge_conn->edge_has_sent_end) && connection_edge_is_rendezvous_stream(edge_conn)) { - rend_client_note_connection_attempt_ended( - edge_conn->rend_data->onion_address); + rend_client_note_connection_attempt_ended(edge_conn->rend_data); } if (base_conn->marked_for_close) { @@ -412,7 +427,7 @@ connection_edge_finished_flushing(edge_connection_t *conn) * that the name resolution that led us to <b>addr</b> will be valid for * <b>ttl</b> seconds. Return -1 on error, or the number of bytes used on * success. */ -/* private */int +STATIC int connected_cell_format_payload(uint8_t *payload_out, const tor_addr_t *addr, uint32_t ttl) @@ -491,6 +506,16 @@ connection_edge_finished_connecting(edge_connection_t *edge_conn) return connection_edge_process_inbuf(edge_conn, 1); } +/** A list of all the entry_connection_t * objects that are not marked + * for close, and are in AP_CONN_STATE_CIRCUIT_WAIT. + * + * (Right now, we check in several places to make sure that this list is + * correct. When it's incorrect, we'll fix it, and log a BUG message.) + */ +static smartlist_t *pending_entry_connections = NULL; + +static int untried_pending_connections = 0; + /** Common code to connection_(ap|exit)_about_to_close. */ static void connection_edge_about_to_close(edge_connection_t *edge_conn) @@ -513,6 +538,8 @@ connection_ap_about_to_close(entry_connection_t *entry_conn) edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn); connection_t *conn = ENTRY_TO_CONN(entry_conn); + connection_edge_about_to_close(edge_conn); + if (entry_conn->socks_request->has_finished == 0) { /* since conn gets removed right after this function finishes, * there's no point trying to send back a reply at this point. */ @@ -531,6 +558,20 @@ connection_ap_about_to_close(entry_connection_t *entry_conn) conn->marked_for_close_file, conn->marked_for_close); dnsserv_reject_request(entry_conn); } + + if (TO_CONN(edge_conn)->state == AP_CONN_STATE_CIRCUIT_WAIT) { + smartlist_remove(pending_entry_connections, entry_conn); + } + +#if 1 + /* Check to make sure that this isn't in pending_entry_connections if it + * didn't actually belong there. */ + if (TO_CONN(edge_conn)->type == CONN_TYPE_AP) { + connection_ap_warn_and_unmark_if_pending_circ(entry_conn, + "about_to_close"); + } +#endif + control_event_stream_bandwidth(edge_conn); control_event_stream_status(entry_conn, STREAM_EVENT_CLOSED, edge_conn->end_reason); @@ -699,26 +740,190 @@ connection_ap_expire_beginning(void) } SMARTLIST_FOREACH_END(base_conn); } -/** Tell any AP streams that are waiting for a new circuit to try again, - * either attaching to an available circ or launching a new one. +/** + * As connection_ap_attach_pending, but first scans the entire connection + * array to see if any elements are missing. */ void -connection_ap_attach_pending(void) +connection_ap_rescan_and_attach_pending(void) { entry_connection_t *entry_conn; smartlist_t *conns = get_connection_array(); + + if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) + pending_entry_connections = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->marked_for_close || conn->type != CONN_TYPE_AP || conn->state != AP_CONN_STATE_CIRCUIT_WAIT) continue; + entry_conn = TO_ENTRY_CONN(conn); + tor_assert(entry_conn); + if (! smartlist_contains(pending_entry_connections, entry_conn)) { + log_warn(LD_BUG, "Found a connection %p that was supposed to be " + "in pending_entry_connections, but wasn't. No worries; " + "adding it.", + pending_entry_connections); + untried_pending_connections = 1; + connection_ap_mark_as_pending_circuit(entry_conn); + } + + } SMARTLIST_FOREACH_END(conn); + + connection_ap_attach_pending(1); +} + +#ifdef DEBUGGING_17659 +#define UNMARK() do { \ + entry_conn->marked_pending_circ_line = 0; \ + entry_conn->marked_pending_circ_file = 0; \ + } while (0) +#else +#define UNMARK() do { } while (0) +#endif + +/** Tell any AP streams that are listed as waiting for a new circuit to try + * again, either attaching to an available circ or launching a new one. + * + * If <b>retry</b> is false, only check the list if it contains at least one + * streams that we have not yet tried to attach to a circuit. + */ +void +connection_ap_attach_pending(int retry) +{ + if (PREDICT_UNLIKELY(!pending_entry_connections)) { + return; + } + + if (untried_pending_connections == 0 && !retry) + return; + + /* Don't allow modifications to pending_entry_connections while we are + * iterating over it. */ + smartlist_t *pending = pending_entry_connections; + pending_entry_connections = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(pending, + entry_connection_t *, entry_conn) { + connection_t *conn = ENTRY_TO_CONN(entry_conn); + tor_assert(conn && entry_conn); + if (conn->marked_for_close) { + UNMARK(); + continue; + } + if (conn->magic != ENTRY_CONNECTION_MAGIC) { + log_warn(LD_BUG, "%p has impossible magic value %u.", + entry_conn, (unsigned)conn->magic); + UNMARK(); + continue; + } + if (conn->state != AP_CONN_STATE_CIRCUIT_WAIT) { + log_warn(LD_BUG, "%p is no longer in circuit_wait. Its current state " + "is %s. Why is it on pending_entry_connections?", + entry_conn, + conn_state_to_string(conn->type, conn->state)); + UNMARK(); + continue; + } + if (connection_ap_handshake_attach_circuit(entry_conn) < 0) { if (!conn->marked_for_close) connection_mark_unattached_ap(entry_conn, END_STREAM_REASON_CANT_ATTACH); } - } SMARTLIST_FOREACH_END(conn); + + if (! conn->marked_for_close && + conn->type == CONN_TYPE_AP && + conn->state == AP_CONN_STATE_CIRCUIT_WAIT) { + if (!smartlist_contains(pending_entry_connections, entry_conn)) { + smartlist_add(pending_entry_connections, entry_conn); + continue; + } + } + + UNMARK(); + } SMARTLIST_FOREACH_END(entry_conn); + + smartlist_free(pending); + untried_pending_connections = 0; +} + +/** Mark <b>entry_conn</b> as needing to get attached to a circuit. + * + * And <b>entry_conn</b> must be in AP_CONN_STATE_CIRCUIT_WAIT, + * should not already be pending a circuit. The circuit will get + * launched or the connection will get attached the next time we + * call connection_ap_attach_pending(). + */ +void +connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn, + const char *fname, int lineno) +{ + connection_t *conn = ENTRY_TO_CONN(entry_conn); + tor_assert(conn->state == AP_CONN_STATE_CIRCUIT_WAIT); + tor_assert(conn->magic == ENTRY_CONNECTION_MAGIC); + if (conn->marked_for_close) + return; + + if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) + pending_entry_connections = smartlist_new(); + + if (PREDICT_UNLIKELY(smartlist_contains(pending_entry_connections, + entry_conn))) { + log_warn(LD_BUG, "What?? pending_entry_connections already contains %p! " + "(Called from %s:%d.)", + entry_conn, fname, lineno); +#ifdef DEBUGGING_17659 + const char *f2 = entry_conn->marked_pending_circ_file; + log_warn(LD_BUG, "(Previously called from %s:%d.)\n", + f2 ? f2 : "<NULL>", + entry_conn->marked_pending_circ_line); +#endif + log_backtrace(LOG_WARN, LD_BUG, "To debug, this may help"); + return; + } + +#ifdef DEBUGGING_17659 + entry_conn->marked_pending_circ_line = (uint16_t) lineno; + entry_conn->marked_pending_circ_file = fname; +#endif + + untried_pending_connections = 1; + smartlist_add(pending_entry_connections, entry_conn); + + /* Work-around for bug 19969: we handle pending_entry_connections at + * the end of run_main_loop_once(), but in many cases that function will + * take a very long time, if ever, to finish its call to event_base_loop(). + * + * So the fix is to tell it right now that it ought to finish its loop at + * its next available opportunity. + */ + tell_event_loop_to_finish(); +} + +/** Mark <b>entry_conn</b> as no longer waiting for a circuit. */ +void +connection_ap_mark_as_non_pending_circuit(entry_connection_t *entry_conn) +{ + if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) + return; + UNMARK(); + smartlist_remove(pending_entry_connections, entry_conn); +} + +/* DOCDOC */ +void +connection_ap_warn_and_unmark_if_pending_circ(entry_connection_t *entry_conn, + const char *where) +{ + if (pending_entry_connections && + smartlist_contains(pending_entry_connections, entry_conn)) { + log_warn(LD_BUG, "What was %p doing in pending_entry_connections in %s?", + entry_conn, where); + connection_ap_mark_as_non_pending_circuit(entry_conn); + } } /** Tell any AP streams that are waiting for a one-hop tunnel to @@ -748,8 +953,9 @@ connection_ap_fail_onehop(const char *failed_digest, /* we don't know the digest; have to compare addr:port */ tor_addr_t addr; if (!build_state || !build_state->chosen_exit || - !entry_conn->socks_request) + !entry_conn->socks_request) { continue; + } if (tor_addr_parse(&addr, entry_conn->socks_request->address)<0 || !tor_addr_eq(&build_state->chosen_exit->addr, &addr) || build_state->chosen_exit->port != entry_conn->socks_request->port) @@ -838,12 +1044,13 @@ connection_ap_detach_retriable(entry_connection_t *conn, * a tunneled directory connection, then just attach it. */ ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_CIRCUIT_WAIT; circuit_detach_stream(TO_CIRCUIT(circ),ENTRY_TO_EDGE_CONN(conn)); - return connection_ap_handshake_attach_circuit(conn); + connection_ap_mark_as_pending_circuit(conn); } else { + CONNECTION_AP_EXPECT_NONPENDING(conn); ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_CONTROLLER_WAIT; circuit_detach_stream(TO_CIRCUIT(circ),ENTRY_TO_EDGE_CONN(conn)); - return 0; } + return 0; } /** Check if <b>conn</b> is using a dangerous port. Then warn and/or @@ -892,84 +1099,109 @@ connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn, const or_options_t *options = get_options(); if (options->LeaveStreamsUnattached) { + CONNECTION_AP_EXPECT_NONPENDING(conn); ENTRY_TO_CONN(conn)->state = AP_CONN_STATE_CONTROLLER_WAIT; return 0; } return connection_ap_handshake_rewrite_and_attach(conn, circ, cpath); } -/** Connection <b>conn</b> just finished its socks handshake, or the - * controller asked us to take care of it. If <b>circ</b> is defined, - * then that's where we'll want to attach it. Otherwise we have to - * figure it out ourselves. - * - * First, parse whether it's a .exit address, remap it, and so on. Then - * if it's for a general circuit, try to attach it to a circuit (or launch - * one as needed), else if it's for a rendezvous circuit, fetch a - * rendezvous descriptor first (or attach/launch a circuit if the - * rendezvous descriptor is already here and fresh enough). - * - * The stream will exit from the hop - * indicated by <b>cpath</b>, or from the last hop in circ's cpath if - * <b>cpath</b> is NULL. +/* Try to perform any map-based rewriting of the target address in + * <b>conn</b>, filling in the fields of <b>out</b> as we go, and modifying + * conn->socks_request.address as appropriate. */ -int -connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath) +STATIC void +connection_ap_handshake_rewrite(entry_connection_t *conn, + rewrite_result_t *out) { socks_request_t *socks = conn->socks_request; - hostname_type_t addresstype; const or_options_t *options = get_options(); tor_addr_t addr_tmp; - /* We set this to true if this is an address we should automatically - * remap to a local address in VirtualAddrNetwork */ - int automap = 0; - char orig_address[MAX_SOCKS_ADDR_LEN]; - time_t map_expires = TIME_MAX; - time_t now = time(NULL); - connection_t *base_conn = ENTRY_TO_CONN(conn); - addressmap_entry_source_t exit_source = ADDRMAPSRC_NONE; - tor_strlower(socks->address); /* normalize it */ - strlcpy(orig_address, socks->address, sizeof(orig_address)); + /* Initialize all the fields of 'out' to reasonable defaults */ + out->automap = 0; + out->exit_source = ADDRMAPSRC_NONE; + out->map_expires = TIME_MAX; + out->end_reason = 0; + out->should_close = 0; + out->orig_address[0] = 0; + + /* We convert all incoming addresses to lowercase. */ + tor_strlower(socks->address); + /* Remember the original address. */ + strlcpy(out->orig_address, socks->address, sizeof(out->orig_address)); log_debug(LD_APP,"Client asked for %s:%d", safe_str_client(socks->address), socks->port); + /* Check for whether this is a .exit address. By default, those are + * disallowed when they're coming straight from the client, but you're + * allowed to have them in MapAddress commands and so forth. */ if (!strcmpend(socks->address, ".exit") && !options->AllowDotExit) { log_warn(LD_APP, "The \".exit\" notation is disabled in Tor due to " "security risks. Set AllowDotExit in your torrc to enable " "it (at your own risk)."); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); - return -1; + out->end_reason = END_STREAM_REASON_TORPROTOCOL; + out->should_close = 1; + return; } - if (! conn->original_dest_address) + /* Remember the original address so we can tell the user about what + * they actually said, not just what it turned into. */ + if (! conn->original_dest_address) { + /* Is the 'if' necessary here? XXXX */ conn->original_dest_address = tor_strdup(conn->socks_request->address); + } + + /* First, apply MapAddress and MAPADDRESS mappings. We need to do + * these only for non-reverse lookups, since they don't exist for those. + * We need to do this before we consider automapping, since we might + * e.g. resolve irc.oftc.net into irconionaddress.onion, at which point + * we'd need to automap it. */ + if (socks->command != SOCKS_COMMAND_RESOLVE_PTR) { + const unsigned rewrite_flags = AMR_FLAG_USE_MAPADDRESS; + if (addressmap_rewrite(socks->address, sizeof(socks->address), + rewrite_flags, &out->map_expires, &out->exit_source)) { + control_event_stream_status(conn, STREAM_EVENT_REMAP, + REMAP_STREAM_SOURCE_CACHE); + } + } + /* Now, handle automapping. Automapping happens when we're asked to + * resolve a hostname, and AutomapHostsOnResolve is set, and + * the hostname has a suffix listed in AutomapHostsSuffixes. + */ if (socks->command == SOCKS_COMMAND_RESOLVE && tor_addr_parse(&addr_tmp, socks->address)<0 && options->AutomapHostsOnResolve) { - automap = addressmap_address_should_automap(socks->address, options); - if (automap) { + /* Check the suffix... */ + out->automap = addressmap_address_should_automap(socks->address, options); + if (out->automap) { + /* If we get here, then we should apply an automapping for this. */ const char *new_addr; + /* We return an IPv4 address by default, or an IPv6 address if we + * are allowed to do so. */ int addr_type = RESOLVED_TYPE_IPV4; if (conn->socks_request->socks_version != 4) { - if (!conn->ipv4_traffic_ok || - (conn->ipv6_traffic_ok && conn->prefer_ipv6_traffic) || - conn->prefer_ipv6_virtaddr) + if (!conn->entry_cfg.ipv4_traffic || + (conn->entry_cfg.ipv6_traffic && conn->entry_cfg.prefer_ipv6) || + conn->entry_cfg.prefer_ipv6_virtaddr) addr_type = RESOLVED_TYPE_IPV6; } + /* Okay, register the target address as automapped, and find the new + * address we're supposed to give as a resolve answer. (Return a cached + * value if we've looked up this address before. + */ new_addr = addressmap_register_virtual_address( addr_type, tor_strdup(socks->address)); if (! new_addr) { log_warn(LD_APP, "Unable to automap address %s", escaped_safe_str(socks->address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_INTERNAL); - return -1; + out->end_reason = END_STREAM_REASON_INTERNAL; + out->should_close = 1; + return; } log_info(LD_APP, "Automapping %s to %s", escaped_safe_str_client(socks->address), @@ -978,28 +1210,35 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, } } + /* Now handle reverse lookups, if they're in the cache. This doesn't + * happen too often, since client-side DNS caching is off by default. */ if (socks->command == SOCKS_COMMAND_RESOLVE_PTR) { unsigned rewrite_flags = 0; - if (conn->use_cached_ipv4_answers) + if (conn->entry_cfg.use_cached_ipv4_answers) rewrite_flags |= AMR_FLAG_USE_IPV4_DNS; - if (conn->use_cached_ipv6_answers) + if (conn->entry_cfg.use_cached_ipv6_answers) rewrite_flags |= AMR_FLAG_USE_IPV6_DNS; if (addressmap_rewrite_reverse(socks->address, sizeof(socks->address), - rewrite_flags, &map_expires)) { + rewrite_flags, &out->map_expires)) { char *result = tor_strdup(socks->address); /* remember _what_ is supposed to have been resolved. */ tor_snprintf(socks->address, sizeof(socks->address), "REVERSE[%s]", - orig_address); + out->orig_address); connection_ap_handshake_socks_resolved(conn, RESOLVED_TYPE_HOSTNAME, strlen(result), (uint8_t*)result, -1, - map_expires); - connection_mark_unattached_ap(conn, - END_STREAM_REASON_DONE | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return 0; + out->map_expires); + tor_free(result); + out->end_reason = END_STREAM_REASON_DONE | + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED; + out->should_close = 1; + return; } + + /* Hang on, did we find an answer saying that this is a reverse lookup for + * an internal address? If so, we should reject it if we're condigured to + * do so. */ if (options->ClientDNSRejectInternalAddresses) { /* Don't let people try to do a reverse lookup on 10.0.0.1. */ tor_addr_t addr; @@ -1009,43 +1248,108 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, if (ok == 1 && tor_addr_is_internal(&addr, 0)) { connection_ap_handshake_socks_resolved(conn, RESOLVED_TYPE_ERROR, 0, NULL, -1, TIME_MAX); - connection_mark_unattached_ap(conn, - END_STREAM_REASON_SOCKSPROTOCOL | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return -1; + out->end_reason = END_STREAM_REASON_SOCKSPROTOCOL | + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED; + out->should_close = 1; + return; } } - } else if (!automap) { - /* For address map controls, remap the address. */ - unsigned rewrite_flags = 0; - if (conn->use_cached_ipv4_answers) + } + + /* If we didn't automap it before, then this is still the address + * that came straight from the user, mapped according to any + * MapAddress/MAPADDRESS commands. Now other mappings, including + * previously registered Automap entries, TrackHostExits entries, + * and client-side DNS cache entries (not recommended). + */ + if (socks->command != SOCKS_COMMAND_RESOLVE_PTR && + !out->automap) { + unsigned rewrite_flags = AMR_FLAG_USE_AUTOMAP | AMR_FLAG_USE_TRACKEXIT; + addressmap_entry_source_t exit_source2; + if (conn->entry_cfg.use_cached_ipv4_answers) rewrite_flags |= AMR_FLAG_USE_IPV4_DNS; - if (conn->use_cached_ipv6_answers) + if (conn->entry_cfg.use_cached_ipv6_answers) rewrite_flags |= AMR_FLAG_USE_IPV6_DNS; if (addressmap_rewrite(socks->address, sizeof(socks->address), - rewrite_flags, &map_expires, &exit_source)) { + rewrite_flags, &out->map_expires, &exit_source2)) { control_event_stream_status(conn, STREAM_EVENT_REMAP, REMAP_STREAM_SOURCE_CACHE); } + if (out->exit_source == ADDRMAPSRC_NONE) { + /* If it wasn't a .exit before, maybe it turned into a .exit. Remember + * the original source of a .exit. */ + out->exit_source = exit_source2; + } } - if (!automap && address_is_in_virtual_range(socks->address)) { - /* This address was probably handed out by client_dns_get_unmapped_address, - * but the mapping was discarded for some reason. We *don't* want to send - * the address through Tor; that's likely to fail, and may leak - * information. + /* Check to see whether we're about to use an address in the virtual + * range without actually having gotten it from an Automap. */ + if (!out->automap && address_is_in_virtual_range(socks->address)) { + /* This address was probably handed out by + * client_dns_get_unmapped_address, but the mapping was discarded for some + * reason. Or the user typed in a virtual address range manually. We + * *don't* want to send the address through Tor; that's likely to fail, + * and may leak information. */ log_warn(LD_APP,"Missing mapping for virtual address '%s'. Refusing.", safe_str_client(socks->address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_INTERNAL); - return -1; + out->end_reason = END_STREAM_REASON_INTERNAL; + out->should_close = 1; + return; } +} + +/** Connection <b>conn</b> just finished its socks handshake, or the + * controller asked us to take care of it. If <b>circ</b> is defined, + * then that's where we'll want to attach it. Otherwise we have to + * figure it out ourselves. + * + * First, parse whether it's a .exit address, remap it, and so on. Then + * if it's for a general circuit, try to attach it to a circuit (or launch + * one as needed), else if it's for a rendezvous circuit, fetch a + * rendezvous descriptor first (or attach/launch a circuit if the + * rendezvous descriptor is already here and fresh enough). + * + * The stream will exit from the hop + * indicated by <b>cpath</b>, or from the last hop in circ's cpath if + * <b>cpath</b> is NULL. + */ +int +connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath) +{ + socks_request_t *socks = conn->socks_request; + const or_options_t *options = get_options(); + connection_t *base_conn = ENTRY_TO_CONN(conn); + time_t now = time(NULL); + rewrite_result_t rr; + + memset(&rr, 0, sizeof(rr)); + connection_ap_handshake_rewrite(conn,&rr); + + if (rr.should_close) { + /* connection_ap_handshake_rewrite told us to close the connection, + * either because it sent back an answer, or because it sent back an + * error */ + connection_mark_unattached_ap(conn, rr.end_reason); + if (END_STREAM_REASON_DONE == (rr.end_reason & END_STREAM_REASON_MASK)) + return 0; + else + return -1; + } + + const time_t map_expires = rr.map_expires; + const int automap = rr.automap; + const addressmap_entry_source_t exit_source = rr.exit_source; /* Parse the address provided by SOCKS. Modify it in-place if it * specifies a hidden-service (.onion) or particular exit node (.exit). */ - addresstype = parse_extended_hostname(socks->address); + const hostname_type_t addresstype = parse_extended_hostname(socks->address); + /* Now see whether the hostname is bogus. This could happen because of an + * onion hostname whose format we don't recognize. */ if (addresstype == BAD_HOSTNAME) { control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); @@ -1053,16 +1357,21 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } + /* If this is a .exit hostname, strip off the .name.exit part, and + * see whether we're going to connect there, and otherwise handle it. + * (The ".exit" part got stripped off by "parse_extended_hostname"). + * + * We'll set chosen_exit_name and/or close the connection as appropriate. + */ if (addresstype == EXIT_HOSTNAME) { - /* foo.exit -- modify conn->chosen_exit_node to specify the exit - * node, and conn->address to hold only the address portion. */ - char *s = strrchr(socks->address,'.'); - - /* If StrictNodes is not set, then .exit overrides ExcludeNodes. */ + /* If StrictNodes is not set, then .exit overrides ExcludeNodes but + * not ExcludeExitNodes. */ routerset_t *excludeset = options->StrictNodes ? options->ExcludeExitNodesUnion_ : options->ExcludeExitNodes; - const node_t *node; + const node_t *node = NULL; + /* If this .exit was added by an AUTOMAP, then it came straight from + * a user. Make sure that options->AllowDotExit permits that. */ if (exit_source == ADDRMAPSRC_AUTOMAP && !options->AllowDotExit) { /* Whoops; this one is stale. It must have gotten added earlier, * when AllowDotExit was on. */ @@ -1075,6 +1384,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } + /* Double-check to make sure there are no .exits coming from + * impossible/weird sources. */ if (exit_source == ADDRMAPSRC_DNS || (exit_source == ADDRMAPSRC_NONE && !options->AllowDotExit)) { /* It shouldn't be possible to get a .exit address from any of these @@ -1089,9 +1400,12 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, } tor_assert(!automap); + /* Now, find the character before the .(name) part. */ + char *s = strrchr(socks->address,'.'); if (s) { /* The address was of the form "(stuff).(name).exit */ if (s[1] != '\0') { + /* Looks like a real .exit one. */ conn->chosen_exit_name = tor_strdup(s+1); node = node_get_by_nickname(conn->chosen_exit_name, 1); @@ -1110,7 +1424,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } } else { - /* It looks like they just asked for "foo.exit". */ + /* It looks like they just asked for "foo.exit". That's a special + * form that means (foo's address).foo.exit. */ conn->chosen_exit_name = tor_strdup(socks->address); node = node_get_by_nickname(conn->chosen_exit_name, 1); @@ -1119,6 +1434,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, node_get_address_string(node, socks->address, sizeof(socks->address)); } } + /* Now make sure that the chosen exit exists... */ if (!node) { log_warn(LD_APP, @@ -1140,8 +1456,12 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, implies no. */ } + /* Now, handle everything that isn't a .onion address. */ if (addresstype != ONION_HOSTNAME) { - /* not a hidden-service request (i.e. normal or .exit) */ + /* Not a hidden-service request. It's either a hostname or an IP, + * possibly with a .exit that we stripped off. */ + + /* Check for funny characters in the address. */ if (address_is_invalid_destination(socks->address, 1)) { control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); @@ -1152,6 +1472,9 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } +#ifdef ENABLE_TOR2WEB_MODE + /* If we're running in Tor2webMode, we don't allow anything BUT .onion + * addresses. */ if (options->Tor2webMode) { log_warn(LD_APP, "Refusing to connect to non-hidden-service hostname %s " "because tor2web mode is enabled.", @@ -1159,13 +1482,17 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY); return -1; } +#endif + /* See if this is a hostname lookup that we can answer immediately. + * (For example, an attempt to look up the IP address for an IP address.) + */ if (socks->command == SOCKS_COMMAND_RESOLVE) { tor_addr_t answer; /* Reply to resolves immediately if we can. */ if (tor_addr_parse(&answer, socks->address) >= 0) {/* is it an IP? */ /* remember _what_ is supposed to have been resolved. */ - strlcpy(socks->address, orig_address, sizeof(socks->address)); + strlcpy(socks->address, rr.orig_address, sizeof(socks->address)); connection_ap_handshake_socks_resolved_addr(conn, &answer, -1, map_expires); connection_mark_unattached_ap(conn, @@ -1176,14 +1503,22 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, tor_assert(!automap); rep_hist_note_used_resolve(now); /* help predict this next time */ } else if (socks->command == SOCKS_COMMAND_CONNECT) { + /* Special handling for attempts to connect */ tor_assert(!automap); + /* Don't allow connections to port 0. */ if (socks->port == 0) { log_notice(LD_APP,"Application asked to connect to port 0. Refusing."); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } + /* You can't make connections to internal addresses, by default. + * Exceptions are begindir requests (where the address is meaningless, + * or cases where you've hand-configured a particular exit, thereby + * making the local address meaningful. */ if (options->ClientRejectInternalAddresses && !conn->use_begindir && !conn->chosen_exit_name && !circ) { + /* If we reach this point then we don't want to allow internal + * addresses. Check if we got one. */ tor_addr_t addr; if (tor_addr_hostname_is_local(socks->address) || (tor_addr_parse(&addr, socks->address) >= 0 && @@ -1218,39 +1553,58 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, connection_mark_unattached_ap(conn, END_STREAM_REASON_PRIVATE_ADDR); return -1; } - } + } /* end "if we should check for internal addresses" */ + /* Okay. We're still doing a CONNECT, and it wasn't a private + * address. Do special handling for literal IP addresses */ { tor_addr_t addr; /* XXX Duplicate call to tor_addr_parse. */ if (tor_addr_parse(&addr, socks->address) >= 0) { + /* If we reach this point, it's an IPv4 or an IPv6 address. */ sa_family_t family = tor_addr_family(&addr); - if ((family == AF_INET && ! conn->ipv4_traffic_ok) || - (family == AF_INET6 && ! conn->ipv4_traffic_ok)) { + + if ((family == AF_INET && ! conn->entry_cfg.ipv4_traffic) || + (family == AF_INET6 && ! conn->entry_cfg.ipv6_traffic)) { + /* You can't do an IPv4 address on a v6-only socks listener, + * or vice versa. */ log_warn(LD_NET, "Rejecting SOCKS request for an IP address " "family that this listener does not support."); connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY); return -1; } else if (family == AF_INET6 && socks->socks_version == 4) { + /* You can't make a socks4 request to an IPv6 address. Socks4 + * doesn't support that. */ log_warn(LD_NET, "Rejecting SOCKS4 request for an IPv6 address."); connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY); return -1; - } else if (socks->socks_version == 4 && !conn->ipv4_traffic_ok) { + } else if (socks->socks_version == 4 && + !conn->entry_cfg.ipv4_traffic) { + /* You can't do any kind of Socks4 request when IPv4 is forbidden. + * + * XXX raise this check outside the enclosing block? */ log_warn(LD_NET, "Rejecting SOCKS4 request on a listener with " "no IPv4 traffic supported."); connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY); return -1; } else if (family == AF_INET6) { - conn->ipv4_traffic_ok = 0; + /* Tell the exit: we won't accept any ipv4 connection to an IPv6 + * address. */ + conn->entry_cfg.ipv4_traffic = 0; } else if (family == AF_INET) { - conn->ipv6_traffic_ok = 0; + /* Tell the exit: we won't accept any ipv6 connection to an IPv4 + * address. */ + conn->entry_cfg.ipv6_traffic = 0; } } } if (socks->socks_version == 4) - conn->ipv6_traffic_ok = 0; + conn->entry_cfg.ipv6_traffic = 0; + /* Still handling CONNECT. Now, check for exit enclaves. (Which we + * don't do on BEGINDIR, or there is a chosen exit.) + */ if (!conn->use_begindir && !conn->chosen_exit_name && !circ) { /* see if we can find a suitable enclave exit */ const node_t *r = @@ -1267,11 +1621,13 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, } } - /* warn or reject if it's using a dangerous port */ + /* Still handling CONNECT: warn or reject if it's using a dangerous + * port. */ if (!conn->use_begindir && !conn->chosen_exit_name && !circ) if (consider_plaintext_ports(conn, socks->port) < 0) return -1; + /* Remember the port so that we do predicted requests there. */ if (!conn->use_begindir) { /* help predict this next time */ rep_hist_note_used_port(now, socks->port); @@ -1280,25 +1636,43 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, rep_hist_note_used_resolve(now); /* help predict this next time */ /* no extra processing needed */ } else { + /* We should only be doing CONNECT or RESOLVE! */ tor_fragile_assert(); } + + /* Okay. At this point we've set chosen_exit_name if needed, rewritten the + * address, and decided not to reject it for any number of reasons. Now + * mark the connection as waiting for a circuit, and try to attach it! + */ base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT; - if ((circ && connection_ap_handshake_attach_chosen_circuit( - conn, circ, cpath) < 0) || - (!circ && - connection_ap_handshake_attach_circuit(conn) < 0)) { + + /* If we were given a circuit to attach to, try to attach. Otherwise, + * try to find a good one and attach to that. */ + int rv; + if (circ) { + rv = connection_ap_handshake_attach_chosen_circuit(conn, circ, cpath); + } else { + connection_ap_mark_as_pending_circuit(conn); + rv = 0; + } + + /* If the above function returned 0 then we're waiting for a circuit. + * if it returned 1, we're attached. Both are okay. But if it returned + * -1, there was an error, so make sure the connection is marked, and + * return -1. */ + if (rv < 0) { if (!base_conn->marked_for_close) connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); return -1; } + return 0; } else { - /* it's a hidden-service request */ - rend_cache_entry_t *entry; - int r; - rend_service_authorization_t *client_auth; - rend_data_t *rend_data; + /* If we get here, it's a request for a .onion address! */ tor_assert(!automap); + + /* Check whether it's RESOLVE or RESOLVE_PTR. We don't handle those + * for hidden service addresses. */ if (SOCKS_COMMAND_IS_RESOLVE(socks->command)) { /* if it's a resolve request, fail it right now, rather than * building all the circuits and then realizing it won't work. */ @@ -1312,6 +1686,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } + /* If we were passed a circuit, then we need to fail. .onion addresses + * only work when we launch our own circuits for now. */ if (circ) { log_warn(LD_CONTROL, "Attachstream to a circuit is not " "supported for .onion addresses currently. Failing."); @@ -1319,51 +1695,77 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } - ENTRY_TO_EDGE_CONN(conn)->rend_data = rend_data = - tor_malloc_zero(sizeof(rend_data_t)); - strlcpy(rend_data->onion_address, socks->address, - sizeof(rend_data->onion_address)); + /* Look up if we have client authorization configured for this hidden + * service. If we do, associate it with the rend_data. */ + rend_service_authorization_t *client_auth = + rend_client_lookup_service_authorization(socks->address); + + const char *cookie = NULL; + rend_auth_type_t auth_type = REND_NO_AUTH; + if (client_auth) { + log_info(LD_REND, "Using previously configured client authorization " + "for hidden service request."); + auth_type = client_auth->auth_type; + cookie = client_auth->descriptor_cookie; + } + + /* Fill in the rend_data field so we can start doing a connection to + * a hidden service. */ + rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data = + rend_data_client_create(socks->address, NULL, cookie, auth_type); + if (rend_data == NULL) { + return -1; + } log_info(LD_REND,"Got a hidden service request for ID '%s'", safe_str_client(rend_data->onion_address)); - /* see if we already have it cached */ - r = rend_cache_lookup_entry(rend_data->onion_address, -1, &entry); - if (r<0) { - log_warn(LD_BUG,"Invalid service name '%s'", - safe_str_client(rend_data->onion_address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); - return -1; + + /* Lookup the given onion address. If invalid, stop right now else we + * might have it in the cache or not, it will be tested later on. */ + unsigned int refetch_desc = 0; + rend_cache_entry_t *entry = NULL; + const int rend_cache_lookup_result = + rend_cache_lookup_entry(rend_data->onion_address, -1, &entry); + if (rend_cache_lookup_result < 0) { + switch (-rend_cache_lookup_result) { + case EINVAL: + /* We should already have rejected this address! */ + log_warn(LD_BUG,"Invalid service name '%s'", + safe_str_client(rend_data->onion_address)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); + return -1; + case ENOENT: + refetch_desc = 1; + break; + default: + log_warn(LD_BUG, "Unknown cache lookup error %d", + rend_cache_lookup_result); + return -1; + } } /* Help predict this next time. We're not sure if it will need * a stable circuit yet, but we know we'll need *something*. */ rep_hist_note_used_internal(now, 0, 1); - /* Look up if we have client authorization for it. */ - client_auth = rend_client_lookup_service_authorization( - rend_data->onion_address); - if (client_auth) { - log_info(LD_REND, "Using previously configured client authorization " - "for hidden service request."); - memcpy(rend_data->descriptor_cookie, - client_auth->descriptor_cookie, REND_DESC_COOKIE_LEN); - rend_data->auth_type = client_auth->auth_type; - } - if (r==0) { + /* Now we have a descriptor but is it usable or not? If not, refetch. + * Also, a fetch could have been requested if the onion address was not + * found in the cache previously. */ + if (refetch_desc || !rend_client_any_intro_points_usable(entry)) { + connection_ap_mark_as_non_pending_circuit(conn); base_conn->state = AP_CONN_STATE_RENDDESC_WAIT; log_info(LD_REND, "Unknown descriptor %s. Fetching.", - safe_str_client(rend_data->onion_address)); + safe_str_client(rend_data->onion_address)); rend_client_refetch_v2_renddesc(rend_data); - } else { /* r > 0 */ - base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT; - log_info(LD_REND, "Descriptor is here. Great."); - if (connection_ap_handshake_attach_circuit(conn) < 0) { - if (!base_conn->marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - return -1; - } + return 0; } + + /* We have the descriptor so launch a connection to the HS. */ + base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT; + log_info(LD_REND, "Descriptor is here. Great."); + connection_ap_mark_as_pending_circuit(conn); return 0; } + return 0; /* unreached but keeps the compiler happy */ } @@ -1395,35 +1797,82 @@ get_pf_socket(void) } #endif -/** Fetch the original destination address and port from a - * system-specific interface and put them into a - * socks_request_t as if they came from a socks request. - * - * Return -1 if an error prevents fetching the destination, - * else return 0. - */ +#if defined(TRANS_NETFILTER) || defined(TRANS_PF) || defined(TRANS_TPROXY) +/** Try fill in the address of <b>req</b> from the socket configured + * with <b>conn</b>. */ static int -connection_ap_get_original_destination(entry_connection_t *conn, - socks_request_t *req) +destination_from_socket(entry_connection_t *conn, socks_request_t *req) { -#ifdef TRANS_NETFILTER - /* Linux 2.4+ */ struct sockaddr_storage orig_dst; socklen_t orig_dst_len = sizeof(orig_dst); tor_addr_t addr; - if (getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IP, SO_ORIGINAL_DST, - (struct sockaddr*)&orig_dst, &orig_dst_len) < 0) { +#ifdef TRANS_TRPOXY + if (options->TransProxyType_parsed == TPT_TPROXY) { + if (getsockname(ENTRY_TO_CONN(conn)->s, (struct sockaddr*)&orig_dst, + &orig_dst_len) < 0) { + int e = tor_socket_errno(ENTRY_TO_CONN(conn)->s); + log_warn(LD_NET, "getsockname() failed: %s", tor_socket_strerror(e)); + return -1; + } + goto done; + } +#endif + +#ifdef TRANS_NETFILTER + int rv = -1; + switch (ENTRY_TO_CONN(conn)->socket_family) { +#ifdef TRANS_NETFILTER_IPV4 + case AF_INET: + rv = getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IP, SO_ORIGINAL_DST, + (struct sockaddr*)&orig_dst, &orig_dst_len); + break; +#endif +#ifdef TRANS_NETFILTER_IPV6 + case AF_INET6: + rv = getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IPV6, IP6T_SO_ORIGINAL_DST, + (struct sockaddr*)&orig_dst, &orig_dst_len); + break; +#endif + default: + log_warn(LD_BUG, + "Received transparent data from an unsuported socket family %d", + ENTRY_TO_CONN(conn)->socket_family); + return -1; + } + if (rv < 0) { int e = tor_socket_errno(ENTRY_TO_CONN(conn)->s); log_warn(LD_NET, "getsockopt() failed: %s", tor_socket_strerror(e)); return -1; } + goto done; +#elif defined(TRANS_PF) + if (getsockname(ENTRY_TO_CONN(conn)->s, (struct sockaddr*)&orig_dst, + &orig_dst_len) < 0) { + int e = tor_socket_errno(ENTRY_TO_CONN(conn)->s); + log_warn(LD_NET, "getsockname() failed: %s", tor_socket_strerror(e)); + return -1; + } + goto done; +#else + (void)conn; + (void)req; + log_warn(LD_BUG, "Unable to determine destination from socket."); + return -1; +#endif + done: tor_addr_from_sockaddr(&addr, (struct sockaddr*)&orig_dst, &req->port); tor_addr_to_str(req->address, &addr, sizeof(req->address), 1); return 0; -#elif defined(TRANS_PF) +} +#endif + +#ifdef TRANS_PF +static int +destination_from_pf(entry_connection_t *conn, socks_request_t *req) +{ struct sockaddr_storage proxy_addr; socklen_t proxy_addr_len = sizeof(proxy_addr); struct sockaddr *proxy_sa = (struct sockaddr*) &proxy_addr; @@ -1439,6 +1888,21 @@ connection_ap_get_original_destination(entry_connection_t *conn, return -1; } +#ifdef __FreeBSD__ + if (get_options()->TransProxyType_parsed == TPT_IPFW) { + /* ipfw(8) is used and in this case getsockname returned the original + destination */ + if (tor_addr_from_sockaddr(&addr, proxy_sa, &req->port) < 0) { + tor_fragile_assert(); + return -1; + } + + tor_addr_to_str(req->address, &addr, sizeof(req->address), 0); + + return 0; + } +#endif + memset(&pnl, 0, sizeof(pnl)); pnl.proto = IPPROTO_TCP; pnl.direction = PF_OUT; @@ -1485,6 +1949,37 @@ connection_ap_get_original_destination(entry_connection_t *conn, req->port = ntohs(pnl.rdport); return 0; +} +#endif + +/** Fetch the original destination address and port from a + * system-specific interface and put them into a + * socks_request_t as if they came from a socks request. + * + * Return -1 if an error prevents fetching the destination, + * else return 0. + */ +static int +connection_ap_get_original_destination(entry_connection_t *conn, + socks_request_t *req) +{ +#ifdef TRANS_NETFILTER + return destination_from_socket(conn, req); +#elif defined(TRANS_PF) + const or_options_t *options = get_options(); + + if (options->TransProxyType_parsed == TPT_PF_DIVERT) + return destination_from_socket(conn, req); + + if (options->TransProxyType_parsed == TPT_DEFAULT || + options->TransProxyType_parsed == TPT_IPFW) + return destination_from_pf(conn, req); + + (void)conn; + (void)req; + log_warn(LD_BUG, "Proxy destination determination mechanism %s unknown.", + options->TransProxyType); + return -1; #else (void)conn; (void)req; @@ -1713,7 +2208,8 @@ connection_ap_supports_optimistic_data(const entry_connection_t *conn) general circuit. */ if (edge_conn->on_circuit == NULL || edge_conn->on_circuit->state != CIRCUIT_STATE_OPEN || - edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_GENERAL) + (edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_GENERAL && + edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_REND_JOINED)) return 0; return conn->may_use_optimistic_data; @@ -1738,19 +2234,19 @@ connection_ap_get_begincell_flags(entry_connection_t *ap_conn) return 0; /* If only IPv4 is supported, no flags */ - if (ap_conn->ipv4_traffic_ok && !ap_conn->ipv6_traffic_ok) + if (ap_conn->entry_cfg.ipv4_traffic && !ap_conn->entry_cfg.ipv6_traffic) return 0; if (! cpath_layer || ! cpath_layer->extend_info) return 0; - if (!ap_conn->ipv4_traffic_ok) + if (!ap_conn->entry_cfg.ipv4_traffic) flags |= BEGIN_FLAG_IPV4_NOT_OK; exitnode = node_get_by_id(cpath_layer->extend_info->identity_digest); - if (ap_conn->ipv6_traffic_ok && exitnode) { + if (ap_conn->entry_cfg.ipv6_traffic && exitnode) { tor_addr_t a; tor_addr_make_null(&a, AF_INET6); if (compare_tor_addr_to_node_policy(&a, ap_conn->socks_request->port, @@ -1765,7 +2261,7 @@ connection_ap_get_begincell_flags(entry_connection_t *ap_conn) if (flags == BEGIN_FLAG_IPV6_OK) { /* When IPv4 and IPv6 are both allowed, consider whether to say we * prefer IPv6. Otherwise there's no point in declaring a preference */ - if (ap_conn->prefer_ipv6_traffic) + if (ap_conn->entry_cfg.prefer_ipv6) flags |= BEGIN_FLAG_IPV6_PREFERRED; } @@ -1826,8 +2322,9 @@ connection_ap_handshake_send_begin(entry_connection_t *ap_conn) } log_info(LD_APP, - "Sending relay cell %d to begin stream %d.", + "Sending relay cell %d on circ %u to begin stream %d.", (int)ap_conn->use_begindir, + (unsigned)circ->base_.n_circ_id, edge_conn->stream_id); begin_type = ap_conn->use_begindir ? @@ -2002,8 +2499,8 @@ connection_ap_make_link(connection_t *partner, /* Populate isolation fields. */ conn->socks_request->listener_type = CONN_TYPE_DIR_LISTENER; conn->original_dest_address = tor_strdup(address); - conn->session_group = session_group; - conn->isolation_flags = isolation_flags; + conn->entry_cfg.session_group = session_group; + conn->entry_cfg.isolation_flags = isolation_flags; base_conn->address = tor_strdup("(Tor_internal)"); tor_addr_make_unspec(&base_conn->addr); @@ -2021,12 +2518,7 @@ connection_ap_make_link(connection_t *partner, control_event_stream_status(conn, STREAM_EVENT_NEW, 0); /* attaching to a dirty circuit is fine */ - if (connection_ap_handshake_attach_circuit(conn) < 0) { - if (!base_conn->marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - return NULL; - } - + connection_ap_mark_as_pending_circuit(conn); log_info(LD_APP,"... application connection created and linked."); return conn; } @@ -2064,7 +2556,7 @@ tell_controller_about_resolved_result(entry_connection_t *conn, * As connection_ap_handshake_socks_resolved, but take a tor_addr_t to send * as the answer. */ -static void +void connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn, const tor_addr_t *answer, int ttl, @@ -2097,13 +2589,13 @@ connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn, **/ /* XXXX the use of the ttl and expires fields is nutty. Let's make this * interface and those that use it less ugly. */ -void -connection_ap_handshake_socks_resolved(entry_connection_t *conn, +MOCK_IMPL(void, +connection_ap_handshake_socks_resolved,(entry_connection_t *conn, int answer_type, size_t answer_len, const uint8_t *answer, int ttl, - time_t expires) + time_t expires)) { char buf[384]; size_t replylen; @@ -2241,13 +2733,21 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, endreason == END_STREAM_REASON_RESOURCELIMIT) { if (!conn->edge_.on_circuit || !CIRCUIT_IS_ORIGIN(conn->edge_.on_circuit)) { - // DNS remaps can trigger this. So can failed hidden service - // lookups. - log_info(LD_BUG, - "No origin circuit for successful SOCKS stream "U64_FORMAT - ". Reason: %d", - U64_PRINTF_ARG(ENTRY_TO_CONN(conn)->global_identifier), - endreason); + if (endreason != END_STREAM_REASON_RESOLVEFAILED) { + log_info(LD_BUG, + "No origin circuit for successful SOCKS stream "U64_FORMAT + ". Reason: %d", + U64_PRINTF_ARG(ENTRY_TO_CONN(conn)->global_identifier), + endreason); + } + /* + * Else DNS remaps and failed hidden service lookups can send us + * here with END_STREAM_REASON_RESOLVEFAILED; ignore it + * + * Perhaps we could make the test more precise; we can tell hidden + * services by conn->edge_.renddata != NULL; anything analogous for + * the DNS remap case? + */ } else { // XXX: Hrmm. It looks like optimistic data can't go through this // codepath, but someone should probably test it and make sure. @@ -2272,13 +2772,24 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, /* leave version, destport, destip zero */ connection_write_to_buf(buf, SOCKS4_NETWORK_LEN, ENTRY_TO_CONN(conn)); } else if (conn->socks_request->socks_version == 5) { - buf[0] = 5; /* version 5 */ - buf[1] = (char)status; - buf[2] = 0; - buf[3] = 1; /* ipv4 addr */ - memset(buf+4,0,6); /* Set external addr/port to 0. - The spec doesn't seem to say what to do here. -RD */ - connection_write_to_buf(buf,10,ENTRY_TO_CONN(conn)); + size_t buf_len; + memset(buf,0,sizeof(buf)); + if (tor_addr_family(&conn->edge_.base_.addr) == AF_INET) { + buf[0] = 5; /* version 5 */ + buf[1] = (char)status; + buf[2] = 0; + buf[3] = 1; /* ipv4 addr */ + /* 4 bytes for the header, 2 bytes for the port, 4 for the address. */ + buf_len = 10; + } else { /* AF_INET6. */ + buf[0] = 5; /* version 5 */ + buf[1] = (char)status; + buf[2] = 0; + buf[3] = 4; /* ipv6 addr */ + /* 4 bytes for the header, 2 bytes for the port, 16 for the address. */ + buf_len = 22; + } + connection_write_to_buf(buf,buf_len,ENTRY_TO_CONN(conn)); } /* If socks_version isn't 4 or 5, don't send anything. * This can happen in the case of AP bridges. */ @@ -2294,7 +2805,7 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, * Return -1 in the case where want to send a RELAY_END cell, and < -1 when * we don't. **/ -/* static */ int +STATIC int begin_cell_parse(const cell_t *cell, begin_cell_t *bcell, uint8_t *end_reason_out) { @@ -2387,7 +2898,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) relay_header_unpack(&rh, cell->payload); if (rh.length > RELAY_PAYLOAD_SIZE) - return -1; + return -END_CIRC_REASON_TORPROTOCOL; /* Note: we have to use relay_send_command_from_edge here, not * connection_edge_end or connection_edge_send_command, since those require @@ -2405,7 +2916,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) r = begin_cell_parse(cell, &bcell, &end_reason); if (r < -1) { - return -1; + return -END_CIRC_REASON_TORPROTOCOL; } else if (r == -1) { tor_free(bcell.address); relay_send_end_cell_from_edge(rh.stream_id, circ, end_reason, NULL); @@ -2449,8 +2960,8 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) return 0; } /* Make sure to get the 'real' address of the previous hop: the - * caller might want to know whether his IP address has changed, and - * we might already have corrected base_.addr[ess] for the relay's + * caller might want to know whether the remote IP address has changed, + * and we might already have corrected base_.addr[ess] for the relay's * canonical IP address. */ if (or_circ && or_circ->p_chan) address = tor_strdup(channel_get_actual_remote_address(or_circ->p_chan)); @@ -2503,15 +3014,31 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) n_stream->rend_data = rend_data_dup(origin_circ->rend_data); tor_assert(connection_edge_is_rendezvous_stream(n_stream)); assert_circuit_ok(circ); - if (rend_service_set_connection_addr_port(n_stream, origin_circ) < 0) { + + const int r = rend_service_set_connection_addr_port(n_stream, origin_circ); + if (r < 0) { log_info(LD_REND,"Didn't find rendezvous service (port %d)", n_stream->base_.port); + /* Send back reason DONE because we want to make hidden service port + * scanning harder thus instead of returning that the exit policy + * didn't match, which makes it obvious that the port is closed, + * return DONE and kill the circuit. That way, a user (malicious or + * not) needs one circuit per bad port unless it matches the policy of + * the hidden service. */ relay_send_end_cell_from_edge(rh.stream_id, circ, - END_STREAM_REASON_EXITPOLICY, + END_STREAM_REASON_DONE, origin_circ->cpath->prev); connection_free(TO_CONN(n_stream)); tor_free(address); - return 0; + + /* Drop the circuit here since it might be someone deliberately + * scanning the hidden service ports. Note that this mitigates port + * scanning by adding more work on the attacker side to successfully + * scan but does not fully solve it. */ + if (r < -1) + return END_CIRC_AT_ORIGIN; + else + return 0; } assert_circuit_ok(circ); log_debug(LD_REND,"Finished assigning addr/port"); @@ -2523,6 +3050,8 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) origin_circ->p_streams = n_stream; assert_circuit_ok(circ); + origin_circ->rend_data->nr_streams++; + connection_exit_connect(n_stream); /* For path bias: This circuit was used successfully */ @@ -2639,7 +3168,7 @@ connection_exit_connect(edge_connection_t *edge_conn) const tor_addr_t *addr; uint16_t port; connection_t *conn = TO_CONN(edge_conn); - int socket_error = 0; + int socket_error = 0, result; if ( (!connection_edge_is_rendezvous_stream(edge_conn) && router_compare_to_my_exit_policy(&edge_conn->base_.addr, @@ -2654,14 +3183,36 @@ connection_exit_connect(edge_connection_t *edge_conn) return; } - addr = &conn->addr; - port = conn->port; +#ifdef HAVE_SYS_UN_H + if (conn->socket_family != AF_UNIX) { +#else + { +#endif /* defined(HAVE_SYS_UN_H) */ + addr = &conn->addr; + port = conn->port; + + if (tor_addr_family(addr) == AF_INET6) + conn->socket_family = AF_INET6; + + log_debug(LD_EXIT, "about to try connecting"); + result = connection_connect(conn, conn->address, + addr, port, &socket_error); +#ifdef HAVE_SYS_UN_H + } else { + /* + * In the AF_UNIX case, we expect to have already had conn->port = 1, + * tor_addr_make_unspec(conn->addr) (cf. the way we mark in the incoming + * case in connection_handle_listener_read()), and conn->address should + * have the socket path to connect to. + */ + tor_assert(conn->address && strlen(conn->address) > 0); - if (tor_addr_family(addr) == AF_INET6) - conn->socket_family = AF_INET6; + log_debug(LD_EXIT, "about to try connecting"); + result = connection_connect_unix(conn, conn->address, &socket_error); +#endif /* defined(HAVE_SYS_UN_H) */ + } - log_debug(LD_EXIT,"about to try connecting"); - switch (connection_connect(conn, conn->address, addr, port, &socket_error)) { + switch (result) { case -1: { int reason = errno_to_stream_end_reason(socket_error); connection_edge_end(edge_conn, reason); @@ -2691,7 +3242,6 @@ connection_exit_connect(edge_connection_t *edge_conn) /* also, deliver a 'connected' cell back through the circuit. */ if (connection_edge_is_rendezvous_stream(edge_conn)) { - /* rendezvous stream */ /* don't send an address back! */ connection_edge_send_command(edge_conn, RELAY_COMMAND_CONNECTED, @@ -2830,10 +3380,10 @@ connection_ap_can_use_exit(const entry_connection_t *conn, const node_t *exit) addr_policy_result_t r; if (0 == tor_addr_parse(&addr, conn->socks_request->address)) { addrp = &addr; - } else if (!conn->ipv4_traffic_ok && conn->ipv6_traffic_ok) { + } else if (!conn->entry_cfg.ipv4_traffic && conn->entry_cfg.ipv6_traffic) { tor_addr_make_null(&addr, AF_INET6); addrp = &addr; - } else if (conn->ipv4_traffic_ok && !conn->ipv6_traffic_ok) { + } else if (conn->entry_cfg.ipv4_traffic && !conn->entry_cfg.ipv6_traffic) { tor_addr_make_null(&addr, AF_INET); addrp = &addr; } @@ -2939,7 +3489,7 @@ int connection_edge_compatible_with_circuit(const entry_connection_t *conn, const origin_circuit_t *circ) { - const uint8_t iso = conn->isolation_flags; + const uint8_t iso = conn->entry_cfg.isolation_flags; const socks_request_t *sr = conn->socks_request; /* If circ has never been used for an isolated connection, we can @@ -2988,7 +3538,8 @@ connection_edge_compatible_with_circuit(const entry_connection_t *conn, if ((iso & ISO_CLIENTADDR) && !tor_addr_eq(&ENTRY_TO_CONN(conn)->addr, &circ->client_addr)) return 0; - if ((iso & ISO_SESSIONGRP) && conn->session_group != circ->session_group) + if ((iso & ISO_SESSIONGRP) && + conn->entry_cfg.session_group != circ->session_group) return 0; if ((iso & ISO_NYM_EPOCH) && conn->nym_epoch != circ->nym_epoch) return 0; @@ -3027,7 +3578,7 @@ connection_edge_update_circuit_isolation(const entry_connection_t *conn, circ->client_proto_type = conn->socks_request->listener_type; circ->client_proto_socksver = conn->socks_request->socks_version; tor_addr_copy(&circ->client_addr, &ENTRY_TO_CONN(conn)->addr); - circ->session_group = conn->session_group; + circ->session_group = conn->entry_cfg.session_group; circ->nym_epoch = conn->nym_epoch; circ->socks_username = sr->username ? tor_memdup(sr->username, sr->usernamelen) : NULL; @@ -3054,7 +3605,7 @@ connection_edge_update_circuit_isolation(const entry_connection_t *conn, mixed |= ISO_CLIENTPROTO; if (!tor_addr_eq(&ENTRY_TO_CONN(conn)->addr, &circ->client_addr)) mixed |= ISO_CLIENTADDR; - if (conn->session_group != circ->session_group) + if (conn->entry_cfg.session_group != circ->session_group) mixed |= ISO_SESSIONGRP; if (conn->nym_epoch != circ->nym_epoch) mixed |= ISO_NYM_EPOCH; @@ -3062,7 +3613,7 @@ connection_edge_update_circuit_isolation(const entry_connection_t *conn, if (dry_run) return mixed; - if ((mixed & conn->isolation_flags) != 0) { + if ((mixed & conn->entry_cfg.isolation_flags) != 0) { log_warn(LD_BUG, "Updating a circuit with seemingly incompatible " "isolation flags."); } @@ -3116,3 +3667,12 @@ circuit_clear_isolation(origin_circuit_t *circ) circ->socks_username_len = circ->socks_password_len = 0; } +/** Free all storage held in module-scoped variables for connection_edge.c */ +void +connection_edge_free_all(void) +{ + untried_pending_connections = 0; + smartlist_free(pending_entry_connections); + pending_entry_connections = NULL; +} + diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index ea284cbcfd..5dfc8af901 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,11 +12,14 @@ #ifndef TOR_CONNECTION_EDGE_H #define TOR_CONNECTION_EDGE_H +#include "testsupport.h" + #define connection_mark_unattached_ap(conn, endreason) \ connection_mark_unattached_ap_((conn), (endreason), __LINE__, SHORT_FILE__) -void connection_mark_unattached_ap_(entry_connection_t *conn, int endreason, - int line, const char *file); +MOCK_DECL(void,connection_mark_unattached_ap_, + (entry_connection_t *conn, int endreason, + int line, const char *file)); int connection_edge_reached_eof(edge_connection_t *conn); int connection_edge_process_inbuf(edge_connection_t *conn, int package_partial); @@ -42,12 +45,17 @@ entry_connection_t *connection_ap_make_link(connection_t *partner, void connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply, size_t replylen, int endreason); -void connection_ap_handshake_socks_resolved(entry_connection_t *conn, - int answer_type, - size_t answer_len, - const uint8_t *answer, - int ttl, - time_t expires); +MOCK_DECL(void,connection_ap_handshake_socks_resolved, + (entry_connection_t *conn, + int answer_type, + size_t answer_len, + const uint8_t *answer, + int ttl, + time_t expires)); +void connection_ap_handshake_socks_resolved_addr(entry_connection_t *conn, + const tor_addr_t *answer, + int ttl, + time_t expires); int connection_exit_begin_conn(cell_t *cell, circuit_t *circ); int connection_exit_begin_resolve(cell_t *cell, or_circuit_t *circ); @@ -56,7 +64,20 @@ int connection_edge_is_rendezvous_stream(edge_connection_t *conn); int connection_ap_can_use_exit(const entry_connection_t *conn, const node_t *exit); void connection_ap_expire_beginning(void); -void connection_ap_attach_pending(void); +void connection_ap_rescan_and_attach_pending(void); +void connection_ap_attach_pending(int retry); +void connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn, + const char *file, int line); +#define connection_ap_mark_as_pending_circuit(c) \ + connection_ap_mark_as_pending_circuit_((c), __FILE__, __LINE__) +void connection_ap_mark_as_non_pending_circuit(entry_connection_t *entry_conn); +#define CONNECTION_AP_EXPECT_NONPENDING(c) do { \ + if (ENTRY_TO_CONN(c)->state == AP_CONN_STATE_CIRCUIT_WAIT) { \ + log_warn(LD_BUG, "At %s:%d: %p was unexpectedly in circuit_wait.", \ + __FILE__, __LINE__, (c)); \ + connection_ap_mark_as_non_pending_circuit(c); \ + } \ + } while (0) void connection_ap_fail_onehop(const char *failed_digest, cpath_build_state_t *build_state); void circuit_discard_optional_exit_enclaves(extend_info_t *info); @@ -92,6 +113,12 @@ int connection_edge_update_circuit_isolation(const entry_connection_t *conn, void circuit_clear_isolation(origin_circuit_t *circ); streamid_t get_unique_stream_id_by_circ(origin_circuit_t *circ); +void connection_edge_free_all(void); + +void connection_ap_warn_and_unmark_if_pending_circ( + entry_connection_t *entry_conn, + const char *where); + /** @name Begin-cell flags * * These flags are used in RELAY_BEGIN cells to change the default behavior @@ -130,11 +157,35 @@ typedef struct begin_cell_t { unsigned is_begindir : 1; } begin_cell_t; -int begin_cell_parse(const cell_t *cell, begin_cell_t *bcell, +STATIC int begin_cell_parse(const cell_t *cell, begin_cell_t *bcell, uint8_t *end_reason_out); -int connected_cell_format_payload(uint8_t *payload_out, +STATIC int connected_cell_format_payload(uint8_t *payload_out, const tor_addr_t *addr, uint32_t ttl); + +typedef struct { + /** Original address, after we lowercased it but before we started + * mapping it. + */ + char orig_address[MAX_SOCKS_ADDR_LEN]; + /** True iff the address has been automatically remapped to a local + * address in VirtualAddrNetwork. (Only set true when we do a resolve + * and get a virtual address; not when we connect to the address.) */ + int automap; + /** If this connection has a .exit address, who put it there? */ + addressmap_entry_source_t exit_source; + /** If we've rewritten the address, when does this map expire? */ + time_t map_expires; + /** If we should close the connection, this is the end_reason to pass + * to connection_mark_unattached_ap */ + int end_reason; + /** True iff we should close the connection, either because of error or + * because of successful early RESOLVED reply. */ + int should_close; +} rewrite_result_t; + +STATIC void connection_ap_handshake_rewrite(entry_connection_t *conn, + rewrite_result_t *out); #endif #endif diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 8e7cd9ea51..9730e1a952 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -30,6 +30,8 @@ #include "entrynodes.h" #include "geoip.h" #include "main.h" +#include "link_handshake.h" +#include "microdesc.h" #include "networkstatus.h" #include "nodelist.h" #include "reasons.h" @@ -37,6 +39,8 @@ #include "rephist.h" #include "router.h" #include "routerlist.h" +#include "ext_orport.h" +#include "scheduler.h" #ifdef USE_BUFFEREVENTS #include <event2/bufferevent_ssl.h> @@ -75,6 +79,10 @@ static void connection_or_handle_event_cb(struct bufferevent *bufev, * they form a linked list, with next_with_same_id as the next pointer. */ static digestmap_t *orconn_identity_map = NULL; +/** Global map between Extended ORPort identifiers and OR + * connections. */ +static digestmap_t *orconn_ext_or_id_map = NULL; + /** If conn is listed in orconn_identity_map, remove it, and clear * conn->identity_digest. Otherwise do nothing. */ void @@ -174,6 +182,71 @@ connection_or_set_identity_digest(or_connection_t *conn, const char *digest) #endif } +/** Remove the Extended ORPort identifier of <b>conn</b> from the + * global identifier list. Also, clear the identifier from the + * connection itself. */ +void +connection_or_remove_from_ext_or_id_map(or_connection_t *conn) +{ + or_connection_t *tmp; + if (!orconn_ext_or_id_map) + return; + if (!conn->ext_or_conn_id) + return; + + tmp = digestmap_remove(orconn_ext_or_id_map, conn->ext_or_conn_id); + if (!tor_digest_is_zero(conn->ext_or_conn_id)) + tor_assert(tmp == conn); + + memset(conn->ext_or_conn_id, 0, EXT_OR_CONN_ID_LEN); +} + +/** Return the connection whose ext_or_id is <b>id</b>. Return NULL if no such + * connection is found. */ +or_connection_t * +connection_or_get_by_ext_or_id(const char *id) +{ + if (!orconn_ext_or_id_map) + return NULL; + return digestmap_get(orconn_ext_or_id_map, id); +} + +/** Deallocate the global Extended ORPort identifier list */ +void +connection_or_clear_ext_or_id_map(void) +{ + digestmap_free(orconn_ext_or_id_map, NULL); + orconn_ext_or_id_map = NULL; +} + +/** Creates an Extended ORPort identifier for <b>conn</b> and deposits + * it into the global list of identifiers. */ +void +connection_or_set_ext_or_identifier(or_connection_t *conn) +{ + char random_id[EXT_OR_CONN_ID_LEN]; + or_connection_t *tmp; + + if (!orconn_ext_or_id_map) + orconn_ext_or_id_map = digestmap_new(); + + /* Remove any previous identifiers: */ + if (conn->ext_or_conn_id && !tor_digest_is_zero(conn->ext_or_conn_id)) + connection_or_remove_from_ext_or_id_map(conn); + + do { + crypto_rand(random_id, sizeof(random_id)); + } while (digestmap_get(orconn_ext_or_id_map, random_id)); + + if (!conn->ext_or_conn_id) + conn->ext_or_conn_id = tor_malloc_zero(EXT_OR_CONN_ID_LEN); + + memcpy(conn->ext_or_conn_id, random_id, EXT_OR_CONN_ID_LEN); + + tmp = digestmap_set(orconn_ext_or_id_map, random_id, conn); + tor_assert(!tmp); +} + /**************************************************************/ /** Map from a string describing what a non-open OR connection was doing when @@ -228,7 +301,7 @@ connection_or_get_state_description(or_connection_t *orconn, const char *conn_state; char tls_state[256]; - tor_assert(conn->type == CONN_TYPE_OR); + tor_assert(conn->type == CONN_TYPE_OR || conn->type == CONN_TYPE_EXT_OR); conn_state = conn_state_to_string(conn->type, conn->state); tor_tls_get_state_description(orconn->tls, tls_state, sizeof(tls_state)); @@ -416,6 +489,28 @@ var_cell_new(uint16_t payload_len) return cell; } +/** + * Copy a var_cell_t + */ + +var_cell_t * +var_cell_copy(const var_cell_t *src) +{ + var_cell_t *copy = NULL; + size_t size = 0; + + if (src != NULL) { + size = STRUCT_OFFSET(var_cell_t, payload) + src->payload_len; + copy = tor_malloc_zero(size); + copy->payload_len = src->payload_len; + copy->command = src->command; + copy->circ_id = src->circ_id; + memcpy(copy->payload, src->payload, copy->payload_len); + } + + return copy; +} + /** Release all space held by <b>cell</b>. */ void var_cell_free(var_cell_t *cell) @@ -505,48 +600,51 @@ connection_or_process_inbuf(or_connection_t *conn) return ret; } -/** When adding cells to an OR connection's outbuf, keep adding until the - * outbuf is at least this long, or we run out of cells. */ -#define OR_CONN_HIGHWATER (32*1024) - -/** Add cells to an OR connection's outbuf whenever the outbuf's data length - * drops below this size. */ -#define OR_CONN_LOWWATER (16*1024) - /** Called whenever we have flushed some data on an or_conn: add more data * from active circuits. */ int connection_or_flushed_some(or_connection_t *conn) { - size_t datalen, temp; - ssize_t n, flushed; - size_t cell_network_size = get_cell_network_size(conn->wide_circ_ids); + size_t datalen; + + /* The channel will want to update its estimated queue size */ + channel_update_xmit_queue_size(TLS_CHAN_TO_BASE(conn->chan)); /* If we're under the low water mark, add cells until we're just over the * high water mark. */ datalen = connection_get_outbuf_len(TO_CONN(conn)); if (datalen < OR_CONN_LOWWATER) { - while ((conn->chan) && channel_tls_more_to_flush(conn->chan)) { - /* Compute how many more cells we want at most */ - n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, cell_network_size); - /* Bail out if we don't want any more */ - if (n <= 0) break; - /* We're still here; try to flush some more cells */ - flushed = channel_tls_flush_some_cells(conn->chan, n); - /* Bail out if it says it didn't flush anything */ - if (flushed <= 0) break; - /* How much in the outbuf now? */ - temp = connection_get_outbuf_len(TO_CONN(conn)); - /* Bail out if we didn't actually increase the outbuf size */ - if (temp <= datalen) break; - /* Update datalen for the next iteration */ - datalen = temp; - } + /* Let the scheduler know */ + scheduler_channel_wants_writes(TLS_CHAN_TO_BASE(conn->chan)); } return 0; } +/** This is for channeltls.c to ask how many cells we could accept if + * they were available. */ +ssize_t +connection_or_num_cells_writeable(or_connection_t *conn) +{ + size_t datalen, cell_network_size; + ssize_t n = 0; + + tor_assert(conn); + + /* + * If we're under the high water mark, we're potentially + * writeable; note this is different from the calculation above + * used to trigger when to start writing after we've stopped. + */ + datalen = connection_get_outbuf_len(TO_CONN(conn)); + if (datalen < OR_CONN_HIGHWATER) { + cell_network_size = get_cell_network_size(conn->wide_circ_ids); + n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, cell_network_size); + } + + return n; +} + /** Connection <b>conn</b> has finished writing and has no bytes left on * its outbuf. * @@ -645,7 +743,8 @@ connection_or_about_to_close(or_connection_t *or_conn) reason); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_problem( - orconn_end_reason_to_control_string(reason), reason); + orconn_end_reason_to_control_string(reason), + reason, or_conn); } } } else if (conn->hold_open_until_flushed) { @@ -756,6 +855,45 @@ connection_or_update_token_buckets(smartlist_t *conns, }); } +/** How long do we wait before killing non-canonical OR connections with no + * circuits? In Tor versions up to 0.2.1.25 and 0.2.2.12-alpha, we waited 15 + * minutes before cancelling these connections, which caused fast relays to + * accrue many many idle connections. Hopefully 3-4.5 minutes is low enough + * that it kills most idle connections, without being so low that we cause + * clients to bounce on and off. + * + * For canonical connections, the limit is higher, at 15-22.5 minutes. + * + * For each OR connection, we randomly add up to 50% extra to its idle_timeout + * field, to avoid exposing when exactly the last circuit closed. Since we're + * storing idle_timeout in a uint16_t, don't let these values get higher than + * 12 hours or so without revising connection_or_set_canonical and/or expanding + * idle_timeout. + */ +#define IDLE_OR_CONN_TIMEOUT_NONCANONICAL 180 +#define IDLE_OR_CONN_TIMEOUT_CANONICAL 900 + +/* Mark <b>or_conn</b> as canonical if <b>is_canonical</b> is set, and + * non-canonical otherwise. Adjust idle_timeout accordingly. + */ +void +connection_or_set_canonical(or_connection_t *or_conn, + int is_canonical) +{ + const unsigned int timeout_base = is_canonical ? + IDLE_OR_CONN_TIMEOUT_CANONICAL : IDLE_OR_CONN_TIMEOUT_NONCANONICAL; + + if (bool_eq(is_canonical, or_conn->is_canonical) && + or_conn->idle_timeout != 0) { + /* Don't recalculate an existing idle_timeout unless the canonical + * status changed. */ + return; + } + + or_conn->is_canonical = !! is_canonical; /* force to a 1-bit boolean */ + or_conn->idle_timeout = timeout_base + crypto_rand_int(timeout_base / 2); +} + /** If we don't necessarily know the router we're connecting to, but we * have an addr/port/id_digest, then fill in as much as we can. Start * by checking to see if this describes a router we know. @@ -780,7 +918,7 @@ connection_or_init_conn_from_address(or_connection_t *conn, /* XXXX proposal 186 is making this more complex. For now, a conn is canonical when it uses the _preferred_ address. */ if (tor_addr_eq(&conn->base_.addr, &node_ap.addr)) - conn->is_canonical = 1; + connection_or_set_canonical(conn, 1); if (!started_here) { /* Override the addr/port, so our log messages will make sense. * This is dangerous, since if we ever try looking up a conn by @@ -799,21 +937,23 @@ connection_or_init_conn_from_address(or_connection_t *conn, tor_free(conn->base_.address); conn->base_.address = tor_dup_addr(&node_ap.addr); } else { - const char *n; - /* If we're an authoritative directory server, we may know a - * nickname for this router. */ - n = dirserv_get_nickname_by_digest(id_digest); - if (n) { - conn->nickname = tor_strdup(n); - } else { - conn->nickname = tor_malloc(HEX_DIGEST_LEN+2); - conn->nickname[0] = '$'; - base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1, - conn->identity_digest, DIGEST_LEN); - } + conn->nickname = tor_malloc(HEX_DIGEST_LEN+2); + conn->nickname[0] = '$'; + base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1, + conn->identity_digest, DIGEST_LEN); + tor_free(conn->base_.address); conn->base_.address = tor_dup_addr(addr); } + + /* + * We have to tell channeltls.c to update the channel marks (local, in + * particular), since we may have changed the address. + */ + + if (conn->chan) { + channel_tls_update_marks(conn); + } } /** These just pass all the is_bad_for_new_circs manipulation on to @@ -1008,7 +1148,7 @@ connection_or_connect_failed(or_connection_t *conn, { control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason); if (!authdir_mode_tests_reachability(get_options())) - control_event_bootstrap_problem(msg, reason); + control_event_bootstrap_problem(msg, reason, conn); } /** <b>conn</b> got an error in connection_handle_read_impl() or @@ -1033,9 +1173,7 @@ connection_or_notify_error(or_connection_t *conn, if (conn->chan) { chan = TLS_CHAN_TO_BASE(conn->chan); /* Don't transition if we're already in closing, closed or error */ - if (!(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(chan)) { channel_close_for_error(chan); } } @@ -1058,10 +1196,10 @@ connection_or_notify_error(or_connection_t *conn, * * Return the launched conn, or NULL if it failed. */ -or_connection_t * -connection_or_connect(const tor_addr_t *_addr, uint16_t port, - const char *id_digest, - channel_tls_t *chan) + +MOCK_IMPL(or_connection_t *, +connection_or_connect, (const tor_addr_t *_addr, uint16_t port, + const char *id_digest, channel_tls_t *chan)) { or_connection_t *conn; const or_options_t *options = get_options(); @@ -1082,7 +1220,7 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, return NULL; } - conn = or_connection_new(tor_addr_family(&addr)); + conn = or_connection_new(CONN_TYPE_OR, tor_addr_family(&addr)); /* * Set up conn so it's got all the data we need to remember for channels @@ -1125,6 +1263,12 @@ connection_or_connect(const tor_addr_t *_addr, uint16_t port, "your pluggable transport proxy stopped running.", fmt_addrport(&TO_CONN(conn)->addr, TO_CONN(conn)->port), transport_name, transport_name); + + control_event_bootstrap_problem( + "Can't connect to bridge", + END_OR_CONN_REASON_PT_MISSING, + conn); + } else { log_warn(LD_GENERAL, "Tried to connect to '%s' through a proxy, but " "the proxy address could not be found.", @@ -1188,9 +1332,7 @@ connection_or_close_normally(or_connection_t *orconn, int flush) if (orconn->chan) { chan = TLS_CHAN_TO_BASE(orconn->chan); /* Don't transition if we're already in closing, closed or error */ - if (!(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(chan)) { channel_close_from_lower_layer(chan); } } @@ -1200,8 +1342,8 @@ connection_or_close_normally(or_connection_t *orconn, int flush) * the error state. */ -void -connection_or_close_for_error(or_connection_t *orconn, int flush) +MOCK_IMPL(void, +connection_or_close_for_error,(or_connection_t *orconn, int flush)) { channel_t *chan = NULL; @@ -1211,9 +1353,7 @@ connection_or_close_for_error(or_connection_t *orconn, int flush) if (orconn->chan) { chan = TLS_CHAN_TO_BASE(orconn->chan); /* Don't transition if we're already in closing, closed or error */ - if (!(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(chan)) { channel_close_for_error(chan); } } @@ -1227,8 +1367,8 @@ connection_or_close_for_error(or_connection_t *orconn, int flush) * * Return -1 if <b>conn</b> is broken, else return 0. */ -int -connection_tls_start_handshake(or_connection_t *conn, int receiving) +MOCK_IMPL(int, +connection_tls_start_handshake,(or_connection_t *conn, int receiving)) { channel_listener_t *chan_listener; channel_t *chan; @@ -1333,17 +1473,12 @@ connection_tls_continue_handshake(or_connection_t *conn) { int result; check_no_tls_errors(); - again: - if (conn->base_.state == OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING) { - // log_notice(LD_OR, "Renegotiate with %p", conn->tls); - result = tor_tls_renegotiate(conn->tls); - // log_notice(LD_OR, "Result: %d", result); - } else { - tor_assert(conn->base_.state == OR_CONN_STATE_TLS_HANDSHAKING); - // log_notice(LD_OR, "Continue handshake with %p", conn->tls); - result = tor_tls_handshake(conn->tls); - // log_notice(LD_OR, "Result: %d", result); - } + + tor_assert(conn->base_.state == OR_CONN_STATE_TLS_HANDSHAKING); + // log_notice(LD_OR, "Continue handshake with %p", conn->tls); + result = tor_tls_handshake(conn->tls); + // log_notice(LD_OR, "Result: %d", result); + switch (result) { CASE_TOR_TLS_ERROR_ANY: log_info(LD_OR,"tls error [%s]. breaking connection.", @@ -1352,23 +1487,10 @@ connection_tls_continue_handshake(or_connection_t *conn) case TOR_TLS_DONE: if (! tor_tls_used_v1_handshake(conn->tls)) { if (!tor_tls_is_server(conn->tls)) { - if (conn->base_.state == OR_CONN_STATE_TLS_HANDSHAKING) { - if (tor_tls_received_v3_certificate(conn->tls)) { - log_info(LD_OR, "Client got a v3 cert! Moving on to v3 " - "handshake with ciphersuite %s", - tor_tls_get_ciphersuite_name(conn->tls)); - return connection_or_launch_v3_or_handshake(conn); - } else { - log_debug(LD_OR, "Done with initial SSL handshake (client-side)." - " Requesting renegotiation."); - connection_or_change_state(conn, - OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING); - goto again; - } - } - // log_notice(LD_OR,"Done. state was %d.", conn->base_.state); + tor_assert(conn->base_.state == OR_CONN_STATE_TLS_HANDSHAKING); + return connection_or_launch_v3_or_handshake(conn); } else { - /* v2/v3 handshake, but not a client. */ + /* v2/v3 handshake, but we are not a client. */ log_debug(LD_OR, "Done with initial SSL handshake (server-side). " "Expecting renegotiation or VERSIONS cell"); tor_tls_set_renegotiate_callback(conn->tls, @@ -1381,6 +1503,7 @@ connection_tls_continue_handshake(or_connection_t *conn) return 0; } } + tor_assert(tor_tls_is_server(conn->tls)); return connection_tls_finish_handshake(conn); case TOR_TLS_WANTWRITE: connection_start_writing(TO_CONN(conn)); @@ -1416,22 +1539,8 @@ connection_or_handle_event_cb(struct bufferevent *bufev, short event, if (! tor_tls_used_v1_handshake(conn->tls)) { if (!tor_tls_is_server(conn->tls)) { if (conn->base_.state == OR_CONN_STATE_TLS_HANDSHAKING) { - if (tor_tls_received_v3_certificate(conn->tls)) { - log_info(LD_OR, "Client got a v3 cert!"); - if (connection_or_launch_v3_or_handshake(conn) < 0) - connection_or_close_for_error(conn, 0); - return; - } else { - connection_or_change_state(conn, - OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING); - tor_tls_unblock_renegotiation(conn->tls); - if (bufferevent_ssl_renegotiate(conn->base_.bufev)<0) { - log_warn(LD_OR, "Start_renegotiating went badly."); - connection_or_close_for_error(conn, 0); - } - tor_tls_unblock_renegotiation(conn->tls); - return; /* ???? */ - } + if (connection_or_launch_v3_or_handshake(conn) < 0) + connection_or_close_for_error(conn, 0); } } else { const int handshakes = tor_tls_get_num_server_handshakes(conn->tls); @@ -1485,7 +1594,8 @@ connection_or_handle_event_cb(struct bufferevent *bufev, short event, int connection_or_nonopen_was_started_here(or_connection_t *conn) { - tor_assert(conn->base_.type == CONN_TYPE_OR); + tor_assert(conn->base_.type == CONN_TYPE_OR || + conn->base_.type == CONN_TYPE_EXT_OR); if (!conn->tls) return 1; /* it's still in proxy states or something */ if (conn->handshake_state) @@ -1494,11 +1604,11 @@ connection_or_nonopen_was_started_here(or_connection_t *conn) } /** <b>Conn</b> just completed its handshake. Return 0 if all is well, and - * return -1 if he is lying, broken, or otherwise something is wrong. + * return -1 if they are lying, broken, or otherwise something is wrong. * * If we initiated this connection (<b>started_here</b> is true), make sure * the other side sent a correctly formed certificate. If I initiated the - * connection, make sure it's the right guy. + * connection, make sure it's the right relay by checking the certificate. * * Otherwise (if we _didn't_ initiate this connection), it's okay for * the certificate to be weird or absent. @@ -1514,7 +1624,7 @@ connection_or_nonopen_was_started_here(or_connection_t *conn) * 1) Set conn->circ_id_type according to tor-spec.txt. * 2) If we're an authdirserver and we initiated the connection: drop all * descriptors that claim to be on that IP/port but that aren't - * this guy; and note that this guy is reachable. + * this relay; and note that this relay is reachable. * 3) If this is a bridge and we didn't configure its identity * fingerprint, remember the keyid we just learned. */ @@ -1589,9 +1699,17 @@ connection_or_check_valid_tls_handshake(or_connection_t *conn, * or renegotiation. For v3 handshakes, this is right after we get a * certificate chain in a CERTS cell. * - * If we want any particular ID before, record the one we got. + * If we did not know the ID before, record the one we got. * - * If we wanted an ID, but we didn't get it, log a warning and return -1. + * If we wanted an ID, but we didn't get the one we expected, log a message + * and return -1. + * On relays: + * - log a protocol warning whenever the fingerprints don't match; + * On clients: + * - if a relay's fingerprint doesn't match, log a warning; + * - if we don't have updated relay fingerprints from a recent consensus, and + * a fallback directory mirror's hard-coded fingerprint has changed, log an + * info explaining that we will try another fallback. * * If we're testing reachability, remember what we learned. * @@ -1602,7 +1720,6 @@ connection_or_client_learned_peer_id(or_connection_t *conn, const uint8_t *peer_id) { const or_options_t *options = get_options(); - int severity = server_mode(options) ? LOG_PROTOCOL_WARN : LOG_WARN; if (tor_digest_is_zero(conn->identity_digest)) { connection_or_set_identity_digest(conn, (const char*)peer_id); @@ -1627,10 +1744,43 @@ connection_or_client_learned_peer_id(or_connection_t *conn, base16_encode(seen, sizeof(seen), (const char*)peer_id, DIGEST_LEN); base16_encode(expected, sizeof(expected), conn->identity_digest, DIGEST_LEN); + const int using_hardcoded_fingerprints = + !networkstatus_get_reasonably_live_consensus(time(NULL), + usable_consensus_flavor()); + const int is_fallback_fingerprint = router_digest_is_fallback_dir( + conn->identity_digest); + const int is_authority_fingerprint = router_digest_is_trusted_dir( + conn->identity_digest); + int severity; + const char *extra_log = ""; + + if (server_mode(options)) { + severity = LOG_PROTOCOL_WARN; + } else { + if (using_hardcoded_fingerprints) { + /* We need to do the checks in this order, because the list of + * fallbacks includes the list of authorities */ + if (is_authority_fingerprint) { + severity = LOG_WARN; + } else if (is_fallback_fingerprint) { + /* we expect a small number of fallbacks to change from their + * hard-coded fingerprints over the life of a release */ + severity = LOG_INFO; + extra_log = " Tor will try a different fallback."; + } else { + /* it's a bridge, it's either a misconfiguration, or unexpected */ + severity = LOG_WARN; + } + } else { + /* a relay has changed its fingerprint from the one in the consensus */ + severity = LOG_WARN; + } + } + log_fn(severity, LD_HANDSHAKE, "Tried connecting to router at %s:%d, but identity key was not " - "as expected: wanted %s but got %s.", - conn->base_.address, conn->base_.port, expected, seen); + "as expected: wanted %s but got %s.%s", + conn->base_.address, conn->base_.port, expected, seen, extra_log); entry_guard_register_connect_status(conn->identity_digest, 0, 1, time(NULL)); control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, @@ -1638,7 +1788,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn, if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_problem( "Unexpected identity in router certificate", - END_OR_CONN_REASON_OR_IDENTITY); + END_OR_CONN_REASON_OR_IDENTITY, + conn); return -1; } if (authdir_mode_tests_reachability(options)) { @@ -1666,7 +1817,7 @@ connection_or_client_used(or_connection_t *conn) * * Make sure we are happy with the person we just handshaked with. * - * If he initiated the connection, make sure he's not already connected, + * If they initiated the connection, make sure they're not already connected, * then initialize conn from the information in router. * * If all is successful, call circuit_n_conn_done() to handle events @@ -1681,6 +1832,8 @@ connection_tls_finish_handshake(or_connection_t *conn) char digest_rcvd[DIGEST_LEN]; int started_here = connection_or_nonopen_was_started_here(conn); + tor_assert(!started_here); + log_debug(LD_HANDSHAKE,"%s tls handshake on %p with %s done, using " "ciphersuite %s. verifying.", started_here?"outgoing":"incoming", @@ -1688,30 +1841,25 @@ connection_tls_finish_handshake(or_connection_t *conn) safe_str_client(conn->base_.address), tor_tls_get_ciphersuite_name(conn->tls)); - directory_set_dirty(); - if (connection_or_check_valid_tls_handshake(conn, started_here, digest_rcvd) < 0) return -1; - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); if (tor_tls_used_v1_handshake(conn->tls)) { conn->link_proto = 1; - if (!started_here) { - connection_or_init_conn_from_address(conn, &conn->base_.addr, - conn->base_.port, digest_rcvd, 0); - } + connection_or_init_conn_from_address(conn, &conn->base_.addr, + conn->base_.port, digest_rcvd, 0); tor_tls_block_renegotiation(conn->tls); + rep_hist_note_negotiated_link_proto(1, started_here); return connection_or_set_state_open(conn); } else { connection_or_change_state(conn, OR_CONN_STATE_OR_HANDSHAKING_V2); if (connection_init_or_handshake_state(conn, started_here) < 0) return -1; - if (!started_here) { - connection_or_init_conn_from_address(conn, &conn->base_.addr, - conn->base_.port, digest_rcvd, 0); - } + connection_or_init_conn_from_address(conn, &conn->base_.addr, + conn->base_.port, digest_rcvd, 0); return connection_or_send_versions(conn, 0); } } @@ -1726,9 +1874,8 @@ static int connection_or_launch_v3_or_handshake(or_connection_t *conn) { tor_assert(connection_or_nonopen_was_started_here(conn)); - tor_assert(tor_tls_received_v3_certificate(conn->tls)); - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); connection_or_change_state(conn, OR_CONN_STATE_OR_HANDSHAKING_V3); if (connection_init_or_handshake_state(conn, 1) < 0) @@ -1762,8 +1909,8 @@ or_handshake_state_free(or_handshake_state_t *state) return; crypto_digest_free(state->digest_sent); crypto_digest_free(state->digest_received); - tor_cert_free(state->auth_cert); - tor_cert_free(state->id_cert); + tor_x509_cert_free(state->auth_cert); + tor_x509_cert_free(state->id_cert); memwipe(state, 0xBE, sizeof(or_handshake_state_t)); tor_free(state); } @@ -1890,18 +2037,15 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn) if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) or_handshake_state_record_cell(conn, conn->handshake_state, cell, 0); - - if (cell->command != CELL_PADDING) - conn->timestamp_last_added_nonpadding = approx_time(); } /** Pack a variable-length <b>cell</b> into wire-format, and write it onto * <b>conn</b>'s outbuf. Right now, this <em>DOES NOT</em> support cells that * affect a circuit. */ -void -connection_or_write_var_cell_to_buf(const var_cell_t *cell, - or_connection_t *conn) +MOCK_IMPL(void, +connection_or_write_var_cell_to_buf,(const var_cell_t *cell, + or_connection_t *conn)) { int n; char hdr[VAR_CELL_MAX_HEADER_SIZE]; @@ -1913,8 +2057,6 @@ connection_or_write_var_cell_to_buf(const var_cell_t *cell, cell->payload_len, TO_CONN(conn)); if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3) or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0); - if (cell->command != CELL_PADDING) - conn->timestamp_last_added_nonpadding = approx_time(); /* Touch the channel's active timestamp if there is one */ if (conn->chan) @@ -1947,6 +2089,19 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) { var_cell_t *var_cell; + /* + * Note on memory management for incoming cells: below the channel layer, + * we shouldn't need to consider its internal queueing/copying logic. It + * is safe to pass cells to it on the stack or on the heap, but in the + * latter case we must be sure we free them later. + * + * The incoming cell queue code in channel.c will (in the common case) + * decide it can pass them to the upper layer immediately, in which case + * those functions may run directly on the cell pointers we pass here, or + * it may decide to queue them, in which case it will allocate its own + * buffer and copy the cell. + */ + while (1) { log_debug(LD_OR, TOR_SOCKET_T_FORMAT": starting, inbuf_datalen %d " @@ -1961,7 +2116,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (conn->chan) channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); channel_tls_handle_var_cell(var_cell, conn); var_cell_free(var_cell); } else { @@ -1977,7 +2132,7 @@ connection_or_process_cells_from_inbuf(or_connection_t *conn) if (conn->chan) channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan)); - circuit_build_times_network_is_live(&circ_times); + circuit_build_times_network_is_live(get_circuit_build_times_mutable()); connection_fetch_from_buf(buf, cell_network_size, TO_CONN(conn)); /* retrieve cell info from buf (create the host-order struct from the @@ -2046,8 +2201,8 @@ connection_or_send_versions(or_connection_t *conn, int v3_plus) /** Send a NETINFO cell on <b>conn</b>, telling the other server what we know * about their address, our address, and the current time. */ -int -connection_or_send_netinfo(or_connection_t *conn) +MOCK_IMPL(int, +connection_or_send_netinfo,(or_connection_t *conn)) { cell_t cell; time_t now = time(NULL); @@ -2116,7 +2271,7 @@ connection_or_send_netinfo(or_connection_t *conn) int connection_or_send_certs_cell(or_connection_t *conn) { - const tor_cert_t *link_cert = NULL, *id_cert = NULL; + const tor_x509_cert_t *link_cert = NULL, *id_cert = NULL; const uint8_t *link_encoded = NULL, *id_encoded = NULL; size_t link_len, id_len; var_cell_t *cell; @@ -2131,8 +2286,8 @@ connection_or_send_certs_cell(or_connection_t *conn) server_mode = ! conn->handshake_state->started_here; if (tor_tls_get_my_certs(server_mode, &link_cert, &id_cert) < 0) return -1; - tor_cert_get_der(link_cert, &link_encoded, &link_len); - tor_cert_get_der(id_cert, &id_encoded, &id_len); + tor_x509_cert_get_der(link_cert, &link_encoded, &link_len); + tor_x509_cert_get_der(id_cert, &id_encoded, &id_len); cell_len = 1 /* 1 byte: num certs in cell */ + 2 * ( 1 + 2 ) /* For each cert: 1 byte for type, 2 for length */ + @@ -2168,28 +2323,36 @@ connection_or_send_certs_cell(or_connection_t *conn) int connection_or_send_auth_challenge_cell(or_connection_t *conn) { - var_cell_t *cell; - uint8_t *cp; - uint8_t challenge[OR_AUTH_CHALLENGE_LEN]; + var_cell_t *cell = NULL; + int r = -1; tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3); if (! conn->handshake_state) return -1; - if (crypto_rand((char*)challenge, OR_AUTH_CHALLENGE_LEN) < 0) - return -1; - cell = var_cell_new(OR_AUTH_CHALLENGE_LEN + 4); + auth_challenge_cell_t *ac = auth_challenge_cell_new(); + + crypto_rand((char*)ac->challenge, sizeof(ac->challenge)); + + auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_TLSSECRET); + auth_challenge_cell_set_n_methods(ac, + auth_challenge_cell_getlen_methods(ac)); + + cell = var_cell_new(auth_challenge_cell_encoded_len(ac)); + ssize_t len = auth_challenge_cell_encode(cell->payload, cell->payload_len, + ac); + if (len != cell->payload_len) + goto done; cell->command = CELL_AUTH_CHALLENGE; - memcpy(cell->payload, challenge, OR_AUTH_CHALLENGE_LEN); - cp = cell->payload + OR_AUTH_CHALLENGE_LEN; - set_uint16(cp, htons(1)); /* We recognize one authentication type. */ - set_uint16(cp+2, htons(AUTHTYPE_RSA_SHA256_TLSSECRET)); connection_or_write_var_cell_to_buf(cell, conn); + r = 0; + + done: var_cell_free(cell); - memwipe(challenge, 0, sizeof(challenge)); + auth_challenge_cell_free(ac); - return 0; + return r; } /** Compute the main body of an AUTHENTICATE cell that a client can use @@ -2216,28 +2379,28 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn, crypto_pk_t *signing_key, int server) { - uint8_t *ptr; + auth1_t *auth = NULL; + auth_ctx_t *ctx = auth_ctx_new(); + int result; /* assert state is reasonable XXXX */ - if (outlen < V3_AUTH_FIXED_PART_LEN || - (!server && outlen < V3_AUTH_BODY_LEN)) - return -1; + ctx->is_ed = 0; - ptr = out; + auth = auth1_new(); /* Type: 8 bytes. */ - memcpy(ptr, "AUTH0001", 8); - ptr += 8; + memcpy(auth1_getarray_type(auth), "AUTH0001", 8); { - const tor_cert_t *id_cert=NULL, *link_cert=NULL; - const digests_t *my_digests, *their_digests; + const tor_x509_cert_t *id_cert=NULL, *link_cert=NULL; + const common_digests_t *my_digests, *their_digests; const uint8_t *my_id, *their_id, *client_id, *server_id; if (tor_tls_get_my_certs(server, &link_cert, &id_cert)) - return -1; - my_digests = tor_cert_get_id_digests(id_cert); - their_digests = tor_cert_get_id_digests(conn->handshake_state->id_cert); + goto err; + my_digests = tor_x509_cert_get_id_digests(id_cert); + their_digests = + tor_x509_cert_get_id_digests(conn->handshake_state->id_cert); tor_assert(my_digests); tor_assert(their_digests); my_id = (uint8_t*)my_digests->d[DIGEST_SHA256]; @@ -2247,12 +2410,10 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn, server_id = server ? my_id : their_id; /* Client ID digest: 32 octets. */ - memcpy(ptr, client_id, 32); - ptr += 32; + memcpy(auth->cid, client_id, 32); /* Server ID digest: 32 octets. */ - memcpy(ptr, server_id, 32); - ptr += 32; + memcpy(auth->sid, server_id, 32); } { @@ -2266,73 +2427,101 @@ connection_or_compute_authenticate_cell_body(or_connection_t *conn, } /* Server log digest : 32 octets */ - crypto_digest_get_digest(server_d, (char*)ptr, 32); - ptr += 32; + crypto_digest_get_digest(server_d, (char*)auth->slog, 32); /* Client log digest : 32 octets */ - crypto_digest_get_digest(client_d, (char*)ptr, 32); - ptr += 32; + crypto_digest_get_digest(client_d, (char*)auth->clog, 32); } { /* Digest of cert used on TLS link : 32 octets. */ - const tor_cert_t *cert = NULL; - tor_cert_t *freecert = NULL; + const tor_x509_cert_t *cert = NULL; + tor_x509_cert_t *freecert = NULL; if (server) { tor_tls_get_my_certs(1, &cert, NULL); } else { freecert = tor_tls_get_peer_cert(conn->tls); cert = freecert; } - if (!cert) - return -1; - memcpy(ptr, tor_cert_get_cert_digests(cert)->d[DIGEST_SHA256], 32); + if (!cert) { + log_warn(LD_OR, "Unable to find cert when making AUTH1 data."); + goto err; + } + + memcpy(auth->scert, + tor_x509_cert_get_cert_digests(cert)->d[DIGEST_SHA256], 32); if (freecert) - tor_cert_free(freecert); - ptr += 32; + tor_x509_cert_free(freecert); } /* HMAC of clientrandom and serverrandom using master key : 32 octets */ - tor_tls_get_tlssecrets(conn->tls, ptr); - ptr += 32; - - tor_assert(ptr - out == V3_AUTH_FIXED_PART_LEN); - - if (server) - return V3_AUTH_FIXED_PART_LEN; // ptr-out + tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets); /* 8 octets were reserved for the current time, but we're trying to get out * of the habit of sending time around willynilly. Fortunately, nothing * checks it. That's followed by 16 bytes of nonce. */ - crypto_rand((char*)ptr, 24); - ptr += 24; + crypto_rand((char*)auth->rand, 24); - tor_assert(ptr - out == V3_AUTH_BODY_LEN); + ssize_t len; + if ((len = auth1_encode(out, outlen, auth, ctx)) < 0) { + log_warn(LD_OR, "Unable to encode signed part of AUTH1 data."); + goto err; + } - if (!signing_key) - return V3_AUTH_BODY_LEN; // ptr - out + if (server) { + auth1_t *tmp = NULL; + ssize_t len2 = auth1_parse(&tmp, out, len, ctx); + if (!tmp) { + log_warn(LD_OR, "Unable to parse signed part of AUTH1 data."); + goto err; + } + result = (int) (tmp->end_of_fixed_part - out); + auth1_free(tmp); + if (len2 != len) { + log_warn(LD_OR, "Mismatched length when re-parsing AUTH1 data."); + goto err; + } + goto done; + } + + if (signing_key) { + auth1_setlen_sig(auth, crypto_pk_keysize(signing_key)); - { - int siglen; char d[32]; - crypto_digest256(d, (char*)out, ptr-out, DIGEST_SHA256); - siglen = crypto_pk_private_sign(signing_key, - (char*)ptr, outlen - (ptr-out), + crypto_digest256(d, (char*)out, len, DIGEST_SHA256); + int siglen = crypto_pk_private_sign(signing_key, + (char*)auth1_getarray_sig(auth), + auth1_getlen_sig(auth), d, 32); - if (siglen < 0) - return -1; + if (siglen < 0) { + log_warn(LD_OR, "Unable to sign AUTH1 data."); + goto err; + } + + auth1_setlen_sig(auth, siglen); - ptr += siglen; - tor_assert(ptr <= out+outlen); - return (int)(ptr - out); + len = auth1_encode(out, outlen, auth, ctx); + if (len < 0) { + log_warn(LD_OR, "Unable to encode signed AUTH1 data."); + goto err; + } } + result = (int) len; + goto done; + + err: + result = -1; + done: + auth1_free(auth); + auth_ctx_free(ctx); + return result; } /** Send an AUTHENTICATE cell on the connection <b>conn</b>. Return 0 on * success, -1 on failure */ -int -connection_or_send_authenticate_cell(or_connection_t *conn, int authtype) +MOCK_IMPL(int, +connection_or_send_authenticate_cell,(or_connection_t *conn, int authtype)) { var_cell_t *cell; crypto_pk_t *pk = tor_tls_get_my_client_auth_key(); diff --git a/src/or/connection_or.h b/src/or/connection_or.h index 85e68f1a33..e2ec47a4f2 100644 --- a/src/or/connection_or.h +++ b/src/or/connection_or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -24,6 +24,7 @@ void connection_or_set_bad_connections(const char *digest, int force); void connection_or_block_renegotiation(or_connection_t *conn); int connection_or_reached_eof(or_connection_t *conn); int connection_or_process_inbuf(or_connection_t *conn); +ssize_t connection_or_num_cells_writeable(or_connection_t *conn); int connection_or_flushed_some(or_connection_t *conn); int connection_or_finished_flushing(or_connection_t *conn); int connection_or_finished_connecting(or_connection_t *conn); @@ -36,17 +37,22 @@ void connection_or_connect_failed(or_connection_t *conn, int reason, const char *msg); void connection_or_notify_error(or_connection_t *conn, int reason, const char *msg); -or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port, - const char *id_digest, - channel_tls_t *chan); +MOCK_DECL(or_connection_t *, + connection_or_connect, + (const tor_addr_t *addr, uint16_t port, + const char *id_digest, channel_tls_t *chan)); void connection_or_close_normally(or_connection_t *orconn, int flush); -void connection_or_close_for_error(or_connection_t *orconn, int flush); +MOCK_DECL(void,connection_or_close_for_error, + (or_connection_t *orconn, int flush)); void connection_or_report_broken_states(int severity, int domain); -int connection_tls_start_handshake(or_connection_t *conn, int receiving); +MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn, + int receiving)); int connection_tls_continue_handshake(or_connection_t *conn); +void connection_or_set_canonical(or_connection_t *or_conn, + int is_canonical); int connection_init_or_handshake_state(or_connection_t *conn, int started_here); @@ -72,17 +78,18 @@ void or_handshake_state_record_var_cell(or_connection_t *conn, int connection_or_set_state_open(or_connection_t *conn); void connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn); -void connection_or_write_var_cell_to_buf(const var_cell_t *cell, - or_connection_t *conn); +MOCK_DECL(void,connection_or_write_var_cell_to_buf,(const var_cell_t *cell, + or_connection_t *conn)); int connection_or_send_versions(or_connection_t *conn, int v3_plus); -int connection_or_send_netinfo(or_connection_t *conn); +MOCK_DECL(int,connection_or_send_netinfo,(or_connection_t *conn)); int connection_or_send_certs_cell(or_connection_t *conn); int connection_or_send_auth_challenge_cell(or_connection_t *conn); int connection_or_compute_authenticate_cell_body(or_connection_t *conn, uint8_t *out, size_t outlen, crypto_pk_t *signing_key, int server); -int connection_or_send_authenticate_cell(or_connection_t *conn, int type); +MOCK_DECL(int,connection_or_send_authenticate_cell, + (or_connection_t *conn, int type)); int is_or_protocol_version_known(uint16_t version); @@ -90,9 +97,10 @@ void cell_pack(packed_cell_t *dest, const cell_t *src, int wide_circ_ids); int var_cell_pack_header(const var_cell_t *cell, char *hdr_out, int wide_circ_ids); var_cell_t *var_cell_new(uint16_t payload_len); +var_cell_t *var_cell_copy(const var_cell_t *src); void var_cell_free(var_cell_t *cell); -/** DOCDOC */ +/* DOCDOC */ #define MIN_LINK_PROTO_FOR_WIDE_CIRC_IDS 4 #endif diff --git a/src/or/control.c b/src/or/control.c index ae9dd69d21..e2ad8cc6dc 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,6 +19,8 @@ #include "circuitlist.h" #include "circuitstats.h" #include "circuituse.h" +#include "command.h" +#include "compat_libevent.h" #include "config.h" #include "confparse.h" #include "connection.h" @@ -36,6 +38,9 @@ #include "nodelist.h" #include "policies.h" #include "reasons.h" +#include "rendclient.h" +#include "rendcommon.h" +#include "rendservice.h" #include "rephist.h" #include "router.h" #include "routerlist.h" @@ -46,52 +51,26 @@ #include <sys/resource.h> #endif +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +#include "crypto_s2k.h" #include "procmon.h" /** Yield true iff <b>s</b> is the state of a control_connection_t that has * finished authentication and is accepting commands. */ #define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN) -/* Recognized asynchronous event types. It's okay to expand this list - * because it is used both as a list of v0 event types, and as indices - * into the bitfield to determine which controllers want which events. - */ -#define EVENT_MIN_ 0x0001 -#define EVENT_CIRCUIT_STATUS 0x0001 -#define EVENT_STREAM_STATUS 0x0002 -#define EVENT_OR_CONN_STATUS 0x0003 -#define EVENT_BANDWIDTH_USED 0x0004 -#define EVENT_CIRCUIT_STATUS_MINOR 0x0005 -#define EVENT_NEW_DESC 0x0006 -#define EVENT_DEBUG_MSG 0x0007 -#define EVENT_INFO_MSG 0x0008 -#define EVENT_NOTICE_MSG 0x0009 -#define EVENT_WARN_MSG 0x000A -#define EVENT_ERR_MSG 0x000B -#define EVENT_ADDRMAP 0x000C -// #define EVENT_AUTHDIR_NEWDESCS 0x000D -#define EVENT_DESCCHANGED 0x000E -// #define EVENT_NS 0x000F -#define EVENT_STATUS_CLIENT 0x0010 -#define EVENT_STATUS_SERVER 0x0011 -#define EVENT_STATUS_GENERAL 0x0012 -#define EVENT_GUARD 0x0013 -#define EVENT_STREAM_BANDWIDTH_USED 0x0014 -#define EVENT_CLIENTS_SEEN 0x0015 -#define EVENT_NEWCONSENSUS 0x0016 -#define EVENT_BUILDTIMEOUT_SET 0x0017 -#define EVENT_SIGNAL 0x0018 -#define EVENT_CONF_CHANGED 0x0019 -#define EVENT_MAX_ 0x0019 -/* If EVENT_MAX_ ever hits 0x0020, we need to make the mask wider. */ - /** Bitfield: The bit 1<<e is set if <b>any</b> open control * connection is interested in events of type <b>e</b>. We use this * so that we can decide to skip generating event messages that nobody * has interest in without having to walk over the global connection * list to find out. **/ -typedef uint32_t event_mask_t; +typedef uint64_t event_mask_t; /** An event mask of all the events that any controller is interested in * receiving. */ @@ -103,7 +82,7 @@ static int disable_log_messages = 0; /** Macro: true if any control connection is interested in events of type * <b>e</b>. */ #define EVENT_IS_INTERESTING(e) \ - (global_event_mask & (1<<(e))) + (!! (global_event_mask & EVENT_MASK_(e))) /** If we're using cookie-type authentication, how long should our cookies be? */ @@ -115,7 +94,7 @@ static int authentication_cookie_is_set = 0; /** If authentication_cookie_is_set, a secret cookie that we've stored to disk * and which we're using to authenticate controllers. (If the controller can * read it off disk, it has permission to connect.) */ -static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; +static uint8_t *authentication_cookie = NULL; #define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \ "Tor safe cookie authentication server-to-controller hash" @@ -123,6 +102,11 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; "Tor safe cookie authentication controller-to-server hash" #define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN +/** The list of onion services that have been added via ADD_ONION that do not + * belong to any particular control connection. + */ +static smartlist_t *detached_onion_services = NULL; + /** A sufficiently large size to record the last bootstrap phase string. */ #define BOOTSTRAP_MSG_LEN 1024 @@ -130,29 +114,20 @@ static char authentication_cookie[AUTHENTICATION_COOKIE_LEN]; * of this so we can respond to getinfo status/bootstrap-phase queries. */ static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; -/** Flag for event_format_t. Indicates that we should use the one standard - format. - */ -#define ALL_FORMATS 1 - -/** Bit field of flags to select how to format a controller event. Recognized - * flag is ALL_FORMATS. */ -typedef int event_format_t; - static void connection_printf_to_buf(control_connection_t *conn, const char *format, ...) CHECK_PRINTF(2,3); -static void send_control_event_impl(uint16_t event, event_format_t which, +static void send_control_event_impl(uint16_t event, const char *format, va_list ap) - CHECK_PRINTF(3,0); + CHECK_PRINTF(2,0); static int control_event_status(int type, int severity, const char *format, va_list args) CHECK_PRINTF(3,0); static void send_control_done(control_connection_t *conn); -static void send_control_event(uint16_t event, event_format_t which, +static void send_control_event(uint16_t event, const char *format, ...) - CHECK_PRINTF(3,4); + CHECK_PRINTF(2,3); static int handle_control_setconf(control_connection_t *conn, uint32_t len, char *body); static int handle_control_resetconf(control_connection_t *conn, uint32_t len, @@ -197,15 +172,27 @@ static int handle_control_resolve(control_connection_t *conn, uint32_t len, static int handle_control_usefeature(control_connection_t *conn, uint32_t len, const char *body); +static int handle_control_hsfetch(control_connection_t *conn, uint32_t len, + const char *body); +static int handle_control_hspost(control_connection_t *conn, uint32_t len, + const char *body); +static int handle_control_add_onion(control_connection_t *conn, uint32_t len, + const char *body); +static int handle_control_del_onion(control_connection_t *conn, uint32_t len, + const char *body); static int write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len); static void orconn_target_get_name(char *buf, size_t len, or_connection_t *conn); -static char *get_cookie_file(void); + +static int get_cached_network_liveness(void); +static void set_cached_network_liveness(int liveness); + +static void flush_queued_events_cb(evutil_socket_t fd, short what, void *arg); /** Given a control event code for a message event, return the corresponding * log severity. */ -static INLINE int +static inline int event_to_log_severity(int event) { switch (event) { @@ -219,7 +206,7 @@ event_to_log_severity(int event) } /** Given a log severity, return the corresponding control event code. */ -static INLINE int +static inline int log_severity_to_event(int severity) { switch (severity) { @@ -232,6 +219,20 @@ log_severity_to_event(int severity) } } +/** Helper: clear bandwidth counters of all origin circuits. */ +static void +clear_circ_bw_fields(void) +{ + origin_circuit_t *ocirc; + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + if (!CIRCUIT_IS_ORIGIN(circ)) + continue; + ocirc = TO_ORIGIN_CIRCUIT(circ); + ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + } + SMARTLIST_FOREACH_END(circ); +} + /** Set <b>global_event_mask*</b> to the bitwise OR of each live control * connection's event_mask field. */ void @@ -257,8 +258,8 @@ control_update_global_event_mask(void) * we want to hear...*/ control_adjust_event_log_severity(); - /* ...then, if we've started logging stream bw, clear the appropriate - * fields. */ + /* ...then, if we've started logging stream or circ bw, clear the + * appropriate fields. */ if (! (old_mask & EVENT_STREAM_BANDWIDTH_USED) && (new_mask & EVENT_STREAM_BANDWIDTH_USED)) { SMARTLIST_FOREACH(conns, connection_t *, conn, @@ -269,6 +270,10 @@ control_update_global_event_mask(void) } }); } + if (! (old_mask & EVENT_CIRC_BANDWIDTH_USED) && + (new_mask & EVENT_CIRC_BANDWIDTH_USED)) { + clear_circ_bw_fields(); + } } /** Adjust the log severities that result in control_event_logmsg being called @@ -320,7 +325,7 @@ control_event_is_interesting(int event) /** Append a NUL-terminated string <b>s</b> to the end of * <b>conn</b>-\>outbuf. */ -static INLINE void +static inline void connection_write_str_to_buf(const char *s, control_connection_t *conn) { size_t len = strlen(s); @@ -334,7 +339,7 @@ connection_write_str_to_buf(const char *s, control_connection_t *conn) * the end. Replace all LF characters sequences with CRLF. Return the number * of bytes in *<b>out</b>. */ -/* static */ size_t +STATIC size_t write_escaped_data(const char *data, size_t len, char **out) { size_t sz_out = len+8; @@ -382,7 +387,7 @@ write_escaped_data(const char *data, size_t len, char **out) * that appears at the start of a line, and replacing all CRLF sequences * with LF. Return the number of * bytes in *<b>out</b>. */ -/* static */ size_t +STATIC size_t read_escaped_data(const char *data, size_t len, char **out) { char *outp; @@ -423,7 +428,7 @@ read_escaped_data(const char *data, size_t len, char **out) /** If the first <b>in_len_max</b> characters in <b>start</b> contain a * double-quoted string with escaped characters, return the length of that * string (as encoded, including quotes). Otherwise return -1. */ -static INLINE int +static inline int get_escaped_string_length(const char *start, size_t in_len_max, int *chars_out) { @@ -582,46 +587,217 @@ send_control_done(control_connection_t *conn) connection_write_str_to_buf("250 OK\r\n", conn); } -/** Send an event to all v1 controllers that are listening for code - * <b>event</b>. The event's body is given by <b>msg</b>. +/** Represents an event that's queued to be sent to one or more + * controllers. */ +typedef struct queued_event_s { + uint16_t event; + char *msg; +} queued_event_t; + +/** Pointer to int. If this is greater than 0, we don't allow new events to be + * queued. */ +static tor_threadlocal_t block_event_queue; + +/** Holds a smartlist of queued_event_t objects that may need to be sent + * to one or more controllers */ +static smartlist_t *queued_control_events = NULL; + +/** True if the flush_queued_events_event is pending. */ +static int flush_queued_event_pending = 0; + +/** Lock to protect the above fields. */ +static tor_mutex_t *queued_control_events_lock = NULL; + +/** An event that should fire in order to flush the contents of + * queued_control_events. */ +static struct event *flush_queued_events_event = NULL; + +void +control_initialize_event_queue(void) +{ + if (queued_control_events == NULL) { + queued_control_events = smartlist_new(); + } + + if (flush_queued_events_event == NULL) { + struct event_base *b = tor_libevent_get_base(); + if (b) { + flush_queued_events_event = tor_event_new(b, + -1, 0, flush_queued_events_cb, + NULL); + tor_assert(flush_queued_events_event); + } + } + + if (queued_control_events_lock == NULL) { + queued_control_events_lock = tor_mutex_new(); + tor_threadlocal_init(&block_event_queue); + } +} + +static int * +get_block_event_queue(void) +{ + int *val = tor_threadlocal_get(&block_event_queue); + if (PREDICT_UNLIKELY(val == NULL)) { + val = tor_malloc_zero(sizeof(int)); + tor_threadlocal_set(&block_event_queue, val); + } + return val; +} + +/** Helper: inserts an event on the list of events queued to be sent to + * one or more controllers, and schedules the events to be flushed if needed. * - * If <b>which</b> & SHORT_NAMES, the event contains short-format names: send - * it to controllers that haven't enabled the VERBOSE_NAMES feature. If - * <b>which</b> & LONG_NAMES, the event contains long-format names: send it - * to controllers that <em>have</em> enabled VERBOSE_NAMES. + * This function takes ownership of <b>msg</b>, and may free it. * - * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with - * respect to the EXTENDED_EVENTS feature. */ + * We queue these events rather than send them immediately in order to break + * the dependency in our callgraph from code that generates events for the + * controller, and the network layer at large. Otherwise, nearly every + * interesting part of Tor would potentially call every other interesting part + * of Tor. + */ +MOCK_IMPL(STATIC void, +queue_control_event_string,(uint16_t event, char *msg)) +{ + /* This is redundant with checks done elsewhere, but it's a last-ditch + * attempt to avoid queueing something we shouldn't have to queue. */ + if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) { + tor_free(msg); + return; + } + + int *block_event_queue = get_block_event_queue(); + if (*block_event_queue) { + tor_free(msg); + return; + } + + queued_event_t *ev = tor_malloc(sizeof(*ev)); + ev->event = event; + ev->msg = msg; + + /* No queueing an event while queueing an event */ + ++*block_event_queue; + + tor_mutex_acquire(queued_control_events_lock); + tor_assert(queued_control_events); + smartlist_add(queued_control_events, ev); + + int activate_event = 0; + if (! flush_queued_event_pending && in_main_thread()) { + activate_event = 1; + flush_queued_event_pending = 1; + } + + tor_mutex_release(queued_control_events_lock); + + --*block_event_queue; + + /* We just put an event on the queue; mark the queue to be + * flushed. We only do this from the main thread for now; otherwise, + * we'd need to incur locking overhead in Libevent or use a socket. + */ + if (activate_event) { + tor_assert(flush_queued_events_event); + event_active(flush_queued_events_event, EV_READ, 1); + } +} + +/** Release all storage held by <b>ev</b>. */ static void -send_control_event_string(uint16_t event, event_format_t which, - const char *msg) +queued_event_free(queued_event_t *ev) { - smartlist_t *conns = get_connection_array(); - (void)which; - tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_); + if (ev == NULL) + return; - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { + tor_free(ev->msg); + tor_free(ev); +} + +/** Send every queued event to every controller that's interested in it, + * and remove the events from the queue. If <b>force</b> is true, + * then make all controllers send their data out immediately, since we + * may be about to shut down. */ +static void +queued_events_flush_all(int force) +{ + if (PREDICT_UNLIKELY(queued_control_events == NULL)) { + return; + } + smartlist_t *all_conns = get_connection_array(); + smartlist_t *controllers = smartlist_new(); + smartlist_t *queued_events; + + int *block_event_queue = get_block_event_queue(); + ++*block_event_queue; + + tor_mutex_acquire(queued_control_events_lock); + /* No queueing an event while flushing events. */ + flush_queued_event_pending = 0; + queued_events = queued_control_events; + queued_control_events = smartlist_new(); + tor_mutex_release(queued_control_events_lock); + + /* Gather all the controllers that will care... */ + SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) { if (conn->type == CONN_TYPE_CONTROL && !conn->marked_for_close && conn->state == CONTROL_CONN_STATE_OPEN) { control_connection_t *control_conn = TO_CONTROL_CONN(conn); - if (control_conn->event_mask & (1<<event)) { - int is_err = 0; - connection_write_to_buf(msg, strlen(msg), TO_CONN(control_conn)); - if (event == EVENT_ERR_MSG) - is_err = 1; - else if (event == EVENT_STATUS_GENERAL) - is_err = !strcmpstart(msg, "STATUS_GENERAL ERR "); - else if (event == EVENT_STATUS_CLIENT) - is_err = !strcmpstart(msg, "STATUS_CLIENT ERR "); - else if (event == EVENT_STATUS_SERVER) - is_err = !strcmpstart(msg, "STATUS_SERVER ERR "); - if (is_err) - connection_flush(TO_CONN(control_conn)); - } + smartlist_add(controllers, control_conn); } } SMARTLIST_FOREACH_END(conn); + + SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) { + const event_mask_t bit = ((event_mask_t)1) << ev->event; + const size_t msg_len = strlen(ev->msg); + SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *, + control_conn) { + if (control_conn->event_mask & bit) { + connection_write_to_buf(ev->msg, msg_len, TO_CONN(control_conn)); + } + } SMARTLIST_FOREACH_END(control_conn); + + queued_event_free(ev); + } SMARTLIST_FOREACH_END(ev); + + if (force) { + SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *, + control_conn) { + connection_flush(TO_CONN(control_conn)); + } SMARTLIST_FOREACH_END(control_conn); + } + + smartlist_free(queued_events); + smartlist_free(controllers); + + --*block_event_queue; +} + +/** Libevent callback: Flushes pending events to controllers that are + * interested in them */ +static void +flush_queued_events_cb(evutil_socket_t fd, short what, void *arg) +{ + (void) fd; + (void) what; + (void) arg; + queued_events_flush_all(0); +} + +/** Send an event to all v1 controllers that are listening for code + * <b>event</b>. The event's body is given by <b>msg</b>. + * + * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with + * respect to the EXTENDED_EVENTS feature. */ +MOCK_IMPL(STATIC void, +send_control_event_string,(uint16_t event, + const char *msg)) +{ + tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_); + queue_control_event_string(event, tor_strdup(msg)); } /** Helper for send_control_event and control_event_status: @@ -629,8 +805,8 @@ send_control_event_string(uint16_t event, event_format_t which, * <b>event</b>. The event's body is created by the printf-style format in * <b>format</b>, and other arguments as provided. */ static void -send_control_event_impl(uint16_t event, event_format_t which, - const char *format, va_list ap) +send_control_event_impl(uint16_t event, + const char *format, va_list ap) { char *buf = NULL; int len; @@ -641,21 +817,19 @@ send_control_event_impl(uint16_t event, event_format_t which, return; } - send_control_event_string(event, which|ALL_FORMATS, buf); - - tor_free(buf); + queue_control_event_string(event, buf); } /** Send an event to all v1 controllers that are listening for code * <b>event</b>. The event's body is created by the printf-style format in * <b>format</b>, and other arguments as provided. */ static void -send_control_event(uint16_t event, event_format_t which, +send_control_event(uint16_t event, const char *format, ...) { va_list ap; va_start(ap, format); - send_control_event_impl(event, which, format, ap); + send_control_event_impl(event, format, ap); va_end(ap); } @@ -956,8 +1130,16 @@ static const struct control_event_t control_event_table[] = { { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" }, { EVENT_NEWCONSENSUS, "NEWCONSENSUS" }, { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" }, - { EVENT_SIGNAL, "SIGNAL" }, + { EVENT_GOT_SIGNAL, "SIGNAL" }, { EVENT_CONF_CHANGED, "CONF_CHANGED"}, + { EVENT_CONN_BW, "CONN_BW" }, + { EVENT_CELL_STATS, "CELL_STATS" }, + { EVENT_TB_EMPTY, "TB_EMPTY" }, + { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" }, + { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" }, + { EVENT_HS_DESC, "HS_DESC" }, + { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" }, + { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" }, { 0, NULL }, }; @@ -967,8 +1149,8 @@ static int handle_control_setevents(control_connection_t *conn, uint32_t len, const char *body) { - int event_code = -1; - uint32_t event_mask = 0; + int event_code; + event_mask_t event_mask = 0; smartlist_t *events = smartlist_new(); (void) len; @@ -981,6 +1163,8 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, continue; } else { int i; + event_code = -1; + for (i = 0; control_event_table[i].event_name != NULL; ++i) { if (!strcasecmp(ev, control_event_table[i].event_name)) { event_code = control_event_table[i].event_code; @@ -996,7 +1180,7 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, return 0; } } - event_mask |= (1 << event_code); + event_mask |= (((event_mask_t)1) << event_code); } SMARTLIST_FOREACH_END(ev); SMARTLIST_FOREACH(events, char *, e, tor_free(e)); @@ -1011,7 +1195,8 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, /** Decode the hashed, base64'd passwords stored in <b>passwords</b>. * Return a smartlist of acceptable passwords (unterminated strings of - * length S2K_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on failure. + * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on + * failure. */ smartlist_t * decode_hashed_passwords(config_line_t *passwords) @@ -1027,16 +1212,17 @@ decode_hashed_passwords(config_line_t *passwords) if (!strcmpstart(hashed, "16:")) { if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))<0 - || strlen(hashed+3) != (S2K_SPECIFIER_LEN+DIGEST_LEN)*2) { + || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) { goto err; } } else { if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed)) - != S2K_SPECIFIER_LEN+DIGEST_LEN) { + != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) { goto err; } } - smartlist_add(sl, tor_memdup(decoded, S2K_SPECIFIER_LEN+DIGEST_LEN)); + smartlist_add(sl, + tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)); } return sl; @@ -1057,7 +1243,7 @@ handle_control_authenticate(control_connection_t *conn, uint32_t len, { int used_quoted_string = 0; const or_options_t *options = get_options(); - const char *errstr = NULL; + const char *errstr = "Unknown error"; char *password; size_t password_len; const char *cp; @@ -1178,22 +1364,27 @@ handle_control_authenticate(control_connection_t *conn, uint32_t len, } if (bad) { if (!also_cookie) { - log_warn(LD_CONTROL, + log_warn(LD_BUG, "Couldn't decode HashedControlPassword: invalid base16"); errstr="Couldn't decode HashedControlPassword value in configuration."; + goto err; } bad_password = 1; SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); smartlist_free(sl); + sl = NULL; } else { SMARTLIST_FOREACH(sl, char *, expected, { - secret_to_key(received,DIGEST_LEN,password,password_len,expected); - if (tor_memeq(expected+S2K_SPECIFIER_LEN, received, DIGEST_LEN)) + secret_to_key_rfc2440(received,DIGEST_LEN, + password,password_len,expected); + if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN, + received, DIGEST_LEN)) goto ok; }); SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); smartlist_free(sl); + sl = NULL; if (used_quoted_string) errstr = "Password did not match HashedControlPassword value from " @@ -1216,9 +1407,12 @@ handle_control_authenticate(control_connection_t *conn, uint32_t len, err: tor_free(password); - connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", - errstr ? errstr : "Unknown reason."); + connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", errstr); connection_mark_for_close(TO_CONN(conn)); + if (sl) { /* clean up */ + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + } return 0; ok: log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT @@ -1268,6 +1462,7 @@ static const struct signal_t signal_table[] = { { SIGTERM, "INT" }, { SIGNEWNYM, "NEWNYM" }, { SIGCLEARDNSCACHE, "CLEARDNSCACHE"}, + { SIGHEARTBEAT, "HEARTBEAT"}, { 0, NULL }, }; @@ -1308,7 +1503,7 @@ handle_control_signal(control_connection_t *conn, uint32_t len, if (sig == SIGTERM || sig == SIGINT) connection_flush(TO_CONN(conn)); - process_signal(sig); + activate_signal(sig); return 0; } @@ -1442,12 +1637,18 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, (void) conn; if (!strcmp(question, "version")) { *answer = tor_strdup(get_version()); + } else if (!strcmp(question, "bw-event-cache")) { + *answer = get_bw_samples(); } else if (!strcmp(question, "config-file")) { - *answer = tor_strdup(get_torrc_fname(0)); + const char *a = get_torrc_fname(0); + if (a) + *answer = tor_strdup(a); } else if (!strcmp(question, "config-defaults-file")) { - *answer = tor_strdup(get_torrc_fname(1)); + const char *a = get_torrc_fname(1); + if (a) + *answer = tor_strdup(a); } else if (!strcmp(question, "config-text")) { - *answer = options_dump(get_options(), 1); + *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL); } else if (!strcmp(question, "info/names")) { *answer = list_getinfo_options(); } else if (!strcmp(question, "dormant")) { @@ -1509,7 +1710,7 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, *answer = tor_strdup(""); #else int myUid = geteuid(); - struct passwd *myPwEntry = getpwuid(myUid); + const struct passwd *myPwEntry = tor_getpwuid(myUid); if (myPwEntry) { *answer = tor_strdup(myPwEntry->pw_name); @@ -1518,9 +1719,11 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, } #endif } else if (!strcmp(question, "process/descriptor-limit")) { - int max_fds=-1; - set_max_file_descriptors(0, &max_fds); + int max_fds = get_max_sockets(); tor_asprintf(answer, "%d", max_fds); + } else if (!strcmp(question, "limits/max-mem-in-queues")) { + tor_asprintf(answer, U64_FORMAT, + U64_PRINTF_ARG(get_options()->MaxMemInQueues)); } else if (!strcmp(question, "dir-usage")) { *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { @@ -1567,12 +1770,13 @@ munge_extrainfo_into_routerinfo(const char *ri_body, outp += router_sig-ri_body; for (i=0; i < 2; ++i) { - const char *kwd = i?"\nwrite-history ":"\nread-history "; + const char *kwd = i ? "\nwrite-history " : "\nread-history "; const char *cp, *eol; if (!(cp = tor_memstr(ei_body, ei_len, kwd))) continue; ++cp; - eol = memchr(cp, '\n', ei_len - (cp-ei_body)); + if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body)))) + continue; memcpy(outp, cp, eol-cp+1); outp += eol-cp+1; } @@ -1707,6 +1911,38 @@ getinfo_helper_dir(control_connection_t *control_conn, *answer = smartlist_join_strings(sl, "", 0, NULL); SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); + } else if (!strcmpstart(question, "hs/client/desc/id/")) { + rend_cache_entry_t *e = NULL; + + question += strlen("hs/client/desc/id/"); + if (strlen(question) != REND_SERVICE_ID_LEN_BASE32) { + *errmsg = "Invalid address"; + return -1; + } + + if (!rend_cache_lookup_entry(question, -1, &e)) { + /* Descriptor found in cache */ + *answer = tor_strdup(e->desc); + } else { + *errmsg = "Not found in cache"; + return -1; + } + } else if (!strcmpstart(question, "hs/service/desc/id/")) { + rend_cache_entry_t *e = NULL; + + question += strlen("hs/service/desc/id/"); + if (strlen(question) != REND_SERVICE_ID_LEN_BASE32) { + *errmsg = "Invalid address"; + return -1; + } + + if (!rend_cache_lookup_v2_desc_as_service(question, &e)) { + /* Descriptor found in cache */ + *answer = tor_strdup(e->desc); + } else { + *errmsg = "Not found in cache"; + return -1; + } } else if (!strcmpstart(question, "md/id/")) { const node_t *node = node_get_by_hex_id(question+strlen("md/id/")); const microdesc_t *md = NULL; @@ -1764,39 +2000,7 @@ getinfo_helper_dir(control_connection_t *control_conn, tor_free(url); smartlist_free(descs); } else if (!strcmpstart(question, "dir/status/")) { - if (directory_permits_controller_requests(get_options())) { - size_t len=0; - char *cp; - smartlist_t *status_list = smartlist_new(); - dirserv_get_networkstatus_v2(status_list, - question+strlen("dir/status/")); - SMARTLIST_FOREACH(status_list, cached_dir_t *, d, len += d->dir_len); - cp = *answer = tor_malloc(len+1); - SMARTLIST_FOREACH(status_list, cached_dir_t *, d, { - memcpy(cp, d->dir, d->dir_len); - cp += d->dir_len; - }); - *cp = '\0'; - smartlist_free(status_list); - } else { - smartlist_t *fp_list = smartlist_new(); - smartlist_t *status_list = smartlist_new(); - dirserv_get_networkstatus_v2_fingerprints( - fp_list, question+strlen("dir/status/")); - SMARTLIST_FOREACH(fp_list, const char *, fp, { - char *s; - char *fname = networkstatus_get_cache_filename(fp); - s = read_file_to_str(fname, 0, NULL); - if (s) - smartlist_add(status_list, s); - tor_free(fname); - }); - SMARTLIST_FOREACH(fp_list, char *, fp, tor_free(fp)); - smartlist_free(fp_list); - *answer = smartlist_join_strings(status_list, "", 0, NULL); - SMARTLIST_FOREACH(status_list, char *, s, tor_free(s)); - smartlist_free(status_list); - } + *answer = tor_strdup(""); } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */ if (directory_caches_dir_info(get_options())) { const cached_dir_t *consensus = dirserv_get_consensus("ns"); @@ -1807,6 +2011,11 @@ getinfo_helper_dir(control_connection_t *control_conn, char *filename = get_datadir_fname("cached-consensus"); *answer = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); tor_free(filename); + if (!*answer) { /* generate an error */ + *errmsg = "Could not open cached consensus. " + "Make sure FetchUselessDescriptors is set to 1."; + return -1; + } } } else if (!strcmp(question, "network-status")) { /* v1 */ routerlist_t *routerlist = router_get_routerlist(); @@ -1908,6 +2117,22 @@ circuit_describe_status_for_controller(origin_circuit_t *circ) smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf); } + // Show username and/or password if available. + if (circ->socks_username_len > 0) { + char* socks_username_escaped = esc_for_log_len(circ->socks_username, + (size_t) circ->socks_username_len); + smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s", + socks_username_escaped); + tor_free(socks_username_escaped); + } + if (circ->socks_password_len > 0) { + char* socks_password_escaped = esc_for_log_len(circ->socks_password, + (size_t) circ->socks_password_len); + smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s", + socks_password_escaped); + tor_free(socks_password_escaped); + } + rv = smartlist_join_strings(descparts, " ", 0, NULL); SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp)); @@ -1923,11 +2148,11 @@ getinfo_helper_events(control_connection_t *control_conn, const char *question, char **answer, const char **errmsg) { + const or_options_t *options = get_options(); (void) control_conn; if (!strcmp(question, "circuit-status")) { - circuit_t *circ_; smartlist_t *status = smartlist_new(); - for (circ_ = circuit_get_global_list_(); circ_; circ_ = circ_->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) { origin_circuit_t *circ; char *circdesc; const char *state; @@ -1949,6 +2174,7 @@ getinfo_helper_events(control_connection_t *control_conn, state, *circdesc ? " " : "", circdesc); tor_free(circdesc); } + SMARTLIST_FOREACH_END(circ_); *answer = smartlist_join_strings(status, "\r\n", 0, NULL); SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); smartlist_free(status); @@ -2048,7 +2274,7 @@ getinfo_helper_events(control_connection_t *control_conn, /* Note that status/ is not a catch-all for events; there's only supposed * to be a status GETINFO if there's a corresponding STATUS event. */ if (!strcmp(question, "status/circuit-established")) { - *answer = tor_strdup(can_complete_circuit ? "1" : "0"); + *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0"); } else if (!strcmp(question, "status/enough-dir-info")) { *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0"); } else if (!strcmp(question, "status/good-server-descriptor") || @@ -2059,17 +2285,19 @@ getinfo_helper_events(control_connection_t *control_conn, *answer = tor_strdup(directories_have_accepted_server_descriptor() ? "1" : "0"); } else if (!strcmp(question, "status/reachability-succeeded/or")) { - *answer = tor_strdup(check_whether_orport_reachable() ? "1" : "0"); + *answer = tor_strdup(check_whether_orport_reachable(options) ? + "1" : "0"); } else if (!strcmp(question, "status/reachability-succeeded/dir")) { - *answer = tor_strdup(check_whether_dirport_reachable() ? "1" : "0"); + *answer = tor_strdup(check_whether_dirport_reachable(options) ? + "1" : "0"); } else if (!strcmp(question, "status/reachability-succeeded")) { tor_asprintf(answer, "OR=%d DIR=%d", - check_whether_orport_reachable() ? 1 : 0, - check_whether_dirport_reachable() ? 1 : 0); + check_whether_orport_reachable(options) ? 1 : 0, + check_whether_dirport_reachable(options) ? 1 : 0); } else if (!strcmp(question, "status/bootstrap-phase")) { *answer = tor_strdup(last_sent_bootstrap_message); } else if (!strcmpstart(question, "status/version/")) { - int is_server = server_mode(get_options()); + int is_server = server_mode(options); networkstatus_t *c = networkstatus_get_latest_consensus(); version_status_t status; const char *recommended; @@ -2110,6 +2338,46 @@ getinfo_helper_events(control_connection_t *control_conn, return -1; } *answer = bridge_stats; + } else if (!strcmp(question, "status/fresh-relay-descs")) { + if (!server_mode(options)) { + *errmsg = "Only relays have descriptors"; + return -1; + } + routerinfo_t *r; + extrainfo_t *e; + if (router_build_fresh_descriptor(&r, &e) < 0) { + *errmsg = "Error generating descriptor"; + return -1; + } + size_t size = r->cache_info.signed_descriptor_len + 1; + if (e) { + size += e->cache_info.signed_descriptor_len + 1; + } + tor_assert(r->cache_info.signed_descriptor_len); + char *descs = tor_malloc(size); + char *cp = descs; + memcpy(cp, signed_descriptor_get_body(&r->cache_info), + r->cache_info.signed_descriptor_len); + cp += r->cache_info.signed_descriptor_len - 1; + if (e) { + if (cp[0] == '\0') { + cp[0] = '\n'; + } else if (cp[0] != '\n') { + cp[1] = '\n'; + cp++; + } + memcpy(cp, signed_descriptor_get_body(&e->cache_info), + e->cache_info.signed_descriptor_len); + cp += e->cache_info.signed_descriptor_len - 1; + } + if (cp[0] == '\n') { + cp[0] = '\0'; + } else if (cp[0] != '\0') { + cp[1] = '\0'; + } + *answer = descs; + routerinfo_free(r); + extrainfo_free(e); } else { return 0; } @@ -2117,6 +2385,55 @@ getinfo_helper_events(control_connection_t *control_conn, return 0; } +/** Implementation helper for GETINFO: knows how to enumerate hidden services + * created via the control port. */ +static int +getinfo_helper_onions(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + smartlist_t *onion_list = NULL; + + if (control_conn && !strcmp(question, "onions/current")) { + onion_list = control_conn->ephemeral_onion_services; + } else if (!strcmp(question, "onions/detached")) { + onion_list = detached_onion_services; + } else { + return 0; + } + if (!onion_list || smartlist_len(onion_list) == 0) { + if (errmsg) { + *errmsg = "No onion services of the specified type."; + } + return -1; + } + if (answer) { + *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL); + } + + return 0; +} + +/** Implementation helper for GETINFO: answers queries about network + * liveness. */ +static int +getinfo_helper_liveness(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + (void)control_conn; + (void)errmsg; + if (strcmp(question, "network-liveness") == 0) { + if (get_cached_network_liveness()) { + *answer = tor_strdup("up"); + } else { + *answer = tor_strdup("down"); + } + } + + return 0; +} + /** Callback function for GETINFO: on a given control connection, try to * answer the question <b>q</b> and store the newly-allocated answer in * *<b>a</b>. If an internal error occurs, return -1 and optionally set @@ -2145,6 +2462,7 @@ typedef struct getinfo_item_t { * to answer them. */ static const getinfo_item_t getinfo_items[] = { ITEM("version", misc, "The current version of Tor."), + ITEM("bw-event-cache", misc, "Cached BW events for a short interval."), ITEM("config-file", misc, "Current location of the \"torrc\" file."), ITEM("config-defaults-file", misc, "Current location of the defaults file."), ITEM("config-text", misc, @@ -2185,6 +2503,10 @@ static const getinfo_item_t getinfo_items[] = { PREFIX("md/id/", dir, "Microdescriptors by ID"), PREFIX("md/name/", dir, "Microdescriptors by name"), PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."), + PREFIX("hs/client/desc/id", dir, + "Hidden Service descriptor in client's cache by onion."), + PREFIX("hs/service/desc/id/", dir, + "Hidden Service descriptor in services's cache by onion."), PREFIX("net/listeners/", listeners, "Bound addresses by type"), ITEM("ns/all", networkstatus, "Brief summary of router status (v2 directory format)"), @@ -2194,8 +2516,12 @@ static const getinfo_item_t getinfo_items[] = { "Brief summary of router status by nickname (v2 directory format)."), PREFIX("ns/purpose/", networkstatus, "Brief summary of router status by purpose (v2 directory format)."), + PREFIX("consensus/", networkstatus, + "Information about and from the ns consensus."), ITEM("network-status", dir, "Brief summary of router status (v1 directory format)"), + ITEM("network-liveness", liveness, + "Current opinion on whether the network is live"), ITEM("circuit-status", events, "List of current circuits originating here."), ITEM("stream-status", events,"List of current streams."), ITEM("orconn-status", events, "A list of current OR connections."), @@ -2217,6 +2543,8 @@ static const getinfo_item_t getinfo_items[] = { "The last bootstrap phase status event that Tor sent."), DOC("status/clients-seen", "Breakdown of client countries seen by a bridge."), + DOC("status/fresh-relay-descs", + "A fresh relay/ei descriptor pair for Tor's current state. Not stored."), DOC("status/version/recommended", "List of currently recommended versions."), DOC("status/version/current", "Status of the current version."), DOC("status/version/num-versioning", "Number of versioning authorities."), @@ -2232,6 +2560,7 @@ static const getinfo_item_t getinfo_items[] = { ITEM("process/user", misc, "Username under which the tor process is running."), ITEM("process/descriptor-limit", misc, "File descriptor limit."), + ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"), ITEM("dir-usage", misc, "Breakdown of bytes transferred over DirPort."), PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."), PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."), @@ -2241,7 +2570,20 @@ static const getinfo_item_t getinfo_items[] = { "v3 Networkstatus consensus as retrieved from a DirPort."), ITEM("exit-policy/default", policies, "The default value appended to the configured exit policy."), + ITEM("exit-policy/reject-private/default", policies, + "The default rules appended to the configured exit policy by" + " ExitPolicyRejectPrivate."), + ITEM("exit-policy/reject-private/relay", policies, + "The relay-specific rules appended to the configured exit policy by" + " ExitPolicyRejectPrivate."), + ITEM("exit-policy/full", policies, "The entire exit policy of onion router"), + ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"), + ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"), PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"), + ITEM("onions/current", onions, + "Onion services owned by the current control connection."), + ITEM("onions/detached", onions, + "Onion services detached from the control connection."), { NULL, NULL, NULL, 0 } }; @@ -2493,6 +2835,14 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, goto done; } + if (smartlist_len(args) < 2) { + connection_printf_to_buf(conn, + "512 syntax error: not enough arguments.\r\n"); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); @@ -2522,12 +2872,26 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, } /* now circ refers to something that is ready to be extended */ + int first_node = zero_circ; SMARTLIST_FOREACH(nodes, const node_t *, node, { - extend_info_t *info = extend_info_from_node(node, 0); - tor_assert(info); /* True, since node_has_descriptor(node) == true */ + extend_info_t *info = extend_info_from_node(node, first_node); + if (first_node && !info) { + log_warn(LD_CONTROL, + "controller tried to connect to a node that doesn't have any " + "addresses that are allowed by the firewall configuration; " + "circuit marked for closing."); + circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED); + connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn); + goto done; + } else { + /* True, since node_has_descriptor(node) == true and we are extending + * to the node's primary address */ + tor_assert(info); + } circuit_append_new_exit(circ, info); extend_info_free(info); + first_node = 0; }); /* now that we've populated the cpath, start extending */ @@ -2669,6 +3033,7 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, edge_conn->end_reason = 0; if (tmpcirc) circuit_detach_stream(tmpcirc, edge_conn); + CONNECTION_AP_EXPECT_NONPENDING(ap_conn); TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT; } @@ -2681,7 +3046,7 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, /* Is this a single hop circuit? */ if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) { const node_t *node = NULL; - char *exit_digest; + char *exit_digest = NULL; if (circ->build_state && circ->build_state->chosen_exit && !tor_digest_is_zero(circ->build_state->chosen_exit->identity_digest)) { @@ -2696,6 +3061,7 @@ handle_control_attachstream(control_connection_t *conn, uint32_t len, "551 Can't attach stream to this one-hop circuit.\r\n", conn); return 0; } + tor_assert(exit_digest); ap_conn->chosen_exit_name = tor_strdup(hex_str(exit_digest, DIGEST_LEN)); } @@ -2727,12 +3093,14 @@ handle_control_postdescriptor(control_connection_t *conn, uint32_t len, uint8_t purpose = ROUTER_PURPOSE_GENERAL; int cache = 0; /* eventually, we may switch this to 1 */ - char *cp = memchr(body, '\n', len); + const char *cp = memchr(body, '\n', len); smartlist_t *args = smartlist_new(); tor_assert(cp); - *cp++ = '\0'; + ++cp; - smartlist_split_string(args, body, " ", + char *cmdline = tor_memdup_nulterm(body, cp-body); + + smartlist_split_string(args, cmdline, " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH_BEGIN(args, char *, option) { if (!strcasecmpstart(option, "purpose=")) { @@ -2781,6 +3149,7 @@ handle_control_postdescriptor(control_connection_t *conn, uint32_t len, done: SMARTLIST_FOREACH(args, char *, arg, tor_free(arg)); smartlist_free(args); + tor_free(cmdline); return 0; } @@ -2921,7 +3290,7 @@ handle_control_resolve(control_connection_t *conn, uint32_t len, int is_reverse = 0; (void) len; /* body is nul-terminated; it's safe to ignore the length */ - if (!(conn->event_mask & ((uint32_t)1L<<EVENT_ADDRMAP))) { + if (!(conn->event_mask & (((event_mask_t)1)<<EVENT_ADDRMAP))) { log_warn(LD_CONTROL, "Controller asked us to resolve an address, but " "isn't listening for ADDRMAP events. It probably won't see " "the answer."); @@ -2985,7 +3354,7 @@ handle_control_protocolinfo(control_connection_t *conn, uint32_t len, } else { const or_options_t *options = get_options(); int cookies = options->CookieAuthentication; - char *cfile = get_cookie_file(); + char *cfile = get_controller_cookie_file_name(); char *abs_cfile; char *esc_cfile; char *methods; @@ -3096,8 +3465,7 @@ handle_control_authchallenge(control_connection_t *conn, uint32_t len, tor_free(client_nonce); return -1; } - - tor_assert(!crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN)); + crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN); /* Now compute and send the server-to-controller response, and the * server's nonce. */ @@ -3181,6 +3549,594 @@ handle_control_usefeature(control_connection_t *conn, return 0; } +/** Implementation for the DROPGUARDS command. */ +static int +handle_control_dropguards(control_connection_t *conn, + uint32_t len, + const char *body) +{ + smartlist_t *args; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + args = smartlist_new(); + smartlist_split_string(args, body, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + + if (smartlist_len(args)) { + connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n"); + } else { + remove_all_entry_guards(); + send_control_done(conn); + } + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + return 0; +} + +/** Implementation for the HSFETCH command. */ +static int +handle_control_hsfetch(control_connection_t *conn, uint32_t len, + const char *body) +{ + int i; + char digest[DIGEST_LEN], *hsaddress = NULL, *arg1 = NULL, *desc_id = NULL; + smartlist_t *args = NULL, *hsdirs = NULL; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + static const char *hsfetch_command = "HSFETCH"; + static const char *v2_str = "v2-"; + const size_t v2_str_len = strlen(v2_str); + rend_data_t *rend_query = NULL; + + /* Make sure we have at least one argument, the HSAddress. */ + args = getargs_helper(hsfetch_command, conn, body, 1, -1); + if (!args) { + goto exit; + } + + /* Extract the first argument (either HSAddress or DescID). */ + arg1 = smartlist_get(args, 0); + /* Test if it's an HS address without the .onion part. */ + if (rend_valid_service_id(arg1)) { + hsaddress = arg1; + } else if (strcmpstart(arg1, v2_str) == 0 && + rend_valid_descriptor_id(arg1 + v2_str_len) && + base32_decode(digest, sizeof(digest), arg1 + v2_str_len, + REND_DESC_ID_V2_LEN_BASE32) == 0) { + /* We have a well formed version 2 descriptor ID. Keep the decoded value + * of the id. */ + desc_id = digest; + } else { + connection_printf_to_buf(conn, "513 Unrecognized \"%s\"\r\n", + arg1); + goto done; + } + + static const char *opt_server = "SERVER="; + + /* Skip first argument because it's the HSAddress or DescID. */ + for (i = 1; i < smartlist_len(args); ++i) { + const char *arg = smartlist_get(args, i); + const node_t *node; + + if (!strcasecmpstart(arg, opt_server)) { + const char *server; + + server = arg + strlen(opt_server); + node = node_get_by_hex_id(server); + if (!node) { + connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", + server); + goto done; + } + if (!hsdirs) { + /* Stores routerstatus_t object for each specified server. */ + hsdirs = smartlist_new(); + } + /* Valid server, add it to our local list. */ + smartlist_add(hsdirs, node->rs); + } else { + connection_printf_to_buf(conn, "513 Unexpected argument \"%s\"\r\n", + arg); + goto done; + } + } + + rend_query = rend_data_client_create(hsaddress, desc_id, NULL, + REND_NO_AUTH); + if (rend_query == NULL) { + connection_printf_to_buf(conn, "551 Error creating the HS query\r\n"); + goto done; + } + + /* Using a descriptor ID, we force the user to provide at least one + * hsdir server using the SERVER= option. */ + if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) { + connection_printf_to_buf(conn, "512 %s option is required\r\n", + opt_server); + goto done; + } + + /* We are about to trigger HSDir fetch so send the OK now because after + * that 650 event(s) are possible so better to have the 250 OK before them + * to avoid out of order replies. */ + send_control_done(conn); + + /* Trigger the fetch using the built rend query and possibly a list of HS + * directory to use. This function ignores the client cache thus this will + * always send a fetch command. */ + rend_client_fetch_v2_desc(rend_query, hsdirs); + + done: + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + /* Contains data pointer that we don't own thus no cleanup. */ + smartlist_free(hsdirs); + rend_data_free(rend_query); + exit: + return 0; +} + +/** Implementation for the HSPOST command. */ +static int +handle_control_hspost(control_connection_t *conn, + uint32_t len, + const char *body) +{ + static const char *opt_server = "SERVER="; + smartlist_t *args = smartlist_new(); + smartlist_t *hs_dirs = NULL; + const char *encoded_desc = body; + size_t encoded_desc_len = len; + + char *cp = memchr(body, '\n', len); + char *argline = tor_strndup(body, cp-body); + + /* If any SERVER= options were specified, try parse the options line */ + if (!strcasecmpstart(argline, opt_server)) { + /* encoded_desc begins after a newline character */ + cp = cp + 1; + encoded_desc = cp; + encoded_desc_len = len-(cp-body); + + smartlist_split_string(args, argline, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SMARTLIST_FOREACH_BEGIN(args, const char *, arg) { + if (!strcasecmpstart(arg, opt_server)) { + const char *server = arg + strlen(opt_server); + const node_t *node = node_get_by_hex_id(server); + + if (!node || !node->rs) { + connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n", + server); + goto done; + } + if (!node->rs->is_hs_dir) { + connection_printf_to_buf(conn, "552 Server \"%s\" is not a HSDir" + "\r\n", server); + goto done; + } + /* Valid server, add it to our local list. */ + if (!hs_dirs) + hs_dirs = smartlist_new(); + smartlist_add(hs_dirs, node->rs); + } else { + connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n", + arg); + goto done; + } + } SMARTLIST_FOREACH_END(arg); + } + + /* Read the dot encoded descriptor, and parse it. */ + rend_encoded_v2_service_descriptor_t *desc = + tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t)); + read_escaped_data(encoded_desc, encoded_desc_len, &desc->desc_str); + + rend_service_descriptor_t *parsed = NULL; + char *intro_content = NULL; + size_t intro_size; + size_t encoded_size; + const char *next_desc; + if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content, + &intro_size, &encoded_size, + &next_desc, desc->desc_str, 1)) { + /* Post the descriptor. */ + char serviceid[REND_SERVICE_ID_LEN_BASE32+1]; + if (!rend_get_service_id(parsed->pk, serviceid)) { + smartlist_t *descs = smartlist_new(); + smartlist_add(descs, desc); + + /* We are about to trigger HS descriptor upload so send the OK now + * because after that 650 event(s) are possible so better to have the + * 250 OK before them to avoid out of order replies. */ + send_control_done(conn); + + /* Trigger the descriptor upload */ + directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0); + smartlist_free(descs); + } + + rend_service_descriptor_free(parsed); + } else { + connection_printf_to_buf(conn, "554 Invalid descriptor\r\n"); + } + + tor_free(intro_content); + rend_encoded_v2_service_descriptor_free(desc); + done: + tor_free(argline); + smartlist_free(hs_dirs); /* Contents belong to the rend service code. */ + SMARTLIST_FOREACH(args, char *, arg, tor_free(arg)); + smartlist_free(args); + return 0; +} + +/** Called when we get a ADD_ONION command; parse the body, and set up + * the new ephemeral Onion Service. */ +static int +handle_control_add_onion(control_connection_t *conn, + uint32_t len, + const char *body) +{ + smartlist_t *args; + size_t arg_len; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + args = getargs_helper("ADD_ONION", conn, body, 2, -1); + if (!args) + return 0; + arg_len = smartlist_len(args); + + /* Parse all of the arguments that do not involve handling cryptographic + * material first, since there's no reason to touch that at all if any of + * the other arguments are malformed. + */ + smartlist_t *port_cfgs = smartlist_new(); + int discard_pk = 0; + int detach = 0; + int max_streams = 0; + int max_streams_close_circuit = 0; + for (size_t i = 1; i < arg_len; i++) { + static const char *port_prefix = "Port="; + static const char *flags_prefix = "Flags="; + static const char *max_s_prefix = "MaxStreams="; + + const char *arg = smartlist_get(args, i); + if (!strcasecmpstart(arg, port_prefix)) { + /* "Port=VIRTPORT[,TARGET]". */ + const char *port_str = arg + strlen(port_prefix); + + rend_service_port_config_t *cfg = + rend_service_parse_port_config(port_str, ",", NULL); + if (!cfg) { + connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); + goto out; + } + smartlist_add(port_cfgs, cfg); + } else if (!strcasecmpstart(arg, max_s_prefix)) { + /* "MaxStreams=[0..65535]". */ + const char *max_s_str = arg + strlen(max_s_prefix); + int ok = 0; + max_streams = (int)tor_parse_long(max_s_str, 10, 0, 65535, &ok, NULL); + if (!ok) { + connection_printf_to_buf(conn, "512 Invalid MaxStreams\r\n"); + goto out; + } + } else if (!strcasecmpstart(arg, flags_prefix)) { + /* "Flags=Flag[,Flag]", where Flag can be: + * * 'DiscardPK' - If tor generates the keypair, do not include it in + * the response. + * * 'Detach' - Do not tie this onion service to any particular control + * connection. + * * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is + * exceeded. + */ + static const char *discard_flag = "DiscardPK"; + static const char *detach_flag = "Detach"; + static const char *max_s_close_flag = "MaxStreamsCloseCircuit"; + + smartlist_t *flags = smartlist_new(); + int bad = 0; + + smartlist_split_string(flags, arg + strlen(flags_prefix), ",", + SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(flags) < 1) { + connection_printf_to_buf(conn, "512 Invalid 'Flags' argument\r\n"); + bad = 1; + } + SMARTLIST_FOREACH_BEGIN(flags, const char *, flag) + { + if (!strcasecmp(flag, discard_flag)) { + discard_pk = 1; + } else if (!strcasecmp(flag, detach_flag)) { + detach = 1; + } else if (!strcasecmp(flag, max_s_close_flag)) { + max_streams_close_circuit = 1; + } else { + connection_printf_to_buf(conn, + "512 Invalid 'Flags' argument: %s\r\n", + escaped(flag)); + bad = 1; + break; + } + } SMARTLIST_FOREACH_END(flag); + SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp)); + smartlist_free(flags); + if (bad) + goto out; + } else { + connection_printf_to_buf(conn, "513 Invalid argument\r\n"); + goto out; + } + } + if (smartlist_len(port_cfgs) == 0) { + connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n"); + goto out; + } + + /* Parse the "keytype:keyblob" argument. */ + crypto_pk_t *pk = NULL; + const char *key_new_alg = NULL; + char *key_new_blob = NULL; + char *err_msg = NULL; + + pk = add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk, + &key_new_alg, &key_new_blob, + &err_msg); + if (!pk) { + if (err_msg) { + connection_write_str_to_buf(err_msg, conn); + tor_free(err_msg); + } + goto out; + } + tor_assert(!err_msg); + + /* Create the HS, using private key pk, and port config port_cfg. + * rend_service_add_ephemeral() will take ownership of pk and port_cfg, + * regardless of success/failure. + */ + char *service_id = NULL; + int ret = rend_service_add_ephemeral(pk, port_cfgs, max_streams, + max_streams_close_circuit, + &service_id); + port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */ + switch (ret) { + case RSAE_OKAY: + { + char *buf = NULL; + tor_assert(service_id); + if (key_new_alg) { + tor_assert(key_new_blob); + tor_asprintf(&buf, + "250-ServiceID=%s\r\n" + "250-PrivateKey=%s:%s\r\n" + "250 OK\r\n", + service_id, + key_new_alg, + key_new_blob); + } else { + tor_asprintf(&buf, + "250-ServiceID=%s\r\n" + "250 OK\r\n", + service_id); + } + if (detach) { + if (!detached_onion_services) + detached_onion_services = smartlist_new(); + smartlist_add(detached_onion_services, service_id); + } else { + if (!conn->ephemeral_onion_services) + conn->ephemeral_onion_services = smartlist_new(); + smartlist_add(conn->ephemeral_onion_services, service_id); + } + + connection_write_str_to_buf(buf, conn); + memwipe(buf, 0, strlen(buf)); + tor_free(buf); + break; + } + case RSAE_BADPRIVKEY: + connection_printf_to_buf(conn, "551 Failed to generate onion address\r\n"); + break; + case RSAE_ADDREXISTS: + connection_printf_to_buf(conn, "550 Onion address collision\r\n"); + break; + case RSAE_BADVIRTPORT: + connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); + break; + case RSAE_INTERNAL: /* FALLSTHROUGH */ + default: + connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n"); + } + if (key_new_blob) { + memwipe(key_new_blob, 0, strlen(key_new_blob)); + tor_free(key_new_blob); + } + + out: + if (port_cfgs) { + SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p, + rend_service_port_config_free(p)); + smartlist_free(port_cfgs); + } + + SMARTLIST_FOREACH(args, char *, cp, { + memwipe(cp, 0, strlen(cp)); + tor_free(cp); + }); + smartlist_free(args); + return 0; +} + +/** Helper function to handle parsing the KeyType:KeyBlob argument to the + * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated + * and the private key not discarded, the algorithm and serialized private key, + * or NULL and an optional control protocol error message on failure. The + * caller is responsible for freeing the returned key_new_blob and err_msg. + * + * Note: The error messages returned are deliberately vague to avoid echoing + * key material. + */ +STATIC crypto_pk_t * +add_onion_helper_keyarg(const char *arg, int discard_pk, + const char **key_new_alg_out, char **key_new_blob_out, + char **err_msg_out) +{ + smartlist_t *key_args = smartlist_new(); + crypto_pk_t *pk = NULL; + const char *key_new_alg = NULL; + char *key_new_blob = NULL; + char *err_msg = NULL; + int ok = 0; + + smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(key_args) != 2) { + err_msg = tor_strdup("512 Invalid key type/blob\r\n"); + goto err; + } + + /* The format is "KeyType:KeyBlob". */ + static const char *key_type_new = "NEW"; + static const char *key_type_best = "BEST"; + static const char *key_type_rsa1024 = "RSA1024"; + + const char *key_type = smartlist_get(key_args, 0); + const char *key_blob = smartlist_get(key_args, 1); + + if (!strcasecmp(key_type_rsa1024, key_type)) { + /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */ + pk = crypto_pk_base64_decode(key_blob, strlen(key_blob)); + if (!pk) { + err_msg = tor_strdup("512 Failed to decode RSA key\r\n"); + goto err; + } + if (crypto_pk_num_bits(pk) != PK_BYTES*8) { + err_msg = tor_strdup("512 Invalid RSA key size\r\n"); + goto err; + } + } else if (!strcasecmp(key_type_new, key_type)) { + /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */ + if (!strcasecmp(key_type_rsa1024, key_blob) || + !strcasecmp(key_type_best, key_blob)) { + /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */ + pk = crypto_pk_new(); + if (crypto_pk_generate_key(pk)) { + tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n", + key_type_rsa1024); + goto err; + } + if (!discard_pk) { + if (crypto_pk_base64_encode(pk, &key_new_blob)) { + tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n", + key_type_rsa1024); + goto err; + } + key_new_alg = key_type_rsa1024; + } + } else { + err_msg = tor_strdup("513 Invalid key type\r\n"); + goto err; + } + } else { + err_msg = tor_strdup("513 Invalid key type\r\n"); + goto err; + } + + /* Succeded in loading or generating a private key. */ + tor_assert(pk); + ok = 1; + + err: + SMARTLIST_FOREACH(key_args, char *, cp, { + memwipe(cp, 0, strlen(cp)); + tor_free(cp); + }); + smartlist_free(key_args); + + if (!ok) { + crypto_pk_free(pk); + pk = NULL; + } + if (err_msg_out) { + *err_msg_out = err_msg; + } else { + tor_free(err_msg); + } + *key_new_alg_out = key_new_alg; + *key_new_blob_out = key_new_blob; + + return pk; +} + +/** Called when we get a DEL_ONION command; parse the body, and remove + * the existing ephemeral Onion Service. */ +static int +handle_control_del_onion(control_connection_t *conn, + uint32_t len, + const char *body) +{ + smartlist_t *args; + (void) len; /* body is nul-terminated; it's safe to ignore the length */ + args = getargs_helper("DEL_ONION", conn, body, 1, 1); + if (!args) + return 0; + + const char *service_id = smartlist_get(args, 0); + if (!rend_valid_service_id(service_id)) { + connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n"); + goto out; + } + + /* Determine if the onion service belongs to this particular control + * connection, or if it is in the global list of detached services. If it + * is in neither, either the service ID is invalid in some way, or it + * explicitly belongs to a different control connection, and an error + * should be returned. + */ + smartlist_t *services[2] = { + conn->ephemeral_onion_services, + detached_onion_services + }; + smartlist_t *onion_services = NULL; + int idx = -1; + for (size_t i = 0; i < ARRAY_LENGTH(services); i++) { + idx = smartlist_string_pos(services[i], service_id); + if (idx != -1) { + onion_services = services[i]; + break; + } + } + if (onion_services == NULL) { + connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n"); + } else { + int ret = rend_service_del_ephemeral(service_id); + if (ret) { + /* This should *NEVER* fail, since the service is on either the + * per-control connection list, or the global one. + */ + log_warn(LD_BUG, "Failed to remove Onion Service %s.", + escaped(service_id)); + tor_fragile_assert(); + } + + /* Remove/scrub the service_id from the appropriate list. */ + char *cp = smartlist_get(onion_services, idx); + smartlist_del(onion_services, idx); + memwipe(cp, 0, strlen(cp)); + tor_free(cp); + + send_control_done(conn); + } + + out: + SMARTLIST_FOREACH(args, char *, cp, { + memwipe(cp, 0, strlen(cp)); + tor_free(cp); + }); + smartlist_free(args); + return 0; +} + /** Called when <b>conn</b> has no more bytes left on its outbuf. */ int connection_control_finished_flushing(control_connection_t *conn) @@ -3205,22 +4161,10 @@ connection_control_reached_eof(control_connection_t *conn) static void lost_owning_controller(const char *owner_type, const char *loss_manner) { - int shutdown_slowly = server_mode(get_options()); - - log_notice(LD_CONTROL, "Owning controller %s has %s -- %s.", - owner_type, loss_manner, - shutdown_slowly ? "shutting down" : "exiting now"); + log_notice(LD_CONTROL, "Owning controller %s has %s -- exiting now.", + owner_type, loss_manner); - /* XXXX Perhaps this chunk of code should be a separate function, - * called here and by process_signal(SIGINT). */ - - if (!shutdown_slowly) { - tor_cleanup(); - exit(0); - } - /* XXXX This will close all listening sockets except control-port - * listeners. Perhaps we should close those too. */ - hibernate_begin_shutdown(); + activate_signal(SIGTERM); } /** Called when <b>conn</b> is being freed. */ @@ -3232,6 +4176,15 @@ connection_control_closed(control_connection_t *conn) conn->event_mask = 0; control_update_global_event_mask(); + /* Close all ephemeral Onion Services if any. + * The list and it's contents are scrubbed/freed in connection_free_. + */ + if (conn->ephemeral_onion_services) { + SMARTLIST_FOREACH(conn->ephemeral_onion_services, char *, cp, { + rend_service_del_ephemeral(cp); + }); + } + if (conn->is_owning_control_connection) { lost_owning_controller("connection", "closed"); } @@ -3480,6 +4433,25 @@ connection_control_process_inbuf(control_connection_t *conn) } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) { if (handle_control_authchallenge(conn, cmd_data_len, args)) return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) { + if (handle_control_dropguards(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "HSFETCH")) { + if (handle_control_hsfetch(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "+HSPOST")) { + if (handle_control_hspost(conn, cmd_data_len, args)) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "ADD_ONION")) { + int ret = handle_control_add_onion(conn, cmd_data_len, args); + memwipe(args, 0, cmd_data_len); /* Scrub the private key. */ + if (ret) + return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DEL_ONION")) { + int ret = handle_control_del_onion(conn, cmd_data_len, args); + memwipe(args, 0, cmd_data_len); /* Scrub the service id/pk. */ + if (ret) + return -1; } else { connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n", conn->incoming_cmd); @@ -3533,7 +4505,7 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, { char *circdesc = circuit_describe_status_for_controller(circ); const char *sp = strlen(circdesc) ? " " : ""; - send_control_event(EVENT_CIRCUIT_STATUS, ALL_FORMATS, + send_control_event(EVENT_CIRCUIT_STATUS, "650 CIRC %lu %s%s%s%s\r\n", (unsigned long)circ->global_identifier, status, sp, @@ -3604,7 +4576,7 @@ control_event_circuit_status_minor(origin_circuit_t *circ, { char *circdesc = circuit_describe_status_for_controller(circ); const char *sp = strlen(circdesc) ? " " : ""; - send_control_event(EVENT_CIRCUIT_STATUS_MINOR, ALL_FORMATS, + send_control_event(EVENT_CIRCUIT_STATUS_MINOR, "650 CIRC_MINOR %lu %s%s%s%s\r\n", (unsigned long)circ->global_identifier, event_desc, sp, @@ -3779,7 +4751,7 @@ control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp, circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn)); if (circ && CIRCUIT_IS_ORIGIN(circ)) origin_circ = TO_ORIGIN_CIRCUIT(circ); - send_control_event(EVENT_STREAM_STATUS, ALL_FORMATS, + send_control_event(EVENT_STREAM_STATUS, "650 STREAM "U64_FORMAT" %s %lu %s%s%s%s\r\n", U64_PRINTF_ARG(ENTRY_TO_CONN(conn)->global_identifier), status, @@ -3847,17 +4819,17 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, } ncircs += connection_or_get_num_circuits(conn); if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) { - tor_snprintf(ncircs_buf, sizeof(ncircs_buf), "%sNCIRCS=%d", - reason ? " " : "", ncircs); + tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs); } orconn_target_get_name(name, sizeof(name), conn); - send_control_event(EVENT_OR_CONN_STATUS, ALL_FORMATS, - "650 ORCONN %s %s %s%s%s\r\n", + send_control_event(EVENT_OR_CONN_STATUS, + "650 ORCONN %s %s%s%s%s ID="U64_FORMAT"\r\n", name, status, - reason ? "REASON=" : "", + reason ? " REASON=" : "", orconn_end_reason_to_control_string(reason), - ncircs_buf); + ncircs_buf, + U64_PRINTF_ARG(conn->base_.global_identifier)); return 0; } @@ -3868,16 +4840,24 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, int control_event_stream_bandwidth(edge_connection_t *edge_conn) { + circuit_t *circ; + origin_circuit_t *ocirc; if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) { if (!edge_conn->n_read && !edge_conn->n_written) return 0; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->base_.global_identifier), (unsigned long)edge_conn->n_read, (unsigned long)edge_conn->n_written); + circ = circuit_get_by_edge_conn(edge_conn); + if (circ && CIRCUIT_IS_ORIGIN(circ)) { + ocirc = TO_ORIGIN_CIRCUIT(circ); + ocirc->n_read_circ_bw += edge_conn->n_read; + ocirc->n_written_circ_bw += edge_conn->n_written; + } edge_conn->n_written = edge_conn->n_read = 0; } @@ -3901,7 +4881,7 @@ control_event_stream_bandwidth_used(void) if (!edge_conn->n_read && !edge_conn->n_written) continue; - send_control_event(EVENT_STREAM_BANDWIDTH_USED, ALL_FORMATS, + send_control_event(EVENT_STREAM_BANDWIDTH_USED, "650 STREAM_BW "U64_FORMAT" %lu %lu\r\n", U64_PRINTF_ARG(edge_conn->base_.global_identifier), (unsigned long)edge_conn->n_read, @@ -3915,13 +4895,258 @@ control_event_stream_bandwidth_used(void) return 0; } +/** A second or more has elapsed: tell any interested control connections + * how much bandwidth origin circuits have used. */ +int +control_event_circ_bandwidth_used(void) +{ + origin_circuit_t *ocirc; + if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED)) + return 0; + + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + if (!CIRCUIT_IS_ORIGIN(circ)) + continue; + ocirc = TO_ORIGIN_CIRCUIT(circ); + if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw) + continue; + send_control_event(EVENT_CIRC_BANDWIDTH_USED, + "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu\r\n", + ocirc->global_identifier, + (unsigned long)ocirc->n_read_circ_bw, + (unsigned long)ocirc->n_written_circ_bw); + ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + } + SMARTLIST_FOREACH_END(circ); + + return 0; +} + +/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset + * bandwidth counters. */ +int +control_event_conn_bandwidth(connection_t *conn) +{ + const char *conn_type_str; + if (!get_options()->TestingEnableConnBwEvent || + !EVENT_IS_INTERESTING(EVENT_CONN_BW)) + return 0; + if (!conn->n_read_conn_bw && !conn->n_written_conn_bw) + return 0; + switch (conn->type) { + case CONN_TYPE_OR: + conn_type_str = "OR"; + break; + case CONN_TYPE_DIR: + conn_type_str = "DIR"; + break; + case CONN_TYPE_EXIT: + conn_type_str = "EXIT"; + break; + default: + return 0; + } + send_control_event(EVENT_CONN_BW, + "650 CONN_BW ID="U64_FORMAT" TYPE=%s " + "READ=%lu WRITTEN=%lu\r\n", + U64_PRINTF_ARG(conn->global_identifier), + conn_type_str, + (unsigned long)conn->n_read_conn_bw, + (unsigned long)conn->n_written_conn_bw); + conn->n_written_conn_bw = conn->n_read_conn_bw = 0; + return 0; +} + +/** A second or more has elapsed: tell any interested control + * connections how much bandwidth connections have used. */ +int +control_event_conn_bandwidth_used(void) +{ + if (get_options()->TestingEnableConnBwEvent && + EVENT_IS_INTERESTING(EVENT_CONN_BW)) { + SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn, + control_event_conn_bandwidth(conn)); + } + return 0; +} + +/** Helper: iterate over cell statistics of <b>circ</b> and sum up added + * cells, removed cells, and waiting times by cell command and direction. + * Store results in <b>cell_stats</b>. Free cell statistics of the + * circuit afterwards. */ +void +sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats) +{ + memset(cell_stats, 0, sizeof(cell_stats_t)); + SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats, + const testing_cell_stats_entry_t *, ent) { + tor_assert(ent->command <= CELL_COMMAND_MAX_); + if (!ent->removed && !ent->exitward) { + cell_stats->added_cells_appward[ent->command] += 1; + } else if (!ent->removed && ent->exitward) { + cell_stats->added_cells_exitward[ent->command] += 1; + } else if (!ent->exitward) { + cell_stats->removed_cells_appward[ent->command] += 1; + cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10; + } else { + cell_stats->removed_cells_exitward[ent->command] += 1; + cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10; + } + } SMARTLIST_FOREACH_END(ent); + circuit_clear_testing_cell_stats(circ); +} + +/** Helper: append a cell statistics string to <code>event_parts</code>, + * prefixed with <code>key</code>=. Statistics consist of comma-separated + * key:value pairs with lower-case command strings as keys and cell + * numbers or total waiting times as values. A key:value pair is included + * if the entry in <code>include_if_non_zero</code> is not zero, but with + * the (possibly zero) entry from <code>number_to_include</code>. Both + * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1. If no + * entry in <code>include_if_non_zero</code> is positive, no string will + * be added to <code>event_parts</code>. */ +void +append_cell_stats_by_command(smartlist_t *event_parts, const char *key, + const uint64_t *include_if_non_zero, + const uint64_t *number_to_include) +{ + smartlist_t *key_value_strings = smartlist_new(); + int i; + for (i = 0; i <= CELL_COMMAND_MAX_; i++) { + if (include_if_non_zero[i] > 0) { + smartlist_add_asprintf(key_value_strings, "%s:"U64_FORMAT, + cell_command_to_string(i), + U64_PRINTF_ARG(number_to_include[i])); + } + } + if (smartlist_len(key_value_strings) > 0) { + char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL); + smartlist_add_asprintf(event_parts, "%s=%s", key, joined); + SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp)); + tor_free(joined); + } + smartlist_free(key_value_strings); +} + +/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a + * CELL_STATS event and write result string to <b>event_string</b>. */ +void +format_cell_stats(char **event_string, circuit_t *circ, + cell_stats_t *cell_stats) +{ + smartlist_t *event_parts = smartlist_new(); + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + smartlist_add_asprintf(event_parts, "ID=%lu", + (unsigned long)ocirc->global_identifier); + } else if (TO_OR_CIRCUIT(circ)->p_chan) { + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + smartlist_add_asprintf(event_parts, "InboundQueue=%lu", + (unsigned long)or_circ->p_circ_id); + smartlist_add_asprintf(event_parts, "InboundConn="U64_FORMAT, + U64_PRINTF_ARG(or_circ->p_chan->global_identifier)); + append_cell_stats_by_command(event_parts, "InboundAdded", + cell_stats->added_cells_appward, + cell_stats->added_cells_appward); + append_cell_stats_by_command(event_parts, "InboundRemoved", + cell_stats->removed_cells_appward, + cell_stats->removed_cells_appward); + append_cell_stats_by_command(event_parts, "InboundTime", + cell_stats->removed_cells_appward, + cell_stats->total_time_appward); + } + if (circ->n_chan) { + smartlist_add_asprintf(event_parts, "OutboundQueue=%lu", + (unsigned long)circ->n_circ_id); + smartlist_add_asprintf(event_parts, "OutboundConn="U64_FORMAT, + U64_PRINTF_ARG(circ->n_chan->global_identifier)); + append_cell_stats_by_command(event_parts, "OutboundAdded", + cell_stats->added_cells_exitward, + cell_stats->added_cells_exitward); + append_cell_stats_by_command(event_parts, "OutboundRemoved", + cell_stats->removed_cells_exitward, + cell_stats->removed_cells_exitward); + append_cell_stats_by_command(event_parts, "OutboundTime", + cell_stats->removed_cells_exitward, + cell_stats->total_time_exitward); + } + *event_string = smartlist_join_strings(event_parts, " ", 0, NULL); + SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp)); + smartlist_free(event_parts); +} + +/** A second or more has elapsed: tell any interested control connection + * how many cells have been processed for a given circuit. */ +int +control_event_circuit_cell_stats(void) +{ + cell_stats_t *cell_stats; + char *event_string; + if (!get_options()->TestingEnableCellStatsEvent || + !EVENT_IS_INTERESTING(EVENT_CELL_STATS)) + return 0; + cell_stats = tor_malloc(sizeof(cell_stats_t));; + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + if (!circ->testing_cell_stats) + continue; + sum_up_cell_stats_by_command(circ, cell_stats); + format_cell_stats(&event_string, circ, cell_stats); + send_control_event(EVENT_CELL_STATS, + "650 CELL_STATS %s\r\n", event_string); + tor_free(event_string); + } + SMARTLIST_FOREACH_END(circ); + tor_free(cell_stats); + return 0; +} + +/** Tokens in <b>bucket</b> have been refilled: the read bucket was empty + * for <b>read_empty_time</b> millis, the write bucket was empty for + * <b>write_empty_time</b> millis, and buckets were last refilled + * <b>milliseconds_elapsed</b> millis ago. Only emit TB_EMPTY event if + * either read or write bucket have been empty before. */ +int +control_event_tb_empty(const char *bucket, uint32_t read_empty_time, + uint32_t write_empty_time, + int milliseconds_elapsed) +{ + if (get_options()->TestingEnableTbEmptyEvent && + EVENT_IS_INTERESTING(EVENT_TB_EMPTY) && + (read_empty_time > 0 || write_empty_time > 0)) { + send_control_event(EVENT_TB_EMPTY, + "650 TB_EMPTY %s READ=%d WRITTEN=%d " + "LAST=%d\r\n", + bucket, read_empty_time, write_empty_time, + milliseconds_elapsed); + } + return 0; +} + +/* about 5 minutes worth. */ +#define N_BW_EVENTS_TO_CACHE 300 +/* Index into cached_bw_events to next write. */ +static int next_measurement_idx = 0; +/* number of entries set in n_measurements */ +static int n_measurements = 0; +static struct cached_bw_event_s { + uint32_t n_read; + uint32_t n_written; +} cached_bw_events[N_BW_EVENTS_TO_CACHE]; + /** A second or more has elapsed: tell any interested control * connections how much bandwidth we used. */ int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) { + cached_bw_events[next_measurement_idx].n_read = n_read; + cached_bw_events[next_measurement_idx].n_written = n_written; + if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE) + next_measurement_idx = 0; + if (n_measurements < N_BW_EVENTS_TO_CACHE) + ++n_measurements; + if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) { - send_control_event(EVENT_BANDWIDTH_USED, ALL_FORMATS, + send_control_event(EVENT_BANDWIDTH_USED, "650 BW %lu %lu\r\n", (unsigned long)n_read, (unsigned long)n_written); @@ -3930,6 +5155,35 @@ control_event_bandwidth_used(uint32_t n_read, uint32_t n_written) return 0; } +STATIC char * +get_bw_samples(void) +{ + int i; + int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements) + % N_BW_EVENTS_TO_CACHE; + tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE); + + smartlist_t *elements = smartlist_new(); + + for (i = 0; i < n_measurements; ++i) { + tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE); + const struct cached_bw_event_s *bwe = &cached_bw_events[idx]; + + smartlist_add_asprintf(elements, "%u,%u", + (unsigned)bwe->n_read, + (unsigned)bwe->n_written); + + idx = (idx + 1) % N_BW_EVENTS_TO_CACHE; + } + + char *result = smartlist_join_strings(elements, " ", 0, NULL); + + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + + return result; +} + /** Called when we are sending a log message to the controllers: suspend * sending further log messages to the controllers until we're done. Used by * CONN_LOG_PROTECT. */ @@ -3991,7 +5245,11 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) default: s = "UnknownLogSeverity"; break; } ++disable_log_messages; - send_control_event(event, ALL_FORMATS, "650 %s %s\r\n", s, b?b:msg); + send_control_event(event, "650 %s %s\r\n", s, b?b:msg); + if (severity == LOG_ERR) { + /* Force a flush, since we may be about to die horribly */ + queued_events_flush_all(1); + } --disable_log_messages; tor_free(b); } @@ -4019,7 +5277,7 @@ control_event_descriptors_changed(smartlist_t *routers) }); ids = smartlist_join_strings(names, " ", 0, NULL); tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids); - send_control_event_string(EVENT_NEW_DESC, ALL_FORMATS, msg); + send_control_event_string(EVENT_NEW_DESC, msg); tor_free(ids); tor_free(msg); SMARTLIST_FOREACH(names, char *, cp, tor_free(cp)); @@ -4041,7 +5299,7 @@ control_event_address_mapped(const char *from, const char *to, time_t expires, return 0; if (expires < 3 || expires == TIME_MAX) - send_control_event(EVENT_ADDRMAP, ALL_FORMATS, + send_control_event(EVENT_ADDRMAP, "650 ADDRMAP %s %s NEVER %s%s" "CACHED=\"%s\"\r\n", from, to, error?error:"", error?" ":"", @@ -4051,7 +5309,7 @@ control_event_address_mapped(const char *from, const char *to, time_t expires, char buf2[ISO_TIME_LEN+1]; format_local_iso_time(buf,expires); format_iso_time(buf2,expires); - send_control_event(EVENT_ADDRMAP, ALL_FORMATS, + send_control_event(EVENT_ADDRMAP, "650 ADDRMAP %s %s \"%s\"" " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n", from, to, buf, @@ -4093,9 +5351,9 @@ control_event_or_authdir_new_descriptor(const char *action, buf = tor_malloc(totallen); strlcpy(buf, firstline, totallen); strlcpy(buf+strlen(firstline), esc, totallen); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, buf); - send_control_event_string(EVENT_AUTHDIR_NEWDESCS, ALL_FORMATS, + send_control_event_string(EVENT_AUTHDIR_NEWDESCS, "650 OK\r\n"); tor_free(esc); tor_free(buf); @@ -4103,6 +5361,52 @@ control_event_or_authdir_new_descriptor(const char *action, return 0; } +/** Cached liveness for network liveness events and GETINFO + */ + +static int network_is_live = 0; + +static int +get_cached_network_liveness(void) +{ + return network_is_live; +} + +static void +set_cached_network_liveness(int liveness) +{ + network_is_live = liveness; +} + +/** The network liveness has changed; this is called from circuitstats.c + * whenever we receive a cell, or when timeout expires and we assume the + * network is down. */ +int +control_event_network_liveness_update(int liveness) +{ + if (liveness > 0) { + if (get_cached_network_liveness() <= 0) { + /* Update cached liveness */ + set_cached_network_liveness(1); + log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP"); + send_control_event_string(EVENT_NETWORK_LIVENESS, + "650 NETWORK_LIVENESS UP\r\n"); + } + /* else was already live, no-op */ + } else { + if (get_cached_network_liveness() > 0) { + /* Update cached liveness */ + set_cached_network_liveness(0); + log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN"); + send_control_event_string(EVENT_NETWORK_LIVENESS, + "650 NETWORK_LIVENESS DOWN\r\n"); + } + /* else was already dead, no-op */ + } + + return 0; +} + /** Helper function for NS-style events. Constructs and sends an event * of type <b>event</b> with string <b>event_string</b> out of the set of * networkstatuses <b>statuses</b>. Currently it is used for NS events @@ -4133,8 +5437,8 @@ control_event_networkstatus_changed_helper(smartlist_t *statuses, SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp)); smartlist_free(strs); tor_free(s); - send_control_event_string(event, ALL_FORMATS, esc); - send_control_event_string(event, ALL_FORMATS, + send_control_event_string(event, esc); + send_control_event_string(event, "650 OK\r\n"); tor_free(esc); @@ -4162,32 +5466,26 @@ control_event_newconsensus(const networkstatus_t *consensus) /** Called when we compute a new circuitbuildtimeout */ int -control_event_buildtimeout_set(const circuit_build_times_t *cbt, - buildtimeout_set_event_t type) +control_event_buildtimeout_set(buildtimeout_set_event_t type, + const char *args) { const char *type_string = NULL; - double qnt; if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET)) return 0; - qnt = circuit_build_times_quantile_cutoff(); - switch (type) { case BUILDTIMEOUT_SET_EVENT_COMPUTED: type_string = "COMPUTED"; break; case BUILDTIMEOUT_SET_EVENT_RESET: type_string = "RESET"; - qnt = 1.0; break; case BUILDTIMEOUT_SET_EVENT_SUSPENDED: type_string = "SUSPENDED"; - qnt = 1.0; break; case BUILDTIMEOUT_SET_EVENT_DISCARD: type_string = "DISCARD"; - qnt = 1.0; break; case BUILDTIMEOUT_SET_EVENT_RESUME: type_string = "RESUME"; @@ -4197,16 +5495,9 @@ control_event_buildtimeout_set(const circuit_build_times_t *cbt, break; } - send_control_event(EVENT_BUILDTIMEOUT_SET, ALL_FORMATS, - "650 BUILDTIMEOUT_SET %s TOTAL_TIMES=%lu " - "TIMEOUT_MS=%lu XM=%lu ALPHA=%f CUTOFF_QUANTILE=%f " - "TIMEOUT_RATE=%f CLOSE_MS=%lu CLOSE_RATE=%f\r\n", - type_string, (unsigned long)cbt->total_build_times, - (unsigned long)cbt->timeout_ms, - (unsigned long)cbt->Xm, cbt->alpha, qnt, - circuit_build_times_timeout_rate(cbt), - (unsigned long)cbt->close_ms, - circuit_build_times_close_rate(cbt)); + send_control_event(EVENT_BUILDTIMEOUT_SET, + "650 BUILDTIMEOUT_SET %s %s\r\n", + type_string, args); return 0; } @@ -4217,7 +5508,7 @@ control_event_signal(uintptr_t signal) { const char *signal_string = NULL; - if (!control_event_is_interesting(EVENT_SIGNAL)) + if (!control_event_is_interesting(EVENT_GOT_SIGNAL)) return 0; switch (signal) { @@ -4236,13 +5527,16 @@ control_event_signal(uintptr_t signal) case SIGCLEARDNSCACHE: signal_string = "CLEARDNSCACHE"; break; + case SIGHEARTBEAT: + signal_string = "HEARTBEAT"; + break; default: log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal", (unsigned long)signal); return -1; } - send_control_event(EVENT_SIGNAL, ALL_FORMATS, "650 SIGNAL %s\r\n", + send_control_event(EVENT_GOT_SIGNAL, "650 SIGNAL %s\r\n", signal_string); return 0; } @@ -4270,7 +5564,7 @@ control_event_networkstatus_changed_single(const routerstatus_t *rs) int control_event_my_descriptor_changed(void) { - send_control_event(EVENT_DESCCHANGED, ALL_FORMATS, "650 DESCCHANGED\r\n"); + send_control_event(EVENT_DESCCHANGED, "650 DESCCHANGED\r\n"); return 0; } @@ -4320,24 +5614,40 @@ control_event_status(int type, int severity, const char *format, va_list args) } tor_vasprintf(&user_buf, format, args); - send_control_event(type, ALL_FORMATS, "%s %s\r\n", format_buf, user_buf); + send_control_event(type, "%s %s\r\n", format_buf, user_buf); tor_free(user_buf); return 0; } +#define CONTROL_EVENT_STATUS_BODY(event, sev) \ + int r; \ + do { \ + va_list ap; \ + if (!EVENT_IS_INTERESTING(event)) \ + return 0; \ + \ + va_start(ap, format); \ + r = control_event_status((event), (sev), format, ap); \ + va_end(ap); \ + } while (0) + /** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained * by formatting the arguments using the printf-style <b>format</b>. */ int control_event_general_status(int severity, const char *format, ...) { - va_list ap; - int r; - if (!EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) - return 0; + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity); + return r; +} - va_start(ap, format); - r = control_event_status(EVENT_STATUS_GENERAL, severity, format, ap); - va_end(ap); +/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the + * controller(s) immediately. */ +int +control_event_general_error(const char *format, ...) +{ + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR); + /* Force a flush, since we may be about to die horribly */ + queued_events_flush_all(1); return r; } @@ -4346,14 +5656,18 @@ control_event_general_status(int severity, const char *format, ...) int control_event_client_status(int severity, const char *format, ...) { - va_list ap; - int r; - if (!EVENT_IS_INTERESTING(EVENT_STATUS_CLIENT)) - return 0; + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity); + return r; +} - va_start(ap, format); - r = control_event_status(EVENT_STATUS_CLIENT, severity, format, ap); - va_end(ap); +/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the + * controller(s) immediately. */ +int +control_event_client_error(const char *format, ...) +{ + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR); + /* Force a flush, since we may be about to die horribly */ + queued_events_flush_all(1); return r; } @@ -4362,14 +5676,18 @@ control_event_client_status(int severity, const char *format, ...) int control_event_server_status(int severity, const char *format, ...) { - va_list ap; - int r; - if (!EVENT_IS_INTERESTING(EVENT_STATUS_SERVER)) - return 0; + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity); + return r; +} - va_start(ap, format); - r = control_event_status(EVENT_STATUS_SERVER, severity, format, ap); - va_end(ap); +/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the + * controller(s) immediately. */ +int +control_event_server_error(const char *format, ...) +{ + CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR); + /* Force a flush, since we may be about to die horribly */ + queued_events_flush_all(1); return r; } @@ -4393,7 +5711,7 @@ control_event_guard(const char *nickname, const char *digest, } else { tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname); } - send_control_event(EVENT_GUARD, ALL_FORMATS, + send_control_event(EVENT_GUARD, "650 GUARD ENTRY %s %s\r\n", buf, status); } return 0; @@ -4424,7 +5742,7 @@ control_event_conf_changed(const smartlist_t *elements) } } result = smartlist_join_strings(lines, "\r\n", 0, NULL); - send_control_event(EVENT_CONF_CHANGED, 0, + send_control_event(EVENT_CONF_CHANGED, "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result); tor_free(result); SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); @@ -4434,8 +5752,8 @@ control_event_conf_changed(const smartlist_t *elements) /** Helper: Return a newly allocated string containing a path to the * file where we store our authentication cookie. */ -static char * -get_cookie_file(void) +char * +get_controller_cookie_file_name(void) { const or_options_t *options = get_options(); if (options->CookieAuthFile && strlen(options->CookieAuthFile)) { @@ -4445,44 +5763,28 @@ get_cookie_file(void) } } -/** Choose a random authentication cookie and write it to disk. - * Anybody who can read the cookie from disk will be considered - * authorized to use the control connection. Return -1 if we can't - * write the file, or 0 on success. */ +/* Initialize the cookie-based authentication system of the + * ControlPort. If <b>enabled</b> is 0, then disable the cookie + * authentication system. */ int -init_cookie_authentication(int enabled) +init_control_cookie_authentication(int enabled) { - char *fname; + char *fname = NULL; + int retval; + if (!enabled) { authentication_cookie_is_set = 0; return 0; } - /* We don't want to generate a new cookie every time we call - * options_act(). One should be enough. */ - if (authentication_cookie_is_set) - return 0; /* all set */ - - fname = get_cookie_file(); - crypto_rand(authentication_cookie, AUTHENTICATION_COOKIE_LEN); - authentication_cookie_is_set = 1; - if (write_bytes_to_file(fname, authentication_cookie, - AUTHENTICATION_COOKIE_LEN, 1)) { - log_warn(LD_FS,"Error writing authentication cookie to %s.", - escaped(fname)); - tor_free(fname); - return -1; - } -#ifndef _WIN32 - if (get_options()->CookieAuthFileGroupReadable) { - if (chmod(fname, 0640)) { - log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname)); - } - } -#endif - + fname = get_controller_cookie_file_name(); + retval = init_cookie_authentication(fname, "", /* no header */ + AUTHENTICATION_COOKIE_LEN, + get_options()->CookieAuthFileGroupReadable, + &authentication_cookie, + &authentication_cookie_is_set); tor_free(fname); - return 0; + return retval; } /** A copy of the process specifier of Tor's owning controller, or @@ -4599,23 +5901,43 @@ bootstrap_status_to_string(bootstrap_status_t s, const char **tag, break; case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: *tag = "requesting_descriptors"; - *summary = "Asking for relay descriptors"; + /* XXXX this appears to incorrectly report internal on most loads */ + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Asking for relay descriptors for internal paths" : + "Asking for relay descriptors"; break; + /* If we're sure there are no exits in the consensus, + * inform the controller by adding "internal" + * to the status summaries. + * (We only check this while loading descriptors, + * so we may not know in the earlier stages.) + * But if there are exits, we can't be sure whether + * we're creating internal or exit paths/circuits. + * XXXX Or should be use different tags or statuses + * for internal and exit/all? */ case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: *tag = "loading_descriptors"; - *summary = "Loading relay descriptors"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Loading relay descriptors for internal paths" : + "Loading relay descriptors"; break; case BOOTSTRAP_STATUS_CONN_OR: *tag = "conn_or"; - *summary = "Connecting to the Tor network"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Connecting to the Tor network internally" : + "Connecting to the Tor network"; break; case BOOTSTRAP_STATUS_HANDSHAKE_OR: *tag = "handshake_or"; - *summary = "Finishing handshake with first hop"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Finishing handshake with first hop of internal circuit" : + "Finishing handshake with first hop"; break; case BOOTSTRAP_STATUS_CIRCUIT_CREATE: *tag = "circuit_create"; - *summary = "Establishing a Tor circuit"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Establishing an internal Tor circuit" : + "Establishing a Tor circuit"; break; case BOOTSTRAP_STATUS_DONE: *tag = "done"; @@ -4636,36 +5958,51 @@ bootstrap_status_to_string(bootstrap_status_t s, const char **tag, * Tor initializes. */ static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; +/** As bootstrap_percent, but holds the bootstrapping level at which we last + * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, + * to avoid flooding the log with a new message every time we get a few more + * microdescriptors */ +static int notice_bootstrap_percent = 0; + /** How many problems have we had getting to the next bootstrapping phase? * These include failure to establish a connection to a Tor relay, * failures to finish the TLS handshake, failures to validate the * consensus document, etc. */ static int bootstrap_problems = 0; -/* We only tell the controller once we've hit a threshold of problems +/** We only tell the controller once we've hit a threshold of problems * for the current phase. */ #define BOOTSTRAP_PROBLEM_THRESHOLD 10 +/** When our bootstrapping progress level changes, but our bootstrapping + * status has not advanced, we only log at NOTICE when we have made at least + * this much progress. + */ +#define BOOTSTRAP_PCT_INCREMENT 5 + /** Called when Tor has made progress at bootstrapping its directory * information and initial circuits. * * <b>status</b> is the new status, that is, what task we will be doing * next. <b>progress</b> is zero if we just started this task, else it - * represents progress on the task. */ -void + * represents progress on the task. + * + * Return true if we logged a message at level NOTICE, and false otherwise. + */ +int control_event_bootstrap(bootstrap_status_t status, int progress) { const char *tag, *summary; char buf[BOOTSTRAP_MSG_LEN]; if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) - return; /* already bootstrapped; nothing to be done here. */ + return 0; /* already bootstrapped; nothing to be done here. */ /* special case for handshaking status, since our TLS handshaking code * can't distinguish what the connection is going to be for. */ if (status == BOOTSTRAP_STATUS_HANDSHAKE) { if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { - status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; + status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; } else { status = BOOTSTRAP_STATUS_HANDSHAKE_OR; } @@ -4673,9 +6010,19 @@ control_event_bootstrap(bootstrap_status_t status, int progress) if (status > bootstrap_percent || (progress && progress > bootstrap_percent)) { + int loglevel = LOG_NOTICE; bootstrap_status_to_string(status, &tag, &summary); - tor_log(status ? LOG_NOTICE : LOG_INFO, LD_CONTROL, - "Bootstrapped %d%%: %s.", progress ? progress : status, summary); + + if (status <= bootstrap_percent && + (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)) { + /* We log the message at info if the status hasn't advanced, and if less + * than BOOTSTRAP_PCT_INCREMENT progress has been made. + */ + loglevel = LOG_INFO; + } + + tor_log(loglevel, LD_CONTROL, + "Bootstrapped %d%%: %s", progress ? progress : status, summary); tor_snprintf(buf, sizeof(buf), "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", progress ? progress : status, tag, summary); @@ -4691,18 +6038,28 @@ control_event_bootstrap(bootstrap_status_t status, int progress) bootstrap_percent = progress; bootstrap_problems = 0; /* Progress! Reset our problem counter. */ } + if (loglevel == LOG_NOTICE && + bootstrap_percent > notice_bootstrap_percent) { + /* Remember that we gave a notice at this level. */ + notice_bootstrap_percent = bootstrap_percent; + } + return loglevel == LOG_NOTICE; } + + return 0; } /** Called when Tor has failed to make bootstrapping progress in a way * that indicates a problem. <b>warn</b> gives a hint as to why, and - * <b>reason</b> provides an "or_conn_end_reason" tag. + * <b>reason</b> provides an "or_conn_end_reason" tag. <b>or_conn</b> + * is the connection that caused this problem. */ -void -control_event_bootstrap_problem(const char *warn, int reason) +MOCK_IMPL(void, + control_event_bootstrap_problem, (const char *warn, int reason, + or_connection_t *or_conn)) { int status = bootstrap_percent; - const char *tag, *summary; + const char *tag = "", *summary = ""; char buf[BOOTSTRAP_MSG_LEN]; const char *recommendation = "ignore"; int severity; @@ -4710,6 +6067,11 @@ control_event_bootstrap_problem(const char *warn, int reason) /* bootstrap_percent must not be in "undefined" state here. */ tor_assert(status >= 0); + if (or_conn->have_noted_bootstrap_problem) + return; + + or_conn->have_noted_bootstrap_problem = 1; + if (bootstrap_percent == 100) return; /* already bootstrapped; nothing to be done here. */ @@ -4721,9 +6083,10 @@ control_event_bootstrap_problem(const char *warn, int reason) if (reason == END_OR_CONN_REASON_NO_ROUTE) recommendation = "warn"; - if (get_options()->UseBridges && - !any_bridge_descriptors_known() && - !any_pending_bridge_descriptor_fetches()) + /* If we are using bridges and all our OR connections are now + closed, it means that we totally failed to connect to our + bridges. Throw a warning. */ + if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) recommendation = "warn"; if (we_are_hibernating()) @@ -4737,19 +6100,26 @@ control_event_bootstrap_problem(const char *warn, int reason) log_fn(severity, LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; " - "count %d; recommendation %s)", + "count %d; recommendation %s; host %s at %s:%d)", status, summary, warn, orconn_end_reason_to_control_string(reason), - bootstrap_problems, recommendation); + bootstrap_problems, recommendation, + hex_str(or_conn->identity_digest, DIGEST_LEN), + or_conn->base_.address, + or_conn->base_.port); connection_or_report_broken_states(severity, LD_HANDSHAKE); tor_snprintf(buf, sizeof(buf), "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " - "COUNT=%d RECOMMENDATION=%s", + "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s:%d\"", bootstrap_percent, tag, summary, warn, orconn_end_reason_to_control_string(reason), bootstrap_problems, - recommendation); + recommendation, + hex_str(or_conn->identity_digest, DIGEST_LEN), + or_conn->base_.address, + (int)or_conn->base_.port); + tor_snprintf(last_sent_bootstrap_message, sizeof(last_sent_bootstrap_message), "WARN %s", buf); @@ -4762,7 +6132,425 @@ control_event_bootstrap_problem(const char *warn, int reason) void control_event_clients_seen(const char *controller_str) { - send_control_event(EVENT_CLIENTS_SEEN, 0, + send_control_event(EVENT_CLIENTS_SEEN, "650 CLIENTS_SEEN %s\r\n", controller_str); } +/** A new pluggable transport called <b>transport_name</b> was + * launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either + * "server" or "client" depending on the mode of the pluggable + * transport. + * "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port + */ +void +control_event_transport_launched(const char *mode, const char *transport_name, + tor_addr_t *addr, uint16_t port) +{ + send_control_event(EVENT_TRANSPORT_LAUNCHED, + "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n", + mode, transport_name, fmt_addr(addr), port); +} + +/** Convert rendezvous auth type to string for HS_DESC control events + */ +const char * +rend_auth_type_to_string(rend_auth_type_t auth_type) +{ + const char *str; + + switch (auth_type) { + case REND_NO_AUTH: + str = "NO_AUTH"; + break; + case REND_BASIC_AUTH: + str = "BASIC_AUTH"; + break; + case REND_STEALTH_AUTH: + str = "STEALTH_AUTH"; + break; + default: + str = "UNKNOWN"; + } + + return str; +} + +/** Return a longname the node whose identity is <b>id_digest</b>. If + * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is + * returned instead. + * + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. + */ +MOCK_IMPL(const char *, +node_describe_longname_by_id,(const char *id_digest)) +{ + static char longname[MAX_VERBOSE_NICKNAME_LEN+1]; + node_get_verbose_nickname_by_id(id_digest, longname); + return longname; +} + +/** Return either the onion address if the given pointer is a non empty + * string else the unknown string. */ +static const char * +rend_hsaddress_str_or_unknown(const char *onion_address) +{ + static const char *str_unknown = "UNKNOWN"; + const char *str_ret = str_unknown; + + /* No valid pointer, unknown it is. */ + if (!onion_address) { + goto end; + } + /* Empty onion address thus we don't know, unknown it is. */ + if (onion_address[0] == '\0') { + goto end; + } + /* All checks are good so return the given onion address. */ + str_ret = onion_address; + + end: + return str_ret; +} + +/** send HS_DESC requested event. + * + * <b>rend_query</b> is used to fetch requested onion address and auth type. + * <b>hs_dir</b> is the description of contacting hs directory. + * <b>desc_id_base32</b> is the ID of requested hs descriptor. + */ +void +control_event_hs_descriptor_requested(const rend_data_t *rend_query, + const char *id_digest, + const char *desc_id_base32) +{ + if (!id_digest || !rend_query || !desc_id_base32) { + log_warn(LD_BUG, "Called with rend_query==%p, " + "id_digest==%p, desc_id_base32==%p", + rend_query, id_digest, desc_id_base32); + return; + } + + send_control_event(EVENT_HS_DESC, + "650 HS_DESC REQUESTED %s %s %s %s\r\n", + rend_hsaddress_str_or_unknown(rend_query->onion_address), + rend_auth_type_to_string(rend_query->auth_type), + node_describe_longname_by_id(id_digest), + desc_id_base32); +} + +/** For an HS descriptor query <b>rend_data</b>, using the + * <b>onion_address</b> and HSDir fingerprint <b>hsdir_fp</b>, find out + * which descriptor ID in the query is the right one. + * + * Return a pointer of the binary descriptor ID found in the query's object + * or NULL if not found. */ +static const char * +get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp) +{ + int replica; + const char *desc_id = NULL; + + /* Possible if the fetch was done using a descriptor ID. This means that + * the HSFETCH command was used. */ + if (!tor_digest_is_zero(rend_data->desc_id_fetch)) { + desc_id = rend_data->desc_id_fetch; + goto end; + } + + /* OK, we have an onion address so now let's find which descriptor ID + * is the one associated with the HSDir fingerprint. */ + for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; + replica++) { + const char *digest = rend_data->descriptor_id[replica]; + + SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) { + if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) { + /* Found it! This descriptor ID is the right one. */ + desc_id = digest; + goto end; + } + } SMARTLIST_FOREACH_END(fingerprint); + } + + end: + return desc_id; +} + +/** send HS_DESC CREATED event when a local service generates a descriptor. + * + * <b>service_id</b> is the descriptor onion address. + * <b>desc_id_base32</b> is the descriptor ID. + * <b>replica</b> is the the descriptor replica number. + */ +void +control_event_hs_descriptor_created(const char *service_id, + const char *desc_id_base32, + int replica) +{ + if (!service_id || !desc_id_base32) { + log_warn(LD_BUG, "Called with service_digest==%p, " + "desc_id_base32==%p", service_id, desc_id_base32); + return; + } + + send_control_event(EVENT_HS_DESC, + "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s " + "REPLICA=%d\r\n", + service_id, + desc_id_base32, + replica); +} + +/** send HS_DESC upload event. + * + * <b>service_id</b> is the descriptor onion address. + * <b>hs_dir</b> is the description of contacting hs directory. + * <b>desc_id_base32</b> is the ID of requested hs descriptor. + */ +void +control_event_hs_descriptor_upload(const char *service_id, + const char *id_digest, + const char *desc_id_base32) +{ + if (!service_id || !id_digest || !desc_id_base32) { + log_warn(LD_BUG, "Called with service_digest==%p, " + "desc_id_base32==%p, id_digest==%p", service_id, + desc_id_base32, id_digest); + return; + } + + send_control_event(EVENT_HS_DESC, + "650 HS_DESC UPLOAD %s UNKNOWN %s %s\r\n", + service_id, + node_describe_longname_by_id(id_digest), + desc_id_base32); +} + +/** send HS_DESC event after got response from hs directory. + * + * NOTE: this is an internal function used by following functions: + * control_event_hs_descriptor_received + * control_event_hs_descriptor_failed + * + * So do not call this function directly. + */ +void +control_event_hs_descriptor_receive_end(const char *action, + const char *onion_address, + const rend_data_t *rend_data, + const char *id_digest, + const char *reason) +{ + char *desc_id_field = NULL; + char *reason_field = NULL; + char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + const char *desc_id = NULL; + + if (!action || !id_digest || !rend_data || !onion_address) { + log_warn(LD_BUG, "Called with action==%p, id_digest==%p, " + "rend_data==%p, onion_address==%p", action, id_digest, + rend_data, onion_address); + return; + } + + desc_id = get_desc_id_from_query(rend_data, id_digest); + if (desc_id != NULL) { + /* Set the descriptor ID digest to base32 so we can send it. */ + base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id, + DIGEST_LEN); + /* Extra whitespace is needed before the value. */ + tor_asprintf(&desc_id_field, " %s", desc_id_base32); + } + + if (reason) { + tor_asprintf(&reason_field, " REASON=%s", reason); + } + + send_control_event(EVENT_HS_DESC, + "650 HS_DESC %s %s %s %s%s%s\r\n", + action, + rend_hsaddress_str_or_unknown(onion_address), + rend_auth_type_to_string(rend_data->auth_type), + node_describe_longname_by_id(id_digest), + desc_id_field ? desc_id_field : "", + reason_field ? reason_field : ""); + + tor_free(desc_id_field); + tor_free(reason_field); +} + +/** send HS_DESC event after got response from hs directory. + * + * NOTE: this is an internal function used by following functions: + * control_event_hs_descriptor_uploaded + * control_event_hs_descriptor_upload_failed + * + * So do not call this function directly. + */ +void +control_event_hs_descriptor_upload_end(const char *action, + const char *onion_address, + const char *id_digest, + const char *reason) +{ + char *reason_field = NULL; + + if (!action || !id_digest) { + log_warn(LD_BUG, "Called with action==%p, id_digest==%p", action, + id_digest); + return; + } + + if (reason) { + tor_asprintf(&reason_field, " REASON=%s", reason); + } + + send_control_event(EVENT_HS_DESC, + "650 HS_DESC %s %s UNKNOWN %s%s\r\n", + action, + rend_hsaddress_str_or_unknown(onion_address), + node_describe_longname_by_id(id_digest), + reason_field ? reason_field : ""); + + tor_free(reason_field); +} + +/** send HS_DESC RECEIVED event + * + * called when we successfully received a hidden service descriptor. + */ +void +control_event_hs_descriptor_received(const char *onion_address, + const rend_data_t *rend_data, + const char *id_digest) +{ + if (!rend_data || !id_digest || !onion_address) { + log_warn(LD_BUG, "Called with rend_data==%p, id_digest==%p, " + "onion_address==%p", rend_data, id_digest, onion_address); + return; + } + control_event_hs_descriptor_receive_end("RECEIVED", onion_address, + rend_data, id_digest, NULL); +} + +/** send HS_DESC UPLOADED event + * + * called when we successfully uploaded a hidden service descriptor. + */ +void +control_event_hs_descriptor_uploaded(const char *id_digest, + const char *onion_address) +{ + if (!id_digest) { + log_warn(LD_BUG, "Called with id_digest==%p", + id_digest); + return; + } + + control_event_hs_descriptor_upload_end("UPLOADED", onion_address, + id_digest, NULL); +} + +/** Send HS_DESC event to inform controller that query <b>rend_query</b> + * failed to retrieve hidden service descriptor identified by + * <b>id_digest</b>. If <b>reason</b> is not NULL, add it to REASON= + * field. + */ +void +control_event_hs_descriptor_failed(const rend_data_t *rend_data, + const char *id_digest, + const char *reason) +{ + if (!rend_data || !id_digest) { + log_warn(LD_BUG, "Called with rend_data==%p, id_digest==%p", + rend_data, id_digest); + return; + } + control_event_hs_descriptor_receive_end("FAILED", + rend_data->onion_address, + rend_data, id_digest, reason); +} + +/** send HS_DESC_CONTENT event after completion of a successful fetch from + * hs directory. */ +void +control_event_hs_descriptor_content(const char *onion_address, + const char *desc_id, + const char *hsdir_id_digest, + const char *content) +{ + static const char *event_name = "HS_DESC_CONTENT"; + char *esc_content = NULL; + + if (!onion_address || !desc_id || !hsdir_id_digest) { + log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, " + "hsdir_id_digest==%p", onion_address, desc_id, hsdir_id_digest); + return; + } + + if (content == NULL) { + /* Point it to empty content so it can still be escaped. */ + content = ""; + } + write_escaped_data(content, strlen(content), &esc_content); + + send_control_event(EVENT_HS_DESC_CONTENT, + "650+%s %s %s %s\r\n%s650 OK\r\n", + event_name, + rend_hsaddress_str_or_unknown(onion_address), + desc_id, + node_describe_longname_by_id(hsdir_id_digest), + esc_content); + tor_free(esc_content); +} + +/** Send HS_DESC event to inform controller upload of hidden service + * descriptor identified by <b>id_digest</b> failed. If <b>reason</b> + * is not NULL, add it to REASON= field. + */ +void +control_event_hs_descriptor_upload_failed(const char *id_digest, + const char *onion_address, + const char *reason) +{ + if (!id_digest) { + log_warn(LD_BUG, "Called with id_digest==%p", + id_digest); + return; + } + control_event_hs_descriptor_upload_end("UPLOAD_FAILED", onion_address, + id_digest, reason); +} + +/** Free any leftover allocated memory of the control.c subsystem. */ +void +control_free_all(void) +{ + if (authentication_cookie) /* Free the auth cookie */ + tor_free(authentication_cookie); + if (detached_onion_services) { /* Free the detached onion services */ + SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp)); + smartlist_free(detached_onion_services); + } + if (queued_control_events) { + SMARTLIST_FOREACH(queued_control_events, queued_event_t *, ev, + queued_event_free(ev)); + smartlist_free(queued_control_events); + queued_control_events = NULL; + } + if (flush_queued_events_event) { + tor_event_free(flush_queued_events_event); + flush_queued_events_event = NULL; + } +} + +#ifdef TOR_UNIT_TESTS +/* For testing: change the value of global_event_mask */ +void +control_testing_set_global_event_mask(uint64_t mask) +{ + global_event_mask = mask; +} +#endif + diff --git a/src/or/control.h b/src/or/control.h index 61062da2c4..008bfb1c3b 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,6 +12,8 @@ #ifndef TOR_CONTROL_H #define TOR_CONTROL_H +void control_initialize_event_queue(void); + void control_update_global_event_mask(void); void control_adjust_event_log_severity(void); @@ -50,6 +52,13 @@ int control_event_or_conn_status(or_connection_t *conn, int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written); int control_event_stream_bandwidth(edge_connection_t *edge_conn); int control_event_stream_bandwidth_used(void); +int control_event_circ_bandwidth_used(void); +int control_event_conn_bandwidth(connection_t *conn); +int control_event_conn_bandwidth_used(void); +int control_event_circuit_cell_stats(void); +int control_event_tb_empty(const char *bucket, uint32_t read_empty_time, + uint32_t write_empty_time, + int milliseconds_elapsed); void control_event_logmsg(int severity, uint32_t domain, const char *msg); int control_event_descriptors_changed(smartlist_t *routers); int control_event_address_mapped(const char *from, const char *to, @@ -60,6 +69,7 @@ int control_event_or_authdir_new_descriptor(const char *action, size_t desclen, const char *msg); int control_event_my_descriptor_changed(void); +int control_event_network_liveness_update(int liveness); int control_event_networkstatus_changed(smartlist_t *statuses); int control_event_newconsensus(const networkstatus_t *consensus); @@ -70,29 +80,185 @@ int control_event_client_status(int severity, const char *format, ...) CHECK_PRINTF(2,3); int control_event_server_status(int severity, const char *format, ...) CHECK_PRINTF(2,3); + +int control_event_general_error(const char *format, ...) + CHECK_PRINTF(1,2); +int control_event_client_error(const char *format, ...) + CHECK_PRINTF(1,2); +int control_event_server_error(const char *format, ...) + CHECK_PRINTF(1,2); + int control_event_guard(const char *nickname, const char *digest, const char *status); int control_event_conf_changed(const smartlist_t *elements); -int control_event_buildtimeout_set(const circuit_build_times_t *cbt, - buildtimeout_set_event_t type); +int control_event_buildtimeout_set(buildtimeout_set_event_t type, + const char *args); int control_event_signal(uintptr_t signal); -int init_cookie_authentication(int enabled); +int init_control_cookie_authentication(int enabled); +char *get_controller_cookie_file_name(void); smartlist_t *decode_hashed_passwords(config_line_t *passwords); void disable_control_logging(void); void enable_control_logging(void); void monitor_owning_controller_process(const char *process_spec); -void control_event_bootstrap(bootstrap_status_t status, int progress); -void control_event_bootstrap_problem(const char *warn, int reason); +int control_event_bootstrap(bootstrap_status_t status, int progress); +MOCK_DECL(void, control_event_bootstrap_problem,(const char *warn, + int reason, + or_connection_t *or_conn)); void control_event_clients_seen(const char *controller_str); +void control_event_transport_launched(const char *mode, + const char *transport_name, + tor_addr_t *addr, uint16_t port); +const char *rend_auth_type_to_string(rend_auth_type_t auth_type); +MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); +void control_event_hs_descriptor_requested(const rend_data_t *rend_query, + const char *desc_id_base32, + const char *hs_dir); +void control_event_hs_descriptor_created(const char *service_id, + const char *desc_id_base32, + int replica); +void control_event_hs_descriptor_upload(const char *service_id, + const char *desc_id_base32, + const char *hs_dir); +void control_event_hs_descriptor_receive_end(const char *action, + const char *onion_address, + const rend_data_t *rend_data, + const char *id_digest, + const char *reason); +void control_event_hs_descriptor_upload_end(const char *action, + const char *onion_address, + const char *hs_dir, + const char *reason); +void control_event_hs_descriptor_received(const char *onion_address, + const rend_data_t *rend_data, + const char *id_digest); +void control_event_hs_descriptor_uploaded(const char *hs_dir, + const char *onion_address); +void control_event_hs_descriptor_failed(const rend_data_t *rend_data, + const char *id_digest, + const char *reason); +void control_event_hs_descriptor_upload_failed(const char *hs_dir, + const char *onion_address, + const char *reason); +void control_event_hs_descriptor_content(const char *onion_address, + const char *desc_id, + const char *hsdir_fp, + const char *content); + +void control_free_all(void); #ifdef CONTROL_PRIVATE +/* Recognized asynchronous event types. It's okay to expand this list + * because it is used both as a list of v0 event types, and as indices + * into the bitfield to determine which controllers want which events. + */ +/* This bitfield has no event zero 0x0000 */ +#define EVENT_MIN_ 0x0001 +#define EVENT_CIRCUIT_STATUS 0x0001 +#define EVENT_STREAM_STATUS 0x0002 +#define EVENT_OR_CONN_STATUS 0x0003 +#define EVENT_BANDWIDTH_USED 0x0004 +#define EVENT_CIRCUIT_STATUS_MINOR 0x0005 +#define EVENT_NEW_DESC 0x0006 +#define EVENT_DEBUG_MSG 0x0007 +#define EVENT_INFO_MSG 0x0008 +#define EVENT_NOTICE_MSG 0x0009 +#define EVENT_WARN_MSG 0x000A +#define EVENT_ERR_MSG 0x000B +#define EVENT_ADDRMAP 0x000C +/* Exposed above */ +// #define EVENT_AUTHDIR_NEWDESCS 0x000D +#define EVENT_DESCCHANGED 0x000E +/* Exposed above */ +// #define EVENT_NS 0x000F +#define EVENT_STATUS_CLIENT 0x0010 +#define EVENT_STATUS_SERVER 0x0011 +#define EVENT_STATUS_GENERAL 0x0012 +#define EVENT_GUARD 0x0013 +#define EVENT_STREAM_BANDWIDTH_USED 0x0014 +#define EVENT_CLIENTS_SEEN 0x0015 +#define EVENT_NEWCONSENSUS 0x0016 +#define EVENT_BUILDTIMEOUT_SET 0x0017 +#define EVENT_GOT_SIGNAL 0x0018 +#define EVENT_CONF_CHANGED 0x0019 +#define EVENT_CONN_BW 0x001A +#define EVENT_CELL_STATS 0x001B +#define EVENT_TB_EMPTY 0x001C +#define EVENT_CIRC_BANDWIDTH_USED 0x001D +#define EVENT_TRANSPORT_LAUNCHED 0x0020 +#define EVENT_HS_DESC 0x0021 +#define EVENT_HS_DESC_CONTENT 0x0022 +#define EVENT_NETWORK_LIVENESS 0x0023 +#define EVENT_MAX_ 0x0023 + +/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ +#define EVENT_CAPACITY_ 0x0040 + +/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a + * different structure, as it can only handle a maximum left shift of 1<<63. */ + +#if EVENT_MAX_ >= EVENT_CAPACITY_ +#error control_connection_t.event_mask has an event greater than its capacity +#endif + +#define EVENT_MASK_(e) (((uint64_t)1)<<(e)) + +#define EVENT_MASK_NONE_ ((uint64_t)0x0) + +#define EVENT_MASK_ABOVE_MIN_ ((~((uint64_t)0x0)) << EVENT_MIN_) +#define EVENT_MASK_BELOW_MAX_ ((~((uint64_t)0x0)) \ + >> (EVENT_CAPACITY_ - EVENT_MAX_ \ + - EVENT_MIN_)) + +#define EVENT_MASK_ALL_ (EVENT_MASK_ABOVE_MIN_ \ + & EVENT_MASK_BELOW_MAX_) + /* Used only by control.c and test.c */ -size_t write_escaped_data(const char *data, size_t len, char **out); -size_t read_escaped_data(const char *data, size_t len, char **out); +STATIC size_t write_escaped_data(const char *data, size_t len, char **out); +STATIC size_t read_escaped_data(const char *data, size_t len, char **out); + +#ifdef TOR_UNIT_TESTS +MOCK_DECL(STATIC void, + send_control_event_string,(uint16_t event, const char *msg)); + +MOCK_DECL(STATIC void, + queue_control_event_string,(uint16_t event, char *msg)); + +void control_testing_set_global_event_mask(uint64_t mask); +#endif + +/** Helper structure: temporarily stores cell statistics for a circuit. */ +typedef struct cell_stats_t { + /** Number of cells added in app-ward direction by command. */ + uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells added in exit-ward direction by command. */ + uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells removed in app-ward direction by command. */ + uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1]; + /** Number of cells removed in exit-ward direction by command. */ + uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1]; + /** Total waiting time of cells in app-ward direction by command. */ + uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1]; + /** Total waiting time of cells in exit-ward direction by command. */ + uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1]; +} cell_stats_t; +void sum_up_cell_stats_by_command(circuit_t *circ, + cell_stats_t *cell_stats); +void append_cell_stats_by_command(smartlist_t *event_parts, + const char *key, + const uint64_t *include_if_non_zero, + const uint64_t *number_to_include); +void format_cell_stats(char **event_string, circuit_t *circ, + cell_stats_t *cell_stats); +STATIC char *get_bw_samples(void); + +STATIC crypto_pk_t *add_onion_helper_keyarg(const char *arg, int discard_pk, + const char **key_new_alg_out, + char **key_new_blob_out, + char **err_msg_out); #endif #endif diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index ecf0d2035d..3109d5a177 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -1,88 +1,103 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file cpuworker.c - * \brief Implements a farm of 'CPU worker' processes to perform - * CPU-intensive tasks in another thread or process, to not - * interrupt the main thread. + * \brief Uses the workqueue/threadpool code to farm CPU-intensive activities + * out to subprocesses. * * Right now, we only use this for processing onionskins. **/ #include "or.h" -#include "buffers.h" #include "channel.h" -#include "channeltls.h" #include "circuitbuild.h" #include "circuitlist.h" -#include "config.h" -#include "connection.h" #include "connection_or.h" +#include "config.h" #include "cpuworker.h" #include "main.h" #include "onion.h" #include "rephist.h" #include "router.h" +#include "workqueue.h" -/** The maximum number of cpuworker processes we will keep around. */ -#define MAX_CPUWORKERS 16 -/** The minimum number of cpuworker processes we will keep around. */ -#define MIN_CPUWORKERS 1 - -/** The tag specifies which circuit this onionskin was from. */ -#define TAG_LEN 12 - -/** How many cpuworkers we have running right now. */ -static int num_cpuworkers=0; -/** How many of the running cpuworkers have an assigned task right now. */ -static int num_cpuworkers_busy=0; -/** We need to spawn new cpuworkers whenever we rotate the onion keys - * on platforms where execution contexts==processes. This variable stores - * the last time we got a key rotation event. */ -static time_t last_rotation_time=0; - -static void cpuworker_main(void *data) ATTR_NORETURN; -static int spawn_cpuworker(void); -static void spawn_enough_cpuworkers(void); -static void process_pending_task(connection_t *cpuworker); - -/** Initialize the cpuworker subsystem. - */ -void -cpu_init(void) +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +static void queue_pending_tasks(void); + +typedef struct worker_state_s { + int generation; + server_onion_keys_t *onion_keys; +} worker_state_t; + +static void * +worker_state_new(void *arg) { - cpuworkers_rotate(); + worker_state_t *ws; + (void)arg; + ws = tor_malloc_zero(sizeof(worker_state_t)); + ws->onion_keys = server_onion_keys_new(); + return ws; } - -/** Called when we're done sending a request to a cpuworker. */ -int -connection_cpu_finished_flushing(connection_t *conn) +static void +worker_state_free(void *arg) { - tor_assert(conn); - tor_assert(conn->type == CONN_TYPE_CPUWORKER); - return 0; + worker_state_t *ws = arg; + server_onion_keys_free(ws->onion_keys); + tor_free(ws); } -/** Pack global_id and circ_id; set *tag to the result. (See note on - * cpuworker_main for wire format.) */ +static replyqueue_t *replyqueue = NULL; +static threadpool_t *threadpool = NULL; +static struct event *reply_event = NULL; + +static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT; + +static int total_pending_tasks = 0; +static int max_pending_tasks = 128; + static void -tag_pack(uint8_t *tag, uint64_t chan_id, circid_t circ_id) +replyqueue_process_cb(evutil_socket_t sock, short events, void *arg) { - /*XXXX RETHINK THIS WHOLE MESS !!!! !NM NM NM NM*/ - /*XXXX DOUBLEPLUSTHIS!!!! AS AS AS AS*/ - set_uint64(tag, chan_id); - set_uint32(tag+8, circ_id); + replyqueue_t *rq = arg; + (void) sock; + (void) events; + replyqueue_process(rq); } -/** Unpack <b>tag</b> into addr, port, and circ_id. +/** Initialize the cpuworker subsystem. It is OK to call this more than once + * during Tor's lifetime. */ -static void -tag_unpack(const uint8_t *tag, uint64_t *chan_id, circid_t *circ_id) +void +cpu_init(void) { - *chan_id = get_uint64(tag); - *circ_id = get_uint32(tag+8); + if (!replyqueue) { + replyqueue = replyqueue_new(0); + } + if (!reply_event) { + reply_event = tor_event_new(tor_libevent_get_base(), + replyqueue_get_socket(replyqueue), + EV_READ|EV_PERSIST, + replyqueue_process_cb, + replyqueue); + event_add(reply_event, NULL); + } + if (!threadpool) { + threadpool = threadpool_new(get_num_cpus(get_options()), + replyqueue, + worker_state_new, + worker_state_free, + NULL); + } + /* Total voodoo. Can we make this more sensible? */ + max_pending_tasks = get_num_cpus(get_options()) * 64; + crypto_seed_weak_rng(&request_sample_rng); } /** Magic numbers to make sure our cpuworker_requests don't grow any @@ -94,10 +109,6 @@ tag_unpack(const uint8_t *tag, uint64_t *chan_id, circid_t *circ_id) typedef struct cpuworker_request_t { /** Magic number; must be CPUWORKER_REQUEST_MAGIC. */ uint32_t magic; - /** Opaque tag to identify the job */ - uint8_t tag[TAG_LEN]; - /** Task code. Must be one of CPUWORKER_TASK_* */ - uint8_t task; /** Flag: Are we timing this request? */ unsigned timed : 1; @@ -114,8 +125,7 @@ typedef struct cpuworker_request_t { typedef struct cpuworker_reply_t { /** Magic number; must be CPUWORKER_REPLY_MAGIC. */ uint32_t magic; - /** Opaque tag to identify the job; matches the request's tag.*/ - uint8_t tag[TAG_LEN]; + /** True iff we got a successful request. */ uint8_t success; @@ -142,42 +152,45 @@ typedef struct cpuworker_reply_t { uint8_t rend_auth_material[DIGEST_LEN]; } cpuworker_reply_t; -/** Called when the onion key has changed and we need to spawn new - * cpuworkers. Close all currently idle cpuworkers, and mark the last - * rotation time as now. - */ -void -cpuworkers_rotate(void) +typedef struct cpuworker_job_u { + or_circuit_t *circ; + union { + cpuworker_request_t request; + cpuworker_reply_t reply; + } u; +} cpuworker_job_t; + +static workqueue_reply_t +update_state_threadfn(void *state_, void *work_) { - connection_t *cpuworker; - while ((cpuworker = connection_get_by_type_state(CONN_TYPE_CPUWORKER, - CPUWORKER_STATE_IDLE))) { - connection_mark_for_close(cpuworker); - --num_cpuworkers; - } - last_rotation_time = time(NULL); - if (server_mode(get_options())) - spawn_enough_cpuworkers(); + worker_state_t *state = state_; + worker_state_t *update = work_; + server_onion_keys_free(state->onion_keys); + state->onion_keys = update->onion_keys; + update->onion_keys = NULL; + ++state->generation; + return WQ_RPL_REPLY; } -/** If the cpuworker closes the connection, - * mark it as closed and spawn a new one as needed. */ -int -connection_cpu_reached_eof(connection_t *conn) +/** Called when the onion key has changed so update all CPU worker(s) with + * new function pointers with which a new state will be generated. + */ +void +cpuworkers_rotate_keyinfo(void) { - log_warn(LD_GENERAL,"Read eof. CPU worker died unexpectedly."); - if (conn->state != CPUWORKER_STATE_IDLE) { - /* the circ associated with this cpuworker will have to wait until - * it gets culled in run_connection_housekeeping(), since we have - * no way to find out which circ it was. */ - log_warn(LD_GENERAL,"...and it left a circuit queued; abandoning circ."); - num_cpuworkers_busy--; + if (!threadpool) { + /* If we're a client, then we won't have cpuworkers, and we won't need + * to tell them to rotate their state. + */ + return; + } + if (threadpool_queue_update(threadpool, + worker_state_new, + update_state_threadfn, + worker_state_free, + NULL)) { + log_warn(LD_OR, "Failed to queue key update for worker threads."); } - num_cpuworkers--; - spawn_enough_cpuworkers(); /* try to regrow. hope we don't end up - spinning. */ - connection_mark_for_close(conn); - return 0; } /** Indexed by handshake type: how many onionskins have we processed and @@ -197,8 +210,6 @@ static uint64_t onionskins_usec_roundtrip[MAX_ONION_HANDSHAKE_TYPE+1]; * time. (microseconds) */ #define MAX_BELIEVABLE_ONIONSKIN_DELAY (2*1000*1000) -static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT; - /** Return true iff we'd like to measure a handshake of type * <b>onionskin_type</b>. Call only from the main thread. */ static int @@ -286,433 +297,275 @@ cpuworker_log_onionskin_overhead(int severity, int onionskin_type, onionskin_type_name, (unsigned)overhead, relative_overhead*100); } -/** Called when we get data from a cpuworker. If the answer is not complete, - * wait for a complete answer. If the answer is complete, - * process it as appropriate. - */ -int -connection_cpu_process_inbuf(connection_t *conn) -{ - uint64_t chan_id; - circid_t circ_id; - channel_t *p_chan = NULL; - circuit_t *circ; - - tor_assert(conn); - tor_assert(conn->type == CONN_TYPE_CPUWORKER); - - if (!connection_get_inbuf_len(conn)) - return 0; - - if (conn->state == CPUWORKER_STATE_BUSY_ONION) { - cpuworker_reply_t rpl; - if (connection_get_inbuf_len(conn) < sizeof(cpuworker_reply_t)) - return 0; /* not yet */ - tor_assert(connection_get_inbuf_len(conn) == sizeof(cpuworker_reply_t)); - - connection_fetch_from_buf((void*)&rpl,sizeof(cpuworker_reply_t),conn); - - tor_assert(rpl.magic == CPUWORKER_REPLY_MAGIC); - - if (rpl.timed && rpl.success && - rpl.handshake_type <= MAX_ONION_HANDSHAKE_TYPE) { - /* Time how long this request took. The handshake_type check should be - needless, but let's leave it in to be safe. */ - struct timeval tv_end, tv_diff; - int64_t usec_roundtrip; - tor_gettimeofday(&tv_end); - timersub(&tv_end, &rpl.started_at, &tv_diff); - usec_roundtrip = ((int64_t)tv_diff.tv_sec)*1000000 + tv_diff.tv_usec; - if (usec_roundtrip >= 0 && - usec_roundtrip < MAX_BELIEVABLE_ONIONSKIN_DELAY) { - ++onionskins_n_processed[rpl.handshake_type]; - onionskins_usec_internal[rpl.handshake_type] += rpl.n_usec; - onionskins_usec_roundtrip[rpl.handshake_type] += usec_roundtrip; - if (onionskins_n_processed[rpl.handshake_type] >= 500000) { - /* Scale down every 500000 handshakes. On a busy server, that's - * less impressive than it sounds. */ - onionskins_n_processed[rpl.handshake_type] /= 2; - onionskins_usec_internal[rpl.handshake_type] /= 2; - onionskins_usec_roundtrip[rpl.handshake_type] /= 2; - } - } - } - /* parse out the circ it was talking about */ - tag_unpack(rpl.tag, &chan_id, &circ_id); - circ = NULL; - log_debug(LD_OR, - "Unpacking cpuworker reply, chan_id is " U64_FORMAT - ", circ_id is %u", - U64_PRINTF_ARG(chan_id), (unsigned)circ_id); - p_chan = channel_find_by_global_id(chan_id); - - if (p_chan) - circ = circuit_get_by_circid_channel(circ_id, p_chan); - - if (rpl.success == 0) { - log_debug(LD_OR, - "decoding onionskin failed. " - "(Old key or bad software.) Closing."); - if (circ) - circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); - goto done_processing; - } - if (!circ) { - /* This happens because somebody sends us a destroy cell and the - * circuit goes away, while the cpuworker is working. This is also - * why our tag doesn't include a pointer to the circ, because we'd - * never know if it's still valid. - */ - log_debug(LD_OR,"processed onion for a circ that's gone. Dropping."); - goto done_processing; - } - tor_assert(! CIRCUIT_IS_ORIGIN(circ)); - if (onionskin_answer(TO_OR_CIRCUIT(circ), - &rpl.created_cell, - (const char*)rpl.keys, - rpl.rend_auth_material) < 0) { - log_warn(LD_OR,"onionskin_answer failed. Closing."); - circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL); - goto done_processing; - } - log_debug(LD_OR,"onionskin_answer succeeded. Yay."); - } else { - tor_assert(0); /* don't ask me to do handshakes yet */ - } - - done_processing: - conn->state = CPUWORKER_STATE_IDLE; - num_cpuworkers_busy--; - if (conn->timestamp_created < last_rotation_time) { - connection_mark_for_close(conn); - num_cpuworkers--; - spawn_enough_cpuworkers(); - } else { - process_pending_task(conn); - } - return 0; -} - -/** Implement a cpuworker. 'data' is an fdarray as returned by socketpair. - * Read and writes from fdarray[1]. Reads requests, writes answers. - * - * Request format: - * cpuworker_request_t. - * Response format: - * cpuworker_reply_t - */ +/** Handle a reply from the worker threads. */ static void -cpuworker_main(void *data) +cpuworker_onion_handshake_replyfn(void *work_) { - /* For talking to the parent thread/process */ - tor_socket_t *fdarray = data; - tor_socket_t fd; - - /* variables for onion processing */ - server_onion_keys_t onion_keys; - cpuworker_request_t req; + cpuworker_job_t *job = work_; cpuworker_reply_t rpl; - - fd = fdarray[1]; /* this side is ours */ -#ifndef TOR_IS_MULTITHREADED - tor_close_socket(fdarray[0]); /* this is the side of the socketpair the - * parent uses */ - tor_free_all(1); /* so the child doesn't hold the parent's fd's open */ - handle_signals(0); /* ignore interrupts from the keyboard, etc */ -#endif - tor_free(data); - - setup_server_onion_keys(&onion_keys); - - for (;;) { - if (read_all(fd, (void *)&req, sizeof(req), 1) != sizeof(req)) { - log_info(LD_OR, "read request failed. Exiting."); - goto end; - } - tor_assert(req.magic == CPUWORKER_REQUEST_MAGIC); - - memset(&rpl, 0, sizeof(rpl)); - - if (req.task == CPUWORKER_TASK_ONION) { - const create_cell_t *cc = &req.create_cell; - created_cell_t *cell_out = &rpl.created_cell; - struct timeval tv_start, tv_end; - int n; - rpl.timed = req.timed; - rpl.started_at = req.started_at; - rpl.handshake_type = cc->handshake_type; - if (req.timed) - tor_gettimeofday(&tv_start); - n = onion_skin_server_handshake(cc->handshake_type, - cc->onionskin, cc->handshake_len, - &onion_keys, - cell_out->reply, - rpl.keys, CPATH_KEY_MATERIAL_LEN, - rpl.rend_auth_material); - if (n < 0) { - /* failure */ - log_debug(LD_OR,"onion_skin_server_handshake failed."); - memset(&rpl, 0, sizeof(rpl)); - memcpy(rpl.tag, req.tag, TAG_LEN); - rpl.success = 0; - } else { - /* success */ - log_debug(LD_OR,"onion_skin_server_handshake succeeded."); - memcpy(rpl.tag, req.tag, TAG_LEN); - cell_out->handshake_len = n; - switch (cc->cell_type) { - case CELL_CREATE: - cell_out->cell_type = CELL_CREATED; break; - case CELL_CREATE2: - cell_out->cell_type = CELL_CREATED2; break; - case CELL_CREATE_FAST: - cell_out->cell_type = CELL_CREATED_FAST; break; - default: - tor_assert(0); - goto end; - } - rpl.success = 1; - } - rpl.magic = CPUWORKER_REPLY_MAGIC; - if (req.timed) { - struct timeval tv_diff; - int64_t usec; - tor_gettimeofday(&tv_end); - timersub(&tv_end, &tv_start, &tv_diff); - usec = ((int64_t)tv_diff.tv_sec)*1000000 + tv_diff.tv_usec; - if (usec < 0 || usec > MAX_BELIEVABLE_ONIONSKIN_DELAY) - rpl.n_usec = MAX_BELIEVABLE_ONIONSKIN_DELAY; - else - rpl.n_usec = (uint32_t) usec; + or_circuit_t *circ = NULL; + + tor_assert(total_pending_tasks > 0); + --total_pending_tasks; + + /* Could avoid this, but doesn't matter. */ + memcpy(&rpl, &job->u.reply, sizeof(rpl)); + + tor_assert(rpl.magic == CPUWORKER_REPLY_MAGIC); + + if (rpl.timed && rpl.success && + rpl.handshake_type <= MAX_ONION_HANDSHAKE_TYPE) { + /* Time how long this request took. The handshake_type check should be + needless, but let's leave it in to be safe. */ + struct timeval tv_end, tv_diff; + int64_t usec_roundtrip; + tor_gettimeofday(&tv_end); + timersub(&tv_end, &rpl.started_at, &tv_diff); + usec_roundtrip = ((int64_t)tv_diff.tv_sec)*1000000 + tv_diff.tv_usec; + if (usec_roundtrip >= 0 && + usec_roundtrip < MAX_BELIEVABLE_ONIONSKIN_DELAY) { + ++onionskins_n_processed[rpl.handshake_type]; + onionskins_usec_internal[rpl.handshake_type] += rpl.n_usec; + onionskins_usec_roundtrip[rpl.handshake_type] += usec_roundtrip; + if (onionskins_n_processed[rpl.handshake_type] >= 500000) { + /* Scale down every 500000 handshakes. On a busy server, that's + * less impressive than it sounds. */ + onionskins_n_processed[rpl.handshake_type] /= 2; + onionskins_usec_internal[rpl.handshake_type] /= 2; + onionskins_usec_roundtrip[rpl.handshake_type] /= 2; } - if (write_all(fd, (void*)&rpl, sizeof(rpl), 1) != sizeof(rpl)) { - log_err(LD_BUG,"writing response buf failed. Exiting."); - goto end; - } - log_debug(LD_OR,"finished writing response."); - } else if (req.task == CPUWORKER_TASK_SHUTDOWN) { - log_info(LD_OR,"Clean shutdown: exiting"); - goto end; } - memwipe(&req, 0, sizeof(req)); - memwipe(&rpl, 0, sizeof(req)); } - end: - memwipe(&req, 0, sizeof(req)); - memwipe(&rpl, 0, sizeof(req)); - release_server_onion_keys(&onion_keys); - tor_close_socket(fd); - crypto_thread_cleanup(); - spawn_exit(); -} -/** Launch a new cpuworker. Return 0 if we're happy, -1 if we failed. - */ -static int -spawn_cpuworker(void) -{ - tor_socket_t *fdarray; - tor_socket_t fd; - connection_t *conn; - int err; - - fdarray = tor_malloc(sizeof(tor_socket_t)*2); - if ((err = tor_socketpair(AF_UNIX, SOCK_STREAM, 0, fdarray)) < 0) { - log_warn(LD_NET, "Couldn't construct socketpair for cpuworker: %s", - tor_socket_strerror(-err)); - tor_free(fdarray); - return -1; - } - - tor_assert(SOCKET_OK(fdarray[0])); - tor_assert(SOCKET_OK(fdarray[1])); + circ = job->circ; - fd = fdarray[0]; - spawn_func(cpuworker_main, (void*)fdarray); - log_debug(LD_OR,"just spawned a cpu worker."); -#ifndef TOR_IS_MULTITHREADED - tor_close_socket(fdarray[1]); /* don't need the worker's side of the pipe */ - tor_free(fdarray); -#endif + log_debug(LD_OR, + "Unpacking cpuworker reply %p, circ=%p, success=%d", + job, circ, rpl.success); - conn = connection_new(CONN_TYPE_CPUWORKER, AF_UNIX); + if (circ->base_.magic == DEAD_CIRCUIT_MAGIC) { + /* The circuit was supposed to get freed while the reply was + * pending. Instead, it got left for us to free so that we wouldn't freak + * out when the job->circ field wound up pointing to nothing. */ + log_debug(LD_OR, "Circuit died while reply was pending. Freeing memory."); + circ->base_.magic = 0; + tor_free(circ); + goto done_processing; + } - /* set up conn so it's got all the data we need to remember */ - conn->s = fd; - conn->address = tor_strdup("localhost"); - tor_addr_make_unspec(&conn->addr); + circ->workqueue_entry = NULL; - if (set_socket_nonblocking(fd) == -1) { - connection_free(conn); /* this closes fd */ - return -1; + if (TO_CIRCUIT(circ)->marked_for_close) { + /* We already marked this circuit; we can't call it open. */ + log_debug(LD_OR,"circuit is already marked."); + goto done_processing; } - if (connection_add(conn) < 0) { /* no space, forget it */ - log_warn(LD_NET,"connection_add for cpuworker failed. Giving up."); - connection_free(conn); /* this closes fd */ - return -1; + if (rpl.success == 0) { + log_debug(LD_OR, + "decoding onionskin failed. " + "(Old key or bad software.) Closing."); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL); + goto done_processing; } - conn->state = CPUWORKER_STATE_IDLE; - connection_start_reading(conn); + if (onionskin_answer(circ, + &rpl.created_cell, + (const char*)rpl.keys, + rpl.rend_auth_material) < 0) { + log_warn(LD_OR,"onionskin_answer failed. Closing."); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); + goto done_processing; + } + log_debug(LD_OR,"onionskin_answer succeeded. Yay."); - return 0; /* success */ + done_processing: + memwipe(&rpl, 0, sizeof(rpl)); + memwipe(job, 0, sizeof(*job)); + tor_free(job); + queue_pending_tasks(); } -/** If we have too few or too many active cpuworkers, try to spawn new ones - * or kill idle ones. - */ -static void -spawn_enough_cpuworkers(void) +/** Implementation function for onion handshake requests. */ +static workqueue_reply_t +cpuworker_onion_handshake_threadfn(void *state_, void *work_) { - int num_cpuworkers_needed = get_num_cpus(get_options()); - int reseed = 0; + worker_state_t *state = state_; + cpuworker_job_t *job = work_; - if (num_cpuworkers_needed < MIN_CPUWORKERS) - num_cpuworkers_needed = MIN_CPUWORKERS; - if (num_cpuworkers_needed > MAX_CPUWORKERS) - num_cpuworkers_needed = MAX_CPUWORKERS; + /* variables for onion processing */ + server_onion_keys_t *onion_keys = state->onion_keys; + cpuworker_request_t req; + cpuworker_reply_t rpl; - while (num_cpuworkers < num_cpuworkers_needed) { - if (spawn_cpuworker() < 0) { - log_warn(LD_GENERAL,"Cpuworker spawn failed. Will try again later."); - return; + memcpy(&req, &job->u.request, sizeof(req)); + + tor_assert(req.magic == CPUWORKER_REQUEST_MAGIC); + memset(&rpl, 0, sizeof(rpl)); + + const create_cell_t *cc = &req.create_cell; + created_cell_t *cell_out = &rpl.created_cell; + struct timeval tv_start = {0,0}, tv_end; + int n; + rpl.timed = req.timed; + rpl.started_at = req.started_at; + rpl.handshake_type = cc->handshake_type; + if (req.timed) + tor_gettimeofday(&tv_start); + n = onion_skin_server_handshake(cc->handshake_type, + cc->onionskin, cc->handshake_len, + onion_keys, + cell_out->reply, + rpl.keys, CPATH_KEY_MATERIAL_LEN, + rpl.rend_auth_material); + if (n < 0) { + /* failure */ + log_debug(LD_OR,"onion_skin_server_handshake failed."); + memset(&rpl, 0, sizeof(rpl)); + rpl.success = 0; + } else { + /* success */ + log_debug(LD_OR,"onion_skin_server_handshake succeeded."); + cell_out->handshake_len = n; + switch (cc->cell_type) { + case CELL_CREATE: + cell_out->cell_type = CELL_CREATED; break; + case CELL_CREATE2: + cell_out->cell_type = CELL_CREATED2; break; + case CELL_CREATE_FAST: + cell_out->cell_type = CELL_CREATED_FAST; break; + default: + tor_assert(0); + return WQ_RPL_SHUTDOWN; } - num_cpuworkers++; - reseed++; + rpl.success = 1; } + rpl.magic = CPUWORKER_REPLY_MAGIC; + if (req.timed) { + struct timeval tv_diff; + int64_t usec; + tor_gettimeofday(&tv_end); + timersub(&tv_end, &tv_start, &tv_diff); + usec = ((int64_t)tv_diff.tv_sec)*1000000 + tv_diff.tv_usec; + if (usec < 0 || usec > MAX_BELIEVABLE_ONIONSKIN_DELAY) + rpl.n_usec = MAX_BELIEVABLE_ONIONSKIN_DELAY; + else + rpl.n_usec = (uint32_t) usec; + } + + memcpy(&job->u.reply, &rpl, sizeof(rpl)); - if (reseed) - crypto_seed_weak_rng(&request_sample_rng); + memwipe(&req, 0, sizeof(req)); + memwipe(&rpl, 0, sizeof(req)); + return WQ_RPL_REPLY; } -/** Take a pending task from the queue and assign it to 'cpuworker'. */ +/** Take pending tasks from the queue and assign them to cpuworkers. */ static void -process_pending_task(connection_t *cpuworker) +queue_pending_tasks(void) { or_circuit_t *circ; create_cell_t *onionskin = NULL; - tor_assert(cpuworker); + while (total_pending_tasks < max_pending_tasks) { + circ = onion_next_task(&onionskin); - /* for now only process onion tasks */ - - circ = onion_next_task(&onionskin); - if (!circ) - return; - if (assign_onionskin_to_cpuworker(cpuworker, circ, onionskin)) - log_warn(LD_OR,"assign_to_cpuworker failed. Ignoring."); -} - -/** How long should we let a cpuworker stay busy before we give - * up on it and decide that we have a bug or infinite loop? - * This value is high because some servers with low memory/cpu - * sometimes spend an hour or more swapping, and Tor starves. */ -#define CPUWORKER_BUSY_TIMEOUT (60*60*12) + if (!circ) + return; -/** We have a bug that I can't find. Sometimes, very rarely, cpuworkers get - * stuck in the 'busy' state, even though the cpuworker process thinks of - * itself as idle. I don't know why. But here's a workaround to kill any - * cpuworker that's been busy for more than CPUWORKER_BUSY_TIMEOUT. - */ -static void -cull_wedged_cpuworkers(void) -{ - time_t now = time(NULL); - smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { - if (!conn->marked_for_close && - conn->type == CONN_TYPE_CPUWORKER && - conn->state == CPUWORKER_STATE_BUSY_ONION && - conn->timestamp_lastwritten + CPUWORKER_BUSY_TIMEOUT < now) { - log_notice(LD_BUG, - "closing wedged cpuworker. Can somebody find the bug?"); - num_cpuworkers_busy--; - num_cpuworkers--; - connection_mark_for_close(conn); - } - } SMARTLIST_FOREACH_END(conn); + if (assign_onionskin_to_cpuworker(circ, onionskin)) + log_warn(LD_OR,"assign_to_cpuworker failed. Ignoring."); + } } /** Try to tell a cpuworker to perform the public key operations necessary to * respond to <b>onionskin</b> for the circuit <b>circ</b>. * - * If <b>cpuworker</b> is defined, assert that he's idle, and use him. Else, - * look for an idle cpuworker and use him. If none idle, queue task onto the - * pending onion list and return. Return 0 if we successfully assign the - * task, or -1 on failure. + * Return 0 if we successfully assign the task, or -1 on failure. */ int -assign_onionskin_to_cpuworker(connection_t *cpuworker, - or_circuit_t *circ, +assign_onionskin_to_cpuworker(or_circuit_t *circ, create_cell_t *onionskin) { + workqueue_entry_t *queue_entry; + cpuworker_job_t *job; cpuworker_request_t req; - time_t now = approx_time(); - static time_t last_culled_cpuworkers = 0; int should_time; - /* Checking for wedged cpuworkers requires a linear search over all - * connections, so let's do it only once a minute. - */ -#define CULL_CPUWORKERS_INTERVAL 60 + tor_assert(threadpool); - if (last_culled_cpuworkers + CULL_CPUWORKERS_INTERVAL <= now) { - cull_wedged_cpuworkers(); - spawn_enough_cpuworkers(); - last_culled_cpuworkers = now; + if (!circ->p_chan) { + log_info(LD_OR,"circ->p_chan gone. Failing circ."); + tor_free(onionskin); + return -1; } - if (1) { - if (num_cpuworkers_busy == num_cpuworkers) { - log_debug(LD_OR,"No idle cpuworkers. Queuing."); - if (onion_pending_add(circ, onionskin) < 0) { - tor_free(onionskin); - return -1; - } - return 0; - } - - if (!cpuworker) - cpuworker = connection_get_by_type_state(CONN_TYPE_CPUWORKER, - CPUWORKER_STATE_IDLE); - - tor_assert(cpuworker); - - if (!circ->p_chan) { - log_info(LD_OR,"circ->p_chan gone. Failing circ."); + if (total_pending_tasks >= max_pending_tasks) { + log_debug(LD_OR,"No idle cpuworkers. Queuing."); + if (onion_pending_add(circ, onionskin) < 0) { tor_free(onionskin); return -1; } + return 0; + } - if (connection_or_digest_is_known_relay(circ->p_chan->identity_digest)) - rep_hist_note_circuit_handshake_completed(onionskin->handshake_type); + if (connection_or_digest_is_known_relay(circ->p_chan->identity_digest)) + rep_hist_note_circuit_handshake_assigned(onionskin->handshake_type); - should_time = should_time_request(onionskin->handshake_type); - memset(&req, 0, sizeof(req)); - req.magic = CPUWORKER_REQUEST_MAGIC; - tag_pack(req.tag, circ->p_chan->global_identifier, - circ->p_circ_id); - req.timed = should_time; + should_time = should_time_request(onionskin->handshake_type); + memset(&req, 0, sizeof(req)); + req.magic = CPUWORKER_REQUEST_MAGIC; + req.timed = should_time; - cpuworker->state = CPUWORKER_STATE_BUSY_ONION; - /* touch the lastwritten timestamp, since that's how we check to - * see how long it's been since we asked the question, and sometimes - * we check before the first call to connection_handle_write(). */ - cpuworker->timestamp_lastwritten = now; - num_cpuworkers_busy++; + memcpy(&req.create_cell, onionskin, sizeof(create_cell_t)); - req.task = CPUWORKER_TASK_ONION; - memcpy(&req.create_cell, onionskin, sizeof(create_cell_t)); + tor_free(onionskin); - tor_free(onionskin); + if (should_time) + tor_gettimeofday(&req.started_at); - if (should_time) - tor_gettimeofday(&req.started_at); + job = tor_malloc_zero(sizeof(cpuworker_job_t)); + job->circ = circ; + memcpy(&job->u.request, &req, sizeof(req)); + memwipe(&req, 0, sizeof(req)); - connection_write_to_buf((void*)&req, sizeof(req), cpuworker); - memwipe(&req, 0, sizeof(req)); + ++total_pending_tasks; + queue_entry = threadpool_queue_work(threadpool, + cpuworker_onion_handshake_threadfn, + cpuworker_onion_handshake_replyfn, + job); + if (!queue_entry) { + log_warn(LD_BUG, "Couldn't queue work on threadpool"); + tor_free(job); + return -1; } + + log_debug(LD_OR, "Queued task %p (qe=%p, circ=%p)", + job, queue_entry, job->circ); + + circ->workqueue_entry = queue_entry; + return 0; } +/** If <b>circ</b> has a pending handshake that hasn't been processed yet, + * remove it from the worker queue. */ +void +cpuworker_cancel_circ_handshake(or_circuit_t *circ) +{ + cpuworker_job_t *job; + if (circ->workqueue_entry == NULL) + return; + + job = workqueue_entry_cancel(circ->workqueue_entry); + if (job) { + /* It successfully cancelled. */ + memwipe(job, 0xe0, sizeof(*job)); + tor_free(job); + tor_assert(total_pending_tasks > 0); + --total_pending_tasks; + /* if (!job), this is done in cpuworker_onion_handshake_replyfn. */ + circ->workqueue_entry = NULL; + } +} + diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h index 317cef43ba..62cf0eb164 100644 --- a/src/or/cpuworker.h +++ b/src/or/cpuworker.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,19 +13,17 @@ #define TOR_CPUWORKER_H void cpu_init(void); -void cpuworkers_rotate(void); -int connection_cpu_finished_flushing(connection_t *conn); -int connection_cpu_reached_eof(connection_t *conn); -int connection_cpu_process_inbuf(connection_t *conn); +void cpuworkers_rotate_keyinfo(void); + struct create_cell_t; -int assign_onionskin_to_cpuworker(connection_t *cpuworker, - or_circuit_t *circ, +int assign_onionskin_to_cpuworker(or_circuit_t *circ, struct create_cell_t *onionskin); uint64_t estimated_usec_for_onionskins(uint32_t n_requests, uint16_t onionskin_type); void cpuworker_log_onionskin_overhead(int severity, int onionskin_type, const char *onionskin_type_name); +void cpuworker_cancel_circ_handshake(or_circuit_t *circ); #endif diff --git a/src/or/dircollate.c b/src/or/dircollate.c new file mode 100644 index 0000000000..3f9d78f02d --- /dev/null +++ b/src/or/dircollate.c @@ -0,0 +1,327 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dircollate.c + * + * \brief Collation code for figuring out which identities to vote for in + * the directory voting process. + */ + +#define DIRCOLLATE_PRIVATE +#include "dircollate.h" +#include "dirvote.h" + +static void dircollator_collate_by_rsa(dircollator_t *dc); +static void dircollator_collate_by_ed25519(dircollator_t *dc); + +/** Hashtable entry mapping a pair of digests (actually an ed25519 key and an + * RSA SHA1 digest) to an array of vote_routerstatus_t. */ +typedef struct ddmap_entry_s { + HT_ENTRY(ddmap_entry_s) node; + uint8_t d[DIGEST_LEN + DIGEST256_LEN]; + /* The nth member of this array corresponds to the vote_routerstatus_t (if + * any) received for this digest pair from the nth voter. */ + vote_routerstatus_t *vrs_lst[FLEXIBLE_ARRAY_MEMBER]; +} ddmap_entry_t; + +/** Release all storage held by e. */ +static void +ddmap_entry_free(ddmap_entry_t *e) +{ + tor_free(e); +} + +/** Return a new empty ddmap_entry, with <b>n_votes</b> elements in + * vrs_list. */ +static ddmap_entry_t * +ddmap_entry_new(int n_votes) +{ + return tor_malloc_zero(STRUCT_OFFSET(ddmap_entry_t, vrs_lst) + + sizeof(vote_routerstatus_t *) * n_votes); +} + +static unsigned +ddmap_entry_hash(const ddmap_entry_t *ent) +{ + return (unsigned) siphash24g(ent->d, sizeof(ent->d)); +} + +static unsigned +ddmap_entry_eq(const ddmap_entry_t *a, const ddmap_entry_t *b) +{ + return fast_memeq(a->d, b->d, sizeof(a->d)); +} + +/** Record the RSA identity of <b>ent</b> as <b>rsa_sha1</b>, and the + * ed25519 identity as <b>ed25519</b>. */ +static void +ddmap_entry_set_digests(ddmap_entry_t *ent, + const uint8_t *rsa_sha1, + const uint8_t *ed25519) +{ + memcpy(ent->d, rsa_sha1, DIGEST_LEN); + memcpy(ent->d + DIGEST_LEN, ed25519, DIGEST256_LEN); +} + +HT_PROTOTYPE(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash, + ddmap_entry_eq); +HT_GENERATE2(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash, + ddmap_entry_eq, 0.6, tor_reallocarray, tor_free_); + +/** Helper: add a single vote_routerstatus_t <b>vrs</b> to the collator + * <b>dc</b>, indexing it by its RSA key digest, and by the 2-tuple of + * its RSA key digest and Ed25519 key. */ +static void +dircollator_add_routerstatus(dircollator_t *dc, + int vote_num, + networkstatus_t *vote, + vote_routerstatus_t *vrs) +{ + const char *id = vrs->status.identity_digest; + + vrs->ed25519_reflects_consensus = 0; + + (void) vote; + vote_routerstatus_t **vrs_lst = digestmap_get(dc->by_rsa_sha1, id); + if (NULL == vrs_lst) { + vrs_lst = tor_calloc(sizeof(vote_routerstatus_t *), dc->n_votes); + digestmap_set(dc->by_rsa_sha1, id, vrs_lst); + } + tor_assert(vrs_lst[vote_num] == NULL); + vrs_lst[vote_num] = vrs; + + const uint8_t *ed = vrs->ed25519_id; + + if (! vrs->has_ed25519_listing) + return; + + ddmap_entry_t search, *found; + memset(&search, 0, sizeof(search)); + ddmap_entry_set_digests(&search, (const uint8_t *)id, ed); + found = HT_FIND(double_digest_map, &dc->by_both_ids, &search); + if (NULL == found) { + found = ddmap_entry_new(dc->n_votes); + ddmap_entry_set_digests(found, (const uint8_t *)id, ed); + HT_INSERT(double_digest_map, &dc->by_both_ids, found); + } + vrs_lst = found->vrs_lst; + tor_assert(vrs_lst[vote_num] == NULL); + vrs_lst[vote_num] = vrs; +} + +/** Create and return a new dircollator object to use when collating + * <b>n_votes</b> out of a total of <b>n_authorities</b>. */ +dircollator_t * +dircollator_new(int n_votes, int n_authorities) +{ + dircollator_t *dc = tor_malloc_zero(sizeof(dircollator_t)); + + tor_assert(n_votes <= n_authorities); + + dc->n_votes = n_votes; + dc->n_authorities = n_authorities; + + dc->by_rsa_sha1 = digestmap_new(); + HT_INIT(double_digest_map, &dc->by_both_ids); + + return dc; +} + +/** Release all storage held by <b>dc</b>. */ +void +dircollator_free(dircollator_t *dc) +{ + if (!dc) + return; + + if (dc->by_collated_rsa_sha1 != dc->by_rsa_sha1) + digestmap_free(dc->by_collated_rsa_sha1, NULL); + + digestmap_free(dc->by_rsa_sha1, tor_free_); + smartlist_free(dc->all_rsa_sha1_lst); + + ddmap_entry_t **e, **next, *this; + for (e = HT_START(double_digest_map, &dc->by_both_ids); + e != NULL; e = next) { + this = *e; + next = HT_NEXT_RMV(double_digest_map, &dc->by_both_ids, e); + ddmap_entry_free(this); + } + HT_CLEAR(double_digest_map, &dc->by_both_ids); + + tor_free(dc); +} + +/** Add a single vote <b>v</b> to a dircollator <b>dc</b>. This function must + * be called exactly once for each vote to be used in the consensus. It may + * only be called before dircollator_collate(). + */ +void +dircollator_add_vote(dircollator_t *dc, networkstatus_t *v) +{ + tor_assert(v->type == NS_TYPE_VOTE); + tor_assert(dc->next_vote_num < dc->n_votes); + tor_assert(!dc->is_collated); + + const int votenum = dc->next_vote_num++; + + SMARTLIST_FOREACH_BEGIN(v->routerstatus_list, vote_routerstatus_t *, vrs) { + dircollator_add_routerstatus(dc, votenum, v, vrs); + } SMARTLIST_FOREACH_END(vrs); +} + +/** Sort the entries in <b>dc</b> according to <b>consensus_method</b>, so + * that the consensus process can iterate over them with + * dircollator_n_routers() and dircollator_get_votes_for_router(). */ +void +dircollator_collate(dircollator_t *dc, int consensus_method) +{ + tor_assert(!dc->is_collated); + dc->all_rsa_sha1_lst = smartlist_new(); + + if (consensus_method < MIN_METHOD_FOR_ED25519_ID_VOTING) + dircollator_collate_by_rsa(dc); + else + dircollator_collate_by_ed25519(dc); + + smartlist_sort_digests(dc->all_rsa_sha1_lst); + dc->is_collated = 1; +} + +/** + * Collation function for RSA-only consensuses: collate the votes for each + * entry in <b>dc</b> by their RSA keys. + * + * The rule is: + * If an RSA identity key is listed by more than half of the authorities, + * include that identity, and treat all descriptors with that RSA identity + * as describing the same router. + */ +static void +dircollator_collate_by_rsa(dircollator_t *dc) +{ + const int total_authorities = dc->n_authorities; + + DIGESTMAP_FOREACH(dc->by_rsa_sha1, k, vote_routerstatus_t **, vrs_lst) { + int n = 0, i; + for (i = 0; i < dc->n_votes; ++i) { + if (vrs_lst[i] != NULL) + ++n; + } + + if (n <= total_authorities / 2) + continue; + + smartlist_add(dc->all_rsa_sha1_lst, (char *)k); + } DIGESTMAP_FOREACH_END; + + dc->by_collated_rsa_sha1 = dc->by_rsa_sha1; +} + +/** + * Collation function for ed25519 consensuses: collate the votes for each + * entry in <b>dc</b> by ed25519 key and by RSA key. + * + * The rule is, approximately: + * If a (ed,rsa) identity is listed by more than half of authorities, + * include it. And include all (rsa)-only votes about that node as + * matching. + * + * Otherwise, if an (*,rsa) or (rsa) identity is listed by more than + * half of the authorities, and no (ed,rsa) pair for the same RSA key + * has been already been included based on the rule above, include + * that RSA identity. + */ +static void +dircollator_collate_by_ed25519(dircollator_t *dc) +{ + const int total_authorities = dc->n_authorities; + digestmap_t *rsa_digests = digestmap_new(); + + ddmap_entry_t **iter; + + /* Go over all <ed,rsa> pairs */ + HT_FOREACH(iter, double_digest_map, &dc->by_both_ids) { + ddmap_entry_t *ent = *iter; + int n = 0, i; + for (i = 0; i < dc->n_votes; ++i) { + if (ent->vrs_lst[i] != NULL) + ++n; + } + + /* If not enough authorties listed this exact <ed,rsa> pair, + * don't include it. */ + if (n <= total_authorities / 2) + continue; + + /* Now consider whether there are any other entries with the same + * RSA key (but with possibly different or missing ed value). */ + vote_routerstatus_t **vrs_lst2 = digestmap_get(dc->by_rsa_sha1, + (char*)ent->d); + tor_assert(vrs_lst2); + + for (i = 0; i < dc->n_votes; ++i) { + if (ent->vrs_lst[i] != NULL) { + ent->vrs_lst[i]->ed25519_reflects_consensus = 1; + } else if (vrs_lst2[i] && ! vrs_lst2[i]->has_ed25519_listing) { + ent->vrs_lst[i] = vrs_lst2[i]; + } + } + + /* Record that we have seen this RSA digest. */ + digestmap_set(rsa_digests, (char*)ent->d, ent->vrs_lst); + smartlist_add(dc->all_rsa_sha1_lst, ent->d); + } + + /* Now look over all entries with an RSA digest, looking for RSA digests + * we didn't put in yet. + */ + DIGESTMAP_FOREACH(dc->by_rsa_sha1, k, vote_routerstatus_t **, vrs_lst) { + if (digestmap_get(rsa_digests, k) != NULL) + continue; /* We already included this RSA digest */ + + int n = 0, i; + for (i = 0; i < dc->n_votes; ++i) { + if (vrs_lst[i] != NULL) + ++n; + } + + if (n <= total_authorities / 2) + continue; /* Not enough votes */ + + digestmap_set(rsa_digests, k, vrs_lst); + smartlist_add(dc->all_rsa_sha1_lst, (char *)k); + } DIGESTMAP_FOREACH_END; + + dc->by_collated_rsa_sha1 = rsa_digests; +} + +/** Return the total number of collated router entries. This function may + * only be called after dircollator_collate. */ +int +dircollator_n_routers(dircollator_t *dc) +{ + tor_assert(dc->is_collated); + return smartlist_len(dc->all_rsa_sha1_lst); +} + +/** Return an array of vote_routerstatus_t entries for the <b>idx</b>th router + * in the collation order. Each array contains n_votes elements, where the + * nth element of the array is the vote_routerstatus_t from the nth voter for + * this identity (or NULL if there is no such entry). + * + * The maximum value for <b>idx</b> is dircollator_n_routers(). + * + * This function may only be called after dircollator_collate. */ +vote_routerstatus_t ** +dircollator_get_votes_for_router(dircollator_t *dc, int idx) +{ + tor_assert(dc->is_collated); + tor_assert(idx < smartlist_len(dc->all_rsa_sha1_lst)); + return digestmap_get(dc->by_collated_rsa_sha1, + smartlist_get(dc->all_rsa_sha1_lst, idx)); +} + diff --git a/src/or/dircollate.h b/src/or/dircollate.h new file mode 100644 index 0000000000..358c730cbb --- /dev/null +++ b/src/or/dircollate.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dircollate.h + * \brief Header file for dircollate.c. + **/ + +#ifndef TOR_DIRCOLLATE_H +#define TOR_DIRCOLLATE_H + +#include "testsupport.h" +#include "or.h" + +typedef struct dircollator_s dircollator_t; + +dircollator_t *dircollator_new(int n_votes, int n_authorities); +void dircollator_free(dircollator_t *obj); +void dircollator_add_vote(dircollator_t *dc, networkstatus_t *v); + +void dircollator_collate(dircollator_t *dc, int consensus_method); + +int dircollator_n_routers(dircollator_t *dc); +vote_routerstatus_t **dircollator_get_votes_for_router(dircollator_t *dc, + int idx); + +#ifdef DIRCOLLATE_PRIVATE +struct ddmap_entry_s; +typedef HT_HEAD(double_digest_map, ddmap_entry_s) double_digest_map_t; +/** A dircollator keeps track of all the routerstatus entries in a + * set of networkstatus votes, and matches them by an appropriate rule. */ +struct dircollator_s { + /** True iff we have run the collation algorithm. */ + int is_collated; + /** The total number of votes that we received. */ + int n_votes; + /** The total number of authorities we acknowledge. */ + int n_authorities; + + /** The index which the next vote to be added to this collator should + * receive. */ + int next_vote_num; + /** Map from RSA-SHA1 identity digest to an array of <b>n_votes</b> + * vote_routerstatus_t* pointers, such that the i'th member of the + * array is the i'th vote's entry for that RSA-SHA1 ID.*/ + digestmap_t *by_rsa_sha1; + /** Map from <ed, RSA-SHA1> pair to an array similar to that used in + * by_rsa_sha1 above. We include <NULL,RSA-SHA1> entries for votes that + * say that there is no Ed key. */ + struct double_digest_map by_both_ids; + + /** One of two outputs created by collation: a map from RSA-SHA1 + * identity digest to an array of the vote_routerstatus_t objects. Entries + * only exist in this map for identities that we should include in the + * consensus. */ + digestmap_t *by_collated_rsa_sha1; + + /** One of two outputs created by collation: a sorted array of RSA-SHA1 + * identity digests .*/ + smartlist_t *all_rsa_sha1_lst; +}; +#endif + +#endif + diff --git a/src/or/directory.c b/src/or/directory.c index 3752367c44..89b08223d2 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1,9 +1,10 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" +#include "backtrace.h" #include "buffers.h" #include "circuitbuild.h" #include "config.h" @@ -20,8 +21,10 @@ #include "networkstatus.h" #include "nodelist.h" #include "policies.h" +#include "relay.h" #include "rendclient.h" #include "rendcommon.h" +#include "rendservice.h" #include "rephist.h" #include "router.h" #include "routerlist.h" @@ -63,19 +66,13 @@ static void directory_send_command(dir_connection_t *conn, time_t if_modified_since); static int directory_handle_command(dir_connection_t *conn); static int body_is_plausible(const char *body, size_t body_len, int purpose); -static int purpose_needs_anonymity(uint8_t dir_purpose, - uint8_t router_purpose); static char *http_get_header(const char *headers, const char *which); static void http_set_address_origin(const char *headers, connection_t *conn); -static void connection_dir_download_v2_networkstatus_failed( - dir_connection_t *conn, int status_code); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn); static void connection_dir_download_cert_failed( dir_connection_t *conn, int status_code); static void connection_dir_retry_bridges(smartlist_t *descs); -static void dir_networkstatus_download_failed(smartlist_t *failed, - int status_code); static void dir_routerdesc_download_failed(smartlist_t *failed, int status_code, int router_purpose, @@ -86,19 +83,21 @@ static void dir_microdesc_download_failed(smartlist_t *failed, static void note_client_request(int purpose, int compressed, size_t bytes); static int client_likes_consensus(networkstatus_t *v, const char *want_url); -static void directory_initiate_command_rend(const char *address, - const tor_addr_t *addr, - uint16_t or_port, - uint16_t dir_port, - const char *digest, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since, - const rend_data_t *rend_query); +static void directory_initiate_command_rend( + const tor_addr_port_t *or_addr_port, + const tor_addr_port_t *dir_addr_port, + const char *digest, + uint8_t dir_purpose, + uint8_t router_purpose, + dir_indirection_t indirection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since, + const rend_data_t *rend_query); + +static void connection_dir_close_consensus_fetches( + dir_connection_t *except_this_one, const char *resource); /********* START VARIABLES **********/ @@ -124,7 +123,7 @@ static void directory_initiate_command_rend(const char *address, /** Return true iff the directory purpose <b>dir_purpose</b> (and if it's * fetching descriptors, it's fetching them for <b>router_purpose</b>) * must use an anonymous connection to a directory. */ -static int +STATIC int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) { if (get_options()->AllDirActionsPrivate) @@ -135,7 +134,6 @@ purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR || dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES || - dir_purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS || dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE || dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES || dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS || @@ -149,21 +147,15 @@ purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) /** Return a newly allocated string describing <b>auth</b>. Only describes * authority features. */ -static char * +STATIC char * authdir_type_to_string(dirinfo_type_t auth) { char *result; smartlist_t *lst = smartlist_new(); - if (auth & V1_DIRINFO) - smartlist_add(lst, (void*)"V1"); - if (auth & V2_DIRINFO) - smartlist_add(lst, (void*)"V2"); if (auth & V3_DIRINFO) smartlist_add(lst, (void*)"V3"); if (auth & BRIDGE_DIRINFO) smartlist_add(lst, (void*)"Bridge"); - if (auth & HIDSERV_DIRINFO) - smartlist_add(lst, (void*)"Hidden service"); if (smartlist_len(lst)) { result = smartlist_join_strings(lst, ", ", 0, NULL); } else { @@ -174,23 +166,17 @@ authdir_type_to_string(dirinfo_type_t auth) } /** Return a string describing a given directory connection purpose. */ -static const char * +STATIC const char * dir_conn_purpose_to_string(int purpose) { switch (purpose) { - case DIR_PURPOSE_FETCH_RENDDESC: - return "hidden-service descriptor fetch"; case DIR_PURPOSE_UPLOAD_DIR: return "server descriptor upload"; - case DIR_PURPOSE_UPLOAD_RENDDESC: - return "hidden-service descriptor upload"; case DIR_PURPOSE_UPLOAD_VOTE: return "server vote upload"; case DIR_PURPOSE_UPLOAD_SIGNATURES: return "consensus signature upload"; - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: - return "network-status fetch"; case DIR_PURPOSE_FETCH_SERVERDESC: return "server descriptor fetch"; case DIR_PURPOSE_FETCH_EXTRAINFO: @@ -215,9 +201,49 @@ dir_conn_purpose_to_string(int purpose) return "(unknown)"; } -/** Return true iff <b>identity_digest</b> is the digest of a router we - * believe to support extrainfo downloads. (If <b>is_authority</b> we do - * additional checking that's only valid for authorities.) */ +/** Return the requisite directory information types. */ +STATIC dirinfo_type_t +dir_fetch_type(int dir_purpose, int router_purpose, const char *resource) +{ + dirinfo_type_t type; + switch (dir_purpose) { + case DIR_PURPOSE_FETCH_EXTRAINFO: + type = EXTRAINFO_DIRINFO; + if (router_purpose == ROUTER_PURPOSE_BRIDGE) + type |= BRIDGE_DIRINFO; + else + type |= V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_SERVERDESC: + if (router_purpose == ROUTER_PURPOSE_BRIDGE) + type = BRIDGE_DIRINFO; + else + type = V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_CERTIFICATE: + type = V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_CONSENSUS: + type = V3_DIRINFO; + if (resource && !strcmp(resource, "microdesc")) + type |= MICRODESC_DIRINFO; + break; + case DIR_PURPOSE_FETCH_MICRODESC: + type = MICRODESC_DIRINFO; + break; + default: + log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose); + type = NO_DIRINFO; + break; + } + return type; +} + +/** Return true iff <b>identity_digest</b> is the digest of a router which + * says that it caches extrainfos. (If <b>is_authority</b> we always + * believe that to be true.) */ int router_supports_extrainfo(const char *identity_digest, int is_authority) { @@ -258,13 +284,13 @@ directories_have_accepted_server_descriptor(void) /** Start a connection to every suitable directory authority, using * connection purpose <b>dir_purpose</b> and uploading <b>payload</b> * (of length <b>payload_len</b>). The dir_purpose should be one of - * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'. + * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'. * * <b>router_purpose</b> describes the type of descriptor we're * publishing, if we're publishing a descriptor -- e.g. general or bridge. * - * <b>type</b> specifies what sort of dir authorities (V1, V2, - * HIDSERV, BRIDGE) we should upload to. + * <b>type</b> specifies what sort of dir authorities (V3, + * BRIDGE, etc) we should upload to. * * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b> @@ -279,7 +305,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, size_t payload_len, size_t extrainfo_len) { const or_options_t *options = get_options(); - int post_via_tor; + dir_indirection_t indirection; const smartlist_t *dirservers = router_get_trusted_dir_servers(); int found = 0; const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE || @@ -291,13 +317,16 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, SMARTLIST_FOREACH_BEGIN(dirservers, dir_server_t *, ds) { routerstatus_t *rs = &(ds->fake_status); size_t upload_len = payload_len; - tor_addr_t ds_addr; if ((type & ds->type) == 0) continue; - if (exclude_self && router_digest_is_me(ds->digest)) + if (exclude_self && router_digest_is_me(ds->digest)) { + /* we don't upload to ourselves, but at least there's now at least + * one authority of this type that has what we wanted to upload. */ + found = 1; continue; + } if (options->StrictNodes && routerset_contains_routerstatus(options->ExcludeNodes, rs, -1)) { @@ -318,12 +347,21 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, log_info(LD_DIR, "Uploading an extrainfo too (length %d)", (int) extrainfo_len); } - tor_addr_from_ipv4h(&ds_addr, ds->addr); - post_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose) || - !fascist_firewall_allows_address_dir(&ds_addr, ds->dir_port); + if (purpose_needs_anonymity(dir_purpose, router_purpose)) { + indirection = DIRIND_ANONYMOUS; + } else if (!fascist_firewall_allows_dir_server(ds, + FIREWALL_DIR_CONNECTION, + 0)) { + if (fascist_firewall_allows_dir_server(ds, FIREWALL_OR_CONNECTION, 0)) + indirection = DIRIND_ONEHOP; + else + indirection = DIRIND_ANONYMOUS; + } else { + indirection = DIRIND_DIRECT_CONN; + } directory_initiate_command_routerstatus(rs, dir_purpose, router_purpose, - post_via_tor, + indirection, NULL, payload, upload_len, 0); } SMARTLIST_FOREACH_END(ds); if (!found) { @@ -336,7 +374,7 @@ directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, /** Return true iff, according to the values in <b>options</b>, we should be * using directory guards for direct downloads of directory information. */ -static int +STATIC int should_use_directory_guards(const or_options_t *options) { /* Public (non-bridge) servers never use directory guards. */ @@ -350,15 +388,12 @@ should_use_directory_guards(const or_options_t *options) /* If we're configured to fetch directory info aggressively or of a * nonstandard type, don't use directory guards. */ if (options->DownloadExtraInfo || options->FetchDirInfoEarly || - options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors || - options->FetchV2Networkstatus) - return 0; - if (! options->PreferTunneledDirConns) + options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors) return 0; return 1; } -/** Pick an unconsetrained directory server from among our guards, the latest +/** Pick an unconstrained directory server from among our guards, the latest * networkstatus, or the fallback dirservers, for use in downloading * information of type <b>type</b>, and return its routerstatus. */ static const routerstatus_t * @@ -394,55 +429,24 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, * Use <b>pds_flags</b> as arguments to router_pick_directory_server() * or router_pick_trusteddirserver(). */ -void -directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource, int pds_flags) +MOCK_IMPL(void, directory_get_from_dirserver, ( + uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource, + int pds_flags, + download_want_authority_t want_authority)) { const routerstatus_t *rs = NULL; const or_options_t *options = get_options(); - int prefer_authority = directory_fetches_from_authorities(options); + int prefer_authority = (directory_fetches_from_authorities(options) + || want_authority == DL_WANT_AUTHORITY); int require_authority = 0; int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose); - dirinfo_type_t type; + dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource); time_t if_modified_since = 0; - /* FFFF we could break this switch into its own function, and call - * it elsewhere in directory.c. -RD */ - switch (dir_purpose) { - case DIR_PURPOSE_FETCH_EXTRAINFO: - type = EXTRAINFO_DIRINFO | - (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO : - V3_DIRINFO); - break; - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: - type = V2_DIRINFO; - prefer_authority = 1; /* Only v2 authorities have these anyway. */ - require_authority = 1; /* Don't fallback to asking a non-authority */ - break; - case DIR_PURPOSE_FETCH_SERVERDESC: - type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO : - V3_DIRINFO); - break; - case DIR_PURPOSE_FETCH_RENDDESC: - type = HIDSERV_DIRINFO; - break; - case DIR_PURPOSE_FETCH_STATUS_VOTE: - case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: - case DIR_PURPOSE_FETCH_CERTIFICATE: - type = V3_DIRINFO; - break; - case DIR_PURPOSE_FETCH_CONSENSUS: - type = V3_DIRINFO; - if (resource && !strcmp(resource,"microdesc")) - type |= MICRODESC_DIRINFO; - break; - case DIR_PURPOSE_FETCH_MICRODESC: - type = MICRODESC_DIRINFO; - break; - default: - log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose); - return; - } + if (type == NO_DIRINFO) + return; if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) { int flav = FLAV_NS; @@ -450,26 +454,41 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, if (resource) flav = networkstatus_parse_flavor_name(resource); + /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus + * period of 1 hour. + */ +#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180) if (flav != -1) { /* IF we have a parsed consensus of this type, we can do an * if-modified-time based on it. */ v = networkstatus_get_latest_consensus_by_flavor(flav); - if (v) - if_modified_since = v->valid_after + 180; + if (v) { + /* In networks with particularly short V3AuthVotingIntervals, + * ask for the consensus if it's been modified since half the + * V3AuthVotingInterval of the most recent consensus. */ + time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (v->fresh_until > v->valid_after + && ims_delay > (v->fresh_until - v->valid_after)/2) { + ims_delay = (v->fresh_until - v->valid_after)/2; + } + if_modified_since = v->valid_after + ims_delay; + } } else { /* Otherwise it might be a consensus we don't parse, but which we * do cache. Look at the cached copy, perhaps. */ cached_dir_t *cd = dirserv_get_consensus(resource); + /* We have no method of determining the voting interval from an + * unparsed consensus, so we use the default. */ if (cd) - if_modified_since = cd->published + 180; + if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; } } - if (!options->FetchServerDescriptors && type != HIDSERV_DIRINFO) + if (!options->FetchServerDescriptors) return; if (!get_via_tor) { - if (options->UseBridges && type != BRIDGE_DIRINFO) { + if (options->UseBridges && !(type & BRIDGE_DIRINFO)) { /* We want to ask a running bridge for which we have a descriptor. * * When we ask choose_random_entry() for a bridge, we specify what @@ -481,11 +500,14 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, const node_t *node = choose_random_dirguard(type); if (node && node->ri) { /* every bridge has a routerinfo. */ - tor_addr_t addr; routerinfo_t *ri = node->ri; - node_get_addr(node, &addr); - directory_initiate_command(ri->address, &addr, - ri->or_port, 0/*no dirport*/, + /* clients always make OR connections to bridges */ + tor_addr_port_t or_ap; + /* we are willing to use a non-preferred address if we need to */ + fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, + &or_ap); + directory_initiate_command(&or_ap.addr, or_ap.port, + NULL, 0, /*no dirport*/ ri->cache_info.identity_digest, dir_purpose, router_purpose, @@ -496,7 +518,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, "nodes are available yet."); return; } else { - if (prefer_authority || type == BRIDGE_DIRINFO) { + if (prefer_authority || (type & BRIDGE_DIRINFO)) { /* only ask authdirservers, and don't ask myself */ rs = router_pick_trusteddirserver(type, pds_flags); if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| @@ -523,33 +545,25 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, return; } } - if (!rs && type != BRIDGE_DIRINFO) { + if (!rs && !(type & BRIDGE_DIRINFO)) { /* */ rs = directory_pick_generic_dirserver(type, pds_flags, dir_purpose); - if (!rs) { - /*XXXX024 I'm pretty sure this can never do any good, since - * rs isn't set. */ + if (!rs) get_via_tor = 1; /* last resort: try routing it via Tor */ - } } } - } else { /* get_via_tor */ + } + + if (get_via_tor) { /* Never use fascistfirewall; we're going via Tor. */ - if (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) { - /* only ask hidserv authorities, any of them will do */ - pds_flags |= PDS_IGNORE_FASCISTFIREWALL|PDS_ALLOW_SELF; - rs = router_pick_trusteddirserver(HIDSERV_DIRINFO, pds_flags); - } else { - /* anybody with a non-zero dirport will do. Disregard firewalls. */ - pds_flags |= PDS_IGNORE_FASCISTFIREWALL; - rs = router_pick_directory_server(type, pds_flags); - /* If we have any hope of building an indirect conn, we know some router - * descriptors. If (rs==NULL), we can't build circuits anyway, so - * there's no point in falling back to the authorities in this case. */ - } + pds_flags |= PDS_IGNORE_FASCISTFIREWALL; + rs = router_pick_directory_server(type, pds_flags); } + /* If we have any hope of building an indirect conn, we know some router + * descriptors. If (rs==NULL), we can't build circuits anyway, so + * there's no point in falling back to the authorities in this case. */ if (rs) { const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP; @@ -602,6 +616,95 @@ dirind_is_anon(dir_indirection_t ind) return ind == DIRIND_ANON_DIRPORT || ind == DIRIND_ANONYMOUS; } +/* Choose reachable OR and Dir addresses and ports from status, copying them + * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're + * connecting via another relay, so choose the primary IPv4 address and ports. + * + * status should have at least one reachable address, if we can't choose a + * reachable address, warn and return -1. Otherwise, return 0. + */ +static int +directory_choose_address_routerstatus(const routerstatus_t *status, + dir_indirection_t indirection, + tor_addr_port_t *use_or_ap, + tor_addr_port_t *use_dir_ap) +{ + tor_assert(status != NULL); + tor_assert(use_or_ap != NULL); + tor_assert(use_dir_ap != NULL); + + const or_options_t *options = get_options(); + int have_or = 0, have_dir = 0; + + /* We expect status to have at least one reachable address if we're + * connecting to it directly. + * + * Therefore, we can simply use the other address if the one we want isn't + * allowed by the firewall. + * + * (When Tor uploads and downloads a hidden service descriptor, it uses + * DIRIND_ANONYMOUS, except for Tor2Web, which uses DIRIND_ONEHOP. + * So this code will only modify the address for Tor2Web's HS descriptor + * fetches. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS, to avoid + * HSDirs denying service by rejecting descriptors.) + */ + + /* Initialise the OR / Dir addresses */ + tor_addr_make_null(&use_or_ap->addr, AF_UNSPEC); + use_or_ap->port = 0; + tor_addr_make_null(&use_dir_ap->addr, AF_UNSPEC); + use_dir_ap->port = 0; + + /* ORPort connections */ + if (indirection == DIRIND_ANONYMOUS) { + if (status->addr) { + /* Since we're going to build a 3-hop circuit and ask the 2nd relay + * to extend to this address, always use the primary (IPv4) OR address */ + tor_addr_from_ipv4h(&use_or_ap->addr, status->addr); + use_or_ap->port = status->or_port; + have_or = 1; + } + } else if (indirection == DIRIND_ONEHOP) { + /* We use an IPv6 address if we have one and we prefer it. + * Use the preferred address and port if they are reachable, otherwise, + * use the alternate address and port (if any). + */ + have_or = fascist_firewall_choose_address_rs(status, + FIREWALL_OR_CONNECTION, 0, + use_or_ap); + } + + /* DirPort connections + * DIRIND_ONEHOP uses ORPort, but may fall back to the DirPort on relays */ + if (indirection == DIRIND_DIRECT_CONN || + indirection == DIRIND_ANON_DIRPORT || + (indirection == DIRIND_ONEHOP + && !directory_must_use_begindir(options))) { + have_dir = fascist_firewall_choose_address_rs(status, + FIREWALL_DIR_CONNECTION, 0, + use_dir_ap); + } + + /* We rejected all addresses in the relay's status. This means we can't + * connect to it. */ + if (!have_or && !have_dir) { + static int logged_backtrace = 0; + log_info(LD_BUG, "Rejected all OR and Dir addresses from %s when " + "launching an outgoing directory connection to: IPv4 %s OR %d " + "Dir %d IPv6 %s OR %d Dir %d", routerstatus_describe(status), + fmt_addr32(status->addr), status->or_port, + status->dir_port, fmt_addr(&status->ipv6_addr), + status->ipv6_orport, status->dir_port); + if (!logged_backtrace) { + log_backtrace(LOG_INFO, LD_BUG, "Addresses came from"); + logged_backtrace = 1; + } + return -1; + } + + return 0; +} + /** Same as directory_initiate_command_routerstatus(), but accepts * rendezvous data to fetch a hidden service descriptor. */ void @@ -617,11 +720,11 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, { const or_options_t *options = get_options(); const node_t *node; - char address_buf[INET_NTOA_BUF_LEN+1]; - struct in_addr in; - const char *address; - tor_addr_t addr; + tor_addr_port_t use_or_ap, use_dir_ap; const int anonymized_connection = dirind_is_anon(indirection); + + tor_assert(status != NULL); + node = node_get_by_id(status->identity_digest); if (!node && anonymized_connection) { @@ -629,15 +732,7 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, "don't have its router descriptor.", routerstatus_describe(status)); return; - } else if (node) { - node_get_address_string(node, address_buf, sizeof(address_buf)); - address = address_buf; - } else { - in.s_addr = htonl(status->addr); - tor_inet_ntoa(&in, address_buf, sizeof(address_buf)); - address = address_buf; } - tor_addr_from_ipv4h(&addr, status->addr); if (options->ExcludeNodes && options->StrictNodes && routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) { @@ -649,20 +744,37 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, return; } - directory_initiate_command_rend(address, &addr, - status->or_port, status->dir_port, - status->identity_digest, - dir_purpose, router_purpose, - indirection, resource, - payload, payload_len, if_modified_since, - rend_query); + /* At this point, if we are a clients making a direct connection to a + * directory server, we have selected a server that has at least one address + * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This + * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if + * possible. (If UseBridges is set, clients always use IPv6, and prefer it + * by default.) + * + * Now choose an address that we can use to connect to the directory server. + */ + if (directory_choose_address_routerstatus(status, indirection, &use_or_ap, + &use_dir_ap) < 0) { + return; + } + + /* We don't retry the alternate OR/Dir address for the same directory if + * the address we choose fails (#6772). + * Instead, we'll retry another directory on failure. */ + + directory_initiate_command_rend(&use_or_ap, &use_dir_ap, + status->identity_digest, + dir_purpose, router_purpose, + indirection, resource, + payload, payload_len, if_modified_since, + rend_query); } /** Launch a new connection to the directory server <b>status</b> to * upload or download a server or rendezvous * descriptor. <b>dir_purpose</b> determines what * kind of directory connection we're launching, and must be one of - * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC|RENDDESC_V2}. <b>router_purpose</b> + * DIR_PURPOSE_{FETCH|UPLOAD}_{DIR|RENDDESC_V2}. <b>router_purpose</b> * specifies the descriptor purposes we have in mind (currently only * used for FETCH_DIR). * @@ -672,15 +784,15 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, * When fetching a rendezvous descriptor, <b>resource</b> is the service ID we * want to fetch. */ -void -directory_initiate_command_routerstatus(const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since) +MOCK_IMPL(void, directory_initiate_command_routerstatus, + (const routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + dir_indirection_t indirection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since)) { directory_initiate_command_routerstatus_rend(status, dir_purpose, router_purpose, @@ -718,12 +830,8 @@ connection_dir_request_failed(dir_connection_t *conn) return; /* this was a test fetch. don't retry. */ } if (!entry_list_is_constrained(get_options())) - router_set_status(conn->identity_digest, 0); /* don't try him again */ - if (conn->base_.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { - log_info(LD_DIR, "Giving up on directory server at '%s'; retrying", - conn->base_.address); - connection_dir_download_v2_networkstatus_failed(conn, -1); - } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || + router_set_status(conn->identity_digest, 0); /* don't try this one again */ + if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) { log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from " "directory server at '%s'; retrying", @@ -747,48 +855,11 @@ connection_dir_request_failed(dir_connection_t *conn) conn->base_.address); } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) { log_info(LD_DIR, "Giving up on downloading microdescriptors from " - " directory server at '%s'; will retry", conn->base_.address); + "directory server at '%s'; will retry", conn->base_.address); connection_dir_download_routerdesc_failed(conn); } } -/** Called when an attempt to download one or more network status - * documents on connection <b>conn</b> failed. Decide whether to - * retry the fetch now, later, or never. - */ -static void -connection_dir_download_v2_networkstatus_failed(dir_connection_t *conn, - int status_code) -{ - if (!conn->requested_resource) { - /* We never reached directory_send_command, which means that we never - * opened a network connection. Either we're out of sockets, or the - * network is down. Either way, retrying would be pointless. */ - return; - } - if (!strcmpstart(conn->requested_resource, "all")) { - /* We're a non-authoritative directory cache; try again. Ignore status - * code, since we don't want to keep trying forever in a tight loop - * if all the authorities are shutting us out. */ - const smartlist_t *trusted_dirs = router_get_trusted_dir_servers(); - SMARTLIST_FOREACH(trusted_dirs, dir_server_t *, ds, - download_status_failed(&ds->v2_ns_dl_status, 0)); - directory_get_from_dirserver(conn->base_.purpose, conn->router_purpose, - "all.z", 0 /* don't retry_if_no_servers */); - } else if (!strcmpstart(conn->requested_resource, "fp/")) { - /* We were trying to download by fingerprint; mark them all as having - * failed, and possibly retry them later.*/ - smartlist_t *failed = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource+3, - failed, NULL, 0); - if (smartlist_len(failed)) { - dir_networkstatus_download_failed(failed, status_code); - SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp)); - } - smartlist_free(failed); - } -} - /** Helper: Attempt to fetch directly the descriptors of each bridge * listed in <b>failed</b>. */ @@ -898,6 +969,16 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) update_certificate_downloads(time(NULL)); } +/* Should this tor instance only use begindir for all its directory requests? + */ +int +directory_must_use_begindir(const or_options_t *options) +{ + /* Clients, onion services, and bridges must use begindir, + * relays and authorities do not have to */ + return !public_server_mode(options); +} + /** Evaluate the situation and decide if we should use an encrypted * "begindir-style" connection for this directory request. * 1) If or_port is 0, or it's a direct conn and or_port is firewalled @@ -905,42 +986,90 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status) * 2) If we prefer to avoid begindir conns, and we're not fetching or * publishing a bridge relay descriptor, no. * 3) Else yes. + * If returning 0, return in *reason why we can't use begindir. + * reason must not be NULL. */ static int directory_command_should_use_begindir(const or_options_t *options, const tor_addr_t *addr, int or_port, uint8_t router_purpose, - dir_indirection_t indirection) + dir_indirection_t indirection, + const char **reason) { - if (!or_port) + (void) router_purpose; + tor_assert(reason); + *reason = NULL; + + /* Reasons why we can't possibly use begindir */ + if (!or_port) { + *reason = "directory with unknown ORPort"; return 0; /* We don't know an ORPort -- no chance. */ - if (indirection == DIRIND_DIRECT_CONN || indirection == DIRIND_ANON_DIRPORT) + } + if (indirection == DIRIND_DIRECT_CONN || + indirection == DIRIND_ANON_DIRPORT) { + *reason = "DirPort connection"; return 0; - if (indirection == DIRIND_ONEHOP) - if (!fascist_firewall_allows_address_or(addr, or_port) || - directory_fetches_from_authorities(options)) - return 0; /* We're firewalled or are acting like a relay -- also no. */ - if (!options->TunnelDirConns && - router_purpose != ROUTER_PURPOSE_BRIDGE) - return 0; /* We prefer to avoid using begindir conns. Fine. */ + } + if (indirection == DIRIND_ONEHOP) { + /* We're firewalled and want a direct OR connection */ + if (!fascist_firewall_allows_address_addr(addr, or_port, + FIREWALL_OR_CONNECTION, 0, 0)) { + *reason = "ORPort not reachable"; + return 0; + } + } + /* Reasons why we want to avoid using begindir */ + if (indirection == DIRIND_ONEHOP) { + if (!directory_must_use_begindir(options)) { + *reason = "in relay mode"; + return 0; + } + } + /* DIRIND_ONEHOP on a client, or DIRIND_ANONYMOUS + */ + *reason = "(using begindir)"; return 1; } -/** Helper for directory_initiate_command_routerstatus: send the - * command to a server whose address is <b>address</b>, whose IP is - * <b>addr</b>, whose directory port is <b>dir_port</b>, whose tor version - * <b>supports_begindir</b>, and whose identity key digest is - * <b>digest</b>. */ +/** Helper for directory_initiate_command_rend: send the + * command to a server whose OR address/port is <b>or_addr</b>/<b>or_port</b>, + * whose directory address/port is <b>dir_addr</b>/<b>dir_port</b>, whose + * identity key digest is <b>digest</b>, with purposes <b>dir_purpose</b> and + * <b>router_purpose</b>, making an (in)direct connection as specified in + * <b>indirection</b>, with command <b>resource</b>, <b>payload</b> of + * <b>payload_len</b>, and asking for a result only <b>if_modified_since</b>. + */ void -directory_initiate_command(const char *address, const tor_addr_t *_addr, - uint16_t or_port, uint16_t dir_port, +directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port, + const tor_addr_t *dir_addr, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, dir_indirection_t indirection, const char *resource, const char *payload, size_t payload_len, time_t if_modified_since) { - directory_initiate_command_rend(address, _addr, or_port, dir_port, + tor_addr_port_t or_ap, dir_ap; + + /* Use the null tor_addr and 0 port if the address or port isn't valid. */ + if (tor_addr_port_is_valid(or_addr, or_port, 0)) { + tor_addr_copy(&or_ap.addr, or_addr); + or_ap.port = or_port; + } else { + /* the family doesn't matter here, so make it IPv4 */ + tor_addr_make_null(&or_ap.addr, AF_INET); + or_ap.port = or_port = 0; + } + + if (tor_addr_port_is_valid(dir_addr, dir_port, 0)) { + tor_addr_copy(&dir_ap.addr, dir_addr); + dir_ap.port = dir_port; + } else { + /* the family doesn't matter here, so make it IPv4 */ + tor_addr_make_null(&dir_ap.addr, AF_INET); + dir_ap.port = dir_port = 0; + } + + directory_initiate_command_rend(&or_ap, &dir_ap, digest, dir_purpose, router_purpose, indirection, resource, payload, payload_len, @@ -954,18 +1083,17 @@ directory_initiate_command(const char *address, const tor_addr_t *_addr, static int is_sensitive_dir_purpose(uint8_t dir_purpose) { - return ((dir_purpose == DIR_PURPOSE_FETCH_RENDDESC) || - (dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC) || - (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC) || + return ((dir_purpose == DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2) || (dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) || (dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2)); } /** Same as directory_initiate_command(), but accepts rendezvous data to - * fetch a hidden service descriptor. */ + * fetch a hidden service descriptor, and takes its address & port arguments + * as tor_addr_port_t. */ static void -directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, - uint16_t or_port, uint16_t dir_port, +directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, + const tor_addr_port_t *dir_addr_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, dir_indirection_t indirection, @@ -974,20 +1102,33 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, time_t if_modified_since, const rend_data_t *rend_query) { + tor_assert(or_addr_port); + tor_assert(dir_addr_port); + tor_assert(or_addr_port->port || dir_addr_port->port); + tor_assert(digest); + dir_connection_t *conn; const or_options_t *options = get_options(); int socket_error = 0; - int use_begindir = directory_command_should_use_begindir(options, _addr, - or_port, router_purpose, indirection); + const char *begindir_reason = NULL; + /* Should the connection be to a relay's OR port (and inside that we will + * send our directory request)? */ + const int use_begindir = directory_command_should_use_begindir(options, + &or_addr_port->addr, or_addr_port->port, + router_purpose, indirection, + &begindir_reason); + /* Will the connection go via a three-hop Tor circuit? Note that this + * is separate from whether it will use_begindir. */ const int anonymized_connection = dirind_is_anon(indirection); - tor_addr_t addr; - tor_assert(address); - tor_assert(_addr); - tor_assert(or_port || dir_port); - tor_assert(digest); - - tor_addr_copy(&addr, _addr); + /* What is the address we want to make the directory request to? If + * we're making a begindir request this is the ORPort of the relay + * we're contacting; if not a begindir request, this is its DirPort. + * Note that if anonymized_connection is true, we won't be initiating + * a connection directly to this address. */ + tor_addr_t addr; + tor_addr_copy(&addr, &(use_begindir ? or_addr_port : dir_addr_port)->addr); + uint16_t port = (use_begindir ? or_addr_port : dir_addr_port)->port; log_debug(LD_DIR, "anonymized %d, use_begindir %d.", anonymized_connection, use_begindir); @@ -1001,6 +1142,14 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, (void)is_sensitive_dir_purpose; #endif + /* use encrypted begindir connections for everything except relays + * this provides better protection for directory fetches */ + if (!use_begindir && directory_must_use_begindir(options)) { + log_warn(LD_BUG, "Client could not use begindir connection: %s", + begindir_reason ? begindir_reason : "(NULL)"); + return; + } + /* ensure that we don't make direct connections when a SOCKS server is * configured. */ if (!anonymized_connection && !use_begindir && !options->HTTPProxy && @@ -1010,12 +1159,26 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, return; } + /* Make sure that the destination addr and port we picked is viable. */ + if (!port || tor_addr_is_null(&addr)) { + static int logged_backtrace = 0; + log_warn(LD_DIR, + "Cannot make an outgoing %sconnection without %sPort.", + use_begindir ? "begindir " : "", + use_begindir ? "an OR" : "a Dir"); + if (!logged_backtrace) { + log_backtrace(LOG_INFO, LD_BUG, "Address came from"); + logged_backtrace = 1; + } + return; + } + conn = dir_connection_new(tor_addr_family(&addr)); /* set up conn so it's got all the data we need to remember */ tor_addr_copy(&conn->base_.addr, &addr); - conn->base_.port = use_begindir ? or_port : dir_port; - conn->base_.address = tor_strdup(address); + conn->base_.port = port; + conn->base_.address = tor_dup_addr(&addr); memcpy(conn->identity_digest, digest, DIGEST_LEN); conn->base_.purpose = dir_purpose; @@ -1037,16 +1200,13 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, if (options->HTTPProxy) { tor_addr_copy(&addr, &options->HTTPProxyAddr); - dir_port = options->HTTPProxyPort; + port = options->HTTPProxyPort; } switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr, - dir_port, &socket_error)) { + port, &socket_error)) { case -1: - connection_dir_request_failed(conn); /* retry if we want */ - /* XXX we only pass 'conn' above, not 'resource', 'payload', - * etc. So in many situations it can't retry! -RD */ - connection_free(TO_CONN(conn)); + connection_mark_for_close(TO_CONN(conn)); return; case 1: /* start flushing conn */ @@ -1061,8 +1221,12 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, /* writable indicates finish, readable indicates broken link, error indicates broken link in windowsland. */ } - } else { /* we want to connect via a tor connection */ + } else { + /* We will use a Tor circuit (maybe 1-hop, maybe 3-hop, maybe with + * begindir, maybe not with begindir) */ + entry_connection_t *linked_conn; + /* Anonymized tunneled connections can never share a circuit. * One-hop directory connections can share circuits with each other * but nothing else. */ @@ -1084,7 +1248,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, conn->base_.address, conn->base_.port, digest, SESSION_GROUP_DIRCONN, iso_flags, - use_begindir, conn->dirconn_direct); + use_begindir, !anonymized_connection); if (!linked_conn) { log_warn(LD_NET,"Making tunnel to dirserver failed."); connection_mark_for_close(TO_CONN(conn)); @@ -1190,6 +1354,23 @@ directory_get_consensus_url(const char *resource) return url; } +/** + * Copies the ipv6 from source to destination, subject to buffer size limit + * size. If decorate is true, makes sure the copied address is decorated. + */ +static void +copy_ipv6_address(char* destination, const char* source, size_t len, + int decorate) { + tor_assert(destination); + tor_assert(source); + + if (decorate && source[0] != '[') { + tor_snprintf(destination, len, "[%s]", source); + } else { + strlcpy(destination, source, len); + } +} + /** Queue an appropriate HTTP command on conn-\>outbuf. The other args * are as in directory_initiate_command(). */ @@ -1201,6 +1382,9 @@ directory_send_command(dir_connection_t *conn, { char proxystring[256]; char hoststring[128]; + /* NEEDS to be the same size hoststring. + Will be decorated with brackets around it if it is ipv6. */ + char decorated_address[128]; smartlist_t *headers = smartlist_new(); char *url; char request[8192]; @@ -1213,12 +1397,20 @@ directory_send_command(dir_connection_t *conn, if (resource) conn->requested_resource = tor_strdup(resource); + /* decorate the ip address if it is ipv6 */ + if (strchr(conn->base_.address, ':')) { + copy_ipv6_address(decorated_address, conn->base_.address, + sizeof(decorated_address), 1); + } else { + strlcpy(decorated_address, conn->base_.address, sizeof(decorated_address)); + } + /* come up with a string for which Host: we want */ if (conn->base_.port == 80) { - strlcpy(hoststring, conn->base_.address, sizeof(hoststring)); + strlcpy(hoststring, decorated_address, sizeof(hoststring)); } else { - tor_snprintf(hoststring, sizeof(hoststring),"%s:%d", - conn->base_.address, conn->base_.port); + tor_snprintf(hoststring, sizeof(hoststring), "%s:%d", + decorated_address, conn->base_.port); } /* Format if-modified-since */ @@ -1250,11 +1442,6 @@ directory_send_command(dir_connection_t *conn, } switch (purpose) { - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: - tor_assert(resource); - httpcommand = "GET"; - tor_asprintf(&url, "/tor/status/%s", resource); - break; case DIR_PURPOSE_FETCH_CONSENSUS: /* resource is optional. If present, it's a flavor name */ tor_assert(!payload); @@ -1326,12 +1513,6 @@ directory_send_command(dir_connection_t *conn, httpcommand = "GET"; tor_asprintf(&url, "/tor/rendezvous2/%s", resource); break; - case DIR_PURPOSE_UPLOAD_RENDDESC: - tor_assert(!resource); - tor_assert(payload); - httpcommand = "POST"; - url = tor_strdup("/tor/rendezvous/publish"); - break; case DIR_PURPOSE_UPLOAD_RENDDESC_V2: tor_assert(!resource); tor_assert(payload); @@ -1343,7 +1524,8 @@ directory_send_command(dir_connection_t *conn, return; } - if (strlen(proxystring) + strlen(url) >= 4096) { + /* warn in the non-tunneled case */ + if (direct && (strlen(proxystring) + strlen(url) >= 4096)) { log_warn(LD_BUG, "Squid does not like URLs longer than 4095 bytes, and this " "one is %d bytes long: %s%s", @@ -1387,7 +1569,7 @@ directory_send_command(dir_connection_t *conn, * so it does. Return 0. * Otherwise, return -1. */ -static int +STATIC int parse_http_url(const char *headers, char **url) { char *s, *start, *tmp; @@ -1416,6 +1598,19 @@ parse_http_url(const char *headers, char **url) } } + /* Check if the header is well formed (next sequence + * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */ + { + unsigned minor_ver; + char ch; + char *e = (char *)eat_whitespace_no_nl(s); + if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) { + return -1; + } + if (ch != '\r') + return -1; + } + if (s-start < 5 || strcmpstart(start,"/tor/")) { /* need to rewrite it */ *url = tor_malloc(s - start + 5); strlcpy(*url,"/tor", s-start+5); @@ -1462,13 +1657,14 @@ http_set_address_origin(const char *headers, connection_t *conn) if (!fwd) fwd = http_get_header(headers, "X-Forwarded-For: "); if (fwd) { - struct in_addr in; - if (!tor_inet_aton(fwd, &in) || is_internal_IP(ntohl(in.s_addr), 0)) { - log_debug(LD_DIR, "Ignoring unrecognized or internal IP %s", - escaped(fwd)); + tor_addr_t toraddr; + if (tor_addr_parse(&toraddr,fwd) == -1 || + tor_addr_is_internal(&toraddr,0)) { + log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd)); tor_free(fwd); return; } + tor_free(conn->address); conn->address = tor_strdup(fwd); tor_free(fwd); @@ -1565,8 +1761,8 @@ parse_http_response(const char *headers, int *code, time_t *date, } /** Return true iff <b>body</b> doesn't start with a plausible router or - * running-list or directory opening. This is a sign of possible compression. - **/ + * network-status or microdescriptor opening. This is a sign of possible + * compression. */ static int body_is_plausible(const char *body, size_t len, int purpose) { @@ -1578,20 +1774,16 @@ body_is_plausible(const char *body, size_t len, int purpose) if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { return (!strcmpstart(body,"onion-key")); } - if (purpose != DIR_PURPOSE_FETCH_RENDDESC) { + if (1) { if (!strcmpstart(body,"router") || - !strcmpstart(body,"signed-directory") || - !strcmpstart(body,"network-status") || - !strcmpstart(body,"running-routers")) - return 1; + !strcmpstart(body,"network-status")) + return 1; for (i=0;i<32;++i) { if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i])) return 0; } - return 1; - } else { - return 1; } + return 1; } /** Called when we've just fetched a bunch of router descriptors in @@ -1626,8 +1818,9 @@ load_downloaded_routers(const char *body, smartlist_t *which, added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which, descriptor_digests, buf); - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); + if (added && general) + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); return added; } @@ -1646,17 +1839,17 @@ connection_dir_client_reached_eof(dir_connection_t *conn) char *body; char *headers; char *reason = NULL; - size_t body_len=0, orig_len=0; + size_t body_len = 0, orig_len = 0; int status_code; - time_t date_header=0; - long delta; + time_t date_header = 0; + long apparent_skew; compress_method_t compression; int plausible; - int skewed=0; + int skewed = 0; int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); - int was_compressed=0; + int was_compressed = 0; time_t now = time(NULL); int src_code; @@ -1708,28 +1901,15 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * and the date header. (We used to check now-date_header, but that's * inaccurate if we spend a lot of time downloading.) */ - delta = conn->base_.timestamp_lastwritten - date_header; - if (labs(delta)>ALLOW_DIRECTORY_TIME_SKEW) { - char dbuf[64]; + apparent_skew = conn->base_.timestamp_lastwritten - date_header; + if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) { int trusted = router_digest_is_trusted_dir(conn->identity_digest); - format_time_interval(dbuf, sizeof(dbuf), delta); - log_fn(trusted ? LOG_WARN : LOG_INFO, - LD_HTTP, - "Received directory with skewed time (server '%s:%d'): " - "It seems that our clock is %s by %s, or that theirs is %s. " - "Tor requires an accurate clock to work: please check your time, " - "timezone, and date settings.", - conn->base_.address, conn->base_.port, - delta>0 ? "ahead" : "behind", dbuf, - delta>0 ? "behind" : "ahead"); + clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP, + "directory", "DIRSERV"); skewed = 1; /* don't check the recommended-versions line */ - if (trusted) - control_event_general_status(LOG_WARN, - "CLOCK_SKEW SKEW=%ld SOURCE=DIRSERV:%s:%d", - delta, conn->base_.address, conn->base_.port); } else { log_debug(LD_HTTP, "Time on received directory is within tolerance; " - "we are %ld seconds skewed. (That's okay.)", delta); + "we are %ld seconds skewed. (That's okay.)", apparent_skew); } } (void) skewed; /* skewed isn't used yet. */ @@ -1810,77 +1990,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) { - smartlist_t *which = NULL; - v2_networkstatus_source_t source; - char *cp; - log_info(LD_DIR,"Received networkstatus objects (size %d) from server " - "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); - if (status_code != 200) { - static ratelim_t warning_limit = RATELIM_INIT(3600); - char *m; - if ((m = rate_limit_log(&warning_limit, now))) { - log_warn(LD_DIR, - "Received http status code %d (%s) from server " - "'%s:%d' while fetching \"/tor/status/%s\". " - "I'll try again soon.%s", - status_code, escaped(reason), conn->base_.address, - conn->base_.port, conn->requested_resource, m); - tor_free(m); - } - tor_free(body); tor_free(headers); tor_free(reason); - connection_dir_download_v2_networkstatus_failed(conn, status_code); - return -1; - } - if (conn->requested_resource && - !strcmpstart(conn->requested_resource,"fp/")) { - source = NS_FROM_DIR_BY_FP; - which = smartlist_new(); - dir_split_resource_into_fingerprints(conn->requested_resource+3, - which, NULL, 0); - } else if (conn->requested_resource && - !strcmpstart(conn->requested_resource, "all")) { - source = NS_FROM_DIR_ALL; - which = smartlist_new(); - SMARTLIST_FOREACH(router_get_trusted_dir_servers(), - dir_server_t *, ds, - { - char *hex = tor_malloc(HEX_DIGEST_LEN+1); - base16_encode(hex, HEX_DIGEST_LEN+1, ds->digest, DIGEST_LEN); - smartlist_add(which, hex); - }); - } else { - /* XXXX Can we even end up here? -- weasel*/ - source = NS_FROM_DIR_BY_FP; - log_warn(LD_BUG, "We received a networkstatus but we didn't ask " - "for it by fp, nor did we ask for all."); - } - cp = body; - while (*cp) { - char *next = strstr(cp, "\nnetwork-status-version"); - if (next) - next[1] = '\0'; - /* learn from it, and then remove it from 'which' */ - if (router_set_networkstatus_v2(cp, now, source, which)<0) - break; - if (next) { - next[1] = 'n'; - cp = next+1; - } else - break; - } - /* launches router downloads as needed */ - routers_update_all_from_networkstatus(now, 2); - directory_info_has_arrived(now, 0); - if (which) { - if (smartlist_len(which)) { - dir_networkstatus_download_failed(which, status_code); - } - SMARTLIST_FOREACH(which, char *, s, tor_free(s)); - smartlist_free(which); - } - } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) { int r; const char *flavname = conn->requested_resource; @@ -1906,11 +2015,15 @@ connection_dir_client_reached_eof(dir_connection_t *conn) networkstatus_consensus_download_failed(0, flavname); return -1; } + + /* If we launched other fetches for this consensus, cancel them. */ + connection_dir_close_consensus_fetches(conn, flavname); + /* launches router downloads as needed */ routers_update_all_from_networkstatus(now, 3); update_microdescs_from_networkstatus(now); update_microdesc_downloads(now); - directory_info_has_arrived(now, 0); + directory_info_has_arrived(now, 0, 0); log_info(LD_DIR, "Successfully loaded consensus."); } @@ -1946,7 +2059,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * ones got flushed to disk so it's safe to call this on them */ connection_dir_download_cert_failed(conn, status_code); } else { - directory_info_has_arrived(now, 0); + directory_info_has_arrived(now, 0, 0); log_info(LD_DIR, "Successfully loaded certificates from fetch."); } } else { @@ -2060,7 +2173,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) if (load_downloaded_routers(body, which, descriptor_digests, conn->router_purpose, conn->base_.address)) - directory_info_has_arrived(now, 0); + directory_info_has_arrived(now, 0, 0); } } if (which) { /* mark remaining ones as failed */ @@ -2111,8 +2224,11 @@ connection_dir_client_reached_eof(dir_connection_t *conn) /* Mark remaining ones as failed. */ dir_microdesc_download_failed(which, status_code); } - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); + if (mds && smartlist_len(mds)) { + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); + directory_info_has_arrived(now, 0, 1); + } SMARTLIST_FOREACH(which, char *, cp, tor_free(cp)); smartlist_free(which); smartlist_free(mds); @@ -2220,87 +2336,70 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * dirservers down just because they don't like us. */ } - if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC) { + if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { + #define SEND_HS_DESC_FAILED_EVENT(reason) ( \ + control_event_hs_descriptor_failed(conn->rend_data, \ + conn->identity_digest, \ + reason) ) + #define SEND_HS_DESC_FAILED_CONTENT() ( \ + control_event_hs_descriptor_content(conn->rend_data->onion_address, \ + conn->requested_resource, \ + conn->identity_digest, \ + NULL) ) tor_assert(conn->rend_data); log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d " "(%s))", (int)body_len, status_code, escaped(reason)); switch (status_code) { case 200: - if (rend_cache_store(body, body_len, 0, - conn->rend_data->onion_address) < -1) { - log_warn(LD_REND,"Failed to parse rendezvous descriptor."); - /* Any pending rendezvous attempts will notice when - * connection_about_to_close_connection() + { + rend_cache_entry_t *entry = NULL; + + if (rend_cache_store_v2_desc_as_client(body, + conn->requested_resource, conn->rend_data, &entry) < 0) { + log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " + "Retrying at another directory."); + /* We'll retry when connection_about_to_close_connection() * cleans this dir conn up. */ - /* We could retry. But since v0 descriptors are going out of - * style, it isn't worth the hassle. We'll do better in v2. */ + SEND_HS_DESC_FAILED_EVENT("BAD_DESC"); + SEND_HS_DESC_FAILED_CONTENT(); } else { - /* Success, or at least there's a v2 descriptor already - * present. Notify pending connections about this. */ - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address); - } - break; - case 404: - /* Not there. Pending connections will be notified when - * connection_about_to_close_connection() cleans this conn up. */ - break; - case 400: - log_warn(LD_REND, - "http status 400 (%s). Dirserver didn't like our " - "rendezvous query?", escaped(reason)); - break; - default: - log_warn(LD_REND,"http status %d (%s) response unexpected while " - "fetching hidden service descriptor (server '%s:%d').", - status_code, escaped(reason), conn->base_.address, - conn->base_.port); - break; - } - } - - if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { - tor_assert(conn->rend_data); - log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d " - "(%s))", - (int)body_len, status_code, escaped(reason)); - switch (status_code) { - case 200: - switch (rend_cache_store_v2_desc_as_client(body, conn->rend_data)) { - case -2: - log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " - "Retrying at another directory."); - /* We'll retry when connection_about_to_close_connection() - * cleans this dir conn up. */ - break; - case -1: - /* We already have a v0 descriptor here. Ignoring this one - * and _not_ performing another request. */ - log_info(LD_REND, "Successfully fetched v2 rendezvous " - "descriptor, but we already have a v0 descriptor."); - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - break; - default: - /* success. notify pending connections about this. */ - log_info(LD_REND, "Successfully fetched v2 rendezvous " - "descriptor."); - conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC; - rend_client_desc_trynow(conn->rend_data->onion_address); - break; + char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; + /* Should never be NULL here if we found the descriptor. */ + tor_assert(entry); + rend_get_service_id(entry->parsed->pk, service_id); + + /* success. notify pending connections about this. */ + log_info(LD_REND, "Successfully fetched v2 rendezvous " + "descriptor."); + control_event_hs_descriptor_received(service_id, + conn->rend_data, + conn->identity_digest); + control_event_hs_descriptor_content(service_id, + conn->requested_resource, + conn->identity_digest, + body); + conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2; + rend_client_desc_trynow(service_id); + memwipe(service_id, 0, sizeof(service_id)); } break; + } case 404: /* Not there. We'll retry when * connection_about_to_close_connection() cleans this conn up. */ log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " "Retrying at another directory."); + SEND_HS_DESC_FAILED_EVENT("NOT_FOUND"); + SEND_HS_DESC_FAILED_CONTENT(); break; case 400: log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " "http status 400 (%s). Dirserver didn't like our " "v2 rendezvous query? Retrying at another directory.", escaped(reason)); + SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED"); + SEND_HS_DESC_FAILED_CONTENT(); break; default: log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " @@ -2309,31 +2408,45 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "Retrying at another directory.", status_code, escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_FAILED_EVENT("UNEXPECTED"); + SEND_HS_DESC_FAILED_CONTENT(); break; } } - if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC || - conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) { + if (conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2) { + #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason) ( \ + control_event_hs_descriptor_upload_failed( \ + conn->identity_digest, \ + conn->rend_data->onion_address, \ + reason) ) log_info(LD_REND,"Uploaded rendezvous descriptor (status %d " "(%s))", status_code, escaped(reason)); + /* Without the rend data, we'll have a problem identifying what has been + * uploaded for which service. */ + tor_assert(conn->rend_data); switch (status_code) { case 200: log_info(LD_REND, "Uploading rendezvous descriptor: finished with status " "200 (%s)", escaped(reason)); + control_event_hs_descriptor_uploaded(conn->identity_digest, + conn->rend_data->onion_address); + rend_service_desc_has_uploaded(conn->rend_data); break; case 400: log_warn(LD_REND,"http status 400 (%s) response from dirserver " "'%s:%d'. Malformed rendezvous descriptor?", escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED"); break; default: log_warn(LD_REND,"http status %d (%s) response unexpected (server " "'%s:%d').", status_code, escaped(reason), conn->base_.address, conn->base_.port); + SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED"); break; } } @@ -2368,12 +2481,15 @@ connection_dir_reached_eof(dir_connection_t *conn) */ #define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20)) +#define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5) + /** Read handler for directory connections. (That's connections <em>to</em> * directory servers and connections <em>at</em> directory servers.) */ int connection_dir_process_inbuf(dir_connection_t *conn) { + size_t max_size; tor_assert(conn); tor_assert(conn->base_.type == CONN_TYPE_DIR); @@ -2392,9 +2508,15 @@ connection_dir_process_inbuf(dir_connection_t *conn) return 0; } - if (connection_get_inbuf_len(TO_CONN(conn)) > MAX_DIRECTORY_OBJECT_SIZE) { - log_warn(LD_HTTP, "Too much data received from directory connection: " - "denial of service attempt, or you need to upgrade?"); + max_size = + (TO_CONN(conn)->purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) ? + MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE; + + if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) { + log_warn(LD_HTTP, + "Too much data received from directory connection (%s): " + "denial of service attempt, or you need to upgrade?", + conn->base_.address); connection_mark_for_close(TO_CONN(conn)); return -1; } @@ -2418,7 +2540,7 @@ connection_dir_about_to_close(dir_connection_t *dir_conn) } /* If we were trying to fetch a v2 rend desc and did not succeed, * retry as needed. (If a fetch is successful, the connection state - * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC to mark that + * is changed to DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 to mark that * refetching is unnecessary.) */ if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 && dir_conn->rend_data && @@ -2439,6 +2561,7 @@ write_http_status_line(dir_connection_t *conn, int status, log_warn(LD_BUG,"status line too long."); return; } + log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase); connection_write_to_buf(buf, strlen(buf), TO_CONN(conn)); } @@ -2492,7 +2615,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, } if (cache_lifetime > 0) { char expbuf[RFC1123_TIME_LEN+1]; - format_rfc1123_time(expbuf, now + cache_lifetime); + format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime)); /* We could say 'Cache-control: max-age=%d' here if we start doing * http/1.1 */ tor_snprintf(cp, sizeof(tmp)-(cp-tmp), @@ -2549,7 +2672,6 @@ note_client_request(int purpose, int compressed, size_t bytes) char *key; const char *kind = NULL; switch (purpose) { - case DIR_PURPOSE_FETCH_V2_NETWORKSTATUS: kind = "dl/status"; break; case DIR_PURPOSE_FETCH_CONSENSUS: kind = "dl/consensus"; break; case DIR_PURPOSE_FETCH_CERTIFICATE: kind = "dl/cert"; break; case DIR_PURPOSE_FETCH_STATUS_VOTE: kind = "dl/vote"; break; @@ -2560,9 +2682,7 @@ note_client_request(int purpose, int compressed, size_t bytes) case DIR_PURPOSE_UPLOAD_DIR: kind = "dl/ul-dir"; break; case DIR_PURPOSE_UPLOAD_VOTE: kind = "dl/ul-vote"; break; case DIR_PURPOSE_UPLOAD_SIGNATURES: kind = "dl/ul-sig"; break; - case DIR_PURPOSE_FETCH_RENDDESC: kind = "dl/rend"; break; case DIR_PURPOSE_FETCH_RENDDESC_V2: kind = "dl/rend2"; break; - case DIR_PURPOSE_UPLOAD_RENDDESC: kind = "dl/ul-rend"; break; case DIR_PURPOSE_UPLOAD_RENDDESC_V2: kind = "dl/ul-rend2"; break; } if (kind) { @@ -2685,7 +2805,7 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) < 0) { log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Failed to decode requested authority digest %s.", d); + "Failed to decode requested authority digest %s.", escaped(d)); continue; }; @@ -2707,12 +2827,30 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) return (have >= need_at_least); } +/** Return the compression level we should use for sending a compressed + * response of size <b>n_bytes</b>. */ +STATIC zlib_compression_level_t +choose_compression_level(ssize_t n_bytes) +{ + if (! have_been_under_memory_pressure()) { + return HIGH_COMPRESSION; /* we have plenty of RAM. */ + } else if (n_bytes < 0) { + return HIGH_COMPRESSION; /* unknown; might be big. */ + } else if (n_bytes < 1024) { + return LOW_COMPRESSION; + } else if (n_bytes < 2048) { + return MEDIUM_COMPRESSION; + } else { + return HIGH_COMPRESSION; + } +} + /** Helper function: called when a dirserver gets a complete HTTP GET * request. Look for a request for a directory or for a rendezvous * service descriptor. On finding one, write a response into * conn-\>outbuf. If the request is unrecognized, send a 400. * Always return 0. */ -static int +STATIC int directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *req_body, size_t req_body_len) { @@ -2738,14 +2876,17 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if ((header = http_get_header(headers, "If-Modified-Since: "))) { struct tm tm; if (parse_http_time(header, &tm) == 0) { - if (tor_timegm(&tm, &if_modified_since)<0) + if (tor_timegm(&tm, &if_modified_since)<0) { if_modified_since = 0; + } else { + log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header)); + } } /* The correct behavior on a malformed If-Modified-Since header is to * act as if no If-Modified-Since header had been given. */ tor_free(header); } - log_debug(LD_DIRSERV,"rewritten url as '%s'.", url); + log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url)); url_mem = url; url_len = strlen(url); @@ -2774,109 +2915,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* if no disclaimer file, fall through and continue */ } - if (!strcmp(url,"/tor/") || !strcmp(url,"/tor/dir")) { /* v1 dir fetch */ - cached_dir_t *d = dirserv_get_directory(); - - if (!d) { - log_info(LD_DIRSERV,"Client asked for the mirrored directory, but we " - "don't have a good one yet. Sending 503 Dir not available."); - write_http_status_line(conn, 503, "Directory unavailable"); - goto done; - } - if (d->published < if_modified_since) { - write_http_status_line(conn, 304, "Not modified"); - goto done; - } - - dlen = compressed ? d->dir_z_len : d->dir_len; - - if (global_write_bucket_low(TO_CONN(conn), dlen, 1)) { - log_debug(LD_DIRSERV, - "Client asked for the mirrored directory, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - goto done; - } - - note_request(url, dlen); - - log_debug(LD_DIRSERV,"Dumping %sdirectory to client.", - compressed?"compressed ":""); - write_http_response_header(conn, dlen, compressed, - FULL_DIR_CACHE_LIFETIME); - conn->cached_dir = d; - conn->cached_dir_offset = 0; - if (!compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD); - ++d->refcnt; - - /* Prime the connection with some data. */ - conn->dir_spool_src = DIR_SPOOL_CACHED_DIR; - connection_dirserv_flushed_some(conn); - goto done; - } - - if (!strcmp(url,"/tor/running-routers")) { /* running-routers fetch */ - cached_dir_t *d = dirserv_get_runningrouters(); - if (!d) { - write_http_status_line(conn, 503, "Directory unavailable"); - goto done; - } - if (d->published < if_modified_since) { - write_http_status_line(conn, 304, "Not modified"); - goto done; - } - dlen = compressed ? d->dir_z_len : d->dir_len; - - if (global_write_bucket_low(TO_CONN(conn), dlen, 1)) { - log_info(LD_DIRSERV, - "Client asked for running-routers, but we've been " - "writing too many bytes lately. Sending 503 Dir busy."); - write_http_status_line(conn, 503, "Directory busy, try again later"); - goto done; - } - note_request(url, dlen); - write_http_response_header(conn, dlen, compressed, - RUNNINGROUTERS_CACHE_LIFETIME); - connection_write_to_buf(compressed ? d->dir_z : d->dir, dlen, - TO_CONN(conn)); - goto done; - } - - if (!strcmpstart(url,"/tor/status/") - || !strcmpstart(url, "/tor/status-vote/current/consensus")) { - /* v2 or v3 network status fetch. */ + if (!strcmpstart(url, "/tor/status-vote/current/consensus")) { + /* v3 network status fetch. */ smartlist_t *dir_fps = smartlist_new(); - int is_v3 = !strcmpstart(url, "/tor/status-vote"); const char *request_type = NULL; - const char *key = url + strlen("/tor/status/"); long lifetime = NETWORKSTATUS_CACHE_LIFETIME; - if (options->DisableV2DirectoryInfo_ && !is_v3) { - static ratelim_t reject_v2_ratelim = RATELIM_INIT(1800); - char *m; - write_http_status_line(conn, 404, "Not found"); - smartlist_free(dir_fps); - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); - if ((m = rate_limit_log(&reject_v2_ratelim, approx_time()))) { - log_notice(LD_DIR, "Rejected a v2 networkstatus request.%s", m); - tor_free(m); - } - goto done; - } - - if (!is_v3) { - dirserv_get_networkstatus_v2_fingerprints(dir_fps, key); - if (!strcmpstart(key, "fp/")) - request_type = compressed?"/tor/status/fp.z":"/tor/status/fp"; - else if (!strcmpstart(key, "authority")) - request_type = compressed?"/tor/status/authority.z": - "/tor/status/authority"; - else if (!strcmpstart(key, "all")) - request_type = compressed?"/tor/status/all.z":"/tor/status/all"; - else - request_type = "/tor/status/?"; - } else { + if (1) { networkstatus_t *v; time_t now = time(NULL); const char *want_fps = NULL; @@ -2929,8 +2974,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */ write_http_status_line(conn, 503, "Network status object unavailable"); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); + geoip_note_ns_response(GEOIP_REJECT_UNAVAILABLE); goto done; } @@ -2938,15 +2982,13 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 404, "Not found"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); goto done; } else if (!smartlist_len(dir_fps)) { write_http_status_line(conn, 304, "Not modified"); SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp)); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); + geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED); goto done; } @@ -2958,17 +3000,17 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 503, "Directory busy, try again later"); SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp)); smartlist_free(dir_fps); - if (is_v3) - geoip_note_ns_response(GEOIP_REJECT_BUSY); + + geoip_note_ns_response(GEOIP_REJECT_BUSY); goto done; } - if (is_v3) { - struct in_addr in; + if (1) { tor_addr_t addr; - if (tor_inet_aton((TO_CONN(conn))->address, &in)) { - tor_addr_from_ipv4h(&addr, ntohl(in.s_addr)); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, time(NULL)); + if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) { + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, + &addr, NULL, + time(NULL)); geoip_note_ns_response(GEOIP_SUCCESS); /* Note that a request for a network status has started, so that we * can measure the download time later on. */ @@ -2986,7 +3028,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, smartlist_len(dir_fps) == 1 ? lifetime : 0); conn->fingerprint_stack = dir_fps; if (! compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); /* Prime the connection with some data. */ conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS; @@ -3066,7 +3108,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, }); if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) { - write_http_status_line(conn, 503, "Directory busy, try again later."); + write_http_status_line(conn, 503, "Directory busy, try again later"); goto vote_done; } write_http_response_header(conn, body_len ? body_len : -1, compressed, @@ -3074,7 +3116,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (smartlist_len(items)) { if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf_zlib(c, strlen(c), conn, 0)); connection_write_to_buf_zlib("", 0, conn, 1); @@ -3123,7 +3166,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, conn->fingerprint_stack = fps; if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(dlen)); connection_dirserv_flushed_some(conn); goto done; @@ -3191,7 +3235,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } write_http_response_header(conn, -1, compressed, cache_lifetime); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(dlen)); /* Prime the connection with some data. */ connection_dirserv_flushed_some(conn); } @@ -3260,13 +3305,14 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, len += c->cache_info.signed_descriptor_len); if (global_write_bucket_low(TO_CONN(conn), compressed?len/2:len, 2)) { - write_http_status_line(conn, 503, "Directory busy, try again later."); + write_http_status_line(conn, 503, "Directory busy, try again later"); goto keys_done; } write_http_response_header(conn, compressed?-1:len, compressed, 60*60); if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, c->cache_info.signed_descriptor_len, @@ -3283,15 +3329,14 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - if (options->HidServDirectoryV2 && - connection_dir_is_encrypted(conn) && + if (connection_dir_is_encrypted(conn) && !strcmpstart(url,"/tor/rendezvous2/")) { /* Handle v2 rendezvous descriptor fetch request. */ const char *descp; const char *query = url + strlen("/tor/rendezvous2/"); - if (strlen(query) == REND_DESC_ID_V2_LEN_BASE32) { + if (rend_valid_descriptor_id(query)) { log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'", - safe_str(query)); + safe_str(escaped(query))); switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) { case 1: /* valid */ write_http_response_header(conn, strlen(descp), 0, 0); @@ -3310,32 +3355,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - if (options->HSAuthoritativeDir && !strcmpstart(url,"/tor/rendezvous/")) { - /* rendezvous descriptor fetch */ - const char *descp; - size_t desc_len; - const char *query = url+strlen("/tor/rendezvous/"); - - log_info(LD_REND, "Handling rendezvous descriptor get"); - switch (rend_cache_lookup_desc(query, 0, &descp, &desc_len)) { - case 1: /* valid */ - write_http_response_header_impl(conn, desc_len, - "application/octet-stream", - NULL, NULL, 0); - note_request("/tor/rendezvous?/", desc_len); - /* need to send descp separately, because it may include NULs */ - connection_write_to_buf(descp, desc_len, TO_CONN(conn)); - break; - case 0: /* well-formed but not present */ - write_http_status_line(conn, 404, "Not found"); - break; - case -1: /* not well-formed */ - write_http_status_line(conn, 400, "Bad request"); - break; - } - goto done; - } - if (options->BridgeAuthoritativeDir && options->BridgePassword_AuthDigest_ && connection_dir_is_encrypted(conn) && @@ -3384,22 +3403,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - if (!strcmp(url,"/tor/dbg-stability.txt")) { - const char *stability; - size_t len; - if (options->BridgeAuthoritativeDir || - ! authdir_mode_tests_reachability(options) || - ! (stability = rep_hist_get_router_stability_doc(time(NULL)))) { - write_http_status_line(conn, 404, "Not found."); - goto done; - } - - len = strlen(stability); - write_http_response_header(conn, len, 0, 0); - connection_write_to_buf(stability, len, TO_CONN(conn)); - goto done; - } - #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) #define ADD_MALLINFO_LINE(x) do { \ smartlist_add_asprintf(lines, "%s %d\n", #x, mi.x); \ @@ -3463,33 +3466,30 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, conn->base_.state = DIR_CONN_STATE_SERVER_WRITING; + if (!public_server_mode(options)) { + log_info(LD_DIR, "Rejected dir post request from %s " + "since we're not a public relay.", conn->base_.address); + write_http_status_line(conn, 503, "Not acting as a public relay"); + goto done; + } + if (parse_http_url(headers, &url) < 0) { write_http_status_line(conn, 400, "Bad request"); return 0; } - log_debug(LD_DIRSERV,"rewritten url as '%s'.", url); + log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url)); /* Handle v2 rendezvous service publish request. */ - if (options->HidServDirectoryV2 && - connection_dir_is_encrypted(conn) && + if (connection_dir_is_encrypted(conn) && !strcmpstart(url,"/tor/rendezvous2/publish")) { - switch (rend_cache_store_v2_desc_as_dir(body)) { - case -2: - log_info(LD_REND, "Rejected v2 rend descriptor (length %d) from %s " - "since we're not currently a hidden service directory.", - (int)body_len, conn->base_.address); - write_http_status_line(conn, 503, "Currently not acting as v2 " - "hidden service directory"); - break; - case -1: - log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.", - (int)body_len, conn->base_.address); - write_http_status_line(conn, 400, - "Invalid v2 service descriptor rejected"); - break; - default: - write_http_status_line(conn, 200, "Service descriptor (v2) stored"); - log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted"); + if (rend_cache_store_v2_desc_as_dir(body) < 0) { + log_warn(LD_REND, "Rejected v2 rend descriptor (length %d) from %s.", + (int)body_len, conn->base_.address); + write_http_status_line(conn, 400, + "Invalid v2 service descriptor rejected"); + } else { + write_http_status_line(conn, 200, "Service descriptor (v2) stored"); + log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted"); } goto done; } @@ -3510,8 +3510,6 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose, conn->base_.address, &msg); tor_assert(msg); - if (WRA_WAS_ADDED(r)) - dirserv_get_directory(); /* rebuild and write to disk */ if (r == ROUTER_ADDED_NOTIFY_GENERATOR) { /* Accepted with a message. */ @@ -3535,22 +3533,6 @@ directory_handle_command_post(dir_connection_t *conn, const char *headers, goto done; } - if (options->HSAuthoritativeDir && - !strcmpstart(url,"/tor/rendezvous/publish")) { - /* rendezvous descriptor post */ - log_info(LD_REND, "Handling rendezvous descriptor post."); - if (rend_cache_store(body, body_len, 1, NULL) < 0) { - log_fn(LOG_PROTOCOL_WARN, LD_DIRSERV, - "Rejected rend descriptor (length %d) from %s.", - (int)body_len, conn->base_.address); - write_http_status_line(conn, 400, - "Invalid v0 service descriptor rejected"); - } else { - write_http_status_line(conn, 200, "Service descriptor (v0) stored"); - } - goto done; - } - if (authdir_mode_v3(options) && !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */ const char *msg = "OK"; @@ -3617,7 +3599,9 @@ directory_handle_command(dir_connection_t *conn) } http_set_address_origin(headers, TO_CONN(conn)); - //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, body); + // we should escape headers here as well, + // but we can't call escaped() twice, as it uses the same buffer + //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body)); if (!strncasecmp(headers,"GET",3)) r = directory_handle_command_get(conn, headers, body, body_len); @@ -3645,7 +3629,7 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn->base_.type == CONN_TYPE_DIR); /* Note that we have finished writing the directory response. For direct - * connections this means we're done, for tunneled connections its only + * connections this means we're done; for tunneled connections it's only * an intermediate step. */ if (conn->dirreq_id) geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED, @@ -3686,8 +3670,38 @@ connection_dir_finished_flushing(dir_connection_t *conn) return 0; } +/* We just got a new consensus! If there are other in-progress requests + * for this consensus flavor (for example because we launched several in + * parallel), cancel them. + * + * We do this check here (not just in + * connection_ap_handshake_attach_circuit()) to handle the edge case where + * a consensus fetch begins and ends before some other one tries to attach to + * a circuit, in which case the other one won't know that we're all happy now. + * + * Don't mark the conn that just gave us the consensus -- otherwise we + * would end up double-marking it when it cleans itself up. + */ +static void +connection_dir_close_consensus_fetches(dir_connection_t *except_this_one, + const char *resource) +{ + smartlist_t *conns_to_close = + connection_dir_list_by_purpose_and_resource(DIR_PURPOSE_FETCH_CONSENSUS, + resource); + SMARTLIST_FOREACH_BEGIN(conns_to_close, dir_connection_t *, d) { + if (d == except_this_one) + continue; + log_info(LD_DIR, "Closing consensus fetch (to %s) since one " + "has just arrived.", TO_CONN(d)->address); + connection_mark_for_close(TO_CONN(d)); + } SMARTLIST_FOREACH_END(d); + smartlist_free(conns_to_close); +} + /** Connected handler for directory connections: begin sending data to the - * server */ + * server, and return 0. + * Only used when connections don't immediately connect. */ int connection_dir_finished_connecting(dir_connection_t *conn) { @@ -3698,138 +3712,229 @@ connection_dir_finished_connecting(dir_connection_t *conn) log_debug(LD_HTTP,"Dir connection to router %s:%u established.", conn->base_.address,conn->base_.port); - conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; /* start flushing conn */ + /* start flushing conn */ + conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING; return 0; } -/** Called when one or more networkstatus fetches have failed (with uppercase - * fingerprints listed in <b>failed</b>). Mark those fingerprints as having - * failed once, unless they failed with status code 503. */ -static void -dir_networkstatus_download_failed(smartlist_t *failed, int status_code) -{ - if (status_code == 503) - return; - SMARTLIST_FOREACH_BEGIN(failed, const char *, fp) { - char digest[DIGEST_LEN]; - dir_server_t *dir; - if (base16_decode(digest, DIGEST_LEN, fp, strlen(fp))<0) { - log_warn(LD_BUG, "Called with bad fingerprint in list: %s", - escaped(fp)); - continue; - } - dir = router_get_fallback_dirserver_by_digest(digest); - - if (dir) - download_status_failed(&dir->v2_ns_dl_status, status_code); - } SMARTLIST_FOREACH_END(fp); -} - -/** Schedule for when servers should download things in general. */ -static const int server_dl_schedule[] = { - 0, 0, 0, 60, 60, 60*2, 60*5, 60*15, INT_MAX -}; -/** Schedule for when clients should download things in general. */ -static const int client_dl_schedule[] = { - 0, 0, 60, 60*5, 60*10, INT_MAX -}; -/** Schedule for when servers should download consensuses. */ -static const int server_consensus_dl_schedule[] = { - 0, 0, 60, 60*5, 60*10, 60*30, 60*30, 60*30, 60*30, 60*30, 60*60, 60*60*2 -}; -/** Schedule for when clients should download consensuses. */ -static const int client_consensus_dl_schedule[] = { - 0, 0, 60, 60*5, 60*10, 60*30, 60*60, 60*60, 60*60, 60*60*3, 60*60*6, 60*60*12 -}; -/** Schedule for when clients should download bridge descriptors. */ -static const int bridge_dl_schedule[] = { - 60*60, 15*60, 15*60, 60*60 -}; - -/** Decide which download schedule we want to use, and then return a - * pointer to it along with a pointer to its length. Helper function for - * download_status_increment_failure() and download_status_reset(). */ -static void -find_dl_schedule_and_len(download_status_t *dls, int server, - const int **schedule, size_t *schedule_len) +/** Decide which download schedule we want to use based on descriptor type + * in <b>dls</b> and <b>options</b>. + * Then return a list of int pointers defining download delays in seconds. + * Helper function for download_status_increment_failure(), + * download_status_reset(), and download_status_increment_attempt(). */ +STATIC const smartlist_t * +find_dl_schedule(download_status_t *dls, const or_options_t *options) { + const int dir_server = dir_server_mode(options); + const int multi_d = networkstatus_consensus_can_use_multiple_directories( + options); + const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping( + time(NULL)); + const int use_fallbacks = networkstatus_consensus_can_use_extra_fallbacks( + options); switch (dls->schedule) { case DL_SCHED_GENERIC: - if (server) { - *schedule = server_dl_schedule; - *schedule_len = sizeof(server_dl_schedule)/sizeof(int); + if (dir_server) { + return options->TestingServerDownloadSchedule; } else { - *schedule = client_dl_schedule; - *schedule_len = sizeof(client_dl_schedule)/sizeof(int); + return options->TestingClientDownloadSchedule; } - break; case DL_SCHED_CONSENSUS: - if (server) { - *schedule = server_consensus_dl_schedule; - *schedule_len = sizeof(server_consensus_dl_schedule)/sizeof(int); + if (!multi_d) { + return options->TestingServerConsensusDownloadSchedule; } else { - *schedule = client_consensus_dl_schedule; - *schedule_len = sizeof(client_consensus_dl_schedule)/sizeof(int); + if (we_are_bootstrapping) { + if (!use_fallbacks) { + /* A bootstrapping client without extra fallback directories */ + return + options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule; + } else if (dls->want_authority) { + /* A bootstrapping client with extra fallback directories, but + * connecting to an authority */ + return + options->ClientBootstrapConsensusAuthorityDownloadSchedule; + } else { + /* A bootstrapping client connecting to extra fallback directories + */ + return + options->ClientBootstrapConsensusFallbackDownloadSchedule; + } + } else { + return options->TestingClientConsensusDownloadSchedule; + } } - break; case DL_SCHED_BRIDGE: - *schedule = bridge_dl_schedule; - *schedule_len = sizeof(bridge_dl_schedule)/sizeof(int); - break; + return options->TestingBridgeDownloadSchedule; default: tor_assert(0); } + + /* Impossible, but gcc will fail with -Werror without a `return`. */ + return NULL; } -/** Called when an attempt to download <b>dls</b> has failed with HTTP status +/* Find the current delay for dls based on schedule. + * Set dls->next_attempt_at based on now, and return the delay. + * Helper for download_status_increment_failure and + * download_status_increment_attempt. */ +STATIC int +download_status_schedule_get_delay(download_status_t *dls, + const smartlist_t *schedule, + time_t now) +{ + tor_assert(dls); + tor_assert(schedule); + + int delay = INT_MAX; + uint8_t dls_schedule_position = (dls->increment_on + == DL_SCHED_INCREMENT_ATTEMPT + ? dls->n_download_attempts + : dls->n_download_failures); + + if (dls_schedule_position < smartlist_len(schedule)) + delay = *(int *)smartlist_get(schedule, dls_schedule_position); + else if (dls_schedule_position == IMPOSSIBLE_TO_DOWNLOAD) + delay = INT_MAX; + else + delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1); + + /* A negative delay makes no sense. Knowing that delay is + * non-negative allows us to safely do the wrapping check below. */ + tor_assert(delay >= 0); + + /* Avoid now+delay overflowing INT_MAX, by comparing with a subtraction + * that won't overflow (since delay is non-negative). */ + if (delay < INT_MAX && now <= INT_MAX - delay) { + dls->next_attempt_at = now+delay; + } else { + dls->next_attempt_at = TIME_MAX; + } + + return delay; +} + +/* Log a debug message about item, which increments on increment_action, has + * incremented dls_n_download_increments times. The message varies based on + * was_schedule_incremented (if not, not_incremented_response is logged), and + * the values of increment, dls_next_attempt_at, and now. + * Helper for download_status_increment_failure and + * download_status_increment_attempt. */ +static void +download_status_log_helper(const char *item, int was_schedule_incremented, + const char *increment_action, + const char *not_incremented_response, + uint8_t dls_n_download_increments, int increment, + time_t dls_next_attempt_at, time_t now) +{ + if (item) { + if (!was_schedule_incremented) + log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.", + item, increment_action, (int)dls_n_download_increments, + not_incremented_response); + else if (increment == 0) + log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.", + item, increment_action, (int)dls_n_download_increments); + else if (dls_next_attempt_at < TIME_MAX) + log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.", + item, increment_action, (int)dls_n_download_increments, + (int)(dls_next_attempt_at-now)); + else + log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.", + item, increment_action, (int)dls_n_download_increments); + } +} + +/** Determine when a failed download attempt should be retried. + * Called when an attempt to download <b>dls</b> has failed with HTTP status * <b>status_code</b>. Increment the failure count (if the code indicates a - * real failure) and set <b>dls</b>-\>next_attempt_at to an appropriate time - * in the future. */ + * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to + * an appropriate time in the future and return it. + * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the + * failure count, and return a time in the far future for the next attempt (to + * avoid an immediate retry). */ time_t download_status_increment_failure(download_status_t *dls, int status_code, const char *item, int server, time_t now) { - const int *schedule; - size_t schedule_len; - int increment; + int increment = -1; tor_assert(dls); + + /* only count the failure if it's permanent, or we're a server */ if (status_code != 503 || server) { if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) ++dls->n_download_failures; } - find_dl_schedule_and_len(dls, server, &schedule, &schedule_len); + if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { + /* We don't find out that a failure-based schedule has attempted a + * connection until that connection fails. + * We'll never find out about successful connections, but this doesn't + * matter, because schedules are reset after a successful download. + */ + if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1) + ++dls->n_download_attempts; - if (dls->n_download_failures < schedule_len) - increment = schedule[dls->n_download_failures]; - else if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD) - increment = INT_MAX; - else - increment = schedule[schedule_len-1]; + /* only return a failure retry time if this schedule increments on failures + */ + const smartlist_t *schedule = find_dl_schedule(dls, get_options()); + increment = download_status_schedule_get_delay(dls, schedule, now); + } - if (increment < INT_MAX) - dls->next_attempt_at = now+increment; - else - dls->next_attempt_at = TIME_MAX; + download_status_log_helper(item, !dls->increment_on, "failed", + "concurrently", dls->n_download_failures, + increment, dls->next_attempt_at, now); - if (item) { - if (increment == 0) - log_debug(LD_DIR, "%s failed %d time(s); I'll try again immediately.", - item, (int)dls->n_download_failures); - else if (dls->next_attempt_at < TIME_MAX) - log_debug(LD_DIR, "%s failed %d time(s); I'll try again in %d seconds.", - item, (int)dls->n_download_failures, - (int)(dls->next_attempt_at-now)); - else - log_debug(LD_DIR, "%s failed %d time(s); Giving up for a while.", - item, (int)dls->n_download_failures); + if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) { + /* stop this schedule retrying on failure, it will launch concurrent + * connections instead */ + return TIME_MAX; + } else { + return dls->next_attempt_at; } +} + +/** Determine when the next download attempt should be made when using an + * attempt-based (potentially concurrent) download schedule. + * Called when an attempt to download <b>dls</b> is being initiated. + * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an + * appropriate time in the future and return it. + * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment + * the attempts, and return a time in the far future (to avoid launching a + * concurrent attempt). */ +time_t +download_status_increment_attempt(download_status_t *dls, const char *item, + time_t now) +{ + int delay = -1; + tor_assert(dls); + + if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { + /* this schedule should retry on failure, and not launch any concurrent + attempts */ + log_info(LD_BUG, "Tried to launch an attempt-based connection on a " + "failure-based schedule."); + return TIME_MAX; + } + + if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1) + ++dls->n_download_attempts; + + const smartlist_t *schedule = find_dl_schedule(dls, get_options()); + delay = download_status_schedule_get_delay(dls, schedule, now); + + download_status_log_helper(item, dls->increment_on, "attempted", + "on failure", dls->n_download_attempts, + delay, dls->next_attempt_at, now); + return dls->next_attempt_at; } /** Reset <b>dls</b> so that it will be considered downloadable * immediately, and/or to show that we don't need it anymore. * + * Must be called to initialise a download schedule, otherwise the zeroth item + * in the schedule will never be used. + * * (We find the zeroth element of the download schedule, and set * next_attempt_at to be the appropriate offset from 'now'. In most * cases this means setting it to 'now', so the item will be immediately @@ -3838,14 +3943,16 @@ download_status_increment_failure(download_status_t *dls, int status_code, void download_status_reset(download_status_t *dls) { - const int *schedule; - size_t schedule_len; + if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD + || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD) + return; /* Don't reset this. */ - find_dl_schedule_and_len(dls, get_options()->DirPort_set, - &schedule, &schedule_len); + const smartlist_t *schedule = find_dl_schedule(dls, get_options()); dls->n_download_failures = 0; - dls->next_attempt_at = time(NULL) + schedule[0]; + dls->n_download_attempts = 0; + dls->next_attempt_at = time(NULL) + *(int *)smartlist_get(schedule, 0); + /* Don't reset dls->want_authority or dls->increment_on */ } /** Return the number of failures on <b>dls</b> since the last success (if @@ -3856,6 +3963,22 @@ download_status_get_n_failures(const download_status_t *dls) return dls->n_download_failures; } +/** Return the number of attempts to download <b>dls</b> since the last success + * (if any). This can differ from download_status_get_n_failures() due to + * outstanding concurrent attempts. */ +int +download_status_get_n_attempts(const download_status_t *dls) +{ + return dls->n_download_attempts; +} + +/** Return the next time to attempt to download <b>dls</b>. */ +time_t +download_status_get_next_attempt_at(const download_status_t *dls) +{ + return dls->next_attempt_at; +} + /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>) * fetches have failed (with uppercase fingerprints listed in <b>failed</b>, * either as descriptor digests or as identity digests based on @@ -3890,7 +4013,8 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, } else { dls = router_get_dl_status_by_descriptor_digest(digest); } - if (!dls || dls->n_download_failures >= MAX_ROUTERDESC_DOWNLOAD_FAILURES) + if (!dls || dls->n_download_failures >= + get_options()->TestingDescriptorMaxDownloadTries) continue; download_status_increment_failure(dls, status_code, cp, server, now); } SMARTLIST_FOREACH_END(cp); @@ -3921,7 +4045,8 @@ dir_microdesc_download_failed(smartlist_t *failed, if (!rs) continue; dls = &rs->dl_status; - if (dls->n_download_failures >= MAX_MICRODESC_DOWNLOAD_FAILURES) + if (dls->n_download_failures >= + get_options()->TestingMicrodescMaxDownloadTries) continue; { char buf[BASE64_DIGEST256_LEN+1]; diff --git a/src/or/directory.h b/src/or/directory.h index 41f18a1725..7646cac03f 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,34 +16,41 @@ int directories_have_accepted_server_descriptor(void); void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, dirinfo_type_t type, const char *payload, size_t payload_len, size_t extrainfo_len); -void directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource, - int pds_flags); +MOCK_DECL(void, directory_get_from_dirserver, ( + uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource, + int pds_flags, + download_want_authority_t want_authority)); void directory_get_from_all_authorities(uint8_t dir_purpose, uint8_t router_purpose, const char *resource); /** Enumeration of ways to connect to a directory server */ typedef enum { - /** Default: connect over a one-hop Tor circuit but fall back to direct - * connection */ + /** Default: connect over a one-hop Tor circuit. Relays fall back to direct + * DirPort connections, clients, onion services, and bridges do not */ DIRIND_ONEHOP=0, /** Connect over a multi-hop anonymizing Tor circuit */ DIRIND_ANONYMOUS=1, - /** Conncet to the DirPort directly */ + /** Connect to the DirPort directly */ DIRIND_DIRECT_CONN, /** Connect over a multi-hop anonymizing Tor circuit to our dirport */ DIRIND_ANON_DIRPORT, } dir_indirection_t; -void directory_initiate_command_routerstatus(const routerstatus_t *status, - uint8_t dir_purpose, - uint8_t router_purpose, - dir_indirection_t indirection, - const char *resource, - const char *payload, - size_t payload_len, - time_t if_modified_since); +int directory_must_use_begindir(const or_options_t *options); + +MOCK_DECL(void, directory_initiate_command_routerstatus, + (const routerstatus_t *status, + uint8_t dir_purpose, + uint8_t router_purpose, + dir_indirection_t indirection, + const char *resource, + const char *payload, + size_t payload_len, + time_t if_modified_since)); + void directory_initiate_command_routerstatus_rend(const routerstatus_t *status, uint8_t dir_purpose, uint8_t router_purpose, @@ -63,8 +70,8 @@ int connection_dir_process_inbuf(dir_connection_t *conn); int connection_dir_finished_flushing(dir_connection_t *conn); int connection_dir_finished_connecting(dir_connection_t *conn); void connection_dir_about_to_close(dir_connection_t *dir_conn); -void directory_initiate_command(const char *address, const tor_addr_t *addr, - uint16_t or_port, uint16_t dir_port, +void directory_initiate_command(const tor_addr_t *or_addr, uint16_t or_port, + const tor_addr_t *dir_addr, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, dir_indirection_t indirection, @@ -89,34 +96,65 @@ int router_supports_extrainfo(const char *identity_digest, int is_authority); time_t download_status_increment_failure(download_status_t *dls, int status_code, const char *item, int server, time_t now); +time_t download_status_increment_attempt(download_status_t *dls, + const char *item, time_t now); /** Increment the failure count of the download_status_t <b>dls</b>, with * the optional status code <b>sc</b>. */ #define download_status_failed(dls, sc) \ download_status_increment_failure((dls), (sc), NULL, \ - get_options()->DirPort_set, time(NULL)) + dir_server_mode(get_options()), \ + time(NULL)) void download_status_reset(download_status_t *dls); static int download_status_is_ready(download_status_t *dls, time_t now, int max_failures); /** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is * ready to get its download reattempted. */ -static INLINE int +static inline int download_status_is_ready(download_status_t *dls, time_t now, int max_failures) { - return (dls->n_download_failures <= max_failures - && dls->next_attempt_at <= now); + int under_failure_limit = (dls->n_download_failures <= max_failures + && dls->n_download_attempts <= max_failures); + return (under_failure_limit && dls->next_attempt_at <= now); } static void download_status_mark_impossible(download_status_t *dl); /** Mark <b>dl</b> as never downloadable. */ -static INLINE void +static inline void download_status_mark_impossible(download_status_t *dl) { dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD; + dl->n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD; } int download_status_get_n_failures(const download_status_t *dls); +int download_status_get_n_attempts(const download_status_t *dls); +time_t download_status_get_next_attempt_at(const download_status_t *dls); + +#ifdef TOR_UNIT_TESTS +/* Used only by directory.c and test_dir.c */ + +STATIC int parse_http_url(const char *headers, char **url); +STATIC int purpose_needs_anonymity(uint8_t dir_purpose, + uint8_t router_purpose); +STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose, + const char *resource); +STATIC int directory_handle_command_get(dir_connection_t *conn, + const char *headers, + const char *req_body, + size_t req_body_len); +STATIC int download_status_schedule_get_delay(download_status_t *dls, + const smartlist_t *schedule, + time_t now); + +STATIC char* authdir_type_to_string(dirinfo_type_t auth); +STATIC const char * dir_conn_purpose_to_string(int purpose); +STATIC int should_use_directory_guards(const or_options_t *options); +STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC const smartlist_t *find_dl_schedule(download_status_t *dls, + const or_options_t *options); +#endif #endif diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 3e46153a55..dafaed8bf2 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE @@ -18,6 +18,7 @@ #include "dirserv.h" #include "dirvote.h" #include "hibernate.h" +#include "keypin.h" #include "microdesc.h" #include "networkstatus.h" #include "nodelist.h" @@ -26,6 +27,8 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "routerset.h" +#include "torcert.h" /** * \file dirserv.c @@ -41,31 +44,10 @@ * directory authorities. */ #define MAX_UNTRUSTED_NETWORKSTATUSES 16 -/** If a v1 directory is older than this, discard it. */ -#define MAX_V1_DIRECTORY_AGE (30*24*60*60) -/** If a v1 running-routers is older than this, discard it. */ -#define MAX_V1_RR_AGE (7*24*60*60) - extern time_t time_of_process_start; /* from main.c */ extern long stats_n_seconds_working; /* from main.c */ -/** Do we need to regenerate the v1 directory when someone asks for it? */ -static time_t the_directory_is_dirty = 1; -/** Do we need to regenerate the v1 runningrouters document when somebody - * asks for it? */ -static time_t runningrouters_is_dirty = 1; -/** Do we need to regenerate our v2 networkstatus document when somebody asks - * for it? */ -static time_t the_v2_networkstatus_is_dirty = 1; - -/** Most recently generated encoded signed v1 directory. (v1 auth dirservers - * only.) */ -static cached_dir_t *the_directory = NULL; - -/** For authoritative directories: the current (v1) network status. */ -static cached_dir_t the_runningrouters; - /** Total number of routers with measured bandwidth; this is set by * dirserv_count_measured_bws() before the loop in * dirserv_generate_networkstatus_vote_obj() and checked by @@ -74,17 +56,13 @@ static cached_dir_t the_runningrouters; static int routers_with_measured_bw = 0; static void directory_remove_invalid(void); -static cached_dir_t *dirserv_regenerate_directory(void); static char *format_versions_list(config_line_t *ln); struct authdir_config_t; -static int add_fingerprint_to_dir(const char *nickname, const char *fp, - struct authdir_config_t *list); static uint32_t dirserv_get_status_impl(const char *fp, const char *nickname, - const char *address, uint32_t addr, uint16_t or_port, - const char *platform, const char *contact, - const char **msg, int should_log); + const char *platform, const char **msg, + int severity); static void clear_cached_dir(cached_dir_t *d); static const signed_descriptor_t *get_signed_descriptor_by_fp( const char *fp, @@ -97,19 +75,19 @@ static uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri); /************** Fingerprint handling code ************/ -#define FP_NAMED 1 /**< Listed in fingerprint file. */ +/* 1 Historically used to indicate Named */ #define FP_INVALID 2 /**< Believed invalid. */ #define FP_REJECT 4 /**< We will not publish this router. */ -#define FP_BADDIR 8 /**< We'll tell clients to avoid using this as a dir. */ +/* 8 Historically used to avoid using this as a dir. */ #define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ -#define FP_UNNAMED 32 /**< Another router has this name in fingerprint file. */ +/* 32 Historically used to indicade Unnamed */ + +/** Target of status_by_digest map. */ +typedef uint32_t router_status_t; -/** Encapsulate a nickname and an FP_* status; target of status_by_digest - * map. */ -typedef struct router_status_t { - char nickname[MAX_NICKNAME_LEN+1]; - uint32_t status; -} router_status_t; +static void add_fingerprint_to_dir(const char *fp, + struct authdir_config_t *list, + router_status_t add_status); /** List of nickname-\>identity fingerprint mappings for all the routers * that we name. Used to prevent router impersonation. */ @@ -131,18 +109,17 @@ authdir_config_new(void) return list; } -/** Add the fingerprint <b>fp</b> for <b>nickname</b> to - * the smartlist of fingerprint_entry_t's <b>list</b>. Return 0 if it's - * new, or 1 if we replaced the old value. +/** Add the fingerprint <b>fp</b> to the smartlist of fingerprint_entry_t's + * <b>list</b>, or-ing the currently set status flags with + * <b>add_status</b>. */ -/* static */ int -add_fingerprint_to_dir(const char *nickname, const char *fp, - authdir_config_t *list) +/* static */ void +add_fingerprint_to_dir(const char *fp, authdir_config_t *list, + router_status_t add_status) { char *fingerprint; char d[DIGEST_LEN]; router_status_t *status; - tor_assert(nickname); tor_assert(fp); tor_assert(list); @@ -152,14 +129,7 @@ add_fingerprint_to_dir(const char *nickname, const char *fp, log_warn(LD_DIRSERV, "Couldn't decode fingerprint \"%s\"", escaped(fp)); tor_free(fingerprint); - return 0; - } - - if (!strcasecmp(nickname, UNNAMED_ROUTER_NICKNAME)) { - log_warn(LD_DIRSERV, "Tried to add a mapping for reserved nickname %s", - UNNAMED_ROUTER_NICKNAME); - tor_free(fingerprint); - return 0; + return; } status = digestmap_get(list->status_by_digest, d); @@ -168,35 +138,15 @@ add_fingerprint_to_dir(const char *nickname, const char *fp, digestmap_set(list->status_by_digest, d, status); } - if (nickname[0] != '!') { - char *old_fp = strmap_get_lc(list->fp_by_name, nickname); - if (old_fp && !strcasecmp(fingerprint, old_fp)) { - tor_free(fingerprint); - } else { - tor_free(old_fp); - strmap_set_lc(list->fp_by_name, nickname, fingerprint); - } - status->status |= FP_NAMED; - strlcpy(status->nickname, nickname, sizeof(status->nickname)); - } else { - tor_free(fingerprint); - if (!strcasecmp(nickname, "!reject")) { - status->status |= FP_REJECT; - } else if (!strcasecmp(nickname, "!invalid")) { - status->status |= FP_INVALID; - } else if (!strcasecmp(nickname, "!baddir")) { - status->status |= FP_BADDIR; - } else if (!strcasecmp(nickname, "!badexit")) { - status->status |= FP_BADEXIT; - } - } - return 0; + tor_free(fingerprint); + *status |= add_status; + return; } -/** Add the nickname and fingerprint for this OR to the - * global list of recognized identity key fingerprints. */ +/** Add the fingerprint for this OR to the global list of recognized + * identity key fingerprints. */ int -dirserv_add_own_fingerprint(const char *nickname, crypto_pk_t *pk) +dirserv_add_own_fingerprint(crypto_pk_t *pk) { char fp[FINGERPRINT_LEN+1]; if (crypto_pk_get_fingerprint(pk, fp, 0)<0) { @@ -205,7 +155,7 @@ dirserv_add_own_fingerprint(const char *nickname, crypto_pk_t *pk) } if (!fingerprint_list) fingerprint_list = authdir_config_new(); - add_fingerprint_to_dir(nickname, fp, fingerprint_list); + add_fingerprint_to_dir(fp, fingerprint_list, 0); return 0; } @@ -223,7 +173,6 @@ dirserv_load_fingerprint_file(void) authdir_config_t *fingerprint_list_new; int result; config_line_t *front=NULL, *list; - const or_options_t *options = get_options(); fname = get_datadir_fname("approved-routers"); log_info(LD_GENERAL, @@ -231,15 +180,9 @@ dirserv_load_fingerprint_file(void) cf = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); if (!cf) { - if (options->NamingAuthoritativeDir) { - log_warn(LD_FS, "Cannot open fingerprint file '%s'. Failing.", fname); - tor_free(fname); - return -1; - } else { - log_info(LD_FS, "Cannot open fingerprint file '%s'. That's ok.", fname); - tor_free(fname); - return 0; - } + log_warn(LD_FS, "Cannot open fingerprint file '%s'. That's ok.", fname); + tor_free(fname); + return 0; } tor_free(fname); @@ -254,22 +197,8 @@ dirserv_load_fingerprint_file(void) for (list=front; list; list=list->next) { char digest_tmp[DIGEST_LEN]; + router_status_t add_status = 0; nickname = list->key; fingerprint = list->value; - if (strlen(nickname) > MAX_NICKNAME_LEN) { - log_notice(LD_CONFIG, - "Nickname '%s' too long in fingerprint file. Skipping.", - nickname); - continue; - } - if (!is_legal_nickname(nickname) && - strcasecmp(nickname, "!reject") && - strcasecmp(nickname, "!invalid") && - strcasecmp(nickname, "!badexit")) { - log_notice(LD_CONFIG, - "Invalid nickname '%s' in fingerprint file. Skipping.", - nickname); - continue; - } tor_strstrip(fingerprint, " "); /* remove spaces */ if (strlen(fingerprint) != HEX_DIGEST_LEN || base16_decode(digest_tmp, sizeof(digest_tmp), @@ -280,26 +209,14 @@ dirserv_load_fingerprint_file(void) nickname, fingerprint); continue; } - if (0==strcasecmp(nickname, DEFAULT_CLIENT_NICKNAME)) { - /* If you approved an OR called "client", then clients who use - * the default nickname could all be rejected. That's no good. */ - log_notice(LD_CONFIG, - "Authorizing nickname '%s' would break " - "many clients; skipping.", - DEFAULT_CLIENT_NICKNAME); - continue; - } - if (0==strcasecmp(nickname, UNNAMED_ROUTER_NICKNAME)) { - /* If you approved an OR called "unnamed", then clients will be - * confused. */ - log_notice(LD_CONFIG, - "Authorizing nickname '%s' is not allowed; skipping.", - UNNAMED_ROUTER_NICKNAME); - continue; + if (!strcasecmp(nickname, "!reject")) { + add_status = FP_REJECT; + } else if (!strcasecmp(nickname, "!badexit")) { + add_status = FP_BADEXIT; + } else if (!strcasecmp(nickname, "!invalid")) { + add_status = FP_INVALID; } - if (add_fingerprint_to_dir(nickname, fingerprint, fingerprint_list_new) - != 0) - log_notice(LD_CONFIG, "Duplicate nickname '%s'.", nickname); + add_fingerprint_to_dir(fingerprint, fingerprint_list_new, add_status); } config_free_lines(front); @@ -310,6 +227,16 @@ dirserv_load_fingerprint_file(void) return 0; } +/* If this is set, then we don't allow routers that have advertised an Ed25519 + * identity to stop doing so. This is going to be essential for good identity + * security: otherwise anybody who can attack RSA-1024 but not Ed25519 could + * just sign fake descriptors missing the Ed25519 key. But we won't actually + * be able to prevent that kind of thing until we're confident that there + * isn't actually a legit reason to downgrade to 0.2.5. So for now, we have + * to leave this #undef. + */ +#undef DISABLE_DISABLING_ED25519 + /** Check whether <b>router</b> has a nickname/identity key combination that * we recognize from the fingerprint list, or an IP we automatically act on * according to our configuration. Return the appropriate router status. @@ -317,9 +244,11 @@ dirserv_load_fingerprint_file(void) * If the status is 'FP_REJECT' and <b>msg</b> is provided, set * *<b>msg</b> to an explanation of why. */ uint32_t -dirserv_router_get_status(const routerinfo_t *router, const char **msg) +dirserv_router_get_status(const routerinfo_t *router, const char **msg, + int severity) { char d[DIGEST_LEN]; + const int key_pinning = get_options()->AuthDirPinKeys; if (crypto_pk_get_digest(router->identity_pkey, d)) { log_warn(LD_BUG,"Error computing fingerprint"); @@ -328,11 +257,45 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg) return FP_REJECT; } + if (router->cache_info.signing_key_cert) { + /* This has an ed25519 identity key. */ + if (KEYPIN_MISMATCH == + keypin_check((const uint8_t*)router->cache_info.identity_digest, + router->cache_info.signing_key_cert->signing_key.pubkey)) { + log_fn(severity, LD_DIR, + "Descriptor from router %s has an Ed25519 key, " + "but the <rsa,ed25519> keys don't match what they were before.", + router_describe(router)); + if (key_pinning) { + if (msg) { + *msg = "Ed25519 identity key or RSA identity key has changed."; + } + return FP_REJECT; + } + } + } else { + /* No ed25519 key */ + if (KEYPIN_MISMATCH == keypin_check_lone_rsa( + (const uint8_t*)router->cache_info.identity_digest)) { + log_fn(severity, LD_DIR, + "Descriptor from router %s has no Ed25519 key, " + "when we previously knew an Ed25519 for it. Ignoring for now, " + "since Ed25519 keys are fairly new.", + router_describe(router)); +#ifdef DISABLE_DISABLING_ED25519 + if (key_pinning) { + if (msg) { + *msg = "Ed25519 identity key has disappeared."; + } + return FP_REJECT; + } +#endif + } + } + return dirserv_get_status_impl(d, router->nickname, - router->address, router->addr, router->or_port, - router->platform, router->contact_info, - msg, 1); + router->platform, msg, severity); } /** Return true if there is no point in downloading the router described by @@ -343,103 +306,46 @@ dirserv_would_reject_router(const routerstatus_t *rs) uint32_t res; res = dirserv_get_status_impl(rs->identity_digest, rs->nickname, - "", /* address is only used in logs */ rs->addr, rs->or_port, - NULL, NULL, - NULL, 0); + NULL, NULL, LOG_DEBUG); return (res & FP_REJECT) != 0; } -/** Helper: Based only on the ID/Nickname combination, - * return FP_UNNAMED (unnamed), FP_NAMED (named), or 0 (neither). - */ -static uint32_t -dirserv_get_name_status(const char *id_digest, const char *nickname) -{ - char fp[HEX_DIGEST_LEN+1]; - char *fp_by_name; - - base16_encode(fp, sizeof(fp), id_digest, DIGEST_LEN); - - if ((fp_by_name = - strmap_get_lc(fingerprint_list->fp_by_name, nickname))) { - if (!strcasecmp(fp, fp_by_name)) { - return FP_NAMED; - } else { - return FP_UNNAMED; /* Wrong fingerprint. */ - } - } - return 0; -} - /** Helper: As dirserv_router_get_status, but takes the router fingerprint * (hex, no spaces), nickname, address (used for logging only), IP address, OR - * port, platform (logging only) and contact info (logging only) as arguments. + * port and platform (logging only) as arguments. * - * If should_log is false, do not log messages. (There's not much point in + * Log messages at 'severity'. (There's not much point in * logging that we're rejecting servers we'll not download.) */ static uint32_t dirserv_get_status_impl(const char *id_digest, const char *nickname, - const char *address, uint32_t addr, uint16_t or_port, - const char *platform, const char *contact, - const char **msg, int should_log) + const char *platform, const char **msg, int severity) { - int reject_unlisted = get_options()->AuthDirRejectUnlisted; - uint32_t result; + uint32_t result = 0; router_status_t *status_by_digest; if (!fingerprint_list) fingerprint_list = authdir_config_new(); - if (should_log) - log_debug(LD_DIRSERV, "%d fingerprints, %d digests known.", - strmap_size(fingerprint_list->fp_by_name), - digestmap_size(fingerprint_list->status_by_digest)); + log_debug(LD_DIRSERV, "%d fingerprints, %d digests known.", + strmap_size(fingerprint_list->fp_by_name), + digestmap_size(fingerprint_list->status_by_digest)); - /* Versions before Tor 0.2.2.35 have known security issues that - * make them unsuitable for the current network. */ - if (platform && !tor_version_as_new_as(platform,"0.2.2.35")) { + /* Versions before Tor 0.2.4.18-rc are too old to support, and are + * missing some important security fixes too. Disable them. */ + if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) { if (msg) *msg = "Tor version is insecure or unsupported. Please upgrade!"; return FP_REJECT; - } else if (platform && tor_version_as_new_as(platform,"0.2.3.0-alpha")) { - /* Versions from 0.2.3-alpha...0.2.3.9-alpha have known security - * issues that make them unusable for the current network */ - if (!tor_version_as_new_as(platform, "0.2.3.10-alpha")) { - if (msg) - *msg = "Tor version is insecure or unsupported. Please upgrade!"; - return FP_REJECT; - } - } - - result = dirserv_get_name_status(id_digest, nickname); - if (result & FP_NAMED) { - if (should_log) - log_debug(LD_DIRSERV,"Good fingerprint for '%s'",nickname); - } - if (result & FP_UNNAMED) { - if (should_log) { - char *esc_contact = esc_for_log(contact); - log_info(LD_DIRSERV, - "Mismatched fingerprint for '%s'. " - "ContactInfo '%s', platform '%s'.)", - nickname, - esc_contact, - platform ? escaped(platform) : ""); - tor_free(esc_contact); - } - if (msg) - *msg = "Rejected: There is already a named server with this nickname " - "and a different fingerprint."; } status_by_digest = digestmap_get(fingerprint_list->status_by_digest, id_digest); if (status_by_digest) - result |= (status_by_digest->status & ~FP_NAMED); + result |= *status_by_digest; if (result & FP_REJECT) { if (msg) @@ -450,61 +356,30 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, *msg = "Fingerprint is marked invalid"; } - if (authdir_policy_baddir_address(addr, or_port)) { - if (should_log) - log_info(LD_DIRSERV, - "Marking '%s' as bad directory because of address '%s'", - nickname, address); - result |= FP_BADDIR; - } - if (authdir_policy_badexit_address(addr, or_port)) { - if (should_log) - log_info(LD_DIRSERV, "Marking '%s' as bad exit because of address '%s'", - nickname, address); + log_fn(severity, LD_DIRSERV, + "Marking '%s' as bad exit because of address '%s'", + nickname, fmt_addr32(addr)); result |= FP_BADEXIT; } - if (!(result & FP_NAMED)) { - if (!authdir_policy_permits_address(addr, or_port)) { - if (should_log) - log_info(LD_DIRSERV, "Rejecting '%s' because of address '%s'", - nickname, address); - if (msg) - *msg = "Authdir is rejecting routers in this range."; - return FP_REJECT; - } - if (!authdir_policy_valid_address(addr, or_port)) { - if (should_log) - log_info(LD_DIRSERV, "Not marking '%s' valid because of address '%s'", - nickname, address); - result |= FP_INVALID; - } - if (reject_unlisted) { - if (msg) - *msg = "Authdir rejects unknown routers."; - return FP_REJECT; - } + if (!authdir_policy_permits_address(addr, or_port)) { + log_fn(severity, LD_DIRSERV, "Rejecting '%s' because of address '%s'", + nickname, fmt_addr32(addr)); + if (msg) + *msg = "Authdir is rejecting routers in this range."; + return FP_REJECT; + } + if (!authdir_policy_valid_address(addr, or_port)) { + log_fn(severity, LD_DIRSERV, + "Not marking '%s' valid because of address '%s'", + nickname, fmt_addr32(addr)); + result |= FP_INVALID; } return result; } -/** If we are an authoritative dirserver, and the list of approved - * servers contains one whose identity key digest is <b>digest</b>, - * return that router's nickname. Otherwise return NULL. */ -const char * -dirserv_get_nickname_by_digest(const char *digest) -{ - router_status_t *status; - if (!fingerprint_list) - return NULL; - tor_assert(digest); - - status = digestmap_get(fingerprint_list->status_by_digest, digest); - return status ? status->nickname : NULL; -} - /** Clear the current fingerprint list. */ void dirserv_free_fingerprint_list(void) @@ -526,26 +401,22 @@ dirserv_free_fingerprint_list(void) static int dirserv_router_has_valid_address(routerinfo_t *ri) { - struct in_addr iaddr; + tor_addr_t addr; if (get_options()->DirAllowPrivateAddresses) return 0; /* whatever it is, we're fine with it */ - if (!tor_inet_aton(ri->address, &iaddr)) { - log_info(LD_DIRSERV,"Router %s published non-IP address '%s'. Refusing.", - router_describe(ri), - ri->address); - return -1; - } - if (is_internal_IP(ntohl(iaddr.s_addr), 0)) { + tor_addr_from_ipv4h(&addr, ri->addr); + + if (tor_addr_is_internal(&addr, 0)) { log_info(LD_DIRSERV, - "Router %s published internal IP address '%s'. Refusing.", - router_describe(ri), ri->address); + "Router %s published internal IP address. Refusing.", + router_describe(ri)); return -1; /* it's a private IP, we should reject it */ } return 0; } /** Check whether we, as a directory server, want to accept <b>ri</b>. If so, - * set its is_valid,named,running fields and return 0. Otherwise, return -1. + * set its is_valid,running fields and return 0. Otherwise, return -1. * * If the router is rejected, set *<b>msg</b> to an explanation of why. * @@ -557,9 +428,9 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, int complain, int *valid_out) { /* Okay. Now check whether the fingerprint is recognized. */ - uint32_t status = dirserv_router_get_status(ri, msg); time_t now; int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO; + uint32_t status = dirserv_router_get_status(ri, msg, severity); tor_assert(msg); if (status & FP_REJECT) return -1; /* msg is already set. */ @@ -590,12 +461,10 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, } if (dirserv_router_has_valid_address(ri) < 0) { log_fn(severity, LD_DIRSERV, - "Router %s has invalid address '%s'. " - "Not adding (%s).", + "Router %s has invalid address. Not adding (%s).", router_describe(ri), - ri->address, esc_router_info(ri)); - *msg = "Rejected: Address is not an IP, or IP is a private address."; + *msg = "Rejected: Address is a private address."; return -1; } @@ -612,7 +481,6 @@ dirserv_set_node_flags_from_authoritative_status(node_t *node, uint32_t authstatus) { node->is_valid = (authstatus & FP_INVALID) ? 0 : 1; - node->is_bad_directory = (authstatus & FP_BADDIR) ? 1 : 0; node->is_bad_exit = (authstatus & FP_BADEXIT) ? 1 : 0; } @@ -658,7 +526,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, s = desc; list = smartlist_new(); if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0, - annotation_buf)) { + annotation_buf, NULL)) { SMARTLIST_FOREACH(list, routerinfo_t *, ri, { msg_out = NULL; tor_assert(ri->purpose == purpose); @@ -674,7 +542,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, s = desc; if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0, - NULL)) { + NULL, NULL)) { SMARTLIST_FOREACH(list, extrainfo_t *, ei, { msg_out = NULL; @@ -692,7 +560,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, if (!n_parsed) { *msg = "No descriptors found in your POST."; if (WRA_WAS_ADDED(r)) - r = ROUTER_WAS_NOT_NEW; + r = ROUTER_IS_ALREADY_KNOWN; } else { *msg = "(no message)"; } @@ -717,7 +585,9 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) was_router_added_t r; routerinfo_t *ri_old; char *desc, *nickname; - size_t desclen = 0; + const size_t desclen = ri->cache_info.signed_descriptor_len + + ri->cache_info.annotations_len; + const int key_pinning = get_options()->AuthDirPinKeys; *msg = NULL; /* If it's too big, refuse it now. Otherwise we'll cache it all over the @@ -731,7 +601,7 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) *msg = "Router descriptor was too large."; control_event_or_authdir_new_descriptor("REJECTED", ri->cache_info.signed_descriptor_body, - ri->cache_info.signed_descriptor_len, *msg); + desclen, *msg); routerinfo_free(ri); return ROUTER_AUTHDIR_REJECTS; } @@ -752,14 +622,36 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) "the last one with this identity."; control_event_or_authdir_new_descriptor("DROPPED", ri->cache_info.signed_descriptor_body, - ri->cache_info.signed_descriptor_len, *msg); + desclen, *msg); routerinfo_free(ri); - return ROUTER_WAS_NOT_NEW; + return ROUTER_IS_ALREADY_KNOWN; + } + + /* Do keypinning again ... this time, to add the pin if appropriate */ + int keypin_status; + if (ri->cache_info.signing_key_cert) { + keypin_status = keypin_check_and_add( + (const uint8_t*)ri->cache_info.identity_digest, + ri->cache_info.signing_key_cert->signing_key.pubkey, + ! key_pinning); + } else { + keypin_status = keypin_check_lone_rsa( + (const uint8_t*)ri->cache_info.identity_digest); +#ifndef DISABLE_DISABLING_ED25519 + if (keypin_status == KEYPIN_MISMATCH) + keypin_status = KEYPIN_NOT_FOUND; +#endif + } + if (keypin_status == KEYPIN_MISMATCH && key_pinning) { + log_info(LD_DIRSERV, "Dropping descriptor from %s (source: %s) because " + "its key did not match an older RSA/Ed25519 keypair", + router_describe(ri), source); + *msg = "Looks like your keypair does not match its older value."; + return ROUTER_AUTHDIR_REJECTS; } /* Make a copy of desc, since router_add_to_routerlist might free * ri and its associated signed_descriptor_t. */ - desclen = ri->cache_info.signed_descriptor_len; desc = tor_strndup(ri->cache_info.signed_descriptor_body, desclen); nickname = tor_strdup(ri->nickname); @@ -799,12 +691,14 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei, const char **msg) { - const routerinfo_t *ri; + routerinfo_t *ri; int r; tor_assert(msg); *msg = NULL; - ri = router_get_by_id_digest(ei->cache_info.identity_digest); + /* Needs to be mutable so routerinfo_incompatible_with_extrainfo + * can mess with some of the flags in ri->cache_info. */ + ri = router_get_mutable_by_digest(ei->cache_info.identity_digest); if (!ri) { *msg = "No corresponding router descriptor for extra-info descriptor"; extrainfo_free(ei); @@ -824,9 +718,10 @@ dirserv_add_extrainfo(extrainfo_t *ei, const char **msg) return ROUTER_BAD_EI; } - if ((r = routerinfo_incompatible_with_extrainfo(ri, ei, NULL, msg))) { + if ((r = routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, + &ri->cache_info, msg))) { extrainfo_free(ei); - return r < 0 ? ROUTER_WAS_NOT_NEW : ROUTER_BAD_EI; + return r < 0 ? ROUTER_IS_ALREADY_KNOWN : ROUTER_BAD_EI; } router_add_extrainfo_to_routerlist(ei, msg, 0, 0); return ROUTER_ADDED_SUCCESSFULLY; @@ -839,99 +734,41 @@ dirserv_add_extrainfo(extrainfo_t *ei, const char **msg) static void directory_remove_invalid(void) { - int changed = 0; routerlist_t *rl = router_get_routerlist(); smartlist_t *nodes = smartlist_new(); smartlist_add_all(nodes, nodelist_get_list()); SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) { - const char *msg; + const char *msg = NULL; routerinfo_t *ent = node->ri; char description[NODE_DESC_BUF_LEN]; uint32_t r; if (!ent) continue; - r = dirserv_router_get_status(ent, &msg); + r = dirserv_router_get_status(ent, &msg, LOG_INFO); router_get_description(description, ent); if (r & FP_REJECT) { log_info(LD_DIRSERV, "Router %s is now rejected: %s", description, msg?msg:""); routerlist_remove(rl, ent, 0, time(NULL)); - changed = 1; continue; } -#if 0 - if (bool_neq((r & FP_NAMED), ent->auth_says_is_named)) { - log_info(LD_DIRSERV, - "Router %s is now %snamed.", description, - (r&FP_NAMED)?"":"un"); - ent->is_named = (r&FP_NAMED)?1:0; - changed = 1; - } - if (bool_neq((r & FP_UNNAMED), ent->auth_says_is_unnamed)) { - log_info(LD_DIRSERV, - "Router '%s' is now %snamed. (FP_UNNAMED)", description, - (r&FP_NAMED)?"":"un"); - ent->is_named = (r&FP_NUNAMED)?0:1; - changed = 1; - } -#endif if (bool_neq((r & FP_INVALID), !node->is_valid)) { log_info(LD_DIRSERV, "Router '%s' is now %svalid.", description, (r&FP_INVALID) ? "in" : ""); node->is_valid = (r&FP_INVALID)?0:1; - changed = 1; - } - if (bool_neq((r & FP_BADDIR), node->is_bad_directory)) { - log_info(LD_DIRSERV, "Router '%s' is now a %s directory", description, - (r & FP_BADDIR) ? "bad" : "good"); - node->is_bad_directory = (r&FP_BADDIR) ? 1: 0; - changed = 1; } if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) { log_info(LD_DIRSERV, "Router '%s' is now a %s exit", description, (r & FP_BADEXIT) ? "bad" : "good"); node->is_bad_exit = (r&FP_BADEXIT) ? 1: 0; - changed = 1; } } SMARTLIST_FOREACH_END(node); - if (changed) - directory_set_dirty(); routerlist_assert_ok(rl); smartlist_free(nodes); } -/** Mark the directory as <b>dirty</b> -- when we're next asked for a - * directory, we will rebuild it instead of reusing the most recently - * generated one. - */ -void -directory_set_dirty(void) -{ - time_t now = time(NULL); - int set_v1_dirty=0; - - /* Regenerate stubs only every 8 hours. - * XXXX It would be nice to generate less often, but these are just - * stubs: it doesn't matter. */ -#define STUB_REGENERATE_INTERVAL (8*60*60) - if (!the_directory || !the_runningrouters.dir) - set_v1_dirty = 1; - else if (the_directory->published < now - STUB_REGENERATE_INTERVAL || - the_runningrouters.published < now - STUB_REGENERATE_INTERVAL) - set_v1_dirty = 1; - - if (set_v1_dirty) { - if (!the_directory_is_dirty) - the_directory_is_dirty = now; - if (!runningrouters_is_dirty) - runningrouters_is_dirty = now; - } - if (!the_v2_networkstatus_is_dirty) - the_v2_networkstatus_is_dirty = now; -} - /** * Allocate and return a description of the status of the server <b>desc</b>, * for use in a v1-style router-status line. The server is listed @@ -963,7 +800,7 @@ list_single_server_status(const routerinfo_t *desc, int is_live) } /* DOCDOC running_long_enough_to_decide_unreachable */ -static INLINE int +static inline int running_long_enough_to_decide_unreachable(void) { return time_of_process_start @@ -971,7 +808,7 @@ running_long_enough_to_decide_unreachable(void) } /** Each server needs to have passed a reachability test no more - * than this number of seconds ago, or he is listed as down in + * than this number of seconds ago, or it is listed as down in * the directory. */ #define REACHABLE_TIMEOUT (45*60) @@ -1118,16 +955,33 @@ format_versions_list(config_line_t *ln) } /** Return 1 if <b>ri</b>'s descriptor is "active" -- running, valid, - * not hibernating, and not too old. Else return 0. + * not hibernating, having observed bw greater 0, and not too old. Else + * return 0. */ static int router_is_active(const routerinfo_t *ri, const node_t *node, time_t now) { time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - if (ri->cache_info.published_on < cutoff) + if (ri->cache_info.published_on < cutoff) { return 0; - if (!node->is_running || !node->is_valid || ri->is_hibernating) + } + if (!node->is_running || !node->is_valid || ri->is_hibernating) { return 0; + } + /* Only require bandwith capacity in non-test networks, or + * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */ + if (!ri->bandwidthcapacity) { + if (get_options()->TestingTorNetwork) { + if (get_options()->TestingMinExitFlagThreshold > 0) { + /* If we're in a TestingTorNetwork, and TestingMinExitFlagThreshold is, + * then require bandwidthcapacity */ + return 0; + } + } else { + /* If we're not in a TestingTorNetwork, then require bandwidthcapacity */ + return 0; + } + } return 1; } @@ -1240,13 +1094,13 @@ directory_fetches_from_authorities(const or_options_t *options) return 1; /* we don't know our IP address; ask an authority. */ refuseunknown = ! router_my_exit_policy_is_reject_star() && should_refuse_unknown_exits(options); - if (!options->DirPort_set && !refuseunknown) + if (!dir_server_mode(options) && !refuseunknown) return 0; if (!server_mode(options) || !advertised_server_mode()) return 0; me = router_get_my_routerinfo(); - if (!me || (!me->dir_port && !refuseunknown)) - return 0; /* if dirport not advertised, return 0 too */ + if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown)) + return 0; /* if we don't service directory requests, return 0 too */ return 1; } @@ -1271,30 +1125,25 @@ directory_fetches_dir_info_later(const or_options_t *options) return options->UseBridges != 0; } -/** Return 1 if we want to cache v2 dir info (each status file). - */ -int -directory_caches_v2_dir_info(const or_options_t *options) -{ - return options->DirPort_set; -} - /** Return true iff we want to fetch and keep certificates for authorities - * that we don't acknowledge as aurthorities ourself. + * that we don't acknowledge as authorities ourself. */ int directory_caches_unknown_auth_certs(const or_options_t *options) { - return options->DirPort_set || options->BridgeRelay; + return dir_server_mode(options) || options->BridgeRelay; } -/** Return 1 if we want to keep descriptors, networkstatuses, etc around - * and we're willing to serve them to others. Else return 0. +/** Return 1 if we want to keep descriptors, networkstatuses, etc around. + * Else return 0. + * Check options->DirPort_set and directory_permits_begindir_requests() + * to see if we are willing to serve these directory documents to others via + * the DirPort and begindir-over-ORPort, respectively. */ int directory_caches_dir_info(const or_options_t *options) { - if (options->BridgeRelay || options->DirPort_set) + if (options->BridgeRelay || dir_server_mode(options)) return 1; if (!server_mode(options) || !advertised_server_mode()) return 0; @@ -1310,16 +1159,7 @@ directory_caches_dir_info(const or_options_t *options) int directory_permits_begindir_requests(const or_options_t *options) { - return options->BridgeRelay != 0 || options->DirPort_set; -} - -/** Return 1 if we want to allow controllers to ask us directory - * requests via the controller interface, which doesn't require - * having any separate port open. */ -int -directory_permits_controller_requests(const or_options_t *options) -{ - return options->DirPort_set; + return options->BridgeRelay != 0 || dir_server_mode(options); } /** Return 1 if we have no need to fetch new descriptors. This generally @@ -1337,55 +1177,10 @@ directory_too_idle_to_fetch_descriptors(const or_options_t *options, /********************************************************************/ -/* Used only by non-v1-auth dirservers: The v1 directory and - * runningrouters we'll serve when requested. */ - -/** The v1 directory we'll serve (as a cache or as an authority) if - * requested. */ -static cached_dir_t *cached_directory = NULL; -/** The v1 runningrouters document we'll serve (as a cache or as an authority) - * if requested. */ -static cached_dir_t cached_runningrouters; - -/** Used for other dirservers' v2 network statuses. Map from hexdigest to - * cached_dir_t. */ -static digestmap_t *cached_v2_networkstatus = NULL; - /** Map from flavor name to the cached_dir_t for the v3 consensuses that we're * currently serving. */ static strmap_t *cached_consensuses = NULL; -/** Possibly replace the contents of <b>d</b> with the value of - * <b>directory</b> published on <b>when</b>, unless <b>when</b> is older than - * the last value, or too far in the future. - * - * Does not copy <b>directory</b>; frees it if it isn't used. - */ -static void -set_cached_dir(cached_dir_t *d, char *directory, time_t when) -{ - time_t now = time(NULL); - if (when<=d->published) { - log_info(LD_DIRSERV, "Ignoring old directory; not caching."); - tor_free(directory); - } else if (when>=now+ROUTER_MAX_AGE_TO_PUBLISH) { - log_info(LD_DIRSERV, "Ignoring future directory; not caching."); - tor_free(directory); - } else { - /* if (when>d->published && when<now+ROUTER_MAX_AGE) */ - log_debug(LD_DIRSERV, "Caching directory."); - tor_free(d->dir); - d->dir = directory; - d->dir_len = strlen(directory); - tor_free(d->dir_z); - if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { - log_warn(LD_BUG,"Error compressing cached directory"); - } - d->published = when; - } -} - /** Decrement the reference count on <b>d</b>, and free it if it no longer has * any references. */ void @@ -1435,93 +1230,13 @@ free_cached_dir_(void *_d) cached_dir_decref(d); } -/** If we have no cached v1 directory, or it is older than <b>published</b>, - * then replace it with <b>directory</b>, published at <b>published</b>. - * - * If <b>published</b> is too old, do nothing. - * - * If <b>is_running_routers</b>, this is really a v1 running_routers - * document rather than a v1 directory. - */ -static void -dirserv_set_cached_directory(const char *directory, time_t published) -{ - - cached_dir_decref(cached_directory); - cached_directory = new_cached_dir(tor_strdup(directory), published); -} - -/** If <b>networkstatus</b> is non-NULL, we've just received a v2 - * network-status for an authoritative directory with identity digest - * <b>identity</b> published at <b>published</b> -- store it so we can - * serve it to others. - * - * If <b>networkstatus</b> is NULL, remove the entry with the given - * identity fingerprint from the v2 cache. - */ -void -dirserv_set_cached_networkstatus_v2(const char *networkstatus, - const char *identity, - time_t published) -{ - cached_dir_t *d, *old_d; - if (!cached_v2_networkstatus) - cached_v2_networkstatus = digestmap_new(); - - old_d = digestmap_get(cached_v2_networkstatus, identity); - if (!old_d && !networkstatus) - return; - - if (networkstatus) { - if (!old_d || published > old_d->published) { - d = new_cached_dir(tor_strdup(networkstatus), published); - digestmap_set(cached_v2_networkstatus, identity, d); - if (old_d) - cached_dir_decref(old_d); - } - } else { - if (old_d) { - digestmap_remove(cached_v2_networkstatus, identity); - cached_dir_decref(old_d); - } - } - - /* Now purge old entries. */ - - if (digestmap_size(cached_v2_networkstatus) > - get_n_authorities(V2_DIRINFO) + MAX_UNTRUSTED_NETWORKSTATUSES) { - /* We need to remove the oldest untrusted networkstatus. */ - const char *oldest = NULL; - time_t oldest_published = TIME_MAX; - digestmap_iter_t *iter; - - for (iter = digestmap_iter_init(cached_v2_networkstatus); - !digestmap_iter_done(iter); - iter = digestmap_iter_next(cached_v2_networkstatus, iter)) { - const char *ident; - void *val; - digestmap_iter_get(iter, &ident, &val); - d = val; - if (d->published < oldest_published && - !router_digest_is_trusted_dir(ident)) { - oldest = ident; - oldest_published = d->published; - } - } - tor_assert(oldest); - d = digestmap_remove(cached_v2_networkstatus, oldest); - if (d) - cached_dir_decref(d); - } -} - /** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that * we're serving with <b>networkstatus</b>, published at <b>published</b>. No * validation is performed. */ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, const char *flavor_name, - const digests_t *digests, + const common_digests_t *digests, time_t published) { cached_dir_t *new_networkstatus; @@ -1530,193 +1245,13 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, cached_consensuses = strmap_new(); new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); - memcpy(&new_networkstatus->digests, digests, sizeof(digests_t)); + memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); old_networkstatus = strmap_set(cached_consensuses, flavor_name, new_networkstatus); if (old_networkstatus) cached_dir_decref(old_networkstatus); } -/** Remove any v2 networkstatus from the directory cache that was published - * before <b>cutoff</b>. */ -void -dirserv_clear_old_networkstatuses(time_t cutoff) -{ - if (!cached_v2_networkstatus) - return; - - DIGESTMAP_FOREACH_MODIFY(cached_v2_networkstatus, id, cached_dir_t *, dir) { - if (dir->published < cutoff) { - char *fname; - fname = networkstatus_get_cache_filename(id); - if (file_status(fname) == FN_FILE) { - log_info(LD_DIR, "Removing too-old untrusted networkstatus in %s", - fname); - unlink(fname); - } - tor_free(fname); - cached_dir_decref(dir); - MAP_DEL_CURRENT(id); - } - } DIGESTMAP_FOREACH_END -} - -/** Remove any v1 info from the directory cache that was published - * too long ago. */ -void -dirserv_clear_old_v1_info(time_t now) -{ - if (cached_directory && - cached_directory->published < (now - MAX_V1_DIRECTORY_AGE)) { - cached_dir_decref(cached_directory); - cached_directory = NULL; - } - if (cached_runningrouters.published < (now - MAX_V1_RR_AGE)) { - clear_cached_dir(&cached_runningrouters); - } -} - -/** Helper: If we're an authority for the right directory version (v1 or v2) - * (based on <b>auth_type</b>), try to regenerate - * auth_src as appropriate and return it, falling back to cache_src on - * failure. If we're a cache, simply return cache_src. - */ -static cached_dir_t * -dirserv_pick_cached_dir_obj(cached_dir_t *cache_src, - cached_dir_t *auth_src, - time_t dirty, cached_dir_t *(*regenerate)(void), - const char *name, - dirinfo_type_t auth_type) -{ - const or_options_t *options = get_options(); - int authority = (auth_type == V1_DIRINFO && authdir_mode_v1(options)) || - (auth_type == V2_DIRINFO && authdir_mode_v2(options)); - - if (!authority || authdir_mode_bridge(options)) { - return cache_src; - } else { - /* We're authoritative. */ - if (regenerate != NULL) { - if (dirty && dirty + DIR_REGEN_SLACK_TIME < time(NULL)) { - if (!(auth_src = regenerate())) { - log_err(LD_BUG, "Couldn't generate %s?", name); - exit(1); - } - } else { - log_info(LD_DIRSERV, "The %s is still clean; reusing.", name); - } - } - return auth_src ? auth_src : cache_src; - } -} - -/** Return the most recently generated encoded signed v1 directory, - * generating a new one as necessary. If not a v1 authoritative directory - * may return NULL if no directory is yet cached. */ -cached_dir_t * -dirserv_get_directory(void) -{ - return dirserv_pick_cached_dir_obj(cached_directory, the_directory, - the_directory_is_dirty, - dirserv_regenerate_directory, - "v1 server directory", V1_DIRINFO); -} - -/** Only called by v1 auth dirservers. - * Generate a fresh v1 directory; set the_directory and return a pointer - * to the new value. - */ -static cached_dir_t * -dirserv_regenerate_directory(void) -{ - /* XXXX 024 Get rid of this function if we can confirm that nobody's - * fetching these any longer */ - char *new_directory=NULL; - - if (dirserv_dump_directory_to_string(&new_directory, - get_server_identity_key())) { - log_warn(LD_BUG, "Error creating directory."); - tor_free(new_directory); - return NULL; - } - cached_dir_decref(the_directory); - the_directory = new_cached_dir(new_directory, time(NULL)); - log_info(LD_DIRSERV,"New directory (size %d) has been built.", - (int)the_directory->dir_len); - log_debug(LD_DIRSERV,"New directory (size %d):\n%s", - (int)the_directory->dir_len, the_directory->dir); - - the_directory_is_dirty = 0; - - /* Save the directory to disk so we re-load it quickly on startup. - */ - dirserv_set_cached_directory(the_directory->dir, time(NULL)); - - return the_directory; -} - -/** Only called by v1 auth dirservers. - * Replace the current running-routers list with a newly generated one. */ -static cached_dir_t * -generate_runningrouters(void) -{ - char *s=NULL; - char digest[DIGEST_LEN]; - char published[ISO_TIME_LEN+1]; - size_t len; - crypto_pk_t *private_key = get_server_identity_key(); - char *identity_pkey; /* Identity key, DER64-encoded. */ - size_t identity_pkey_len; - - if (crypto_pk_write_public_key_to_string(private_key,&identity_pkey, - &identity_pkey_len)<0) { - log_warn(LD_BUG,"write identity_pkey to string failed!"); - goto err; - } - format_iso_time(published, time(NULL)); - - len = 2048; - s = tor_malloc_zero(len); - tor_snprintf(s, len, - "network-status\n" - "published %s\n" - "router-status %s\n" - "dir-signing-key\n%s" - "directory-signature %s\n", - published, "", identity_pkey, - get_options()->Nickname); - tor_free(identity_pkey); - if (router_get_runningrouters_hash(s,digest)) { - log_warn(LD_BUG,"couldn't compute digest"); - goto err; - } - note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(s, len, digest, DIGEST_LEN, - private_key)<0) - goto err; - - set_cached_dir(&the_runningrouters, s, time(NULL)); - runningrouters_is_dirty = 0; - - return &the_runningrouters; - err: - tor_free(s); - return NULL; -} - -/** Set *<b>rr</b> to the most recently generated encoded signed - * running-routers list, generating a new one as necessary. Return the - * size of the directory on success, and 0 on failure. */ -cached_dir_t * -dirserv_get_runningrouters(void) -{ - return dirserv_pick_cached_dir_obj( - &cached_runningrouters, &the_runningrouters, - runningrouters_is_dirty, - generate_runningrouters, - "v1 network status list", V1_DIRINFO); -} - /** Return the latest downloaded consensus networkstatus in encoded, signed, * optionally compressed format, suitable for sending to clients. */ cached_dir_t * @@ -1727,19 +1262,6 @@ dirserv_get_consensus(const char *flavor_name) return strmap_get(cached_consensuses, flavor_name); } -/** For authoritative directories: the current (v2) network status. */ -static cached_dir_t *the_v2_networkstatus = NULL; - -/** Return true iff our opinion of the routers has been stale for long - * enough that we should generate a new v2 network status doc. */ -static int -should_generate_v2_networkstatus(void) -{ - return authdir_mode_v2(get_options()) && - the_v2_networkstatus_is_dirty && - the_v2_networkstatus_is_dirty + DIR_REGEN_SLACK_TIME < time(NULL); -} - /** If a router's uptime is at least this value, then it is always * considered stable, regardless of the rest of the network. This * way we resist attacks where an attacker doubles the size of the @@ -1786,7 +1308,7 @@ static uint32_t guard_bandwidth_excluding_exits_kb = 0; /** Helper: estimate the uptime of a router given its stated uptime and the * amount of time since it last stated its stated uptime. */ -static INLINE long +static inline long real_uptime(const routerinfo_t *router, time_t now) { if (now < router->cache_info.published_on) @@ -1834,8 +1356,10 @@ dirserv_thinks_router_is_unreliable(time_t now, } /** Return true iff <b>router</b> should be assigned the "HSDir" flag. - * Right now this means it advertises support for it, it has a high - * uptime, it has a DirPort open, and it's currently considered Running. + * + * Right now this means it advertises support for it, it has a high uptime, + * it's a directory cache, it has the Stable and Fast flags, and it's currently + * considered Running. * * This function needs to be called after router-\>is_running has * been set. @@ -1861,16 +1385,11 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router, else uptime = real_uptime(router, now); - /* XXX We shouldn't need to check dir_port, but we do because of - * bug 1693. In the future, once relays set wants_to_be_hs_dir - * correctly, we can revert to only checking dir_port if router's - * version is too old. */ - /* XXX Unfortunately, we need to keep checking dir_port until all - * *clients* suffering from bug 2722 are obsolete. The first version - * to fix the bug was 0.2.2.25-alpha. */ - return (router->wants_to_be_hs_dir && router->dir_port && + return (router->wants_to_be_hs_dir && + router->supports_tunnelled_dir_requests && + node->is_stable && node->is_fast && uptime >= get_options()->MinUptimeHidServDirectoryV2 && - node->is_running); + router_is_active(router, node, now)); } /** Don't consider routers with less bandwidth than this when computing @@ -1907,17 +1426,17 @@ router_counts_toward_thresholds(const node_t *node, time_t now, * the Weighted Fractional Uptime history, and use them to set thresholds for * the Stable, Fast, and Guard flags. Update the fields stable_uptime, * stable_mtbf, enough_mtbf_info, guard_wfu, guard_tk, fast_bandwidth, - * guard_bandwidh_including_exits, guard_bandwidth_excluding_exits, + * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits. * * Also, set the is_exit flag of each router appropriately. */ static void -dirserv_compute_performance_thresholds(routerlist_t *rl, - digestmap_t *omit_as_sybil) +dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil) { int n_active, n_active_nonexit, n_familiar; uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb; long *tks; double *mtbfs, *wfus; + smartlist_t *nodelist; time_t now = time(NULL); const or_options_t *options = get_options(); @@ -1935,32 +1454,39 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, guard_tk = 0; guard_wfu = 0; + nodelist_assert_ok(); + nodelist = nodelist_get_list(); + /* Initialize arrays that will hold values for each router. We'll * sort them and use that to compute thresholds. */ n_active = n_active_nonexit = 0; /* Uptime for every active router. */ - uptimes = tor_malloc(sizeof(uint32_t)*smartlist_len(rl->routers)); + uptimes = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t)); /* Bandwidth for every active router. */ - bandwidths_kb = tor_malloc(sizeof(uint32_t)*smartlist_len(rl->routers)); + bandwidths_kb = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t)); /* Bandwidth for every active non-exit router. */ bandwidths_excluding_exits_kb = - tor_malloc(sizeof(uint32_t)*smartlist_len(rl->routers)); + tor_calloc(smartlist_len(nodelist), sizeof(uint32_t)); /* Weighted mean time between failure for each active router. */ - mtbfs = tor_malloc(sizeof(double)*smartlist_len(rl->routers)); + mtbfs = tor_calloc(smartlist_len(nodelist), sizeof(double)); /* Time-known for each active router. */ - tks = tor_malloc(sizeof(long)*smartlist_len(rl->routers)); + tks = tor_calloc(smartlist_len(nodelist), sizeof(long)); /* Weighted fractional uptime for each active router. */ - wfus = tor_malloc(sizeof(double)*smartlist_len(rl->routers)); - - nodelist_assert_ok(); + wfus = tor_calloc(smartlist_len(nodelist), sizeof(double)); /* Now, fill in the arrays. */ - SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { + SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) { + if (options->BridgeAuthoritativeDir && + node->ri && + node->ri->purpose != ROUTER_PURPOSE_BRIDGE) + continue; if (router_counts_toward_thresholds(node, now, omit_as_sybil, require_mbw)) { routerinfo_t *ri = node->ri; const char *id = node->identity; uint32_t bw_kb; + /* resolve spurious clang shallow analysis null pointer errors */ + tor_assert(ri); node->is_exit = (!router_exit_policy_rejects_all(ri) && exit_policy_is_general_exit(ri->exit_policy)); uptimes[n_active] = (uint32_t)real_uptime(ri, now); @@ -1984,9 +1510,10 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, /* The 12.5th percentile bandwidth is fast. */ fast_bandwidth_kb = find_nth_uint32(bandwidths_kb, n_active, n_active/8); /* (Now bandwidths is sorted.) */ - if (fast_bandwidth_kb < ROUTER_REQUIRED_MIN_BANDWIDTH/(2 * 1000)) + if (fast_bandwidth_kb < RELAY_REQUIRED_MIN_BANDWIDTH/(2 * 1000)) fast_bandwidth_kb = bandwidths_kb[n_active/4]; - guard_bandwidth_including_exits_kb = bandwidths_kb[(n_active-1)/2]; + guard_bandwidth_including_exits_kb = + third_quartile_uint32(bandwidths_kb, n_active); guard_tk = find_nth_long(tks, n_active, n_active/8); } @@ -2024,7 +1551,7 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, * fill wfus with the wfu of every such "familiar" router. */ n_familiar = 0; - SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) { + SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) { if (router_counts_toward_thresholds(node, now, omit_as_sybil, require_mbw)) { routerinfo_t *ri = node->ri; @@ -2044,7 +1571,8 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, if (n_active_nonexit) { guard_bandwidth_excluding_exits_kb = - median_uint32(bandwidths_excluding_exits_kb, n_active_nonexit); + find_nth_uint32(bandwidths_excluding_exits_kb, + n_active_nonexit, n_active_nonexit*3/4); } log_info(LD_DIRSERV, @@ -2060,7 +1588,7 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, (unsigned long)guard_tk, (unsigned long)guard_bandwidth_including_exits_kb, (unsigned long)guard_bandwidth_excluding_exits_kb, - enough_mtbf_info ? "" : " don't "); + enough_mtbf_info ? "" : " don't"); tor_free(uptimes); tor_free(mtbfs); @@ -2070,6 +1598,20 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, tor_free(wfus); } +/* Use dirserv_compute_performance_thresholds() to compute the thresholds + * for the status flags, specifically for bridges. + * + * This is only called by a Bridge Authority from + * networkstatus_getinfo_by_purpose(). + */ +void +dirserv_compute_bridge_flag_thresholds(void) +{ + digestmap_t *omit_as_sybil = digestmap_new(); + dirserv_compute_performance_thresholds(omit_as_sybil); + digestmap_free(omit_as_sybil, NULL); +} + /** Measured bandwidth cache entry */ typedef struct mbw_cache_entry_s { long mbw_kb; @@ -2082,7 +1624,7 @@ static digestmap_t *mbw_cache = NULL; /** Store a measured bandwidth cache entry when reading the measured * bandwidths file. */ -void +STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, time_t as_of) { @@ -2112,7 +1654,7 @@ dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, } /** Clear and free the measured bandwidth cache */ -void +STATIC void dirserv_clear_measured_bw_cache(void) { if (mbw_cache) { @@ -2123,7 +1665,7 @@ dirserv_clear_measured_bw_cache(void) } /** Scan the measured bandwidth cache and remove expired entries */ -void +STATIC void dirserv_expire_measured_bw_cache(time_t now) { @@ -2145,7 +1687,7 @@ dirserv_expire_measured_bw_cache(time_t now) } /** Get the current size of the measured bandwidth cache */ -int +STATIC int dirserv_get_measured_bw_cache_size(void) { if (mbw_cache) return digestmap_size(mbw_cache); @@ -2155,7 +1697,7 @@ dirserv_get_measured_bw_cache_size(void) /** Query the cache by identity digest, return value indicates whether * we found it. The bw_out and as_of_out pointers receive the cached * bandwidth value and the time it was cached if not NULL. */ -int +STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, time_t *as_of_out) { @@ -2176,7 +1718,7 @@ dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, } /** Predicate wrapper for dirserv_query_measured_bw_cache() */ -int +STATIC int dirserv_has_measured_bw(const char *node_id) { return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL); @@ -2219,16 +1761,13 @@ dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri) * how many measured bandwidths we know. This is used to decide whether we * ever trust advertised bandwidths for purposes of assigning flags. */ static void -dirserv_count_measured_bws(routerlist_t *rl) +dirserv_count_measured_bws(const smartlist_t *routers) { /* Initialize this first */ routers_with_measured_bw = 0; - tor_assert(rl); - tor_assert(rl->routers); - /* Iterate over the routerlist and count measured bandwidths */ - SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) { + SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { /* Check if we know a measured bandwidth for this one */ if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) { ++routers_with_measured_bw; @@ -2341,13 +1880,12 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, char published[ISO_TIME_LEN+1]; char identity64[BASE64_DIGEST_LEN+1]; char digest64[BASE64_DIGEST_LEN+1]; - smartlist_t *chunks = NULL; + smartlist_t *chunks = smartlist_new(); format_iso_time(published, rs->published_on); digest_to_base64(identity64, rs->identity_digest); digest_to_base64(digest64, rs->descriptor_digest); - chunks = smartlist_new(); smartlist_add_asprintf(chunks, "r %s %s %s%s%s %s %d %d\n", rs->nickname, @@ -2378,19 +1916,16 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, goto done; smartlist_add_asprintf(chunks, - "s%s%s%s%s%s%s%s%s%s%s%s%s%s\n", + "s%s%s%s%s%s%s%s%s%s%s\n", /* These must stay in alphabetical order. */ rs->is_authority?" Authority":"", - rs->is_bad_directory?" BadDirectory":"", rs->is_bad_exit?" BadExit":"", rs->is_exit?" Exit":"", rs->is_fast?" Fast":"", rs->is_possible_guard?" Guard":"", rs->is_hs_dir?" HSDir":"", - rs->is_named?" Named":"", rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", - rs->is_unnamed?" Unnamed":"", rs->is_v2_dir?" V2Dir":"", rs->is_valid?" Valid":""); @@ -2459,6 +1994,13 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, smartlist_add_asprintf(chunks, " Measured=%d", vrs->measured_bw_kb); } + /* Write down guardfraction information if we have it. */ + if (format == NS_V3_VOTE && vrs && vrs->status.has_guardfraction) { + smartlist_add_asprintf(chunks, + " GuardFraction=%d", + vrs->status.guardfraction_percentage); + } + smartlist_add(chunks, tor_strdup("\n")); if (desc) { @@ -2466,16 +2008,24 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, smartlist_add_asprintf(chunks, "p %s\n", summary); tor_free(summary); } + + if (format == NS_V3_VOTE && vrs) { + if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) { + smartlist_add(chunks, tor_strdup("id ed25519 none\n")); + } else { + char ed_b64[BASE64_DIGEST256_LEN+1]; + digest256_to_base64(ed_b64, (const char*)vrs->ed25519_id); + smartlist_add_asprintf(chunks, "id ed25519 %s\n", ed_b64); + } + } } done: result = smartlist_join_strings(chunks, "", 0, NULL); err: - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); - smartlist_free(chunks); - } + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); return result; } @@ -2581,78 +2131,52 @@ get_possible_sybil_list(const smartlist_t *routers) return omit_as_sybil; } -/** Return non-zero iff a relay running the Tor version specified in - * <b>platform</b> is suitable for use as a potential entry guard. */ -static int -is_router_version_good_for_possible_guard(const char *platform) +/** If there are entries in <b>routers</b> with exactly the same ed25519 keys, + * remove the older one. If they are exactly the same age, remove the one + * with the greater descriptor digest. May alter the order of the list. */ +static void +routers_make_ed_keys_unique(smartlist_t *routers) { - static int parsed_versions_initialized = 0; - static tor_version_t first_good_0_2_1_guard_version; - static tor_version_t first_good_0_2_2_guard_version; - static tor_version_t first_good_later_guard_version; - - tor_version_t router_version; - - /* XXX024 This block should be extracted into its own function. */ - /* XXXX Begin code copied from tor_version_as_new_as (in routerparse.c) */ - { - char *s, *s2, *start; - char tmp[128]; + routerinfo_t *ri2; + digest256map_t *by_ed_key = digest256map_new(); - tor_assert(platform); - - /* nonstandard Tor; be safe and say yes */ - if (strcmpstart(platform,"Tor ")) - return 1; - - start = (char *)eat_whitespace(platform+3); - if (!*start) return 0; - s = (char *)find_whitespace(start); /* also finds '\0', which is fine */ - s2 = (char*)eat_whitespace(s); - if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-")) - s = (char*)find_whitespace(s2); + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { + ri->omit_from_vote = 0; + if (ri->cache_info.signing_key_cert == NULL) + continue; /* No ed key */ + const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey; + if ((ri2 = digest256map_get(by_ed_key, pk))) { + /* Duplicate; must omit one. Set the omit_from_vote flag in whichever + * one has the earlier published_on. */ + const time_t ri_pub = ri->cache_info.published_on; + const time_t ri2_pub = ri2->cache_info.published_on; + if (ri2_pub < ri_pub || + (ri2_pub == ri_pub && + memcmp(ri->cache_info.signed_descriptor_digest, + ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) { + digest256map_set(by_ed_key, pk, ri); + ri2->omit_from_vote = 1; + } else { + ri->omit_from_vote = 1; + } + } else { + /* Add to map */ + digest256map_set(by_ed_key, pk, ri); + } + } SMARTLIST_FOREACH_END(ri); - if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */ - return 0; - strlcpy(tmp, start, s-start+1); + digest256map_free(by_ed_key, NULL); - if (tor_version_parse(tmp, &router_version)<0) { - log_info(LD_DIR,"Router version '%s' unparseable.",tmp); - return 1; /* be safe and say yes */ + /* Now remove every router where the omit_from_vote flag got set. */ + SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { + if (ri->omit_from_vote) { + SMARTLIST_DEL_CURRENT(routers, ri); } - } - /* XXXX End code copied from tor_version_as_new_as (in routerparse.c) */ - - if (!parsed_versions_initialized) { - /* CVE-2011-2769 was fixed on the relay side in Tor versions - * 0.2.1.31, 0.2.2.34, and 0.2.3.6-alpha. */ - tor_assert(tor_version_parse("0.2.1.31", - &first_good_0_2_1_guard_version)>=0); - tor_assert(tor_version_parse("0.2.2.34", - &first_good_0_2_2_guard_version)>=0); - tor_assert(tor_version_parse("0.2.3.6-alpha", - &first_good_later_guard_version)>=0); - - /* Don't parse these constant version strings once for every relay - * for every vote. */ - parsed_versions_initialized = 1; - } - - return ((tor_version_same_series(&first_good_0_2_1_guard_version, - &router_version) && - tor_version_compare(&first_good_0_2_1_guard_version, - &router_version) <= 0) || - (tor_version_same_series(&first_good_0_2_2_guard_version, - &router_version) && - tor_version_compare(&first_good_0_2_2_guard_version, - &router_version) <= 0) || - (tor_version_compare(&first_good_later_guard_version, - &router_version) <= 0)); + } SMARTLIST_FOREACH_END(ri); } /** Extract status information from <b>ri</b> and from other authority - * functions and store it in <b>rs</b>>. If <b>naming</b>, consider setting - * the named flag in <b>rs</b>. + * functions and store it in <b>rs</b>>. * * We assume that ri-\>is_running has already been set, e.g. by * dirserv_set_router_is_running(ri, now); @@ -2662,8 +2186,7 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, routerinfo_t *ri, time_t now, - int naming, int listbadexits, - int listbaddirs, int vote_on_hsdirs) + int listbadexits) { const or_options_t *options = get_options(); uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri); @@ -2676,27 +2199,18 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, /* Already set by compute_performance_thresholds. */ rs->is_exit = node->is_exit; rs->is_stable = node->is_stable = - router_is_active(ri, node, now) && !dirserv_thinks_router_is_unreliable(now, ri, 1, 0); rs->is_fast = node->is_fast = - router_is_active(ri, node, now) && !dirserv_thinks_router_is_unreliable(now, ri, 0, 1); rs->is_flagged_running = node->is_running; /* computed above */ - if (naming) { - uint32_t name_status = dirserv_get_name_status( - node->identity, ri->nickname); - rs->is_named = (naming && (name_status & FP_NAMED)) ? 1 : 0; - rs->is_unnamed = (naming && (name_status & FP_UNNAMED)) ? 1 : 0; - } rs->is_valid = node->is_valid; if (node->is_fast && ((options->AuthDirGuardBWGuarantee && routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) || routerbw_kb >= MIN(guard_bandwidth_including_exits_kb, - guard_bandwidth_excluding_exits_kb)) && - is_router_version_good_for_possible_guard(ri->platform)) { + guard_bandwidth_excluding_exits_kb))) { long tk = rep_hist_get_weighted_time_known( node->identity, now); double wfu = rep_hist_get_weighted_fractional_uptime( @@ -2706,14 +2220,11 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->is_possible_guard = 0; } - rs->is_bad_directory = listbaddirs && node->is_bad_directory; rs->is_bad_exit = listbadexits && node->is_bad_exit; - node->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, node, now); - rs->is_hs_dir = vote_on_hsdirs && node->is_hs_dir; - rs->is_v2_dir = ri->dir_port != 0; + rs->is_hs_dir = node->is_hs_dir = + dirserv_thinks_router_is_hs_dir(ri, node, now); - if (!strcasecmp(ri->nickname, UNNAMED_ROUTER_NICKNAME)) - rs->is_named = rs->is_unnamed = 0; + rs->is_named = rs->is_unnamed = 0; rs->published_on = ri->cache_info.published_on; memcpy(rs->identity_digest, node->identity, DIGEST_LEN); @@ -2723,6 +2234,7 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname)); rs->or_port = ri->or_port; rs->dir_port = ri->dir_port; + rs->is_v2_dir = ri->supports_tunnelled_dir_requests; if (options->AuthDirHasIPv6Connectivity == 1 && !tor_addr_is_null(&ri->ipv6_addr) && node->last_reachable6 >= now - REACHABLE_TIMEOUT) { @@ -2731,6 +2243,43 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr); rs->ipv6_orport = ri->ipv6_orport; } + + if (options->TestingTorNetwork) { + dirserv_set_routerstatus_testing(rs); + } +} + +/** Use TestingDirAuthVoteExit, TestingDirAuthVoteGuard, and + * TestingDirAuthVoteHSDir to give out the Exit, Guard, and HSDir flags, + * respectively. But don't set the corresponding node flags. + * Should only be called if TestingTorNetwork is set. */ +STATIC void +dirserv_set_routerstatus_testing(routerstatus_t *rs) +{ + const or_options_t *options = get_options(); + + tor_assert(options->TestingTorNetwork); + + if (routerset_contains_routerstatus(options->TestingDirAuthVoteExit, + rs, 0)) { + rs->is_exit = 1; + } else if (options->TestingDirAuthVoteExitIsStrict) { + rs->is_exit = 0; + } + + if (routerset_contains_routerstatus(options->TestingDirAuthVoteGuard, + rs, 0)) { + rs->is_possible_guard = 1; + } else if (options->TestingDirAuthVoteGuardIsStrict) { + rs->is_possible_guard = 0; + } + + if (routerset_contains_routerstatus(options->TestingDirAuthVoteHSDir, + rs, 0)) { + rs->is_hs_dir = 1; + } else if (options->TestingDirAuthVoteHSDirIsStrict) { + rs->is_hs_dir = 0; + } } /** Routerstatus <b>rs</b> is part of a group of routers that are on @@ -2741,20 +2290,332 @@ static void clear_status_flags_on_sybil(routerstatus_t *rs) { rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_flagged_running = rs->is_named = rs->is_valid = rs->is_v2_dir = - rs->is_hs_dir = rs->is_possible_guard = rs->is_bad_exit = - rs->is_bad_directory = 0; + rs->is_flagged_running = rs->is_named = rs->is_valid = + rs->is_hs_dir = rs->is_possible_guard = rs->is_bad_exit = 0; /* FFFF we might want some mechanism to check later on if we * missed zeroing any flags: it's easy to add a new flag but * forget to add it to this clause. */ } +/** The guardfraction of the guard with identity fingerprint <b>guard_id</b> + * is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for + * this guard in <b>vote_routerstatuses</b>, and if we do, register the + * information to it. + * + * Return 1 if we applied the information and 0 if we couldn't find a + * matching guard. + * + * Requires that <b>vote_routerstatuses</b> be sorted. + */ +static int +guardfraction_line_apply(const char *guard_id, + uint32_t guardfraction_percentage, + smartlist_t *vote_routerstatuses) +{ + vote_routerstatus_t *vrs = NULL; + + tor_assert(vote_routerstatuses); + + vrs = smartlist_bsearch(vote_routerstatuses, guard_id, + compare_digest_to_vote_routerstatus_entry); + + if (!vrs) { + return 0; + } + + vrs->status.has_guardfraction = 1; + vrs->status.guardfraction_percentage = guardfraction_percentage; + + return 1; +} + +/* Given a guard line from a guardfraction file, parse it and register + * its information to <b>vote_routerstatuses</b>. + * + * Return: + * * 1 if the line was proper and its information got registered. + * * 0 if the line was proper but no currently active guard was found + * to register the guardfraction information to. + * * -1 if the line could not be parsed and set <b>err_msg</b> to a + newly allocated string containing the error message. + */ +static int +guardfraction_file_parse_guard_line(const char *guard_line, + smartlist_t *vote_routerstatuses, + char **err_msg) +{ + char guard_id[DIGEST_LEN]; + uint32_t guardfraction; + char *inputs_tmp = NULL; + int num_ok = 1; + + smartlist_t *sl = smartlist_new(); + int retval = -1; + + tor_assert(err_msg); + + /* guard_line should contain something like this: + <hex digest> <guardfraction> <appearances> */ + smartlist_split_string(sl, guard_line, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3); + if (smartlist_len(sl) < 3) { + tor_asprintf(err_msg, "bad line '%s'", guard_line); + goto done; + } + + inputs_tmp = smartlist_get(sl, 0); + if (strlen(inputs_tmp) != HEX_DIGEST_LEN || + base16_decode(guard_id, DIGEST_LEN, inputs_tmp, HEX_DIGEST_LEN)) { + tor_asprintf(err_msg, "bad digest '%s'", inputs_tmp); + goto done; + } + + inputs_tmp = smartlist_get(sl, 1); + /* Guardfraction is an integer in [0, 100]. */ + guardfraction = + (uint32_t) tor_parse_long(inputs_tmp, 10, 0, 100, &num_ok, NULL); + if (!num_ok) { + tor_asprintf(err_msg, "wrong percentage '%s'", inputs_tmp); + goto done; + } + + /* If routerstatuses were provided, apply this info to actual routers. */ + if (vote_routerstatuses) { + retval = guardfraction_line_apply(guard_id, guardfraction, + vote_routerstatuses); + } else { + retval = 0; /* If we got this far, line was correctly formatted. */ + } + + done: + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + + return retval; +} + +/** Given an inputs line from a guardfraction file, parse it and + * register its information to <b>total_consensuses</b> and + * <b>total_days</b>. + * + * Return 0 if it parsed well. Return -1 if there was an error, and + * set <b>err_msg</b> to a newly allocated string containing the + * error message. + */ +static int +guardfraction_file_parse_inputs_line(const char *inputs_line, + int *total_consensuses, + int *total_days, + char **err_msg) +{ + int retval = -1; + char *inputs_tmp = NULL; + int num_ok = 1; + smartlist_t *sl = smartlist_new(); + + tor_assert(err_msg); + + /* Second line is inputs information: + * n-inputs <total_consensuses> <total_days>. */ + smartlist_split_string(sl, inputs_line, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3); + if (smartlist_len(sl) < 2) { + tor_asprintf(err_msg, "incomplete line '%s'", inputs_line); + goto done; + } + + inputs_tmp = smartlist_get(sl, 0); + *total_consensuses = + (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL); + if (!num_ok) { + tor_asprintf(err_msg, "unparseable consensus '%s'", inputs_tmp); + goto done; + } + + inputs_tmp = smartlist_get(sl, 1); + *total_days = + (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL); + if (!num_ok) { + tor_asprintf(err_msg, "unparseable days '%s'", inputs_tmp); + goto done; + } + + retval = 0; + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + + return retval; +} + +/* Maximum age of a guardfraction file that we are willing to accept. */ +#define MAX_GUARDFRACTION_FILE_AGE (7*24*60*60) /* approx a week */ + +/** Static strings of guardfraction files. */ +#define GUARDFRACTION_DATE_STR "written-at" +#define GUARDFRACTION_INPUTS "n-inputs" +#define GUARDFRACTION_GUARD "guard-seen" +#define GUARDFRACTION_VERSION "guardfraction-file-version" + +/** Given a guardfraction file in a string, parse it and register the + * guardfraction information to the provided vote routerstatuses. + * + * This is the rough format of the guardfraction file: + * + * guardfraction-file-version 1 + * written-at <date and time> + * n-inputs <number of consesuses parsed> <number of days considered> + * + * guard-seen <fpr 1> <guardfraction percentage> <consensus appearances> + * guard-seen <fpr 2> <guardfraction percentage> <consensus appearances> + * guard-seen <fpr 3> <guardfraction percentage> <consensus appearances> + * guard-seen <fpr 4> <guardfraction percentage> <consensus appearances> + * guard-seen <fpr 5> <guardfraction percentage> <consensus appearances> + * ... + * + * Return -1 if the parsing failed and 0 if it went smoothly. Parsing + * should tolerate errors in all lines but the written-at header. + */ +STATIC int +dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str, + smartlist_t *vote_routerstatuses) +{ + config_line_t *front=NULL, *line; + int ret_tmp; + int retval = -1; + int current_line_n = 0; /* line counter for better log messages */ + + /* Guardfraction info to be parsed */ + int total_consensuses = 0; + int total_days = 0; + + /* Stats */ + int guards_read_n = 0; + int guards_applied_n = 0; + + /* Parse file and split it in lines */ + ret_tmp = config_get_lines(guardfraction_file_str, &front, 0); + if (ret_tmp < 0) { + log_warn(LD_CONFIG, "Error reading from guardfraction file"); + goto done; + } + + /* Sort routerstatuses (needed later when applying guardfraction info) */ + if (vote_routerstatuses) + smartlist_sort(vote_routerstatuses, compare_vote_routerstatus_entries); + + for (line = front; line; line=line->next) { + current_line_n++; + + if (!strcmp(line->key, GUARDFRACTION_VERSION)) { + int num_ok = 1; + unsigned int version; + + version = + (unsigned int) tor_parse_long(line->value, + 10, 0, INT_MAX, &num_ok, NULL); + + if (!num_ok || version != 1) { + log_warn(LD_GENERAL, "Got unknown guardfraction version %d.", version); + goto done; + } + } else if (!strcmp(line->key, GUARDFRACTION_DATE_STR)) { + time_t file_written_at; + time_t now = time(NULL); + + /* First line is 'written-at <date>' */ + if (parse_iso_time(line->value, &file_written_at) < 0) { + log_warn(LD_CONFIG, "Guardfraction:%d: Bad date '%s'. Ignoring", + current_line_n, line->value); + goto done; /* don't tolerate failure here. */ + } + if (file_written_at < now - MAX_GUARDFRACTION_FILE_AGE) { + log_warn(LD_CONFIG, "Guardfraction:%d: was written very long ago '%s'", + current_line_n, line->value); + goto done; /* don't tolerate failure here. */ + } + } else if (!strcmp(line->key, GUARDFRACTION_INPUTS)) { + char *err_msg = NULL; + + if (guardfraction_file_parse_inputs_line(line->value, + &total_consensuses, + &total_days, + &err_msg) < 0) { + log_warn(LD_CONFIG, "Guardfraction:%d: %s", + current_line_n, err_msg); + tor_free(err_msg); + continue; + } + + } else if (!strcmp(line->key, GUARDFRACTION_GUARD)) { + char *err_msg = NULL; + + ret_tmp = guardfraction_file_parse_guard_line(line->value, + vote_routerstatuses, + &err_msg); + if (ret_tmp < 0) { /* failed while parsing the guard line */ + log_warn(LD_CONFIG, "Guardfraction:%d: %s", + current_line_n, err_msg); + tor_free(err_msg); + continue; + } + + /* Successfully parsed guard line. Check if it was applied properly. */ + guards_read_n++; + if (ret_tmp > 0) { + guards_applied_n++; + } + } else { + log_warn(LD_CONFIG, "Unknown guardfraction line %d (%s %s)", + current_line_n, line->key, line->value); + } + } + + retval = 0; + + log_info(LD_CONFIG, + "Successfully parsed guardfraction file with %d consensuses over " + "%d days. Parsed %d nodes and applied %d of them%s.", + total_consensuses, total_days, guards_read_n, guards_applied_n, + vote_routerstatuses ? "" : " (no routerstatus provided)" ); + + done: + config_free_lines(front); + + if (retval < 0) { + return retval; + } else { + return guards_read_n; + } +} + +/** Read a guardfraction file at <b>fname</b> and load all its + * information to <b>vote_routerstatuses</b>. */ +int +dirserv_read_guardfraction_file(const char *fname, + smartlist_t *vote_routerstatuses) +{ + char *guardfraction_file_str; + + /* Read file to a string */ + guardfraction_file_str = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + if (!guardfraction_file_str) { + log_warn(LD_FS, "Cannot open guardfraction file '%s'. Failing.", fname); + return -1; + } + + return dirserv_read_guardfraction_file_from_str(guardfraction_file_str, + vote_routerstatuses); +} + /** * Helper function to parse out a line in the measured bandwidth file * into a measured_bw_line_t output structure. Returns -1 on failure * or 0 on success. */ -int +STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) { char *line = tor_strdup(orig_line); @@ -2835,7 +2696,7 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) * of bandwidth statuses. Returns true if a line is found, * false otherwise. */ -int +STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line, smartlist_t *routerstatuses) { @@ -2865,7 +2726,7 @@ int dirserv_read_measured_bandwidths(const char *from_file, smartlist_t *routerstatuses) { - char line[256]; + char line[512]; FILE *fp = tor_fopen_cloexec(from_file, "r"); int applied_lines = 0; time_t file_time, now; @@ -2886,7 +2747,7 @@ dirserv_read_measured_bandwidths(const char *from_file, } line[strlen(line)-1] = '\0'; - file_time = tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); + file_time = (time_t)tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL); if (!ok) { log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", escaped(line)); @@ -2941,10 +2802,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, smartlist_t *routers, *routerstatuses; char identity_digest[DIGEST_LEN]; char signing_key_digest[DIGEST_LEN]; - int naming = options->NamingAuthoritativeDir; int listbadexits = options->AuthDirListBadExits; - int listbaddirs = options->AuthDirListBadDirs; - int vote_on_hsdirs = options->VoteOnHidServDirectoriesV2; routerlist_t *rl = router_get_routerlist(); time_t now = time(NULL); time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; @@ -2957,14 +2815,6 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, tor_assert(private_key); tor_assert(cert); - if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { - log_warn(LD_NET, "Couldn't resolve my hostname"); - return NULL; - } - if (!hostname || !strchr(hostname, '.')) { - tor_free(hostname); - hostname = tor_dup_ip(addr); - } if (crypto_pk_get_digest(private_key, signing_key_digest)<0) { log_err(LD_BUG, "Error computing signing key digest"); return NULL; @@ -2973,6 +2823,14 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, log_err(LD_BUG, "Error computing identity key digest"); return NULL; } + if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { + log_warn(LD_NET, "Couldn't resolve my hostname"); + return NULL; + } + if (!hostname || !strchr(hostname, '.')) { + tor_free(hostname); + hostname = tor_dup_ip(addr); + } if (options->VersioningAuthoritativeDir) { client_versions = format_versions_list(options->RecommendedClientVersions); @@ -3007,6 +2865,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, routers = smartlist_new(); smartlist_add_all(routers, rl->routers); + routers_make_ed_keys_unique(routers); + /* After this point, don't use rl->routers; use 'routers' instead. */ routers_sort_by_identity(routers); omit_as_sybil = get_possible_sybil_list(routers); @@ -3017,9 +2877,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, /* Count how many have measured bandwidths so we know how to assign flags; * this must come before dirserv_compute_performance_thresholds() */ - dirserv_count_measured_bws(rl); + dirserv_count_measured_bws(routers); - dirserv_compute_performance_thresholds(rl, omit_as_sybil); + dirserv_compute_performance_thresholds(omit_as_sybil); routerstatuses = smartlist_new(); microdescriptors = smartlist_new(); @@ -3035,8 +2895,13 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; set_routerstatus_from_routerinfo(rs, node, ri, now, - naming, listbadexits, listbaddirs, - vote_on_hsdirs); + listbadexits); + + if (ri->cache_info.signing_key_cert) { + memcpy(vrs->ed25519_id, + ri->cache_info.signing_key_cert->signing_key.pubkey, + ED25519_PUBKEY_LEN); + } if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) clear_status_flags_on_sybil(rs); @@ -3063,6 +2928,12 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); + /* Apply guardfraction information to routerstatuses. */ + if (options->GuardfractionFile) { + dirserv_read_guardfraction_file(options->GuardfractionFile, + routerstatuses); + } + /* This pass through applies the measured bw lines to the routerstatuses */ if (options->V3BandwidthsFile) { dirserv_read_measured_bandwidths(options->V3BandwidthsFile, @@ -3093,7 +2964,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, else last_consensus_interval = options->TestingV3AuthInitialVotingInterval; v3_out->valid_after = - dirvote_get_start_of_next_interval(now, (int)last_consensus_interval); + dirvote_get_start_of_next_interval(now, (int)last_consensus_interval, + options->TestingV3AuthVotingStartOffset); format_iso_time(tbuf, v3_out->valid_after); log_notice(LD_DIR,"Choosing valid-after time in vote as %s: " "consensus_set=%d, last_interval=%d", @@ -3110,22 +2982,23 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, v3_out->client_versions = client_versions; v3_out->server_versions = server_versions; + v3_out->package_lines = smartlist_new(); + { + config_line_t *cl; + for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) { + if (validate_recommended_package_line(cl->value)) + smartlist_add(v3_out->package_lines, tor_strdup(cl->value)); + } + } + v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid", + "Authority Exit Fast Guard Stable V2Dir Valid HSDir", 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add(v3_out->known_flags, tor_strdup("Running")); - if (listbaddirs) - smartlist_add(v3_out->known_flags, tor_strdup("BadDirectory")); if (listbadexits) smartlist_add(v3_out->known_flags, tor_strdup("BadExit")); - if (naming) { - smartlist_add(v3_out->known_flags, tor_strdup("Named")); - smartlist_add(v3_out->known_flags, tor_strdup("Unnamed")); - } - if (vote_on_hsdirs) - smartlist_add(v3_out->known_flags, tor_strdup("HSDir")); smartlist_sort_strings(v3_out->known_flags); if (options->ConsensusParams) { @@ -3164,270 +3037,6 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, return v3_out; } -/** For v2 authoritative directories only: Replace the contents of - * <b>the_v2_networkstatus</b> with a newly generated network status - * object. */ -cached_dir_t * -generate_v2_networkstatus_opinion(void) -{ - cached_dir_t *r = NULL; - size_t identity_pkey_len; - char *status = NULL, *client_versions = NULL, *server_versions = NULL, - *identity_pkey = NULL, *hostname = NULL; - const or_options_t *options = get_options(); - char fingerprint[FINGERPRINT_LEN+1]; - char published[ISO_TIME_LEN+1]; - char digest[DIGEST_LEN]; - uint32_t addr; - crypto_pk_t *private_key; - routerlist_t *rl = router_get_routerlist(); - time_t now = time(NULL); - time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - int naming = options->NamingAuthoritativeDir; - int versioning = options->VersioningAuthoritativeDir; - int listbaddirs = options->AuthDirListBadDirs; - int listbadexits = options->AuthDirListBadExits; - int vote_on_hsdirs = options->VoteOnHidServDirectoriesV2; - const char *contact; - char *version_lines = NULL; - smartlist_t *routers = NULL; - digestmap_t *omit_as_sybil = NULL; - smartlist_t *chunks = NULL; - - private_key = get_server_identity_key(); - - if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { - log_warn(LD_NET, "Couldn't resolve my hostname"); - goto done; - } - if (!hostname) - hostname = tor_dup_ip(addr); - - format_iso_time(published, now); - - client_versions = format_versions_list(options->RecommendedClientVersions); - server_versions = format_versions_list(options->RecommendedServerVersions); - - if (crypto_pk_write_public_key_to_string(private_key, &identity_pkey, - &identity_pkey_len)<0) { - log_warn(LD_BUG,"Writing public key to string failed."); - goto done; - } - - if (crypto_pk_get_fingerprint(private_key, fingerprint, 0)<0) { - log_err(LD_BUG, "Error computing fingerprint"); - goto done; - } - - contact = options->ContactInfo; - if (!contact) - contact = "(none)"; - - if (versioning) { - tor_asprintf(&version_lines, - "client-versions %s\nserver-versions %s\n", - client_versions, server_versions); - } else { - version_lines = tor_strdup(""); - } - - chunks = smartlist_new(); - smartlist_add_asprintf(chunks, - "network-status-version 2\n" - "dir-source %s %s %d\n" - "fingerprint %s\n" - "contact %s\n" - "published %s\n" - "dir-options%s%s%s%s\n" - "%s" /* client version line, server version line. */ - "dir-signing-key\n%s", - hostname, fmt_addr32(addr), - (int)router_get_advertised_dir_port(options, 0), - fingerprint, - contact, - published, - naming ? " Names" : "", - listbaddirs ? " BadDirectories" : "", - listbadexits ? " BadExits" : "", - versioning ? " Versions" : "", - version_lines, - identity_pkey); - - /* precompute this part, since we need it to decide what "stable" - * means. */ - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { - dirserv_set_router_is_running(ri, now); - }); - - routers = smartlist_new(); - smartlist_add_all(routers, rl->routers); - routers_sort_by_identity(routers); - omit_as_sybil = get_possible_sybil_list(routers); - - dirserv_compute_performance_thresholds(rl, omit_as_sybil); - - SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { - if (ri->cache_info.published_on >= cutoff) { - routerstatus_t rs; - char *version = version_from_platform(ri->platform); - node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest); - if (!node) { - tor_free(version); - continue; - } - set_routerstatus_from_routerinfo(&rs, node, ri, now, - naming, listbadexits, listbaddirs, - vote_on_hsdirs); - - if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) - clear_status_flags_on_sybil(&rs); - - { - char *rsf = routerstatus_format_entry(&rs, version, NS_V2, NULL); - if (rsf) - smartlist_add(chunks, rsf); - } - tor_free(version); - } - } SMARTLIST_FOREACH_END(ri); - - smartlist_add_asprintf(chunks, "directory-signature %s\n", - options->Nickname); - - crypto_digest_smartlist(digest, DIGEST_LEN, chunks, "", DIGEST_SHA1); - - note_crypto_pk_op(SIGN_DIR); - { - char *sig; - if (!(sig = router_get_dirobj_signature(digest,DIGEST_LEN, - private_key))) { - log_warn(LD_BUG, "Unable to sign router status."); - goto done; - } - smartlist_add(chunks, sig); - } - - status = smartlist_join_strings(chunks, "", 0, NULL); - - { - networkstatus_v2_t *ns; - if (!(ns = networkstatus_v2_parse_from_string(status))) { - log_err(LD_BUG,"Generated a networkstatus we couldn't parse."); - goto done; - } - networkstatus_v2_free(ns); - } - - { - cached_dir_t **ns_ptr = &the_v2_networkstatus; - if (*ns_ptr) - cached_dir_decref(*ns_ptr); - *ns_ptr = new_cached_dir(status, now); - status = NULL; /* So it doesn't get double-freed. */ - the_v2_networkstatus_is_dirty = 0; - router_set_networkstatus_v2((*ns_ptr)->dir, now, NS_GENERATED, NULL); - r = *ns_ptr; - } - - done: - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); - smartlist_free(chunks); - } - tor_free(client_versions); - tor_free(server_versions); - tor_free(version_lines); - tor_free(status); - tor_free(hostname); - tor_free(identity_pkey); - smartlist_free(routers); - digestmap_free(omit_as_sybil, NULL); - return r; -} - -/** Given the portion of a networkstatus request URL after "tor/status/" in - * <b>key</b>, append to <b>result</b> the digests of the identity keys of the - * networkstatus objects that the client has requested. */ -void -dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, - const char *key) -{ - tor_assert(result); - - if (!cached_v2_networkstatus) - cached_v2_networkstatus = digestmap_new(); - - if (should_generate_v2_networkstatus()) - generate_v2_networkstatus_opinion(); - - if (!strcmp(key,"authority")) { - if (authdir_mode_v2(get_options())) { - const routerinfo_t *me = router_get_my_routerinfo(); - if (me) - smartlist_add(result, - tor_memdup(me->cache_info.identity_digest, DIGEST_LEN)); - } - } else if (!strcmp(key, "all")) { - if (digestmap_size(cached_v2_networkstatus)) { - digestmap_iter_t *iter; - iter = digestmap_iter_init(cached_v2_networkstatus); - while (!digestmap_iter_done(iter)) { - const char *ident; - void *val; - digestmap_iter_get(iter, &ident, &val); - smartlist_add(result, tor_memdup(ident, DIGEST_LEN)); - iter = digestmap_iter_next(cached_v2_networkstatus, iter); - } - } else { - SMARTLIST_FOREACH(router_get_trusted_dir_servers(), - dir_server_t *, ds, - if (ds->type & V2_DIRINFO) - smartlist_add(result, tor_memdup(ds->digest, DIGEST_LEN))); - } - smartlist_sort_digests(result); - if (smartlist_len(result) == 0) - log_info(LD_DIRSERV, - "Client requested 'all' network status objects; we have none."); - } else if (!strcmpstart(key, "fp/")) { - dir_split_resource_into_fingerprints(key+3, result, NULL, - DSR_HEX|DSR_SORT_UNIQ); - } -} - -/** Look for a network status object as specified by <b>key</b>, which should - * be either "authority" (to find a network status generated by us), a hex - * identity digest (to find a network status generated by given directory), or - * "all" (to return all the v2 network status objects we have). - */ -void -dirserv_get_networkstatus_v2(smartlist_t *result, - const char *key) -{ - cached_dir_t *cached; - smartlist_t *fingerprints = smartlist_new(); - tor_assert(result); - - if (!cached_v2_networkstatus) - cached_v2_networkstatus = digestmap_new(); - - dirserv_get_networkstatus_v2_fingerprints(fingerprints, key); - SMARTLIST_FOREACH_BEGIN(fingerprints, const char *, fp) { - if (router_digest_is_me(fp) && should_generate_v2_networkstatus()) - generate_v2_networkstatus_opinion(); - cached = digestmap_get(cached_v2_networkstatus, fp); - if (cached) { - smartlist_add(result, cached); - } else { - char hexbuf[HEX_DIGEST_LEN+1]; - base16_encode(hexbuf, sizeof(hexbuf), fp, DIGEST_LEN); - log_info(LD_DIRSERV, "Don't know about any network status with " - "fingerprint '%s'", hexbuf); - } - } SMARTLIST_FOREACH_END(fp); - SMARTLIST_FOREACH(fingerprints, char *, cp, tor_free(cp)); - smartlist_free(fingerprints); -} - /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t * pointers, adds copies of digests to fps_out, and doesn't use the * /tor/server/ prefix. For a /d/ request, adds descriptor digests; for other @@ -3544,7 +3153,7 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, DSR_HEX|DSR_SORT_UNIQ); SMARTLIST_FOREACH_BEGIN(digests, const char *, d) { if (router_digest_is_me(d)) { - /* make sure desc_routerinfo exists */ + /* calling router_get_my_routerinfo() to make sure it exists */ const routerinfo_t *ri = router_get_my_routerinfo(); if (ri) smartlist_add(descs_out, (void*) &(ri->cache_info)); @@ -3576,7 +3185,7 @@ dirserv_get_routerdescs(smartlist_t *descs_out, const char *key, * router listening at <b>address</b>:<b>or_port</b>, and has yielded * a certificate with digest <b>digest_rcvd</b>. * - * Inform the reachability checker that we could get to this guy. + * Inform the reachability checker that we could get to this relay. */ void dirserv_orconn_tls_done(const tor_addr_t *addr, @@ -3661,7 +3270,7 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router) /* IPv4. */ log_debug(LD_OR,"Testing reachability of %s at %s:%u.", - router->nickname, router->address, router->or_port); + router->nickname, fmt_addr32(router->addr), router->or_port); tor_addr_from_ipv4h(&router_addr, router->addr); chan = channel_tls_connect(&router_addr, router->or_port, router->cache_info.identity_digest); @@ -3727,15 +3336,12 @@ static cached_dir_t * lookup_cached_dir_by_fp(const char *fp) { cached_dir_t *d = NULL; - if (tor_digest_is_zero(fp) && cached_consensuses) + if (tor_digest_is_zero(fp) && cached_consensuses) { d = strmap_get(cached_consensuses, "ns"); - else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && + } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && (d = strmap_get(cached_consensuses, fp))) { /* this here interface is a nasty hack XXXX024 */; - } else if (router_digest_is_me(fp) && the_v2_networkstatus) - d = the_v2_networkstatus; - else if (cached_v2_networkstatus) - d = digestmap_get(cached_v2_networkstatus, fp); + } return d; } @@ -3941,8 +3547,6 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) } body = signed_descriptor_get_body(sd); if (conn->zlib_state) { - /* XXXX024 This 'last' business should actually happen on the last - * routerinfo, not on the last fingerprint. */ int last = ! smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(body, sd->signed_descriptor_len, conn, last); @@ -3959,6 +3563,11 @@ connection_dirserv_add_servers_to_outbuf(dir_connection_t *conn) if (!smartlist_len(conn->fingerprint_stack)) { /* We just wrote the last one; finish up. */ + if (conn->zlib_state) { + connection_write_to_buf_zlib("", 0, conn, 1); + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } conn->dir_spool_src = DIR_SPOOL_NONE; smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; @@ -3984,8 +3593,6 @@ connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) if (!md || !md->body) continue; if (conn->zlib_state) { - /* XXXX024 This 'last' business should actually happen on the last - * routerinfo, not on the last fingerprint. */ int last = !smartlist_len(conn->fingerprint_stack); connection_write_to_buf_zlib(md->body, md->bodylen, conn, last); if (last) { @@ -3997,6 +3604,11 @@ connection_dirserv_add_microdescs_to_outbuf(dir_connection_t *conn) } } if (!smartlist_len(conn->fingerprint_stack)) { + if (conn->zlib_state) { + connection_write_to_buf_zlib("", 0, conn, 1); + tor_zlib_free(conn->zlib_state); + conn->zlib_state = NULL; + } conn->dir_spool_src = DIR_SPOOL_NONE; smartlist_free(conn->fingerprint_stack); conn->fingerprint_stack = NULL; @@ -4069,7 +3681,7 @@ connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) if (uncompressing && ! conn->zlib_state && conn->fingerprint_stack && smartlist_len(conn->fingerprint_stack)) { - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); } } if (r) return r; @@ -4122,20 +3734,88 @@ connection_dirserv_flushed_some(dir_connection_t *conn) } } +/** Return true iff <b>line</b> is a valid RecommendedPackages line. + */ +/* + The grammar is: + + "package" SP PACKAGENAME SP VERSION SP URL SP DIGESTS NL + + PACKAGENAME = NONSPACE + VERSION = NONSPACE + URL = NONSPACE + DIGESTS = DIGEST | DIGESTS SP DIGEST + DIGEST = DIGESTTYPE "=" DIGESTVAL + + NONSPACE = one or more non-space printing characters + + DIGESTVAL = DIGESTTYPE = one or more non-=, non-" " characters. + + SP = " " + NL = a newline + + */ +int +validate_recommended_package_line(const char *line) +{ + const char *cp = line; + +#define WORD() \ + do { \ + if (*cp == ' ') \ + return 0; \ + cp = strchr(cp, ' '); \ + if (!cp) \ + return 0; \ + } while (0) + + WORD(); /* skip packagename */ + ++cp; + WORD(); /* skip version */ + ++cp; + WORD(); /* Skip URL */ + ++cp; + + /* Skip digesttype=digestval + */ + int n_entries = 0; + while (1) { + const char *start_of_word = cp; + const char *end_of_word = strchr(cp, ' '); + if (! end_of_word) + end_of_word = cp + strlen(cp); + + if (start_of_word == end_of_word) + return 0; + + const char *eq = memchr(start_of_word, '=', end_of_word - start_of_word); + + if (!eq) + return 0; + if (eq == start_of_word) + return 0; + if (eq == end_of_word - 1) + return 0; + if (memchr(eq+1, '=', end_of_word - (eq+1))) + return 0; + + ++n_entries; + if (0 == *end_of_word) + break; + + cp = end_of_word + 1; + } + + /* If we reach this point, we have at least 1 entry. */ + tor_assert(n_entries > 0); + return 1; +} + /** Release all storage used by the directory server. */ void dirserv_free_all(void) { dirserv_free_fingerprint_list(); - cached_dir_decref(the_directory); - clear_cached_dir(&the_runningrouters); - cached_dir_decref(the_v2_networkstatus); - cached_dir_decref(cached_directory); - clear_cached_dir(&cached_runningrouters); - - digestmap_free(cached_v2_networkstatus, free_cached_dir_); - cached_v2_networkstatus = NULL; strmap_free(cached_consensuses, free_cached_dir_); cached_consensuses = NULL; diff --git a/src/or/dirserv.h b/src/or/dirserv.h index f9d36d760f..9a9725ad6f 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,6 +12,8 @@ #ifndef TOR_DIRSERV_H #define TOR_DIRSERV_H +#include "testsupport.h" + /** What fraction (1 over this number) of the relay ID space do we * (as a directory authority) launch connections to at each reachability * test? */ @@ -32,10 +34,9 @@ int connection_dirserv_flushed_some(dir_connection_t *conn); -int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_t *pk); +int dirserv_add_own_fingerprint(crypto_pk_t *pk); int dirserv_load_fingerprint_file(void); void dirserv_free_fingerprint_list(void); -const char *dirserv_get_nickname_by_digest(const char *digest); enum was_router_added_t dirserv_add_multiple_descriptors( const char *desc, uint8_t purpose, const char *source, @@ -49,34 +50,23 @@ int list_server_status_v1(smartlist_t *routers, char **router_status_out, int dirserv_dump_directory_to_string(char **dir_out, crypto_pk_t *private_key); char *dirserv_get_flag_thresholds_line(void); +void dirserv_compute_bridge_flag_thresholds(void); int directory_fetches_from_authorities(const or_options_t *options); int directory_fetches_dir_info_early(const or_options_t *options); int directory_fetches_dir_info_later(const or_options_t *options); -int directory_caches_v2_dir_info(const or_options_t *options); int directory_caches_unknown_auth_certs(const or_options_t *options); int directory_caches_dir_info(const or_options_t *options); int directory_permits_begindir_requests(const or_options_t *options); -int directory_permits_controller_requests(const or_options_t *options); int directory_too_idle_to_fetch_descriptors(const or_options_t *options, time_t now); -void directory_set_dirty(void); -cached_dir_t *dirserv_get_directory(void); -cached_dir_t *dirserv_get_runningrouters(void); cached_dir_t *dirserv_get_consensus(const char *flavor_name); -void dirserv_set_cached_networkstatus_v2(const char *directory, - const char *identity, - time_t published); void dirserv_set_cached_consensus_networkstatus(const char *consensus, - const char *flavor_name, - const digests_t *digests, - time_t published); + const char *flavor_name, + const common_digests_t *digests, + time_t published); void dirserv_clear_old_networkstatuses(time_t cutoff); -void dirserv_clear_old_v1_info(time_t now); -void dirserv_get_networkstatus_v2(smartlist_t *result, const char *key); -void dirserv_get_networkstatus_v2_fingerprints(smartlist_t *result, - const char *key); int dirserv_get_routerdesc_fingerprints(smartlist_t *fps_out, const char *key, const char **msg, int for_unencrypted_conn, @@ -94,7 +84,8 @@ int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, int complain, int *valid_out); uint32_t dirserv_router_get_status(const routerinfo_t *router, - const char **msg); + const char **msg, + int severity); void dirserv_set_node_flags_from_authoritative_status(node_t *node, uint32_t authstatus); @@ -114,29 +105,40 @@ void dirserv_free_all(void); void cached_dir_decref(cached_dir_t *d); cached_dir_t *new_cached_dir(char *s, time_t published); +int validate_recommended_package_line(const char *line); + #ifdef DIRSERV_PRIVATE +STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs); + /* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */ #define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ -int measured_bw_line_parse(measured_bw_line_t *out, const char *line); +STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line); -int measured_bw_line_apply(measured_bw_line_t *parsed_line, +STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line, smartlist_t *routerstatuses); -void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, +STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, time_t as_of); -void dirserv_clear_measured_bw_cache(void); -void dirserv_expire_measured_bw_cache(time_t now); -int dirserv_get_measured_bw_cache_size(void); -int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_out, - time_t *as_of_out); -int dirserv_has_measured_bw(const char *node_id); -cached_dir_t *generate_v2_networkstatus_opinion(void); +STATIC void dirserv_clear_measured_bw_cache(void); +STATIC void dirserv_expire_measured_bw_cache(time_t now); +STATIC int dirserv_get_measured_bw_cache_size(void); +STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id, + long *bw_out, + time_t *as_of_out); +STATIC int dirserv_has_measured_bw(const char *node_id); + +STATIC int +dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str, + smartlist_t *vote_routerstatuses); #endif int dirserv_read_measured_bandwidths(const char *from_file, smartlist_t *routerstatuses); +int dirserv_read_guardfraction_file(const char *fname, + smartlist_t *vote_routerstatuses); + #endif diff --git a/src/or/dirvote.c b/src/or/dirvote.c index c6d1244902..62f85877fe 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -1,11 +1,12 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRVOTE_PRIVATE #include "or.h" #include "config.h" +#include "dircollate.h" #include "directory.h" #include "dirserv.h" #include "dirvote.h" @@ -16,6 +17,8 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "entrynodes.h" /* needed for guardfraction methods */ +#include "torcert.h" /** * \file dirvote.c @@ -51,7 +54,6 @@ static int dirvote_perform_vote(void); static void dirvote_clear_votes(int all_votes); static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); -static char *make_consensus_method_list(int low, int high, const char *sep); /* ===== * Voting @@ -60,12 +62,13 @@ static char *make_consensus_method_list(int low, int high, const char *sep); /** Return a new string containing the string representation of the vote in * <b>v3_ns</b>, signed with our v3 signing key <b>private_signing_key</b>. * For v3 authorities. */ -char * +STATIC char * format_networkstatus_vote(crypto_pk_t *private_signing_key, networkstatus_t *v3_ns) { - smartlist_t *chunks; + smartlist_t *chunks = smartlist_new(); const char *client_versions = NULL, *server_versions = NULL; + char *packages = NULL; char fingerprint[FINGERPRINT_LEN+1]; char digest[DIGEST_LEN]; uint32_t addr; @@ -98,7 +101,18 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, server_versions_line = tor_strdup(""); } - chunks = smartlist_new(); + if (v3_ns->package_lines) { + smartlist_t *tmp = smartlist_new(); + SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p, + if (validate_recommended_package_line(p)) + smartlist_add_asprintf(tmp, "package %s\n", p)); + packages = smartlist_join_strings(tmp, "", 0, NULL); + SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp)); + smartlist_free(tmp); + } else { + packages = tor_strdup(""); + } + { char published[ISO_TIME_LEN+1]; char va[ISO_TIME_LEN+1]; @@ -110,7 +124,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, char *params; authority_cert_t *cert = v3_ns->cert; char *methods = - make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD, " "); + make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD, + MAX_SUPPORTED_CONSENSUS_METHOD, " "); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); @@ -132,6 +147,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, "valid-until %s\n" "voting-delay %d %d\n" "%s%s" /* versions */ + "%s" /* packages */ "known-flags %s\n" "flag-thresholds %s\n" "params %s\n" @@ -143,6 +159,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, v3_ns->vote_seconds, v3_ns->dist_seconds, client_versions_line, server_versions_line, + packages, flags, flag_thresholds, params, @@ -230,10 +247,10 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, done: tor_free(client_versions_line); tor_free(server_versions_line); - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); - smartlist_free(chunks); - } + tor_free(packages); + + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); return status; } @@ -335,6 +352,9 @@ static int compare_vote_rs(const vote_routerstatus_t *a, const vote_routerstatus_t *b) { int r; + tor_assert(a); + tor_assert(b); + if ((r = fast_memcmp(a->status.identity_digest, b->status.identity_digest, DIGEST_LEN))) return r; @@ -432,6 +452,7 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, const tor_addr_port_t *most_alt_orport = NULL; SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { + tor_assert(rs); if (compare_vote_rs(most, rs) == 0 && !tor_addr_is_null(&rs->status.ipv6_addr) && rs->status.ipv6_orport) { @@ -454,10 +475,9 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, smartlist_free(alt_orports); } - if (consensus_method >= MIN_METHOD_FOR_MICRODESC && - microdesc_digest256_out) { + if (microdesc_digest256_out) { smartlist_t *digests = smartlist_new(); - const char *best_microdesc_digest; + const uint8_t *best_microdesc_digest; SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { char d[DIGEST256_LEN]; if (compare_vote_rs(rs, most)) @@ -537,12 +557,20 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - return (method >= 1) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); + if (method == MIN_METHOD_FOR_ED25519_ID_IN_MD) { + /* This method was broken due to buggy code accidently left in + * dircollate.c; do not actually use it. + */ + return 0; + } + + return (method >= MIN_SUPPORTED_CONSENSUS_METHOD) && + (method <= MAX_SUPPORTED_CONSENSUS_METHOD); } /** Return a newly allocated string holding the numbers between low and high * (inclusive) that are supported consensus methods. */ -static char * +STATIC char * make_consensus_method_list(int low, int high, const char *separator) { char *list; @@ -587,7 +615,7 @@ compute_consensus_versions_list(smartlist_t *lst, int n_versioning) /** Helper: given a list of valid networkstatus_t, return a new string * containing the contents of the consensus network parameter set. */ -/* private */ char * +STATIC char * dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) { int i; @@ -601,13 +629,14 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) const int n_votes = smartlist_len(votes); smartlist_t *output; smartlist_t *param_list = smartlist_new(); + (void) method; /* We require that the parameter lists in the votes are well-formed: that is, that their keywords are unique and sorted, and that their values are between INT32_MIN and INT32_MAX inclusive. This should be guaranteed by the parsing code. */ - vals = tor_malloc(sizeof(int)*n_votes); + vals = tor_calloc(n_votes, sizeof(int)); SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { if (!v->net_params) @@ -643,12 +672,13 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) next_param = NULL; else next_param = smartlist_get(param_list, param_sl_idx+1); + /* resolve spurious clang shallow analysis null pointer errors */ + tor_assert(param); if (!next_param || strncmp(next_param, param, cur_param_len)) { /* We've reached the end of a series. */ /* Make sure enough authorities voted on this param, unless the * the consensus method we use is too old for that. */ - if (method < MIN_METHOD_FOR_MAJORITY_PARAMS || - i > total_authorities/2 || + if (i > total_authorities/2 || i >= MIN_VOTES_FOR_PARAM) { int32_t median = median_int32(vals, i); char *out_string = tor_malloc(64+cur_param_len); @@ -1001,299 +1031,87 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G, I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); return 1; } -/** - * This function computes the bandwidth weights for consensus method 9. - * - * It has been obsoleted in favor of consensus method 10. - */ + +/** Update total bandwidth weights (G/M/E/D/T) with the bandwidth of + * the router in <b>rs</b>. */ static void -networkstatus_compute_bw_weights_v9(smartlist_t *chunks, int64_t G, int64_t M, - int64_t E, int64_t D, int64_t T, - int64_t weight_scale) +update_total_bandwidth_weights(const routerstatus_t *rs, + int is_exit, int is_guard, + int64_t *G, int64_t *M, int64_t *E, int64_t *D, + int64_t *T) { - int64_t Wgg = -1, Wgd = -1; - int64_t Wmg = -1, Wme = -1, Wmd = -1; - int64_t Wed = -1, Wee = -1; - const char *casename; + int default_bandwidth = rs->bandwidth_kb; + int guardfraction_bandwidth = 0; - if (G <= 0 || M <= 0 || E <= 0 || D <= 0) { - log_warn(LD_DIR, "Consensus with empty bandwidth: " - "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT - " D="I64_FORMAT" T="I64_FORMAT, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + if (!rs->has_bandwidth) { + log_info(LD_BUG, "Missing consensus bandwidth for router %s", + rs->nickname); return; } - /* - * Computed from cases in 3.4.3 of dir-spec.txt + /* If this routerstatus represents a guard that we have + * guardfraction information on, use it to calculate its actual + * bandwidth. From proposal236: * - * 1. Neither are scarce - * 2. Both Guard and Exit are scarce - * a. R+D <= S - * b. R+D > S - * 3. One of Guard or Exit is scarce - * a. S+D < T/3 - * b. S+D >= T/3 + * Similarly, when calculating the bandwidth-weights line as in + * section 3.8.3 of dir-spec.txt, directory authorities should treat N + * as if fraction F of its bandwidth has the guard flag and (1-F) does + * not. So when computing the totals G,M,E,D, each relay N with guard + * visibility fraction F and bandwidth B should be added as follows: + * + * G' = G + F*B, if N does not have the exit flag + * M' = M + (1-F)*B, if N does not have the exit flag + * + * or + * + * D' = D + F*B, if N has the exit flag + * E' = E + (1-F)*B, if N has the exit flag + * + * In this block of code, we prepare the bandwidth values by setting + * the default_bandwidth to F*B and guardfraction_bandwidth to (1-F)*B. */ - if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3 - bw_weights_error_t berr = 0; - /* Case 1: Neither are scarce. - * - * Attempt to ensure that we have a large amount of exit bandwidth - * in the middle position. - */ - casename = "Case 1 (Wme*E = Wmd*D)"; - Wgg = (weight_scale*(D+E+G+M))/(3*G); - if (D==0) Wmd = 0; - else Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); - Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); - Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); - Wgd = 0; - Wmg = weight_scale - Wgg; - Wed = weight_scale - Wmd; + if (rs->has_guardfraction) { + guardfraction_bandwidth_t guardfraction_bw; - berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, - weight_scale, G, M, E, D, T, 10, 1); + tor_assert(is_guard); - if (berr) { - log_warn(LD_DIR, "Bw Weights error %d for case %s. " - "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT - " D="I64_FORMAT" T="I64_FORMAT, - berr, casename, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - } - } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3 - int64_t R = MIN(E, G); - int64_t S = MAX(E, G); - /* - * Case 2: Both Guards and Exits are scarce - * Balance D between E and G, depending upon - * D capacity and scarcity. - */ - if (R+D < S) { // Subcase a - Wgg = weight_scale; - Wee = weight_scale; - Wmg = 0; - Wme = 0; - Wmd = 0; - if (E < G) { - casename = "Case 2a (E scarce)"; - Wed = weight_scale; - Wgd = 0; - } else { /* E >= G */ - casename = "Case 2a (G scarce)"; - Wed = 0; - Wgd = weight_scale; - } - } else { // Subcase b: R+D > S - bw_weights_error_t berr = 0; - casename = "Case 2b (Wme*E == Wmd*D)"; - if (D != 0) { - Wgg = weight_scale; - Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); // T/3 >= G (Ok) - Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); // T/3 >= M - Wme = (weight_scale*(D + E + G - 2*M))/(6*E); - Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); // 2E+M >= T/3 - Wmg = 0; - Wed = weight_scale - Wgd - Wmd; + guard_get_guardfraction_bandwidth(&guardfraction_bw, + rs->bandwidth_kb, + rs->guardfraction_percentage); - berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, - weight_scale, G, M, E, D, T, 10, 1); - } + default_bandwidth = guardfraction_bw.guard_bw; + guardfraction_bandwidth = guardfraction_bw.non_guard_bw; + } - if (D == 0 || berr) { // Can happen if M > T/3 - casename = "Case 2b (E=G)"; - Wgg = weight_scale; - Wee = weight_scale; - Wmg = 0; - Wme = 0; - Wmd = 0; - if (D == 0) Wgd = 0; - else Wgd = (weight_scale*(D+E-G))/(2*D); - Wed = weight_scale - Wgd; - berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, - Wed, weight_scale, G, M, E, D, T, 10, 1); - } - if (berr != BW_WEIGHTS_NO_ERROR && - berr != BW_WEIGHTS_BALANCE_MID_ERROR) { - log_warn(LD_DIR, "Bw Weights error %d for case %s. " - "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT - " D="I64_FORMAT" T="I64_FORMAT, - berr, casename, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - } - } - } else { // if (E < T/3 || G < T/3) { - int64_t S = MIN(E, G); - // Case 3: Exactly one of Guard or Exit is scarce - if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) { - log_warn(LD_BUG, - "Bw-Weights Case 3 but with G="I64_FORMAT" M=" - I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + /* Now calculate the total bandwidth weights with or without + * guardfraction. Depending on the flags of the relay, add its + * bandwidth to the appropriate weight pool. If it's a guard and + * guardfraction is enabled, add its bandwidth to both pools as + * indicated by the previous comment. + */ + *T += default_bandwidth; + if (is_exit && is_guard) { + + *D += default_bandwidth; + if (rs->has_guardfraction) { + *E += guardfraction_bandwidth; } - if (3*(S+D) < T) { // Subcase a: S+D < T/3 - if (G < E) { - casename = "Case 3a (G scarce)"; - Wgg = Wgd = weight_scale; - Wmd = Wed = Wmg = 0; - // Minor subcase, if E is more scarce than M, - // keep its bandwidth in place. - if (E < M) Wme = 0; - else Wme = (weight_scale*(E-M))/(2*E); - Wee = weight_scale-Wme; - } else { // G >= E - casename = "Case 3a (E scarce)"; - Wee = Wed = weight_scale; - Wmd = Wgd = Wme = 0; - // Minor subcase, if G is more scarce than M, - // keep its bandwidth in place. - if (G < M) Wmg = 0; - else Wmg = (weight_scale*(G-M))/(2*G); - Wgg = weight_scale-Wmg; - } - } else { // Subcase b: S+D >= T/3 - bw_weights_error_t berr = 0; - // D != 0 because S+D >= T/3 - if (G < E) { - casename = "Case 3b (G scarce, Wme*E == Wmd*D)"; - Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); - Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); - Wme = (weight_scale*(D + E + G - 2*M))/(6*E); - Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); - Wgg = weight_scale; - Wmg = 0; - Wed = weight_scale - Wgd - Wmd; + } else if (is_exit) { - berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, - Wed, weight_scale, G, M, E, D, T, 10, 1); - } else { // G >= E - casename = "Case 3b (E scarce, Wme*E == Wmd*D)"; - Wgg = (weight_scale*(D + E + G + M))/(3*G); - Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); - Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); - Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); - Wgd = 0; - Wmg = weight_scale - Wgg; - Wed = weight_scale - Wmd; + *E += default_bandwidth; - berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, - Wed, weight_scale, G, M, E, D, T, 10, 1); - } - if (berr) { - log_warn(LD_DIR, "Bw Weights error %d for case %s. " - "G="I64_FORMAT" M="I64_FORMAT - " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, - berr, casename, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - } - } - } + } else if (is_guard) { - /* We cast down the weights to 32 bit ints on the assumption that - * weight_scale is ~= 10000. We need to ensure a rogue authority - * doesn't break this assumption to rig our weights */ - tor_assert(0 < weight_scale && weight_scale <= INT32_MAX); + *G += default_bandwidth; + if (rs->has_guardfraction) { + *M += guardfraction_bandwidth; + } - if (Wgg < 0 || Wgg > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wgg="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wgg), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } else { - Wgg = MAX(MIN(Wgg, weight_scale), 0); - } - if (Wgd < 0 || Wgd > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wgd="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wgd), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wgd = MAX(MIN(Wgd, weight_scale), 0); - } - if (Wmg < 0 || Wmg > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wmg="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wmg), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wmg = MAX(MIN(Wmg, weight_scale), 0); + *M += default_bandwidth; } - if (Wme < 0 || Wme > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wme="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wme), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wme = MAX(MIN(Wme, weight_scale), 0); - } - if (Wmd < 0 || Wmd > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wmd="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wmd), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wmd = MAX(MIN(Wmd, weight_scale), 0); - } - if (Wee < 0 || Wee > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wee="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wee), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wee = MAX(MIN(Wee, weight_scale), 0); - } - if (Wed < 0 || Wed > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wed="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wed), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wed = MAX(MIN(Wed, weight_scale), 0); - } - - // Add consensus weight keywords - smartlist_add(chunks, tor_strdup("bandwidth-weights ")); - /* - * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine - * that middle nodes need different bandwidth weights for dirport traffic, - * or that weird exit policies need special weight, or that bridges - * need special weight. - * - * NOTE: This list is sorted. - */ - smartlist_add_asprintf(chunks, - "Wbd=%d Wbe=%d Wbg=%d Wbm=%d " - "Wdb=%d " - "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d " - "Wgb=%d Wgd=%d Wgg=%d Wgm=%d " - "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n", - (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale, - (int)weight_scale, - (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee, - (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg, - (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale); - - log_notice(LD_CIRC, "Computed bandwidth weights for %s with v9: " - "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); } /** Given a list of vote networkstatus_t in <b>votes</b>, our public @@ -1326,9 +1144,12 @@ networkstatus_compute_consensus(smartlist_t *votes, const routerstatus_format_type_t rs_format = flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; char *params = NULL; + char *packages = NULL; int added_weights = 0; + dircollator_t *collator = NULL; tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); + tor_assert(total_authorities > 0); flavor_name = networkstatus_get_flavor_name(flavor); @@ -1346,18 +1167,18 @@ networkstatus_compute_consensus(smartlist_t *votes, log_warn(LD_DIR, "The other authorities will use consensus method %d, " "which I don't support. Maybe I should upgrade!", consensus_method); - consensus_method = 1; + consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD; } /* Compute medians of time-related things, and figure out how many * routers we might need to talk about. */ { int n_votes = smartlist_len(votes); - time_t *va_times = tor_malloc(n_votes * sizeof(time_t)); - time_t *fu_times = tor_malloc(n_votes * sizeof(time_t)); - time_t *vu_times = tor_malloc(n_votes * sizeof(time_t)); - int *votesec_list = tor_malloc(n_votes * sizeof(int)); - int *distsec_list = tor_malloc(n_votes * sizeof(int)); + time_t *va_times = tor_calloc(n_votes, sizeof(time_t)); + time_t *fu_times = tor_calloc(n_votes, sizeof(time_t)); + time_t *vu_times = tor_calloc(n_votes, sizeof(time_t)); + int *votesec_list = tor_calloc(n_votes, sizeof(int)); + int *distsec_list = tor_calloc(n_votes, sizeof(int)); int n_versioning_clients = 0, n_versioning_servers = 0; smartlist_t *combined_client_versions = smartlist_new(); smartlist_t *combined_server_versions = smartlist_new(); @@ -1396,8 +1217,12 @@ networkstatus_compute_consensus(smartlist_t *votes, vote_seconds = median_int(votesec_list, n_votes); dist_seconds = median_int(distsec_list, n_votes); - tor_assert(valid_after+MIN_VOTE_INTERVAL <= fresh_until); - tor_assert(fresh_until+MIN_VOTE_INTERVAL <= valid_until); + tor_assert(valid_after + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) <= fresh_until); + tor_assert(fresh_until + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) <= valid_until); tor_assert(vote_seconds >= MIN_VOTE_SECONDS); tor_assert(dist_seconds >= MIN_DIST_SECONDS); @@ -1405,12 +1230,20 @@ networkstatus_compute_consensus(smartlist_t *votes, n_versioning_servers); client_versions = compute_consensus_versions_list(combined_client_versions, n_versioning_clients); + if (consensus_method >= MIN_METHOD_FOR_PACKAGE_LINES) { + packages = compute_consensus_package_lines(votes); + } else { + packages = tor_strdup(""); + } SMARTLIST_FOREACH(combined_server_versions, char *, cp, tor_free(cp)); SMARTLIST_FOREACH(combined_client_versions, char *, cp, tor_free(cp)); smartlist_free(combined_server_versions); smartlist_free(combined_client_versions); + if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING) + smartlist_add(flags, tor_strdup("NoEdConsensus")); + smartlist_sort_strings(flags); smartlist_uniq_strings(flags); @@ -1437,10 +1270,8 @@ networkstatus_compute_consensus(smartlist_t *votes, flavor == FLAV_NS ? "" : " ", flavor == FLAV_NS ? "" : flavor_name); - if (consensus_method >= 2) { - smartlist_add_asprintf(chunks, "consensus-method %d\n", - consensus_method); - } + smartlist_add_asprintf(chunks, "consensus-method %d\n", + consensus_method); smartlist_add_asprintf(chunks, "valid-after %s\n" @@ -1449,22 +1280,23 @@ networkstatus_compute_consensus(smartlist_t *votes, "voting-delay %d %d\n" "client-versions %s\n" "server-versions %s\n" + "%s" /* packages */ "known-flags %s\n", va_buf, fu_buf, vu_buf, vote_seconds, dist_seconds, - client_versions, server_versions, flaglist); + client_versions, server_versions, + packages, + flaglist); tor_free(flaglist); } - if (consensus_method >= MIN_METHOD_FOR_PARAMS) { - params = dirvote_compute_params(votes, consensus_method, - total_authorities); - if (params) { - smartlist_add(chunks, tor_strdup("params ")); - smartlist_add(chunks, params); - smartlist_add(chunks, tor_strdup("\n")); - } + params = dirvote_compute_params(votes, consensus_method, + total_authorities); + if (params) { + smartlist_add(chunks, tor_strdup("params ")); + smartlist_add(chunks, params); + smartlist_add(chunks, tor_strdup("\n")); } /* Sort the votes. */ @@ -1478,8 +1310,7 @@ networkstatus_compute_consensus(smartlist_t *votes, e->digest = get_voter(v)->identity_digest; e->is_legacy = 0; smartlist_add(dir_sources, e); - if (consensus_method >= 3 && - !tor_digest_is_zero(get_voter(v)->legacy_id_digest)) { + if (!tor_digest_is_zero(get_voter(v)->legacy_id_digest)) { dir_src_ent_t *e_legacy = tor_malloc_zero(sizeof(dir_src_ent_t)); e_legacy->v = v; e_legacy->digest = get_voter(v)->legacy_id_digest; @@ -1495,9 +1326,6 @@ networkstatus_compute_consensus(smartlist_t *votes, networkstatus_t *v = e->v; networkstatus_voter_info_t *voter = get_voter(v); - if (e->is_legacy) - tor_assert(consensus_method >= 2); - base16_encode(fingerprint, sizeof(fingerprint), e->digest, DIGEST_LEN); base16_encode(votedigest, sizeof(votedigest), voter->vote_digest, DIGEST_LEN); @@ -1555,12 +1383,15 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_t *chosen_flags = smartlist_new(); smartlist_t *versions = smartlist_new(); smartlist_t *exitsummaries = smartlist_new(); - uint32_t *bandwidths_kb = tor_malloc(sizeof(uint32_t) * - smartlist_len(votes)); - uint32_t *measured_bws_kb = tor_malloc(sizeof(uint32_t) * - smartlist_len(votes)); + uint32_t *bandwidths_kb = tor_calloc(smartlist_len(votes), + sizeof(uint32_t)); + uint32_t *measured_bws_kb = tor_calloc(smartlist_len(votes), + sizeof(uint32_t)); + uint32_t *measured_guardfraction = tor_calloc(smartlist_len(votes), + sizeof(uint32_t)); int num_bandwidths; int num_mbws; + int num_guardfraction_inputs; int *n_voter_flags; /* n_voter_flags[j] is the number of flags that * votes[j] knows about. */ @@ -1570,7 +1401,6 @@ networkstatus_compute_consensus(smartlist_t *votes, * is the same flag as votes[j]->known_flags[b]. */ int *named_flag; /* Index of the flag "Named" for votes[j] */ int *unnamed_flag; /* Index of the flag "Unnamed" for votes[j] */ - int chosen_named_idx; int n_authorities_measuring_bandwidth; strmap_t *name_to_id_map = strmap_new(); @@ -1579,16 +1409,15 @@ networkstatus_compute_consensus(smartlist_t *votes, memset(conflict, 0, sizeof(conflict)); memset(unknown, 0xff, sizeof(conflict)); - index = tor_malloc_zero(sizeof(int)*smartlist_len(votes)); - size = tor_malloc_zero(sizeof(int)*smartlist_len(votes)); - n_voter_flags = tor_malloc_zero(sizeof(int) * smartlist_len(votes)); - n_flag_voters = tor_malloc_zero(sizeof(int) * smartlist_len(flags)); - flag_map = tor_malloc_zero(sizeof(int*) * smartlist_len(votes)); - named_flag = tor_malloc_zero(sizeof(int) * smartlist_len(votes)); - unnamed_flag = tor_malloc_zero(sizeof(int) * smartlist_len(votes)); + index = tor_calloc(smartlist_len(votes), sizeof(int)); + size = tor_calloc(smartlist_len(votes), sizeof(int)); + n_voter_flags = tor_calloc(smartlist_len(votes), sizeof(int)); + n_flag_voters = tor_calloc(smartlist_len(flags), sizeof(int)); + flag_map = tor_calloc(smartlist_len(votes), sizeof(int *)); + named_flag = tor_calloc(smartlist_len(votes), sizeof(int)); + unnamed_flag = tor_calloc(smartlist_len(votes), sizeof(int)); for (i = 0; i < smartlist_len(votes); ++i) unnamed_flag[i] = named_flag[i] = -1; - chosen_named_idx = smartlist_string_pos(flags, "Named"); /* Build the flag indexes. Note that no vote can have more than 64 members * for known_flags, so no value will be greater than 63, so it's safe to @@ -1597,8 +1426,8 @@ networkstatus_compute_consensus(smartlist_t *votes, * that they're actually set before doing U64_LITERAL(1) << index with * them.*/ SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { - flag_map[v_sl_idx] = tor_malloc_zero( - sizeof(int)*smartlist_len(v->known_flags)); + flag_map[v_sl_idx] = tor_calloc(smartlist_len(v->known_flags), + sizeof(int)); if (smartlist_len(v->known_flags) > MAX_KNOWN_FLAGS_IN_VOTE) { log_warn(LD_BUG, "Somehow, a vote has %d entries in known_flags", smartlist_len(v->known_flags)); @@ -1618,7 +1447,7 @@ networkstatus_compute_consensus(smartlist_t *votes, } SMARTLIST_FOREACH_END(v); /* Named and Unnamed get treated specially */ - if (consensus_method >= 2) { + { SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { uint64_t nf; if (named_flag[v_sl_idx]<0) @@ -1671,18 +1500,30 @@ networkstatus_compute_consensus(smartlist_t *votes, /* We need to know how many votes measure bandwidth. */ n_authorities_measuring_bandwidth = 0; - SMARTLIST_FOREACH(votes, networkstatus_t *, v, + SMARTLIST_FOREACH(votes, const networkstatus_t *, v, if (v->has_measured_bws) { ++n_authorities_measuring_bandwidth; } ); + /* Populate the collator */ + collator = dircollator_new(smartlist_len(votes), total_authorities); + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { + dircollator_add_vote(collator, v); + } SMARTLIST_FOREACH_END(v); + + dircollator_collate(collator, consensus_method); + /* Now go through all the votes */ - flag_counts = tor_malloc(sizeof(int) * smartlist_len(flags)); - while (1) { + flag_counts = tor_calloc(smartlist_len(flags), sizeof(int)); + const int num_routers = dircollator_n_routers(collator); + for (i = 0; i < num_routers; ++i) { + vote_routerstatus_t **vrs_lst = + dircollator_get_votes_for_router(collator, i); + vote_routerstatus_t *rs; routerstatus_t rs_out; - const char *lowest_id = NULL; + const char *current_rsa_id = NULL; const char *chosen_version; const char *chosen_name = NULL; int exitsummary_disagreement = 0; @@ -1690,54 +1531,39 @@ networkstatus_compute_consensus(smartlist_t *votes, int is_guard = 0, is_exit = 0, is_bad_exit = 0; int naming_conflict = 0; int n_listing = 0; - int i; char microdesc_digest[DIGEST256_LEN]; tor_addr_port_t alt_orport = {TOR_ADDR_NULL, 0}; - /* Of the next-to-be-considered digest in each voter, which is first? */ - SMARTLIST_FOREACH(votes, networkstatus_t *, v, { - if (index[v_sl_idx] < size[v_sl_idx]) { - rs = smartlist_get(v->routerstatus_list, index[v_sl_idx]); - if (!lowest_id || - fast_memcmp(rs->status.identity_digest, - lowest_id, DIGEST_LEN) < 0) - lowest_id = rs->status.identity_digest; - } - }); - if (!lowest_id) /* we're out of routers. */ - break; - memset(flag_counts, 0, sizeof(int)*smartlist_len(flags)); smartlist_clear(matching_descs); smartlist_clear(chosen_flags); smartlist_clear(versions); num_bandwidths = 0; num_mbws = 0; + num_guardfraction_inputs = 0; + int ed_consensus = 0; + const uint8_t *ed_consensus_val = NULL; /* Okay, go through all the entries for this digest. */ - SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { - if (index[v_sl_idx] >= size[v_sl_idx]) - continue; /* out of entries. */ - rs = smartlist_get(v->routerstatus_list, index[v_sl_idx]); - if (fast_memcmp(rs->status.identity_digest, lowest_id, DIGEST_LEN)) - continue; /* doesn't include this router. */ - /* At this point, we know that we're looking at a routerstatus with - * identity "lowest". - */ - ++index[v_sl_idx]; + for (int voter_idx = 0; voter_idx < smartlist_len(votes); ++voter_idx) { + if (vrs_lst[voter_idx] == NULL) + continue; /* This voter had nothing to say about this entry. */ + rs = vrs_lst[voter_idx]; ++n_listing; + current_rsa_id = rs->status.identity_digest; + smartlist_add(matching_descs, rs); if (rs->version && rs->version[0]) smartlist_add(versions, rs->version); /* Tally up all the flags. */ - for (i = 0; i < n_voter_flags[v_sl_idx]; ++i) { - if (rs->flags & (U64_LITERAL(1) << i)) - ++flag_counts[flag_map[v_sl_idx][i]]; + for (int flag = 0; flag < n_voter_flags[voter_idx]; ++flag) { + if (rs->flags & (U64_LITERAL(1) << flag)) + ++flag_counts[flag_map[voter_idx][flag]]; } - if (named_flag[v_sl_idx] >= 0 && - (rs->flags & (U64_LITERAL(1) << named_flag[v_sl_idx]))) { + if (named_flag[voter_idx] >= 0 && + (rs->flags & (U64_LITERAL(1) << named_flag[voter_idx]))) { if (chosen_name && strcmp(chosen_name, rs->status.nickname)) { log_notice(LD_DIR, "Conflict on naming for router: %s vs %s", chosen_name, rs->status.nickname); @@ -1746,19 +1572,48 @@ networkstatus_compute_consensus(smartlist_t *votes, chosen_name = rs->status.nickname; } + /* Count guardfraction votes and note down the values. */ + if (rs->status.has_guardfraction) { + measured_guardfraction[num_guardfraction_inputs++] = + rs->status.guardfraction_percentage; + } + /* count bandwidths */ if (rs->has_measured_bw) measured_bws_kb[num_mbws++] = rs->measured_bw_kb; if (rs->status.has_bandwidth) bandwidths_kb[num_bandwidths++] = rs->status.bandwidth_kb; - } SMARTLIST_FOREACH_END(v); + + /* Count number for which ed25519 is canonical. */ + if (rs->ed25519_reflects_consensus) { + ++ed_consensus; + if (ed_consensus_val) { + tor_assert(fast_memeq(ed_consensus_val, rs->ed25519_id, + ED25519_PUBKEY_LEN)); + } else { + ed_consensus_val = rs->ed25519_id; + } + } + } /* We don't include this router at all unless more than half of * the authorities we believe in list it. */ if (n_listing <= total_authorities/2) continue; + if (ed_consensus > 0) { + tor_assert(consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING); + if (ed_consensus <= total_authorities / 2) { + log_warn(LD_BUG, "Not enough entries had ed_consensus set; how " + "can we have a consensus of %d?", ed_consensus); + } + } + + /* The clangalyzer can't figure out that this will never be NULL + * if n_listing is at least 1 */ + tor_assert(current_rsa_id); + /* Figure out the most popular opinion of what the most recent * routerinfo and its contents are. */ memset(microdesc_digest, 0, sizeof(microdesc_digest)); @@ -1766,8 +1621,9 @@ networkstatus_compute_consensus(smartlist_t *votes, microdesc_digest, &alt_orport); /* Copy bits of that into rs_out. */ memset(&rs_out, 0, sizeof(rs_out)); - tor_assert(fast_memeq(lowest_id, rs->status.identity_digest,DIGEST_LEN)); - memcpy(rs_out.identity_digest, lowest_id, DIGEST_LEN); + tor_assert(fast_memeq(current_rsa_id, + rs->status.identity_digest,DIGEST_LEN)); + memcpy(rs_out.identity_digest, current_rsa_id, DIGEST_LEN); memcpy(rs_out.descriptor_digest, rs->status.descriptor_digest, DIGEST_LEN); rs_out.addr = rs->status.addr; @@ -1787,14 +1643,11 @@ networkstatus_compute_consensus(smartlist_t *votes, strlcpy(rs_out.nickname, rs->status.nickname, sizeof(rs_out.nickname)); } - if (consensus_method == 1) { - is_named = chosen_named_idx >= 0 && - (!naming_conflict && flag_counts[chosen_named_idx]); - } else { + { const char *d = strmap_get_lc(name_to_id_map, rs_out.nickname); if (!d) { is_named = is_unnamed = 0; - } else if (fast_memeq(d, lowest_id, DIGEST_LEN)) { + } else if (fast_memeq(d, current_rsa_id, DIGEST_LEN)) { is_named = 1; is_unnamed = 0; } else { is_named = 0; is_unnamed = 1; @@ -1807,9 +1660,13 @@ networkstatus_compute_consensus(smartlist_t *votes, if (!strcmp(fl, "Named")) { if (is_named) smartlist_add(chosen_flags, (char*)fl); - } else if (!strcmp(fl, "Unnamed") && consensus_method >= 2) { + } else if (!strcmp(fl, "Unnamed")) { if (is_unnamed) smartlist_add(chosen_flags, (char*)fl); + } else if (!strcmp(fl, "NoEdConsensus") && + consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING) { + if (ed_consensus <= total_authorities/2) + smartlist_add(chosen_flags, (char*)fl); } else { if (flag_counts[fl_sl_idx] > n_flag_voters[fl_sl_idx]/2) { smartlist_add(chosen_flags, (char*)fl); @@ -1827,7 +1684,7 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Starting with consensus method 4 we do not list servers * that are not running in a consensus. See Proposal 138 */ - if (consensus_method >= 4 && !is_running) + if (!is_running) continue; /* Pick the version. */ @@ -1838,12 +1695,23 @@ networkstatus_compute_consensus(smartlist_t *votes, chosen_version = NULL; } + /* If it's a guard and we have enough guardfraction votes, + calculate its consensus guardfraction value. */ + if (is_guard && num_guardfraction_inputs > 2 && + consensus_method >= MIN_METHOD_FOR_GUARDFRACTION) { + rs_out.has_guardfraction = 1; + rs_out.guardfraction_percentage = median_uint32(measured_guardfraction, + num_guardfraction_inputs); + /* final value should be an integer percentage! */ + tor_assert(rs_out.guardfraction_percentage <= 100); + } + /* Pick a bandwidth */ - if (consensus_method >= 6 && num_mbws > 2) { + if (num_mbws > 2) { rs_out.has_bandwidth = 1; rs_out.bw_is_unmeasured = 0; rs_out.bandwidth_kb = median_uint32(measured_bws_kb, num_mbws); - } else if (consensus_method >= 5 && num_bandwidths > 0) { + } else if (num_bandwidths > 0) { rs_out.has_bandwidth = 1; rs_out.bw_is_unmeasured = 1; rs_out.bandwidth_kb = median_uint32(bandwidths_kb, num_bandwidths); @@ -1857,25 +1725,13 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Fix bug 2203: Do not count BadExit nodes as Exits for bw weights */ - if (consensus_method >= MIN_METHOD_TO_CUT_BADEXIT_WEIGHT) { - is_exit = is_exit && !is_bad_exit; - } + is_exit = is_exit && !is_bad_exit; - if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { - if (rs_out.has_bandwidth) { - T += rs_out.bandwidth_kb; - if (is_exit && is_guard) - D += rs_out.bandwidth_kb; - else if (is_exit) - E += rs_out.bandwidth_kb; - else if (is_guard) - G += rs_out.bandwidth_kb; - else - M += rs_out.bandwidth_kb; - } else { - log_warn(LD_BUG, "Missing consensus bandwidth for router %s", - rs_out.nickname); - } + /* Update total bandwidth weights with the bandwidths of this router. */ + { + update_total_bandwidth_weights(&rs_out, + is_exit, is_guard, + &G, &M, &E, &D, &T); } /* Ok, we already picked a descriptor digest we want to list @@ -1892,7 +1748,7 @@ networkstatus_compute_consensus(smartlist_t *votes, * the policy that was most often listed in votes, again breaking * ties like in the previous case. */ - if (consensus_method >= 5) { + { /* Okay, go through all the votes for this router. We prepared * that list previously */ const char *chosen_exitsummary = NULL; @@ -1963,7 +1819,6 @@ networkstatus_compute_consensus(smartlist_t *votes, } if (flavor == FLAV_MICRODESC && - consensus_method >= MIN_METHOD_FOR_MANDATORY_MICRODESC && tor_digest256_is_zero(microdesc_digest)) { /* With no microdescriptor digest, we omit the entry entirely. */ continue; @@ -1995,11 +1850,21 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add(chunks, tor_strdup("\n")); /* Now the weight line. */ if (rs_out.has_bandwidth) { + char *guardfraction_str = NULL; int unmeasured = rs_out.bw_is_unmeasured && consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW; - smartlist_add_asprintf(chunks, "w Bandwidth=%d%s\n", + + /* If we have guardfraction info, include it in the 'w' line. */ + if (rs_out.has_guardfraction) { + tor_asprintf(&guardfraction_str, + " GuardFraction=%u", rs_out.guardfraction_percentage); + } + smartlist_add_asprintf(chunks, "w Bandwidth=%d%s%s\n", rs_out.bandwidth_kb, - unmeasured?" Unmeasured=1":""); + unmeasured?" Unmeasured=1":"", + guardfraction_str ? guardfraction_str : ""); + + tor_free(guardfraction_str); } /* Now the exitpolicy summary line. */ @@ -2027,15 +1892,13 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(exitsummaries); tor_free(bandwidths_kb); tor_free(measured_bws_kb); + tor_free(measured_guardfraction); } - if (consensus_method >= MIN_METHOD_FOR_FOOTER) { - /* Starting with consensus method 9, we clearly mark the directory - * footer region */ - smartlist_add(chunks, tor_strdup("directory-footer\n")); - } + /* Mark the directory footer region */ + smartlist_add(chunks, tor_strdup("directory-footer\n")); - if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + { int64_t weight_scale = BW_WEIGHT_SCALE; char *bw_weight_param = NULL; @@ -2068,13 +1931,8 @@ networkstatus_compute_consensus(smartlist_t *votes, } } - if (consensus_method < 10) { - networkstatus_compute_bw_weights_v9(chunks, G, M, E, D, T, weight_scale); - added_weights = 1; - } else { - added_weights = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, - T, weight_scale); - } + added_weights = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, + T, weight_scale); } /* Add a signature. */ @@ -2115,7 +1973,7 @@ networkstatus_compute_consensus(smartlist_t *votes, } smartlist_add(chunks, signature); - if (legacy_id_key_digest && legacy_signing_key && consensus_method >= 3) { + if (legacy_id_key_digest && legacy_signing_key) { smartlist_add(chunks, tor_strdup("directory-signature ")); base16_encode(fingerprint, sizeof(fingerprint), legacy_id_key_digest, DIGEST_LEN); @@ -2151,7 +2009,7 @@ networkstatus_compute_consensus(smartlist_t *votes, goto done; } // Verify balancing parameters - if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS && added_weights) { + if (added_weights) { networkstatus_verify_bw_weights(c, consensus_method); } networkstatus_vote_free(c); @@ -2159,8 +2017,10 @@ networkstatus_compute_consensus(smartlist_t *votes, done: + dircollator_free(collator); tor_free(client_versions); tor_free(server_versions); + tor_free(packages); SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp)); smartlist_free(flags); SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); @@ -2169,6 +2029,78 @@ networkstatus_compute_consensus(smartlist_t *votes, return result; } +/** Given a list of networkstatus_t for each vote, return a newly allocated + * string containing the "package" lines for the vote. */ +STATIC char * +compute_consensus_package_lines(smartlist_t *votes) +{ + const int n_votes = smartlist_len(votes); + + /* This will be a map from "packagename version" strings to arrays + * of const char *, with the i'th member of the array corresponding to the + * package line from the i'th vote. + */ + strmap_t *package_status = strmap_new(); + + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { + if (! v->package_lines) + continue; + SMARTLIST_FOREACH_BEGIN(v->package_lines, const char *, line) { + if (! validate_recommended_package_line(line)) + continue; + + /* Skip 'cp' to the second space in the line. */ + const char *cp = strchr(line, ' '); + if (!cp) continue; + ++cp; + cp = strchr(cp, ' '); + if (!cp) continue; + + char *key = tor_strndup(line, cp - line); + + const char **status = strmap_get(package_status, key); + if (!status) { + status = tor_calloc(n_votes, sizeof(const char *)); + strmap_set(package_status, key, status); + } + status[v_sl_idx] = line; /* overwrite old value */ + tor_free(key); + } SMARTLIST_FOREACH_END(line); + } SMARTLIST_FOREACH_END(v); + + smartlist_t *entries = smartlist_new(); /* temporary */ + smartlist_t *result_list = smartlist_new(); /* output */ + STRMAP_FOREACH(package_status, key, const char **, values) { + int i, count=-1; + for (i = 0; i < n_votes; ++i) { + if (values[i]) + smartlist_add(entries, (void*) values[i]); + } + smartlist_sort_strings(entries); + int n_voting_for_entry = smartlist_len(entries); + const char *most_frequent = + smartlist_get_most_frequent_string_(entries, &count); + + if (n_voting_for_entry >= 3 && count > n_voting_for_entry / 2) { + smartlist_add_asprintf(result_list, "package %s\n", most_frequent); + } + + smartlist_clear(entries); + + } STRMAP_FOREACH_END; + + smartlist_sort_strings(result_list); + + char *result = smartlist_join_strings(result_list, "", 0, NULL); + + SMARTLIST_FOREACH(result_list, char *, cp, tor_free(cp)); + smartlist_free(result_list); + smartlist_free(entries); + strmap_free(package_status, tor_free_); + + return result; +} + /** Given a consensus vote <b>target</b> and a set of detached signatures in * <b>sigs</b> that correspond to the same consensus, check whether there are * any new signatures in <b>src_voter_list</b> that should be added to @@ -2217,14 +2149,14 @@ networkstatus_add_detached_signatures(networkstatus_t *target, /** Make sure all the digests we know match, and at least one matches. */ { - digests_t *digests = strmap_get(sigs->digests, flavor); + common_digests_t *digests = strmap_get(sigs->digests, flavor); int n_matches = 0; int alg; if (!digests) { *msg_out = "No digests for given consensus flavor"; return -1; } - for (alg = DIGEST_SHA1; alg < N_DIGEST_ALGORITHMS; ++alg) { + for (alg = DIGEST_SHA1; alg < N_COMMON_DIGEST_ALGORITHMS; ++alg) { if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { if (fast_memeq(target->digests.d[alg], digests->d[alg], DIGEST256_LEN)) { @@ -2275,8 +2207,11 @@ networkstatus_add_detached_signatures(networkstatus_t *target, if (!sig->good_signature && !sig->bad_signature) { cert = authority_cert_get_by_digests(sig->identity_digest, sig->signing_key_digest); - if (cert) - networkstatus_check_document_signature(target, sig, cert); + if (cert) { + /* Not checking the return value here, since we are going to look + * at the status of sig->good_signature in a moment. */ + (void) networkstatus_check_document_signature(target, sig, cert); + } } /* If this signature is good, or we don't have any signature yet, @@ -2347,7 +2282,8 @@ networkstatus_format_signatures(networkstatus_t *consensus, for_detached_signatures ? flavor_name : "", digest_name, id, sk); } - base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len); + base64_encode(buf, sizeof(buf), sig->signature, sig->signature_len, + BASE64_ENCODE_MULTILINE); strlcat(buf, "-----END SIGNATURE-----\n", sizeof(buf)); smartlist_add(elements, tor_strdup(buf)); } SMARTLIST_FOREACH_END(sig); @@ -2413,7 +2349,7 @@ networkstatus_get_detached_signatures(smartlist_t *consensuses) /* start with SHA256; we don't include SHA1 for anything but the basic * consensus. */ - for (alg = DIGEST_SHA256; alg < N_DIGEST_ALGORITHMS; ++alg) { + for (alg = DIGEST_SHA256; alg < N_COMMON_DIGEST_ALGORITHMS; ++alg) { char d[HEX_DIGEST256_LEN+1]; const char *alg_name = crypto_digest_algorithm_get_name(alg); @@ -2533,12 +2469,13 @@ dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out) timing_out->dist_delay = options->V3AuthDistDelay; } -/** Return the start of the next interval of size <b>interval</b> (in seconds) - * after <b>now</b>. Midnight always starts a fresh interval, and if the last - * interval of a day would be truncated to less than half its size, it is - * rolled into the previous interval. */ +/** Return the start of the next interval of size <b>interval</b> (in + * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always + * starts a fresh interval, and if the last interval of a day would be + * truncated to less than half its size, it is rolled into the + * previous interval. */ time_t -dirvote_get_start_of_next_interval(time_t now, int interval) +dirvote_get_start_of_next_interval(time_t now, int interval, int offset) { struct tm tm; time_t midnight_today=0; @@ -2566,6 +2503,10 @@ dirvote_get_start_of_next_interval(time_t now, int interval) if (next + interval/2 > midnight_tomorrow) next = midnight_tomorrow; + next += offset; + if (next - interval > now) + next -= interval; + return next; } @@ -2629,8 +2570,10 @@ dirvote_recalculate_timing(const or_options_t *options, time_t now) vote_delay = dist_delay = interval / 4; start = voting_schedule.interval_starts = - dirvote_get_start_of_next_interval(now,interval); - end = dirvote_get_start_of_next_interval(start+1, interval); + dirvote_get_start_of_next_interval(now,interval, + options->TestingV3AuthVotingStartOffset); + end = dirvote_get_start_of_next_interval(start+1, interval, + options->TestingV3AuthVotingStartOffset); tor_assert(end > start); @@ -3009,7 +2952,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) goto discard; } else if (v->vote->published < vote->published) { log_notice(LD_DIR, "Replacing an older pending vote from this " - "directory."); + "directory (%s)", vi->address); cached_dir_decref(v->vote_body); networkstatus_vote_free(v->vote); v->vote_body = new_cached_dir(tor_strndup(vote_body, @@ -3136,7 +3079,7 @@ dirvote_compute_consensuses(void) }); votefile = get_datadir_fname("v3-status-votes"); - write_chunks_to_file(votefile, votestrings, 0); + write_chunks_to_file(votefile, votestrings, 0, 0); tor_free(votefile); SMARTLIST_FOREACH(votestrings, sized_chunk_t *, c, tor_free(c)); smartlist_free(votestrings); @@ -3464,8 +3407,8 @@ dirvote_free_all(void) * ==== */ /** Return the body of the consensus that we're currently trying to build. */ -const char * -dirvote_get_pending_consensus(consensus_flavor_t flav) +MOCK_IMPL(const char *, +dirvote_get_pending_consensus, (consensus_flavor_t flav)) { tor_assert(((int)flav) >= 0 && (int)flav < N_CONSENSUS_FLAVORS); return pending_consensuses[flav].body; @@ -3473,8 +3416,8 @@ dirvote_get_pending_consensus(consensus_flavor_t flav) /** Return the signatures that we know for the consensus that we're currently * trying to build. */ -const char * -dirvote_get_pending_detached_signatures(void) +MOCK_IMPL(const char *, +dirvote_get_pending_detached_signatures, (void)) { return pending_consensus_signatures; } @@ -3555,7 +3498,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) char kbuf[128]; base64_encode(kbuf, sizeof(kbuf), (const char*)ri->onion_curve25519_pkey->public_key, - CURVE25519_PUBKEY_LEN); + CURVE25519_PUBKEY_LEN, BASE64_ENCODE_MULTILINE); smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf); } @@ -3581,12 +3524,28 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) tor_free(p6); } + if (consensus_method >= MIN_METHOD_FOR_ID_HASH_IN_MD) { + char idbuf[ED25519_BASE64_LEN+1]; + const char *keytype; + if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_IN_MD && + ri->cache_info.signing_key_cert && + ri->cache_info.signing_key_cert->signing_key_included) { + keytype = "ed25519"; + ed25519_public_to_base64(idbuf, + &ri->cache_info.signing_key_cert->signing_key); + } else { + keytype = "rsa1024"; + digest_to_base64(idbuf, ri->cache_info.identity_digest); + } + smartlist_add_asprintf(chunks, "id %s %s\n", keytype, idbuf); + } + output = smartlist_join_strings(chunks, "", 0, NULL); { smartlist_t *lst = microdescs_parse_from_string(output, - output+strlen(output), 0, - SAVED_NOWHERE); + output+strlen(output), 0, + SAVED_NOWHERE, NULL); if (smartlist_len(lst) != 1) { log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse."); SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md)); @@ -3647,10 +3606,12 @@ static const struct consensus_method_range_t { int low; int high; } microdesc_consensus_methods[] = { - {MIN_METHOD_FOR_MICRODESC, MIN_METHOD_FOR_A_LINES - 1}, + {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_A_LINES - 1}, {MIN_METHOD_FOR_A_LINES, MIN_METHOD_FOR_P6_LINES - 1}, {MIN_METHOD_FOR_P6_LINES, MIN_METHOD_FOR_NTOR_KEY - 1}, - {MIN_METHOD_FOR_NTOR_KEY, MAX_SUPPORTED_CONSENSUS_METHOD}, + {MIN_METHOD_FOR_NTOR_KEY, MIN_METHOD_FOR_ID_HASH_IN_MD - 1}, + {MIN_METHOD_FOR_ID_HASH_IN_MD, MIN_METHOD_FOR_ED25519_ID_IN_MD - 1}, + {MIN_METHOD_FOR_ED25519_ID_IN_MD, MAX_SUPPORTED_CONSENSUS_METHOD}, {-1, -1} }; diff --git a/src/or/dirvote.h b/src/or/dirvote.h index b236452122..0b1d284060 100644 --- a/src/or/dirvote.h +++ b/src/or/dirvote.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,34 +12,50 @@ #ifndef TOR_DIRVOTE_H #define TOR_DIRVOTE_H -/** Lowest allowable value for VoteSeconds. */ -#define MIN_VOTE_SECONDS 20 -/** Lowest allowable value for DistSeconds. */ -#define MIN_DIST_SECONDS 20 -/** Smallest allowable voting interval. */ -#define MIN_VOTE_INTERVAL 300 - -/** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 17 - -/** Lowest consensus method that contains a 'directory-footer' marker */ -#define MIN_METHOD_FOR_FOOTER 9 +#include "testsupport.h" -/** Lowest consensus method that contains bandwidth weights */ -#define MIN_METHOD_FOR_BW_WEIGHTS 9 +/* + * Ideally, assuming synced clocks, we should only need 1 second for each of: + * - Vote + * - Distribute + * - Consensus Publication + * As we can gather descriptors continuously. + * (Could we even go as far as publishing the previous consensus, + * in the same second that we vote for the next one?) + * But we're not there yet: these are the lowest working values at this time. + */ -/** Lowest consensus method that contains consensus params */ -#define MIN_METHOD_FOR_PARAMS 7 +/** Lowest allowable value for VoteSeconds. */ +#define MIN_VOTE_SECONDS 2 +/** Lowest allowable value for VoteSeconds when TestingTorNetwork is 1 */ +#define MIN_VOTE_SECONDS_TESTING 2 -/** Lowest consensus method that generates microdescriptors */ -#define MIN_METHOD_FOR_MICRODESC 8 +/** Lowest allowable value for DistSeconds. */ +#define MIN_DIST_SECONDS 2 +/** Lowest allowable value for DistSeconds when TestingTorNetwork is 1 */ +#define MIN_DIST_SECONDS_TESTING 2 -/** Lowest consensus method that doesn't count bad exits as exits for weight */ -#define MIN_METHOD_TO_CUT_BADEXIT_WEIGHT 11 +/** Lowest allowable voting interval. */ +#define MIN_VOTE_INTERVAL 300 +/** Lowest allowable voting interval when TestingTorNetwork is 1: + * Voting Interval can be: + * 10, 12, 15, 18, 20, 24, 25, 30, 36, 40, 45, 50, 60, ... + * Testing Initial Voting Interval can be: + * 5, 6, 8, 9, or any of the possible values for Voting Interval, + * as they both need to evenly divide 30 minutes. + * If clock desynchronisation is an issue, use an interval of at least: + * 18 * drift in seconds, to allow for a clock slop factor */ +#define MIN_VOTE_INTERVAL_TESTING \ + (((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)*2) + +#define MIN_VOTE_INTERVAL_TESTING_INITIAL \ + ((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1) + +/** The lowest consensus method that we currently support. */ +#define MIN_SUPPORTED_CONSENSUS_METHOD 13 -/** Lowest consensus method that ensures a majority of authorities voted - * for a param. */ -#define MIN_METHOD_FOR_MAJORITY_PARAMS 12 +/** The highest consensus method that we currently support. */ +#define MAX_SUPPORTED_CONSENSUS_METHOD 22 /** Lowest consensus method where microdesc consensuses omit any entry * with no microdesc. */ @@ -59,8 +75,28 @@ * Unmeasured=1 flag for unmeasured bandwidths */ #define MIN_METHOD_TO_CLIP_UNMEASURED_BW 17 +/** Lowest consensus method where authorities may include an "id" line in + * microdescriptors. */ +#define MIN_METHOD_FOR_ID_HASH_IN_MD 18 + +/** Lowest consensus method where we include "package" lines*/ +#define MIN_METHOD_FOR_PACKAGE_LINES 19 + +/** Lowest consensus method where authorities may include + * GuardFraction information in microdescriptors. */ +#define MIN_METHOD_FOR_GUARDFRACTION 20 + +/** Lowest consensus method where authorities may include an "id" line for + * ed25519 identities in microdescriptors. (Broken; see + * consensus_method_is_supported() for more info.) */ +#define MIN_METHOD_FOR_ED25519_ID_IN_MD 21 +/** Lowest consensus method where authorities vote on ed25519 ids and ensure + * ed25519 id consistency. */ +#define MIN_METHOD_FOR_ED25519_ID_VOTING 22 + /** Default bandwidth to clip unmeasured bandwidths to using method >= - * MIN_METHOD_TO_CLIP_UNMEASURED_BW */ + * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not + * get confused with the above macros.) */ #define DEFAULT_MAX_UNMEASURED_BW_KB 20 void dirvote_free_all(void); @@ -86,7 +122,9 @@ authority_cert_t *authority_cert_dup(authority_cert_t *cert); /* vote scheduling */ void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); -time_t dirvote_get_start_of_next_interval(time_t now, int interval); +time_t dirvote_get_start_of_next_interval(time_t now, + int interval, + int offset); void dirvote_recalculate_timing(const or_options_t *options, time_t now); void dirvote_act(const or_options_t *options, time_t now); @@ -99,8 +137,10 @@ int dirvote_add_signatures(const char *detached_signatures_body, const char **msg_out); /* Item access */ -const char *dirvote_get_pending_consensus(consensus_flavor_t flav); -const char *dirvote_get_pending_detached_signatures(void); +MOCK_DECL(const char*, dirvote_get_pending_consensus, + (consensus_flavor_t flav)); +MOCK_DECL(const char*, dirvote_get_pending_detached_signatures, (void)); + #define DGV_BY_ID 1 #define DGV_INCLUDE_PENDING 2 #define DGV_INCLUDE_PREVIOUS 4 @@ -108,8 +148,7 @@ const cached_dir_t *dirvote_get_vote(const char *fp, int flags); void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, routerinfo_t *ri, time_t now, - int naming, int listbadexits, - int listbaddirs, int vote_on_hsdirs); + int listbadexits); networkstatus_t * dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, authority_cert_t *cert); @@ -134,10 +173,12 @@ document_signature_t *voter_get_sig_by_algorithm( digest_algorithm_t alg); #ifdef DIRVOTE_PRIVATE -char *format_networkstatus_vote(crypto_pk_t *private_key, +STATIC char *format_networkstatus_vote(crypto_pk_t *private_key, networkstatus_t *v3_ns); -char *dirvote_compute_params(smartlist_t *votes, int method, +STATIC char *dirvote_compute_params(smartlist_t *votes, int method, int total_authorities); +STATIC char *compute_consensus_package_lines(smartlist_t *votes); +STATIC char *make_consensus_method_list(int low, int high, const char *sep); #endif #endif diff --git a/src/or/dns.c b/src/or/dns.c index 8b6e3b0543..c7adfbc971 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,6 +11,8 @@ * be nonblocking.) **/ +#define DNS_PRIVATE + #include "or.h" #include "circuitlist.h" #include "circuituse.h" @@ -24,6 +26,7 @@ #include "relay.h" #include "router.h" #include "ht.h" +#include "sandbox.h" #ifdef HAVE_EVENT2_DNS_H #include <event2/event.h> #include <event2/dns.h> @@ -80,9 +83,6 @@ struct evdns_request; #endif -/** Longest hostname we're willing to resolve. */ -#define MAX_ADDRESSLEN 256 - /** How long will we wait for an answer from the resolver before we decide * that the resolver is wedged? */ #define RESOLVE_MAX_TIMEOUT 300 @@ -101,107 +101,15 @@ static char *resolv_conf_fname = NULL; * the nameservers? Used to check whether we need to reconfigure. */ static time_t resolv_conf_mtime = 0; -/** Linked list of connections waiting for a DNS answer. */ -typedef struct pending_connection_t { - edge_connection_t *conn; - struct pending_connection_t *next; -} pending_connection_t; - -/** Value of 'magic' field for cached_resolve_t. Used to try to catch bad - * pointers and memory stomping. */ -#define CACHED_RESOLVE_MAGIC 0x1234F00D - -/* Possible states for a cached resolve_t */ -/** We are waiting for the resolver system to tell us an answer here. - * When we get one, or when we time out, the state of this cached_resolve_t - * will become "DONE" and we'll possibly add a CACHED - * entry. This cached_resolve_t will be in the hash table so that we will - * know not to launch more requests for this addr, but rather to add more - * connections to the pending list for the addr. */ -#define CACHE_STATE_PENDING 0 -/** This used to be a pending cached_resolve_t, and we got an answer for it. - * Now we're waiting for this cached_resolve_t to expire. This should - * have no pending connections, and should not appear in the hash table. */ -#define CACHE_STATE_DONE 1 -/** We are caching an answer for this address. This should have no pending - * connections, and should appear in the hash table. */ -#define CACHE_STATE_CACHED 2 - -/** @name status values for a single DNS request. - * - * @{ */ -/** The DNS request is in progress. */ -#define RES_STATUS_INFLIGHT 1 -/** The DNS request finished and gave an answer */ -#define RES_STATUS_DONE_OK 2 -/** The DNS request finished and gave an error */ -#define RES_STATUS_DONE_ERR 3 -/**@}*/ - -/** A DNS request: possibly completed, possibly pending; cached_resolve - * structs are stored at the OR side in a hash table, and as a linked - * list from oldest to newest. - */ -typedef struct cached_resolve_t { - HT_ENTRY(cached_resolve_t) node; - uint32_t magic; /**< Must be CACHED_RESOLVE_MAGIC */ - char address[MAX_ADDRESSLEN]; /**< The hostname to be resolved. */ - - union { - uint32_t addr_ipv4; /**< IPv4 addr for <b>address</b>, if successful. - * (In host order.) */ - int err_ipv4; /**< One of DNS_ERR_*, if IPv4 lookup failed. */ - } result_ipv4; /**< Outcome of IPv4 lookup */ - union { - struct in6_addr addr_ipv6; /**< IPv6 addr for <b>address</b>, if - * successful */ - int err_ipv6; /**< One of DNS_ERR_*, if IPv6 lookup failed. */ - } result_ipv6; /**< Outcome of IPv6 lookup, if any */ - union { - char *hostname; /** A hostname, if PTR lookup happened successfully*/ - int err_hostname; /** One of DNS_ERR_*, if PTR lookup failed. */ - } result_ptr; - /** @name Status fields - * - * These take one of the RES_STATUS_* values, depending on the state - * of the corresponding lookup. - * - * @{ */ - unsigned int res_status_ipv4 : 2; - unsigned int res_status_ipv6 : 2; - unsigned int res_status_hostname : 2; - /**@}*/ - uint8_t state; /**< Is this cached entry pending/done/informative? */ - - time_t expire; /**< Remove items from cache after this time. */ - uint32_t ttl_ipv4; /**< What TTL did the nameserver tell us? */ - uint32_t ttl_ipv6; /**< What TTL did the nameserver tell us? */ - uint32_t ttl_hostname; /**< What TTL did the nameserver tell us? */ - /** Connections that want to know when we get an answer for this resolve. */ - pending_connection_t *pending_connections; - /** Position of this element in the heap*/ - int minheap_idx; -} cached_resolve_t; - static void purge_expired_resolves(time_t now); static void dns_found_answer(const char *address, uint8_t query_type, int dns_answer, const tor_addr_t *addr, const char *hostname, uint32_t ttl); -static void send_resolved_cell(edge_connection_t *conn, uint8_t answer_type, - const cached_resolve_t *resolve); -static int launch_resolve(cached_resolve_t *resolve); static void add_wildcarded_test_address(const char *address); static int configure_nameservers(int force); static int answer_is_wildcarded(const char *ip); -static int dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, - or_circuit_t *oncirc, char **resolved_to_hostname, - int *made_connection_pending_out, - cached_resolve_t **resolve_out); -static int set_exitconn_info_from_resolve(edge_connection_t *exitconn, - const cached_resolve_t *resolve, - char **hostname_out); static int evdns_err_is_transient(int err); static void inform_pending_connections(cached_resolve_t *resolve); static void make_pending_resolve_cached(cached_resolve_t *cached); @@ -226,7 +134,7 @@ static int dns_is_broken_for_ipv6 = 0; /** Function to compare hashed resolves on their addresses; used to * implement hash tables. */ -static INLINE int +static inline int cached_resolves_eq(cached_resolve_t *a, cached_resolve_t *b) { /* make this smarter one day? */ @@ -235,16 +143,16 @@ cached_resolves_eq(cached_resolve_t *a, cached_resolve_t *b) } /** Hash function for cached_resolve objects */ -static INLINE unsigned int +static inline unsigned int cached_resolve_hash(cached_resolve_t *a) { - return ht_string_hash(a->address); + return (unsigned) siphash24g((const uint8_t*)a->address, strlen(a->address)); } HT_PROTOTYPE(cache_map, cached_resolve_t, node, cached_resolve_hash, cached_resolves_eq) -HT_GENERATE(cache_map, cached_resolve_t, node, cached_resolve_hash, - cached_resolves_eq, 0.6, malloc, realloc, free) +HT_GENERATE2(cache_map, cached_resolve_t, node, cached_resolve_hash, + cached_resolves_eq, 0.6, tor_reallocarray_, tor_free_) /** Initialize the DNS cache. */ static void @@ -366,7 +274,7 @@ dns_clip_ttl(uint32_t ttl) /** Helper: Given a TTL from a DNS response, determine how long to hold it in * our cache. */ -static uint32_t +STATIC uint32_t dns_get_expiry_ttl(uint32_t ttl) { if (ttl < MIN_DNS_TTL) @@ -604,9 +512,9 @@ purge_expired_resolves(time_t now) * answer back along circ; otherwise, send the answer back along * <b>conn</b>'s attached circuit. */ -static void -send_resolved_cell(edge_connection_t *conn, uint8_t answer_type, - const cached_resolve_t *resolved) +MOCK_IMPL(STATIC void, +send_resolved_cell,(edge_connection_t *conn, uint8_t answer_type, + const cached_resolve_t *resolved)) { char buf[RELAY_PAYLOAD_SIZE], *cp = buf; size_t buflen = 0; @@ -670,8 +578,9 @@ send_resolved_cell(edge_connection_t *conn, uint8_t answer_type, * answer back along circ; otherwise, send the answer back along * <b>conn</b>'s attached circuit. */ -static void -send_resolved_hostname_cell(edge_connection_t *conn, const char *hostname) +MOCK_IMPL(STATIC void, +send_resolved_hostname_cell,(edge_connection_t *conn, + const char *hostname)) { char buf[RELAY_PAYLOAD_SIZE]; size_t buflen; @@ -799,11 +708,11 @@ dns_resolve(edge_connection_t *exitconn) * * Set *<b>resolve_out</b> to a cached resolve, if we found one. */ -static int -dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, +MOCK_IMPL(STATIC int, +dns_resolve_impl,(edge_connection_t *exitconn, int is_resolve, or_circuit_t *oncirc, char **hostname_out, int *made_connection_pending_out, - cached_resolve_t **resolve_out) + cached_resolve_t **resolve_out)) { cached_resolve_t *resolve; cached_resolve_t search; @@ -946,10 +855,10 @@ dns_resolve_impl(edge_connection_t *exitconn, int is_resolve, * Return -2 on a transient error, -1 on a permenent error, and 1 on * a successful lookup. */ -static int -set_exitconn_info_from_resolve(edge_connection_t *exitconn, - const cached_resolve_t *resolve, - char **hostname_out) +MOCK_IMPL(STATIC int, +set_exitconn_info_from_resolve,(edge_connection_t *exitconn, + const cached_resolve_t *resolve, + char **hostname_out)) { int ipv4_ok, ipv6_ok, answer_with_ipv4, r; uint32_t begincell_flags; @@ -1144,8 +1053,8 @@ connection_dns_remove(edge_connection_t *conn) * the resolve for <b>address</b> itself, and remove any cached results for * <b>address</b> from the cache. */ -void -dns_cancel_pending_resolve(const char *address) +MOCK_IMPL(void, +dns_cancel_pending_resolve,(const char *address)) { pending_connection_t *pend; cached_resolve_t search; @@ -1217,7 +1126,7 @@ dns_cancel_pending_resolve(const char *address) /** Return true iff <b>address</b> is one of the addresses we use to verify * that well-known sites aren't being hijacked by our DNS servers. */ -static INLINE int +static inline int is_test_address(const char *address) { const or_options_t *options = get_options(); @@ -1448,13 +1357,14 @@ configure_nameservers(int force) const or_options_t *options; const char *conf_fname; struct stat st; - int r; + int r, flags; options = get_options(); conf_fname = options->ServerDNSResolvConfFile; #ifndef _WIN32 if (!conf_fname) conf_fname = "/etc/resolv.conf"; #endif + flags = DNS_OPTIONS_ALL; if (!the_evdns_base) { if (!(the_evdns_base = evdns_base_new(tor_libevent_get_base(), 0))) { @@ -1482,7 +1392,8 @@ configure_nameservers(int force) evdns_set_log_fn(evdns_log_cb); if (conf_fname) { - if (stat(conf_fname, &st)) { + log_debug(LD_FS, "stat()ing %s", conf_fname); + if (stat(sandbox_intern_string(conf_fname), &st)) { log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s", conf_fname, strerror(errno)); goto err; @@ -1496,9 +1407,17 @@ configure_nameservers(int force) evdns_base_search_clear(the_evdns_base); evdns_base_clear_nameservers_and_suspend(the_evdns_base); } +#if defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) + if (flags & DNS_OPTION_HOSTSFILE) { + flags ^= DNS_OPTION_HOSTSFILE; + log_debug(LD_FS, "Loading /etc/hosts"); + evdns_base_load_hosts(the_evdns_base, + sandbox_intern_string("/etc/hosts")); + } +#endif log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_base_resolv_conf_parse(the_evdns_base, - DNS_OPTIONS_ALL, conf_fname))) { + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, + sandbox_intern_string(conf_fname)))) { log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", conf_fname, conf_fname, r); goto err; @@ -1741,8 +1660,8 @@ launch_one_resolve(const char *address, uint8_t query_type, /** For eventdns: start resolving as necessary to find the target for * <b>exitconn</b>. Returns -1 on error, -2 on transient error, * 0 on "resolve launched." */ -static int -launch_resolve(cached_resolve_t *resolve) +MOCK_IMPL(STATIC int, +launch_resolve,(cached_resolve_t *resolve)) { tor_addr_t a; int r; @@ -2167,7 +2086,7 @@ static void assert_cache_ok_(void) { cached_resolve_t **resolve; - int bad_rep = _cache_map_HT_REP_IS_BAD(&cache_root); + int bad_rep = HT_REP_IS_BAD_(cache_map, &cache_root); if (bad_rep) { log_err(LD_BUG, "Bad rep type %d on dns cache hash table", bad_rep); tor_assert(!bad_rep); @@ -2195,5 +2114,18 @@ assert_cache_ok_(void) } }); } + #endif +cached_resolve_t +*dns_get_cache_entry(cached_resolve_t *query) +{ + return HT_FIND(cache_map, &cache_root, query); +} + +void +dns_insert_cache_entry(cached_resolve_t *new_entry) +{ + HT_INSERT(cache_map, &cache_root, new_entry); +} + diff --git a/src/or/dns.h b/src/or/dns.h index 022cd4ac63..b14f7dd29c 100644 --- a/src/or/dns.h +++ b/src/or/dns.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,7 +20,7 @@ int dns_reset(void); void connection_dns_remove(edge_connection_t *conn); void assert_connection_edge_not_dns_pending(edge_connection_t *conn); void assert_all_pending_dns_resolves_ok(void); -void dns_cancel_pending_resolve(const char *question); +MOCK_DECL(void,dns_cancel_pending_resolve,(const char *question)); int dns_resolve(edge_connection_t *exitconn); void dns_launch_correctness_checks(void); int dns_seems_to_be_broken(void); @@ -28,5 +28,33 @@ int dns_seems_to_be_broken_for_ipv6(void); void dns_reset_correctness_checks(void); void dump_dns_mem_usage(int severity); +#ifdef DNS_PRIVATE +#include "dns_structs.h" + +STATIC uint32_t dns_get_expiry_ttl(uint32_t ttl); + +MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn, +int is_resolve,or_circuit_t *oncirc, char **hostname_out, +int *made_connection_pending_out, cached_resolve_t **resolve_out)); + +MOCK_DECL(STATIC void,send_resolved_cell,(edge_connection_t *conn, +uint8_t answer_type,const cached_resolve_t *resolved)); + +MOCK_DECL(STATIC void,send_resolved_hostname_cell,(edge_connection_t *conn, +const char *hostname)); + +cached_resolve_t *dns_get_cache_entry(cached_resolve_t *query); +void dns_insert_cache_entry(cached_resolve_t *new_entry); + +MOCK_DECL(STATIC int, +set_exitconn_info_from_resolve,(edge_connection_t *exitconn, + const cached_resolve_t *resolve, + char **hostname_out)); + +MOCK_DECL(STATIC int, +launch_resolve,(cached_resolve_t *resolve)); + +#endif + #endif diff --git a/src/or/dns_structs.h b/src/or/dns_structs.h new file mode 100644 index 0000000000..bb67459d7b --- /dev/null +++ b/src/or/dns_structs.h @@ -0,0 +1,90 @@ +#ifndef TOR_DNS_STRUCTS_H +#define TOR_DNS_STRUCTS_H + +/** Longest hostname we're willing to resolve. */ +#define MAX_ADDRESSLEN 256 + +/** Linked list of connections waiting for a DNS answer. */ +typedef struct pending_connection_t { + edge_connection_t *conn; + struct pending_connection_t *next; +} pending_connection_t; + +/** Value of 'magic' field for cached_resolve_t. Used to try to catch bad + * pointers and memory stomping. */ +#define CACHED_RESOLVE_MAGIC 0x1234F00D + +/* Possible states for a cached resolve_t */ +/** We are waiting for the resolver system to tell us an answer here. + * When we get one, or when we time out, the state of this cached_resolve_t + * will become "DONE" and we'll possibly add a CACHED + * entry. This cached_resolve_t will be in the hash table so that we will + * know not to launch more requests for this addr, but rather to add more + * connections to the pending list for the addr. */ +#define CACHE_STATE_PENDING 0 +/** This used to be a pending cached_resolve_t, and we got an answer for it. + * Now we're waiting for this cached_resolve_t to expire. This should + * have no pending connections, and should not appear in the hash table. */ +#define CACHE_STATE_DONE 1 +/** We are caching an answer for this address. This should have no pending + * connections, and should appear in the hash table. */ +#define CACHE_STATE_CACHED 2 + +/** @name status values for a single DNS request. + * + * @{ */ +/** The DNS request is in progress. */ +#define RES_STATUS_INFLIGHT 1 +/** The DNS request finished and gave an answer */ +#define RES_STATUS_DONE_OK 2 +/** The DNS request finished and gave an error */ +#define RES_STATUS_DONE_ERR 3 +/**@}*/ + +/** A DNS request: possibly completed, possibly pending; cached_resolve + * structs are stored at the OR side in a hash table, and as a linked + * list from oldest to newest. + */ +typedef struct cached_resolve_t { + HT_ENTRY(cached_resolve_t) node; + uint32_t magic; /**< Must be CACHED_RESOLVE_MAGIC */ + char address[MAX_ADDRESSLEN]; /**< The hostname to be resolved. */ + + union { + uint32_t addr_ipv4; /**< IPv4 addr for <b>address</b>, if successful. + * (In host order.) */ + int err_ipv4; /**< One of DNS_ERR_*, if IPv4 lookup failed. */ + } result_ipv4; /**< Outcome of IPv4 lookup */ + union { + struct in6_addr addr_ipv6; /**< IPv6 addr for <b>address</b>, if + * successful */ + int err_ipv6; /**< One of DNS_ERR_*, if IPv6 lookup failed. */ + } result_ipv6; /**< Outcome of IPv6 lookup, if any */ + union { + char *hostname; /** A hostname, if PTR lookup happened successfully*/ + int err_hostname; /** One of DNS_ERR_*, if PTR lookup failed. */ + } result_ptr; + /** @name Status fields + * + * These take one of the RES_STATUS_* values, depending on the state + * of the corresponding lookup. + * + * @{ */ + unsigned int res_status_ipv4 : 2; + unsigned int res_status_ipv6 : 2; + unsigned int res_status_hostname : 2; + /**@}*/ + uint8_t state; /**< Is this cached entry pending/done/informative? */ + + time_t expire; /**< Remove items from cache after this time. */ + uint32_t ttl_ipv4; /**< What TTL did the nameserver tell us? */ + uint32_t ttl_ipv6; /**< What TTL did the nameserver tell us? */ + uint32_t ttl_hostname; /**< What TTL did the nameserver tell us? */ + /** Connections that want to know when we get an answer for this resolve. */ + pending_connection_t *pending_connections; + /** Position of this element in the heap*/ + int minheap_idx; +} cached_resolve_t; + +#endif + diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index ebff7b524c..74f17ce78c 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -1,8 +1,9 @@ -/* Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** - * \file dnsserv.c \brief Implements client-side DNS proxy server code. Note: + * \file dnsserv.c + * \brief Implements client-side DNS proxy server code. Note: * this is the DNS Server code, not the Server DNS code. Confused? This code * runs on client-side, and acts as a DNS server. The code in dns.c, on the * other hand, runs on Tor servers, and acts as a DNS client. @@ -35,7 +36,7 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) entry_connection_t *entry_conn; edge_connection_t *conn; int i = 0; - struct evdns_server_question *q = NULL; + struct evdns_server_question *q = NULL, *supported_q = NULL; struct sockaddr_storage addr; struct sockaddr *sa; int addrlen; @@ -91,27 +92,31 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) case EVDNS_TYPE_A: case EVDNS_TYPE_AAAA: case EVDNS_TYPE_PTR: - q = req->questions[i]; + /* We always pick the first one of these questions, if there is + one. */ + if (! supported_q) + supported_q = req->questions[i]; + break; default: break; } } + if (supported_q) + q = supported_q; if (!q) { log_info(LD_APP, "None of the questions we got were ones we're willing " "to support. Sending NOTIMPL."); evdns_server_request_respond(req, DNS_ERR_NOTIMPL); return; } - if (q->type != EVDNS_TYPE_A && q->type != EVDNS_TYPE_AAAA) { - tor_assert(q->type == EVDNS_TYPE_PTR); - } /* Make sure the name isn't too long: This should be impossible, I think. */ if (err == DNS_ERR_NONE && strlen(q->name) > MAX_SOCKS_ADDR_LEN-1) err = DNS_ERR_FORMAT; - if (err != DNS_ERR_NONE) { - /* We got an error? Then send back an answer immediately; we're done. */ + if (err != DNS_ERR_NONE || !supported_q) { + /* We got an error? There's no question we're willing to answer? Then + * send back an answer immediately; we're done. */ evdns_server_request_respond(req, err); return; } @@ -119,6 +124,7 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) /* Make a new dummy AP connection, and attach the request to it. */ entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET); conn = ENTRY_TO_EDGE_CONN(entry_conn); + CONNECTION_AP_EXPECT_NONPENDING(entry_conn); TO_CONN(conn)->state = AP_CONN_STATE_RESOLVE_WAIT; conn->is_dns_request = 1; @@ -126,18 +132,31 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) TO_CONN(conn)->port = port; TO_CONN(conn)->address = tor_dup_addr(&tor_addr); - if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA) + if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA || + q->type == EVDNS_QTYPE_ALL) { entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE; - else + } else { + tor_assert(q->type == EVDNS_TYPE_PTR); entry_conn->socks_request->command = SOCKS_COMMAND_RESOLVE_PTR; + } + + if (q->type == EVDNS_TYPE_A || q->type == EVDNS_QTYPE_ALL) { + entry_conn->entry_cfg.ipv4_traffic = 1; + entry_conn->entry_cfg.ipv6_traffic = 0; + entry_conn->entry_cfg.prefer_ipv6 = 0; + } else if (q->type == EVDNS_TYPE_AAAA) { + entry_conn->entry_cfg.ipv4_traffic = 0; + entry_conn->entry_cfg.ipv6_traffic = 1; + entry_conn->entry_cfg.prefer_ipv6 = 1; + } strlcpy(entry_conn->socks_request->address, q->name, sizeof(entry_conn->socks_request->address)); entry_conn->socks_request->listener_type = listener->base_.type; entry_conn->dns_server_request = req; - entry_conn->isolation_flags = listener->isolation_flags; - entry_conn->session_group = listener->session_group; + entry_conn->entry_cfg.isolation_flags = listener->entry_cfg.isolation_flags; + entry_conn->entry_cfg.session_group = listener->entry_cfg.session_group; entry_conn->nym_epoch = get_signewnym_epoch(); if (connection_add(ENTRY_TO_CONN(entry_conn)) < 0) { @@ -180,6 +199,7 @@ dnsserv_launch_request(const char *name, int reverse, /* Make a new dummy AP connection, and attach the request to it. */ entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET); conn = ENTRY_TO_EDGE_CONN(entry_conn); + CONNECTION_AP_EXPECT_NONPENDING(entry_conn); conn->base_.state = AP_CONN_STATE_RESOLVE_WAIT; tor_addr_copy(&TO_CONN(conn)->addr, &control_conn->base_.addr); @@ -213,9 +233,9 @@ dnsserv_launch_request(const char *name, int reverse, entry_conn->socks_request->listener_type = CONN_TYPE_CONTROL_LISTENER; entry_conn->original_dest_address = tor_strdup(name); - entry_conn->session_group = SESSION_GROUP_CONTROL_RESOLVE; + entry_conn->entry_cfg.session_group = SESSION_GROUP_CONTROL_RESOLVE; entry_conn->nym_epoch = get_signewnym_epoch(); - entry_conn->isolation_flags = ISO_DEFAULT; + entry_conn->entry_cfg.isolation_flags = ISO_DEFAULT; if (connection_add(TO_CONN(conn))<0) { log_warn(LD_APP, "Couldn't register dummy connection for RESOLVE request"); diff --git a/src/or/dnsserv.h b/src/or/dnsserv.h index 687a77e59e..ad0e248c83 100644 --- a/src/or/dnsserv.h +++ b/src/or/dnsserv.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 484b88dbf8..310a948b35 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,7 +12,10 @@ * circumvention). **/ +#define ENTRYNODES_PRIVATE + #include "or.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitstats.h" #include "config.h" @@ -54,6 +57,10 @@ typedef struct { /** When should we next try to fetch a descriptor for this bridge? */ download_status_t fetch_status; + + /** A smartlist of k=v values to be passed to the SOCKS proxy, if + transports are used for this bridge. */ + smartlist_t *socks_args; } bridge_info_t; /** A list of our chosen entry guards. */ @@ -65,7 +72,9 @@ static int entry_guards_dirty = 0; static void bridge_free(bridge_info_t *bridge); static const node_t *choose_random_entry_impl(cpath_build_state_t *state, int for_directory, - dirinfo_type_t dirtype); + dirinfo_type_t dirtype, + int *n_options_out); +static int num_bridges_usable(void); /** Return the list of entry guards, creating it if necessary. */ const smartlist_t * @@ -78,7 +87,7 @@ get_entry_guards(void) /** Check whether the entry guard <b>e</b> is usable, given the directory * authorities' opinion about the router (stored in <b>ri</b>) and the user's - * configuration (in <b>options</b>). Set <b>e</b>->bad_since + * configuration (in <b>options</b>). Set <b>e</b>->bad_since * accordingly. Return true iff the entry guard's status changes. * * If it's not usable, set *<b>reason</b> to a static string explaining why. @@ -108,6 +117,9 @@ entry_guard_set_status(entry_guard_t *e, const node_t *node, *reason = "not recommended as a guard"; else if (routerset_contains_node(options->ExcludeNodes, node)) *reason = "excluded"; + /* We only care about OR connection connectivity for entry guards. */ + else if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0)) + *reason = "unreachable by config"; else if (e->path_bias_disabled) *reason = "path-biased"; @@ -132,8 +144,7 @@ entry_guard_set_status(entry_guard_t *e, const node_t *node, } if (node) { - int is_dir = node_is_dir(node) && node->rs && - node->rs->version_supports_microdesc_cache; + int is_dir = node_is_dir(node); if (options->UseBridges && node_is_a_configured_bridge(node)) is_dir = 1; if (e->is_dir_cache != is_dir) { @@ -147,21 +158,41 @@ entry_guard_set_status(entry_guard_t *e, const node_t *node, /** Return true iff enough time has passed since we last tried to connect * to the unreachable guard <b>e</b> that we're willing to try again. */ -static int -entry_is_time_to_retry(entry_guard_t *e, time_t now) +STATIC int +entry_is_time_to_retry(const entry_guard_t *e, time_t now) { - long diff; + struct guard_retry_period_s { + time_t period_duration; + time_t interval_during_period; + }; + + struct guard_retry_period_s periods[] = { + { 6*60*60, 60*60 }, /* For first 6 hrs., retry hourly; */ + { 3*24*60*60, 4*60*60 }, /* Then retry every 4 hrs. until the + 3-day mark; */ + { 7*24*60*60, 18*60*60 }, /* After 3 days, retry every 18 hours until + 1 week mark. */ + { TIME_MAX, 36*60*60 } /* After 1 week, retry every 36 hours. */ + }; + + time_t ith_deadline_for_retry; + time_t unreachable_for; + unsigned i; + if (e->last_attempted < e->unreachable_since) return 1; - diff = now - e->unreachable_since; - if (diff < 6*60*60) - return now > (e->last_attempted + 60*60); - else if (diff < 3*24*60*60) - return now > (e->last_attempted + 4*60*60); - else if (diff < 7*24*60*60) - return now > (e->last_attempted + 18*60*60); - else - return now > (e->last_attempted + 36*60*60); + + unreachable_for = now - e->unreachable_since; + + for (i = 0; i < ARRAY_LENGTH(periods); i++) { + if (unreachable_for <= periods[i].period_duration) { + ith_deadline_for_retry = e->last_attempted + + periods[i].interval_during_period; + + return (now > ith_deadline_for_retry); + } + } + return 0; } /** Return the node corresponding to <b>e</b>, if <b>e</b> is @@ -181,12 +212,17 @@ entry_is_time_to_retry(entry_guard_t *e, time_t now) * If need_descriptor is true, only return the node if we currently have * a descriptor (routerinfo or microdesc) for it. */ -static INLINE const node_t * -entry_is_live(entry_guard_t *e, int need_uptime, int need_capacity, - int assume_reachable, int need_descriptor, const char **msg) +STATIC const node_t * +entry_is_live(const entry_guard_t *e, entry_is_live_flags_t flags, + const char **msg) { const node_t *node; const or_options_t *options = get_options(); + int need_uptime = (flags & ENTRY_NEED_UPTIME) != 0; + int need_capacity = (flags & ENTRY_NEED_CAPACITY) != 0; + const int assume_reachable = (flags & ENTRY_ASSUME_REACHABLE) != 0; + const int need_descriptor = (flags & ENTRY_NEED_DESCRIPTOR) != 0; + tor_assert(msg); if (e->path_bias_disabled) { @@ -235,7 +271,7 @@ entry_is_live(entry_guard_t *e, int need_uptime, int need_capacity, *msg = "not fast/stable"; return NULL; } - if (!fascist_firewall_allows_node(node)) { + if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, 0)) { *msg = "unreachable by config"; return NULL; } @@ -248,12 +284,18 @@ num_live_entry_guards(int for_directory) { int n = 0; const char *msg; + /* Set the entry node attributes we are interested in. */ + entry_is_live_flags_t entry_flags = ENTRY_NEED_CAPACITY; + if (!for_directory) { + entry_flags |= ENTRY_NEED_DESCRIPTOR; + } + if (! entry_guards) return 0; SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { if (for_directory && !entry->is_dir_cache) continue; - if (entry_is_live(entry, 0, 1, 0, !for_directory, &msg)) + if (entry_is_live(entry, entry_flags, &msg)) ++n; } SMARTLIST_FOREACH_END(entry); return n; @@ -282,7 +324,7 @@ log_entry_guards(int severity) SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { const char *msg = NULL; - if (entry_is_live(e, 0, 1, 0, 0, &msg)) + if (entry_is_live(e, ENTRY_NEED_CAPACITY, &msg)) smartlist_add_asprintf(elements, "%s [%s] (up %s)", e->nickname, hex_str(e->identity, DIGEST_LEN), @@ -343,7 +385,7 @@ control_event_guard_deferred(void) * If <b>chosen</b> is defined, use that one, and if it's not * already in our entry_guards list, put it at the *beginning*. * Else, put the one we pick at the end of the list. */ -static const node_t * +STATIC const node_t * add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, int for_discovery, int for_directory) { @@ -358,10 +400,10 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, entry->bad_since = 0; entry->can_retry = 1; } - entry->is_dir_cache = node->rs && - node->rs->version_supports_microdesc_cache; + entry->is_dir_cache = node_is_dir(node); if (get_options()->UseBridges && node_is_a_configured_bridge(node)) entry->is_dir_cache = 1; + return NULL; } } else if (!for_directory) { @@ -371,7 +413,7 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, } else { const routerstatus_t *rs; rs = router_pick_directory_server(MICRODESC_DIRINFO|V3_DIRINFO, - PDS_PREFER_TUNNELED_DIR_CONNS_|PDS_FOR_GUARD); + PDS_FOR_GUARD); if (!rs) return NULL; node = node_get_by_id(rs->identity_digest); @@ -392,8 +434,7 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, node_describe(node)); strlcpy(entry->nickname, node_get_nickname(node), sizeof(entry->nickname)); memcpy(entry->identity, node->identity, DIGEST_LEN); - entry->is_dir_cache = node_is_dir(node) && - node->rs && node->rs->version_supports_microdesc_cache; + entry->is_dir_cache = node_is_dir(node); if (get_options()->UseBridges && node_is_a_configured_bridge(node)) entry->is_dir_cache = 1; @@ -402,7 +443,8 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, * don't all select them on the same day, and b) avoid leaving a * precise timestamp in the state file about when we first picked * this guard. For details, see the Jan 2010 or-dev thread. */ - entry->chosen_on_date = time(NULL) - crypto_rand_int(3600*24*30); + time_t now = time(NULL); + entry->chosen_on_date = crypto_rand_time_range(now - 3600*24*30, now); entry->chosen_by_version = tor_strdup(VERSION); /* Are we picking this guard because all of our current guards are @@ -430,7 +472,7 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, /** Choose how many entry guards or directory guards we'll use. If * <b>for_directory</b> is true, we return how many directory guards to * use; else we return how many entry guards to use. */ -static int +STATIC int decide_num_guards(const or_options_t *options, int for_directory) { if (for_directory) { @@ -531,22 +573,6 @@ remove_obsolete_entry_guards(time_t now) } else if (tor_version_parse(ver, &v)) { msg = "does not seem to be from any recognized version of Tor"; version_is_bad = 1; - } else { - char *tor_ver = NULL; - tor_asprintf(&tor_ver, "Tor %s", ver); - if ((tor_version_as_new_as(tor_ver, "0.1.0.10-alpha") && - !tor_version_as_new_as(tor_ver, "0.1.2.16-dev")) || - (tor_version_as_new_as(tor_ver, "0.2.0.0-alpha") && - !tor_version_as_new_as(tor_ver, "0.2.0.6-alpha")) || - /* above are bug 440; below are bug 1217 */ - (tor_version_as_new_as(tor_ver, "0.2.1.3-alpha") && - !tor_version_as_new_as(tor_ver, "0.2.1.23")) || - (tor_version_as_new_as(tor_ver, "0.2.2.0-alpha") && - !tor_version_as_new_as(tor_ver, "0.2.2.7-alpha"))) { - msg = "was selected without regard for guard bandwidth"; - version_is_bad = 1; - } - tor_free(tor_ver); } if (!version_is_bad && entry->chosen_on_date + guard_lifetime < now) { /* It's been too long since the date listed in our state file. */ @@ -605,6 +631,25 @@ remove_dead_entry_guards(time_t now) return changed ? 1 : 0; } +/** Remove all currently listed entry guards. So new ones will be chosen. */ +void +remove_all_entry_guards(void) +{ + char dbuf[HEX_DIGEST_LEN+1]; + + while (smartlist_len(entry_guards)) { + entry_guard_t *entry = smartlist_get(entry_guards, 0); + base16_encode(dbuf, sizeof(dbuf), entry->identity, DIGEST_LEN); + log_info(LD_CIRC, "Entry guard '%s' (%s) has been dropped.", + entry->nickname, dbuf); + control_event_guard(entry->nickname, entry->identity, "DROPPED"); + entry_guard_free(entry); + smartlist_del(entry_guards, 0); + } + log_entry_guards(LOG_INFO); + entry_guards_changed(); +} + /** A new directory or router-status has arrived; update the down/listed * status of the entry guards. * @@ -650,7 +695,7 @@ entry_guards_compute_status(const or_options_t *options, time_t now) SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { const char *reason = digestmap_get(reasons, entry->identity); const char *live_msg = ""; - const node_t *r = entry_is_live(entry, 0, 1, 0, 0, &live_msg); + const node_t *r = entry_is_live(entry, ENTRY_NEED_CAPACITY, &live_msg); log_info(LD_CIRC, "Summary: Entry %s [%s] is %s, %s%s%s, and %s%s.", entry->nickname, hex_str(entry->identity, DIGEST_LEN), @@ -768,7 +813,9 @@ entry_guard_register_connect_status(const char *digest, int succeeded, break; if (e->made_contact) { const char *msg; - const node_t *r = entry_is_live(e, 0, 1, 1, 0, &msg); + const node_t *r = entry_is_live(e, + ENTRY_NEED_CAPACITY | ENTRY_ASSUME_REACHABLE, + &msg); if (r && e->unreachable_since) { refuse_conn = 1; e->can_retry = 1; @@ -821,7 +868,7 @@ update_node_guard_status(void) /** Adjust the entry guards list so that it only contains entries from * EntryNodes, adding new entries from EntryNodes to the list as needed. */ -static void +STATIC void entry_guards_set_from_config(const or_options_t *options) { smartlist_t *entry_nodes, *worse_entry_nodes, *entry_fps; @@ -874,7 +921,8 @@ entry_guards_set_from_config(const or_options_t *options) } else if (routerset_contains_node(options->ExcludeNodes, node)) { SMARTLIST_DEL_CURRENT(entry_nodes, node); continue; - } else if (!fascist_firewall_allows_node(node)) { + } else if (!fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, + 0)) { SMARTLIST_DEL_CURRENT(entry_nodes, node); continue; } else if (! node->is_possible_guard) { @@ -928,49 +976,17 @@ entry_list_is_constrained(const or_options_t *options) return 0; } -/** Return true iff this node can answer directory questions about - * microdescriptors. */ -static int -node_understands_microdescriptors(const node_t *node) -{ - tor_assert(node); - if (node->rs && node->rs->version_supports_microdesc_cache) - return 1; - if (node->ri && tor_version_supports_microdescriptors(node->ri->platform)) - return 1; - return 0; -} - -/** Return true iff <b>node</b> is able to answer directory questions - * of type <b>dirinfo</b>. */ -static int -node_can_handle_dirinfo(const node_t *node, dirinfo_type_t dirinfo) -{ - /* Checking dirinfo for any type other than microdescriptors isn't required - yet, since we only choose directory guards that can support microdescs, - routerinfos, and networkstatuses, AND we don't use directory guards if - we're configured to do direct downloads of anything else. The only case - where we might have a guard that doesn't know about a type of directory - information is when we're retrieving directory information from a - bridge. */ - - if ((dirinfo & MICRODESC_DIRINFO) && - !node_understands_microdescriptors(node)) - return 0; - return 1; -} - /** Pick a live (up and listed) entry guard from entry_guards. If * <b>state</b> is non-NULL, this is for a specific circuit -- * make sure not to pick this circuit's exit or any node in the * exit's family. If <b>state</b> is NULL, we're looking for a random - * guard (likely a bridge). If <b>dirinfo</b> is not NO_DIRINFO, then - * only select from nodes that know how to answer directory questions + * guard (likely a bridge). If <b>dirinfo</b> is not NO_DIRINFO (zero), + * then only select from nodes that know how to answer directory questions * of that type. */ const node_t * choose_random_entry(cpath_build_state_t *state) { - return choose_random_entry_impl(state, 0, 0); + return choose_random_entry_impl(state, 0, NO_DIRINFO, NULL); } /** Pick a live (up and listed) directory guard from entry_guards for @@ -978,47 +994,66 @@ choose_random_entry(cpath_build_state_t *state) const node_t * choose_random_dirguard(dirinfo_type_t type) { - return choose_random_entry_impl(NULL, 1, type); + return choose_random_entry_impl(NULL, 1, type, NULL); } -/** Helper for choose_random{entry,dirguard}. */ -static const node_t * -choose_random_entry_impl(cpath_build_state_t *state, int for_directory, - dirinfo_type_t dirinfo_type) +/** Filter <b>all_entry_guards</b> for usable entry guards and put them + * in <b>live_entry_guards</b>. We filter based on whether the node is + * currently alive, and on whether it satisfies the restrictions + * imposed by the other arguments of this function. + * + * We don't place more guards than NumEntryGuards in <b>live_entry_guards</b>. + * + * If <b>chosen_exit</b> is set, it contains the exit node of this + * circuit. Make sure to not use it or its family as an entry guard. + * + * If <b>need_uptime</b> is set, we are looking for a stable entry guard. + * if <b>need_capacity</b> is set, we are looking for a fast entry guard. + * + * The rest of the arguments are the same as in choose_random_entry_impl(). + * + * Return 1 if we should choose a guard right away. Return 0 if we + * should try to add more nodes to our list before deciding on a + * guard. + */ +STATIC int +populate_live_entry_guards(smartlist_t *live_entry_guards, + const smartlist_t *all_entry_guards, + const node_t *chosen_exit, + dirinfo_type_t dirinfo_type, + int for_directory, + int need_uptime, int need_capacity) { const or_options_t *options = get_options(); - smartlist_t *live_entry_guards = smartlist_new(); - smartlist_t *exit_family = smartlist_new(); - const node_t *chosen_exit = - state?build_state_get_exit_node(state) : NULL; const node_t *node = NULL; - int need_uptime = state ? state->need_uptime : 0; - int need_capacity = state ? state->need_capacity : 0; - int preferred_min, consider_exit_family = 0; - int need_descriptor = !for_directory; const int num_needed = decide_num_guards(options, for_directory); + smartlist_t *exit_family = smartlist_new(); + int retval = 0; + entry_is_live_flags_t entry_flags = 0; - if (chosen_exit) { - nodelist_add_node_and_family(exit_family, chosen_exit); - consider_exit_family = 1; - } + (void) dirinfo_type; - if (!entry_guards) - entry_guards = smartlist_new(); + { /* Set the flags we want our entry node to have */ + if (need_uptime) { + entry_flags |= ENTRY_NEED_UPTIME; + } + if (need_capacity) { + entry_flags |= ENTRY_NEED_CAPACITY; + } + if (!for_directory) { + entry_flags |= ENTRY_NEED_DESCRIPTOR; + } + } - if (should_add_entry_nodes) - entry_guards_set_from_config(options); + tor_assert(all_entry_guards); - if (!entry_list_is_constrained(options) && - smartlist_len(entry_guards) < num_needed) - pick_entry_guards(options, for_directory); + if (chosen_exit) { + nodelist_add_node_and_family(exit_family, chosen_exit); + } - retry: - smartlist_clear(live_entry_guards); - SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { + SMARTLIST_FOREACH_BEGIN(all_entry_guards, const entry_guard_t *, entry) { const char *msg; - node = entry_is_live(entry, need_uptime, need_capacity, 0, - need_descriptor, &msg); + node = entry_is_live(entry, entry_flags, &msg); if (!node) continue; /* down, no point */ if (for_directory) { @@ -1027,39 +1062,93 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, } if (node == chosen_exit) continue; /* don't pick the same node for entry and exit */ - if (consider_exit_family && smartlist_contains(exit_family, node)) + if (smartlist_contains(exit_family, node)) continue; /* avoid relays that are family members of our exit */ - if (dirinfo_type != NO_DIRINFO && - !node_can_handle_dirinfo(node, dirinfo_type)) - continue; /* this node won't be able to answer our dir questions */ -#if 0 /* since EntryNodes is always strict now, this clause is moot */ - if (options->EntryNodes && - !routerset_contains_node(options->EntryNodes, node)) { - /* We've come to the end of our preferred entry nodes. */ - if (smartlist_len(live_entry_guards)) - goto choose_and_finish; /* only choose from the ones we like */ - if (options->StrictNodes) { - /* in theory this case should never happen, since - * entry_guards_set_from_config() drops unwanted relays */ - tor_fragile_assert(); - } else { - log_info(LD_CIRC, - "No relays from EntryNodes available. Using others."); - } - } -#endif smartlist_add(live_entry_guards, (void*)node); if (!entry->made_contact) { /* Always start with the first not-yet-contacted entry * guard. Otherwise we might add several new ones, pick * the second new one, and now we've expanded our entry * guard list without needing to. */ - goto choose_and_finish; + retval = 1; + goto done; + } + if (smartlist_len(live_entry_guards) >= num_needed) { + retval = 1; + goto done; /* We picked enough entry guards. Done! */ } - if (smartlist_len(live_entry_guards) >= num_needed) - goto choose_and_finish; /* we have enough */ } SMARTLIST_FOREACH_END(entry); + done: + smartlist_free(exit_family); + + return retval; +} + +/** Pick a node to be used as the entry guard of a circuit. + * + * If <b>state</b> is set, it contains the information we know about + * the upcoming circuit. + * + * If <b>for_directory</b> is set, we are looking for a directory guard. + * + * <b>dirinfo_type</b> contains the kind of directory information we + * are looking for in our node, or NO_DIRINFO (zero) if we are not + * looking for any particular directory information (when set to + * NO_DIRINFO, the <b>dirinfo_type</b> filter is ignored). + * + * If <b>n_options_out</b> is set, we set it to the number of + * candidate guard nodes we had before picking a specific guard node. + * + * On success, return the node that should be used as the entry guard + * of the circuit. Return NULL if no such node could be found. + * + * Helper for choose_random{entry,dirguard}. +*/ +static const node_t * +choose_random_entry_impl(cpath_build_state_t *state, int for_directory, + dirinfo_type_t dirinfo_type, int *n_options_out) +{ + const or_options_t *options = get_options(); + smartlist_t *live_entry_guards = smartlist_new(); + const node_t *chosen_exit = + state?build_state_get_exit_node(state) : NULL; + const node_t *node = NULL; + int need_uptime = state ? state->need_uptime : 0; + int need_capacity = state ? state->need_capacity : 0; + int preferred_min = 0; + const int num_needed = decide_num_guards(options, for_directory); + int retval = 0; + + if (n_options_out) + *n_options_out = 0; + + if (!entry_guards) + entry_guards = smartlist_new(); + + if (should_add_entry_nodes) + entry_guards_set_from_config(options); + + if (!entry_list_is_constrained(options) && + smartlist_len(entry_guards) < num_needed) + pick_entry_guards(options, for_directory); + + retry: + smartlist_clear(live_entry_guards); + + /* Populate the list of live entry guards so that we pick one of + them. */ + retval = populate_live_entry_guards(live_entry_guards, + entry_guards, + chosen_exit, + dirinfo_type, + for_directory, + need_uptime, need_capacity); + + if (retval == 1) { /* We should choose a guard right now. */ + goto choose_and_finish; + } + if (entry_list_is_constrained(options)) { /* If we prefer the entry nodes we've got, and we have at least * one choice, that's great. Use it. */ @@ -1067,7 +1156,7 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, } else { /* Try to have at least 2 choices available. This way we don't * get stuck with a single live-but-crummy entry and just keep - * using him. + * using it. * (We might get 2 live-but-crummy entry guards, but so be it.) */ preferred_min = 2; } @@ -1098,18 +1187,7 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, need_capacity = 0; goto retry; } -#if 0 - /* Removing this retry logic: if we only allow one exit, and it is in the - same family as all our entries, then we are just plain not going to win - here. */ - if (!node && entry_list_is_constrained(options) && consider_exit_family) { - /* still no? if we're using bridges or have strictentrynodes - * set, and our chosen exit is in the same family as all our - * bridges/entry guards, then be flexible about families. */ - consider_exit_family = 0; - goto retry; - } -#endif + /* live_entry_guards may be empty below. Oh well, we tried. */ } @@ -1124,8 +1202,9 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, * *double*-weight our guard selection. */ node = smartlist_choose(live_entry_guards); } + if (n_options_out) + *n_options_out = smartlist_len(live_entry_guards); smartlist_free(live_entry_guards); - smartlist_free(exit_family); return node; } @@ -1193,7 +1272,7 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) "EntryGuardDownSince/UnlistedSince without EntryGuard"); break; } - if (parse_iso_time(line->value, &when)<0) { + if (parse_iso_time_(line->value, &when, 0)<0) { *msg = tor_strdup("Unable to parse entry nodes: " "Bad time in EntryGuardDownSince/UnlistedSince"); break; @@ -1365,8 +1444,9 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) } } else { if (state_version) { + time_t now = time(NULL); + e->chosen_on_date = crypto_rand_time_range(now - 3600*24*30, now); e->chosen_by_version = tor_strdup(state_version); - e->chosen_on_date = time(NULL) - crypto_rand_int(3600*24*30); } } if (e->path_bias_disabled && !e->bad_since) @@ -1397,6 +1477,13 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) return *msg ? -1 : 0; } +/** How long will we let a change in our guard nodes stay un-saved + * when we are trying to avoid disk writes? */ +#define SLOW_GUARD_STATE_FLUSH_TIME 600 +/** How long will we let a change in our guard nodes stay un-saved + * when we are not trying to avoid disk writes? */ +#define FAST_GUARD_STATE_FLUSH_TIME 30 + /** Our list of entry guards has changed, or some element of one * of our entry guards has changed. Write the changes to disk within * the next few minutes. @@ -1407,8 +1494,12 @@ entry_guards_changed(void) time_t when; entry_guards_dirty = 1; + if (get_options()->AvoidDiskWrites) + when = time(NULL) + SLOW_GUARD_STATE_FLUSH_TIME; + else + when = time(NULL) + FAST_GUARD_STATE_FLUSH_TIME; + /* or_state_save() will call entry_guards_update_state(). */ - when = get_options()->AvoidDiskWrites ? time(NULL) + 3600 : time(NULL)+600; or_state_mark_dirty(get_or_state(), when); } @@ -1529,6 +1620,9 @@ getinfo_helper_entry_guards(control_connection_t *conn, } else if (e->bad_since) { when = e->bad_since; status = "unusable"; + } else if (e->unreachable_since) { + when = e->unreachable_since; + status = "down"; } else { status = "up"; } @@ -1557,6 +1651,63 @@ getinfo_helper_entry_guards(control_connection_t *conn, return 0; } +/** Return 0 if we should apply guardfraction information found in the + * consensus. A specific consensus can be specified with the + * <b>ns</b> argument, if NULL the most recent one will be picked.*/ +int +should_apply_guardfraction(const networkstatus_t *ns) +{ + /* We need to check the corresponding torrc option and the consensus + * parameter if we need to. */ + const or_options_t *options = get_options(); + + /* If UseGuardFraction is 'auto' then check the same-named consensus + * parameter. If the consensus parameter is not present, default to + * "off". */ + if (options->UseGuardFraction == -1) { + return networkstatus_get_param(ns, "UseGuardFraction", + 0, /* default to "off" */ + 0, 1); + } + + return options->UseGuardFraction; +} + +/* Given the original bandwidth of a guard and its guardfraction, + * calculate how much bandwidth the guard should have as a guard and + * as a non-guard. + * + * Quoting from proposal236: + * + * Let Wpf denote the weight from the 'bandwidth-weights' line a + * client would apply to N for position p if it had the guard + * flag, Wpn the weight if it did not have the guard flag, and B the + * measured bandwidth of N in the consensus. Then instead of choosing + * N for position p proportionally to Wpf*B or Wpn*B, clients should + * choose N proportionally to F*Wpf*B + (1-F)*Wpn*B. + * + * This function fills the <b>guardfraction_bw</b> structure. It sets + * <b>guard_bw</b> to F*B and <b>non_guard_bw</b> to (1-F)*B. + */ +void +guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw, + int orig_bandwidth, + uint32_t guardfraction_percentage) +{ + double guardfraction_fraction; + + /* Turn the percentage into a fraction. */ + tor_assert(guardfraction_percentage <= 100); + guardfraction_fraction = guardfraction_percentage / 100.0; + + long guard_bw = tor_lround(guardfraction_fraction * orig_bandwidth); + tor_assert(guard_bw <= INT_MAX); + + guardfraction_bw->guard_bw = (int) guard_bw; + + guardfraction_bw->non_guard_bw = orig_bandwidth - (int) guard_bw; +} + /** A list of configured bridges. Whenever we actually get a descriptor * for one, we add it as an entry guard. Note that the order of bridges * in this list does not necessarily correspond to the order of bridges @@ -1607,6 +1758,11 @@ bridge_free(bridge_info_t *bridge) return; tor_free(bridge->transport_name); + if (bridge->socks_args) { + SMARTLIST_FOREACH(bridge->socks_args, char*, s, tor_free(s)); + smartlist_free(bridge->socks_args); + } + tor_free(bridge); } @@ -1639,8 +1795,9 @@ get_configured_bridge_by_orports_digest(const char *digest, } /** If we have a bridge configured whose digest matches <b>digest</b>, or a - * bridge with no known digest whose address matches <b>addr</b>:<b>/port</b>, - * return that bridge. Else return NULL. */ + * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>, + * return that bridge. Else return NULL. If <b>digest</b> is NULL, check for + * address/port matches only. */ static bridge_info_t * get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr, uint16_t port, @@ -1650,7 +1807,7 @@ get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr, return NULL; SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, bridge) { - if (tor_digest_is_zero(bridge->identity) && + if ((tor_digest_is_zero(bridge->identity) || digest == NULL) && !tor_addr_compare(&bridge->addr, addr, CMP_EXACT) && bridge->port == port) return bridge; @@ -1661,6 +1818,30 @@ get_configured_bridge_by_addr_port_digest(const tor_addr_t *addr, return NULL; } +/** If we have a bridge configured whose digest matches <b>digest</b>, or a + * bridge with no known digest whose address matches <b>addr</b>:<b>port</b>, + * return 1. Else return 0. If <b>digest</b> is NULL, check for + * address/port matches only. */ +int +addr_is_a_configured_bridge(const tor_addr_t *addr, + uint16_t port, + const char *digest) +{ + tor_assert(addr); + return get_configured_bridge_by_addr_port_digest(addr, port, digest) ? 1 : 0; +} + +/** If we have a bridge configured whose digest matches + * <b>ei->identity_digest</b>, or a bridge with no known digest whose address + * matches <b>ei->addr</b>:<b>ei->port</b>, return 1. Else return 0. + * If <b>ei->onion_key</b> is NULL, check for address/port matches only. */ +int +extend_info_is_a_configured_bridge(const extend_info_t *ei) +{ + const char *digest = ei->onion_key ? ei->identity_digest : NULL; + return addr_is_a_configured_bridge(&ei->addr, ei->port, digest); +} + /** Wrapper around get_configured_bridge_by_addr_port_digest() to look * it up via router descriptor <b>ri</b>. */ static bridge_info_t * @@ -1785,30 +1966,68 @@ bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port, } SMARTLIST_FOREACH_END(bridge); } -/** Remember a new bridge at <b>addr</b>:<b>port</b>. If <b>digest</b> - * is set, it tells us the identity key too. If we already had the - * bridge in our list, unmark it, and don't actually add anything new. - * If <b>transport_name</b> is non-NULL - the bridge is associated with a - * pluggable transport - we assign the transport to the bridge. */ +/** Return True if we have a bridge that uses a transport with name + * <b>transport_name</b>. */ +MOCK_IMPL(int, +transport_is_needed, (const char *transport_name)) +{ + if (!bridge_list) + return 0; + + SMARTLIST_FOREACH_BEGIN(bridge_list, const bridge_info_t *, bridge) { + if (bridge->transport_name && + !strcmp(bridge->transport_name, transport_name)) + return 1; + } SMARTLIST_FOREACH_END(bridge); + + return 0; +} + +/** Register the bridge information in <b>bridge_line</b> to the + * bridge subsystem. Steals reference of <b>bridge_line</b>. */ void -bridge_add_from_config(const tor_addr_t *addr, uint16_t port, - const char *digest, const char *transport_name) +bridge_add_from_config(bridge_line_t *bridge_line) { bridge_info_t *b; - bridge_resolve_conflicts(addr, port, digest, transport_name); + { /* Log the bridge we are about to register: */ + log_debug(LD_GENERAL, "Registering bridge at %s (transport: %s) (%s)", + fmt_addrport(&bridge_line->addr, bridge_line->port), + bridge_line->transport_name ? + bridge_line->transport_name : "no transport", + tor_digest_is_zero(bridge_line->digest) ? + "no key listed" : hex_str(bridge_line->digest, DIGEST_LEN)); + + if (bridge_line->socks_args) { /* print socks arguments */ + int i = 0; + + tor_assert(smartlist_len(bridge_line->socks_args) > 0); + + log_debug(LD_GENERAL, "Bridge uses %d SOCKS arguments:", + smartlist_len(bridge_line->socks_args)); + SMARTLIST_FOREACH(bridge_line->socks_args, const char *, arg, + log_debug(LD_CONFIG, "%d: %s", ++i, arg)); + } + } + + bridge_resolve_conflicts(&bridge_line->addr, + bridge_line->port, + bridge_line->digest, + bridge_line->transport_name); b = tor_malloc_zero(sizeof(bridge_info_t)); - tor_addr_copy(&b->addr, addr); - b->port = port; - if (digest) - memcpy(b->identity, digest, DIGEST_LEN); - if (transport_name) - b->transport_name = tor_strdup(transport_name); + tor_addr_copy(&b->addr, &bridge_line->addr); + b->port = bridge_line->port; + memcpy(b->identity, bridge_line->digest, DIGEST_LEN); + if (bridge_line->transport_name) + b->transport_name = bridge_line->transport_name; b->fetch_status.schedule = DL_SCHED_BRIDGE; + b->socks_args = bridge_line->socks_args; if (!bridge_list) bridge_list = smartlist_new(); + tor_free(bridge_line); /* Deallocate bridge_line now. */ + smartlist_add(bridge_list, b); } @@ -1869,7 +2088,7 @@ find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port) * transport, but the transport could not be found. */ int -find_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, +get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, const transport_t **transport) { *transport = NULL; @@ -1896,11 +2115,21 @@ find_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, return 0; } +/** Return a smartlist containing all the SOCKS arguments that we + * should pass to the SOCKS proxy. */ +const smartlist_t * +get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port) +{ + bridge_info_t *bridge = get_configured_bridge_by_addr_port_digest(addr, + port, + NULL); + return bridge ? bridge->socks_args : NULL; +} + /** We need to ask <b>bridge</b> for its server descriptor. */ static void launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) { - char *address; const or_options_t *options = get_options(); if (connection_get_by_type_addr_port_purpose( @@ -1915,15 +2144,22 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; } - address = tor_dup_addr(&bridge->addr); + /* Until we get a descriptor for the bridge, we only know one address for + * it. */ + if (!fascist_firewall_allows_address_addr(&bridge->addr, bridge->port, + FIREWALL_OR_CONNECTION, 0, 0)) { + log_notice(LD_CONFIG, "Tried to fetch a descriptor directly from a " + "bridge, but that bridge is not reachable through our " + "firewall."); + return; + } - directory_initiate_command(address, &bridge->addr, - bridge->port, 0/*no dirport*/, + directory_initiate_command(&bridge->addr, bridge->port, + NULL, 0, /*no dirport*/ bridge->identity, DIR_PURPOSE_FETCH_SERVERDESC, ROUTER_PURPOSE_BRIDGE, DIRIND_ONEHOP, "authority.z", NULL, 0, 0); - tor_free(address); } /** Fetching the bridge descriptor from the bridge authority returned a @@ -1980,7 +2216,9 @@ fetch_bridge_descriptors(const or_options_t *options, time_t now) !options->UpdateBridgesFromAuthority, !num_bridge_auths); if (ask_bridge_directly && - !fascist_firewall_allows_address_or(&bridge->addr, bridge->port)) { + !fascist_firewall_allows_address_addr(&bridge->addr, bridge->port, + FIREWALL_OR_CONNECTION, 0, + 0)) { log_notice(LD_DIR, "Bridge at '%s' isn't reachable by our " "firewall policy. %s.", fmt_addrport(&bridge->addr, bridge->port), @@ -2007,7 +2245,7 @@ fetch_bridge_descriptors(const or_options_t *options, time_t now) log_info(LD_DIR, "Fetching bridge info '%s' from bridge authority.", resource); directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC, - ROUTER_PURPOSE_BRIDGE, resource, 0); + ROUTER_PURPOSE_BRIDGE, resource, 0, DL_WANT_AUTHORITY); } } SMARTLIST_FOREACH_END(bridge); @@ -2028,6 +2266,7 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) * does so through an address from any source other than node_get_addr(). */ tor_addr_t addr; + const or_options_t *options = get_options(); if (node->ri) { routerinfo_t *ri = node->ri; @@ -2041,13 +2280,11 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } else { if (tor_addr_family(&bridge->addr) == AF_INET) { ri->addr = tor_addr_to_ipv4h(&bridge->addr); - tor_free(ri->address); - ri->address = tor_dup_ip(ri->addr); ri->or_port = bridge->port; log_info(LD_DIR, "Adjusted bridge routerinfo for '%s' to match configured " "address %s:%d.", - ri->nickname, ri->address, ri->or_port); + ri->nickname, fmt_addr32(ri->addr), ri->or_port); } else if (tor_addr_family(&bridge->addr) == AF_INET6) { tor_addr_copy(&ri->ipv6_addr, &bridge->addr); ri->ipv6_orport = bridge->port; @@ -2062,9 +2299,15 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } } - /* Mark which address to use based on which bridge_t we got. */ - node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 && - !tor_addr_is_null(&node->ri->ipv6_addr)); + if (options->ClientPreferIPv6ORPort == -1) { + /* Mark which address to use based on which bridge_t we got. */ + node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 && + !tor_addr_is_null(&node->ri->ipv6_addr)); + } else { + /* Mark which address to use based on user preference */ + node->ipv6_preferred = (fascist_firewall_prefer_ipv6_orport(options) && + !tor_addr_is_null(&node->ri->ipv6_addr)); + } /* XXXipv6 we lack support for falling back to another address for the same relay, warn the user */ @@ -2073,10 +2316,13 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) node_get_pref_orport(node, &ap); log_notice(LD_CONFIG, "Bridge '%s' has both an IPv4 and an IPv6 address. " - "Will prefer using its %s address (%s).", + "Will prefer using its %s address (%s) based on %s.", ri->nickname, - tor_addr_family(&ap.addr) == AF_INET6 ? "IPv6" : "IPv4", - fmt_addrport(&ap.addr, ap.port)); + node->ipv6_preferred ? "IPv6" : "IPv4", + fmt_addrport(&ap.addr, ap.port), + options->ClientPreferIPv6ORPort == -1 ? + "the configured Bridge address" : + "ClientPreferIPv6ORPort"); } } if (node->rs) { @@ -2105,7 +2351,7 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) tor_assert(ri); tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE); if (get_options()->UseBridges) { - int first = !any_bridge_descriptors_known(); + int first = num_bridges_usable() <= 1; bridge_info_t *bridge = get_configured_bridge_by_routerinfo(ri); time_t now = time(NULL); router_set_status(ri->cache_info.identity_digest, 1); @@ -2119,6 +2365,13 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) node = node_get_mutable_by_id(ri->cache_info.identity_digest); tor_assert(node); rewrite_node_address_for_bridge(bridge, node); + if (tor_digest_is_zero(bridge->identity)) { + memcpy(bridge->identity,ri->cache_info.identity_digest, DIGEST_LEN); + log_notice(LD_DIR, "Learned identity %s for bridge at %s:%d", + hex_str(bridge->identity, DIGEST_LEN), + fmt_and_decorate_addr(&bridge->addr), + (int) bridge->port); + } add_an_entry_guard(node, 1, 1, 0, 0); log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname, @@ -2128,17 +2381,14 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) entry_guard_register_connect_status(ri->cache_info.identity_digest, 1, 0, now); if (first) { - /* XXXX apparently, this is never called. See bug #9229. */ routerlist_retry_directory_downloads(now); } - - update_networkstatus_downloads(now); } } } -/** Return 1 if any of our entry guards have descriptors that - * are marked with purpose 'bridge' and are running. Else return 0. +/** Return the number of bridges that have descriptors that + * are marked with purpose 'bridge' and are running. * * We use this function to decide if we're ready to start building * circuits through our bridges, or if we need to wait until the @@ -2150,25 +2400,16 @@ any_bridge_descriptors_known(void) return choose_random_entry(NULL) != NULL; } -/** Return 1 if there are any directory conns fetching bridge descriptors - * that aren't marked for close. We use this to guess if we should tell - * the controller that we have a problem. */ -int -any_pending_bridge_descriptor_fetches(void) -{ - smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { - if (conn->type == CONN_TYPE_DIR && - conn->purpose == DIR_PURPOSE_FETCH_SERVERDESC && - TO_DIR_CONN(conn)->router_purpose == ROUTER_PURPOSE_BRIDGE && - !conn->marked_for_close && - conn->linked && - conn->linked_conn && !conn->linked_conn->marked_for_close) { - log_debug(LD_DIR, "found one: %s", conn->address); - return 1; - } - } SMARTLIST_FOREACH_END(conn); - return 0; +/** Return the number of bridges that have descriptors that are marked with + * purpose 'bridge' and are running. + */ +static int +num_bridges_usable(void) +{ + int n_options = 0; + tor_assert(get_options()->UseBridges); + (void) choose_random_entry_impl(NULL, 0, 0, &n_options); + return n_options; } /** Return 1 if we have at least one descriptor for an entry guard @@ -2187,7 +2428,9 @@ entries_retry_helper(const or_options_t *options, int act) SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { node = node_get_by_id(e->identity); if (node && node_has_descriptor(node) && - node_is_bridge(node) == need_bridges) { + node_is_bridge(node) == need_bridges && + (!need_bridges || (!e->bad_since && + node_is_a_configured_bridge(node)))) { any_known = 1; if (node->is_running) any_running = 1; /* some entry is both known and running */ @@ -2241,11 +2484,9 @@ any_bridge_supports_microdescriptors(void) SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { node = node_get_by_id(e->identity); if (node && node->is_running && - node_is_bridge(node) && node_is_a_configured_bridge(node) && - node_understands_microdescriptors(node)) { + node_is_bridge(node) && node_is_a_configured_bridge(node)) { /* This is one of our current bridges, and we know enough about - * it to know that it will be able to answer our microdescriptor - * questions. */ + * it to know that it will be able to answer our questions. */ return 1; } } SMARTLIST_FOREACH_END(e); @@ -2266,6 +2507,6 @@ entry_guards_free_all(void) clear_bridge_list(); smartlist_free(bridge_list); bridge_list = NULL; - circuit_build_times_free_timeouts(&circ_times); + circuit_build_times_free_timeouts(get_circuit_build_times_mutable()); } diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h index 52b8dc00e4..247c80940e 100644 --- a/src/or/entrynodes.h +++ b/src/or/entrynodes.h @@ -1,11 +1,11 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** - * \file guardnodes.h + * \file entrynodes.h * \brief Header file for circuitbuild.c. **/ @@ -77,6 +77,40 @@ int num_live_entry_guards(int for_directory); #endif +#ifdef ENTRYNODES_PRIVATE +STATIC const node_t *add_an_entry_guard(const node_t *chosen, + int reset_status, int prepend, + int for_discovery, int for_directory); + +STATIC int populate_live_entry_guards(smartlist_t *live_entry_guards, + const smartlist_t *all_entry_guards, + const node_t *chosen_exit, + dirinfo_type_t dirinfo_type, + int for_directory, + int need_uptime, int need_capacity); +STATIC int decide_num_guards(const or_options_t *options, int for_directory); + +STATIC void entry_guards_set_from_config(const or_options_t *options); + +/** Flags to be passed to entry_is_live() to indicate what kind of + * entry nodes we are looking for. */ +typedef enum { + ENTRY_NEED_UPTIME = 1<<0, + ENTRY_NEED_CAPACITY = 1<<1, + ENTRY_ASSUME_REACHABLE = 1<<2, + ENTRY_NEED_DESCRIPTOR = 1<<3, +} entry_is_live_flags_t; + +STATIC const node_t *entry_is_live(const entry_guard_t *e, + entry_is_live_flags_t flags, + const char **msg); + +STATIC int entry_is_time_to_retry(const entry_guard_t *e, time_t now); + +#endif + +void remove_all_entry_guards(void); + void entry_guards_compute_status(const or_options_t *options, time_t now); int entry_guard_register_connect_status(const char *digest, int succeeded, int mark_relay_status, time_t now); @@ -93,35 +127,57 @@ int getinfo_helper_entry_guards(control_connection_t *conn, void mark_bridge_list(void); void sweep_bridge_list(void); +int addr_is_a_configured_bridge(const tor_addr_t *addr, uint16_t port, + const char *digest); +int extend_info_is_a_configured_bridge(const extend_info_t *ei); int routerinfo_is_a_configured_bridge(const routerinfo_t *ri); int node_is_a_configured_bridge(const node_t *node); void learned_router_identity(const tor_addr_t *addr, uint16_t port, const char *digest); -void bridge_add_from_config(const tor_addr_t *addr, uint16_t port, - const char *digest, - const char *transport_name); +struct bridge_line_t; +void bridge_add_from_config(struct bridge_line_t *bridge_line); void retry_bridge_descriptor_fetch_directly(const char *digest); void fetch_bridge_descriptors(const or_options_t *options, time_t now); void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); int any_bridge_descriptors_known(void); -int any_pending_bridge_descriptor_fetches(void); int entries_known_but_down(const or_options_t *options); void entries_retry_all(const or_options_t *options); int any_bridge_supports_microdescriptors(void); +const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr, + uint16_t port); + +int any_bridges_dont_support_microdescriptors(void); void entry_guards_free_all(void); const char *find_transport_name_by_bridge_addrport(const tor_addr_t *addr, uint16_t port); struct transport_t; -int find_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, +int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, const struct transport_t **transport); +MOCK_DECL(int, transport_is_needed, (const char *transport_name)); int validate_pluggable_transports_config(void); double pathbias_get_close_success_count(entry_guard_t *guard); double pathbias_get_use_success_count(entry_guard_t *guard); +/** Contains the bandwidth of a relay as a guard and as a non-guard + * after the guardfraction has been considered. */ +typedef struct guardfraction_bandwidth_t { + /** Bandwidth as a guard after guardfraction has been considered. */ + int guard_bw; + /** Bandwidth as a non-guard after guardfraction has been considered. */ + int non_guard_bw; +} guardfraction_bandwidth_t; + +int should_apply_guardfraction(const networkstatus_t *ns); + +void +guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw, + int orig_bandwidth, + uint32_t guardfraction_percentage); + #endif diff --git a/src/or/eventdns_tor.h b/src/or/eventdns_tor.h index 69662281bc..5db09ae043 100644 --- a/src/or/eventdns_tor.h +++ b/src/or/eventdns_tor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_EVENTDNS_TOR_H @@ -12,9 +12,6 @@ typedef unsigned int uint; #ifndef HAVE_U_CHAR typedef unsigned char u_char; #endif -#ifdef _WIN32 -#define inline __inline -#endif #include "torint.h" /* These are for debugging possible memory leaks. */ diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c new file mode 100644 index 0000000000..aa1b3e26fe --- /dev/null +++ b/src/or/ext_orport.c @@ -0,0 +1,648 @@ +/* Copyright (c) 2012-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ext_orport.c + * \brief Code implementing the Extended ORPort. +*/ + +#define EXT_ORPORT_PRIVATE +#include "or.h" +#include "connection.h" +#include "connection_or.h" +#include "ext_orport.h" +#include "control.h" +#include "config.h" +#include "util.h" +#include "main.h" + +/** Allocate and return a structure capable of holding an Extended + * ORPort message of body length <b>len</b>. */ +ext_or_cmd_t * +ext_or_cmd_new(uint16_t len) +{ + size_t size = STRUCT_OFFSET(ext_or_cmd_t, body) + len; + ext_or_cmd_t *cmd = tor_malloc(size); + cmd->len = len; + return cmd; +} + +/** Deallocate the Extended ORPort message in <b>cmd</b>. */ +void +ext_or_cmd_free(ext_or_cmd_t *cmd) +{ + tor_free(cmd); +} + +/** Get an Extended ORPort message from <b>conn</b>, and place it in + * <b>out</b>. Return -1 on fail, 0 if we need more data, and 1 if we + * successfully extracted an Extended ORPort command from the + * buffer. */ +static int +connection_fetch_ext_or_cmd_from_buf(connection_t *conn, ext_or_cmd_t **out) +{ + IF_HAS_BUFFEREVENT(conn, { + struct evbuffer *input = bufferevent_get_input(conn->bufev); + return fetch_ext_or_command_from_evbuffer(input, out); + }) ELSE_IF_NO_BUFFEREVENT { + return fetch_ext_or_command_from_buf(conn->inbuf, out); + } +} + +/** Write an Extended ORPort message to <b>conn</b>. Use + * <b>command</b> as the command type, <b>bodylen</b> as the body + * length, and <b>body</b>, if it's present, as the body of the + * message. */ +STATIC int +connection_write_ext_or_command(connection_t *conn, + uint16_t command, + const char *body, + size_t bodylen) +{ + char header[4]; + if (bodylen > UINT16_MAX) + return -1; + set_uint16(header, htons(command)); + set_uint16(header+2, htons(bodylen)); + connection_write_to_buf(header, 4, conn); + if (bodylen) { + tor_assert(body); + connection_write_to_buf(body, bodylen, conn); + } + return 0; +} + +/** Transition from an Extended ORPort which accepts Extended ORPort + * messages, to an Extended ORport which accepts OR traffic. */ +static void +connection_ext_or_transition(or_connection_t *conn) +{ + tor_assert(conn->base_.type == CONN_TYPE_EXT_OR); + + conn->base_.type = CONN_TYPE_OR; + TO_CONN(conn)->state = 0; // set the state to a neutral value + control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0); + connection_tls_start_handshake(conn, 1); +} + +/** Length of authentication cookie. */ +#define EXT_OR_PORT_AUTH_COOKIE_LEN 32 +/** Length of the header of the cookie file. */ +#define EXT_OR_PORT_AUTH_COOKIE_HEADER_LEN 32 +/** Static cookie file header. */ +#define EXT_OR_PORT_AUTH_COOKIE_HEADER "! Extended ORPort Auth Cookie !\x0a" +/** Length of safe-cookie protocol hashes. */ +#define EXT_OR_PORT_AUTH_HASH_LEN DIGEST256_LEN +/** Length of safe-cookie protocol nonces. */ +#define EXT_OR_PORT_AUTH_NONCE_LEN 32 +/** Safe-cookie protocol constants. */ +#define EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST \ + "ExtORPort authentication server-to-client hash" +#define EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST \ + "ExtORPort authentication client-to-server hash" + +/* Code to indicate cookie authentication */ +#define EXT_OR_AUTHTYPE_SAFECOOKIE 0x01 + +/** If true, we've set ext_or_auth_cookie to a secret code and stored + * it to disk. */ +STATIC int ext_or_auth_cookie_is_set = 0; +/** If ext_or_auth_cookie_is_set, a secret cookie that we've stored to disk + * and which we're using to authenticate controllers. (If the controller can + * read it off disk, it has permission to connect.) */ +STATIC uint8_t *ext_or_auth_cookie = NULL; + +/** Helper: Return a newly allocated string containing a path to the + * file where we store our authentication cookie. */ +char * +get_ext_or_auth_cookie_file_name(void) +{ + const or_options_t *options = get_options(); + if (options->ExtORPortCookieAuthFile && + strlen(options->ExtORPortCookieAuthFile)) { + return tor_strdup(options->ExtORPortCookieAuthFile); + } else { + return get_datadir_fname("extended_orport_auth_cookie"); + } +} + +/* Initialize the cookie-based authentication system of the + * Extended ORPort. If <b>is_enabled</b> is 0, then disable the cookie + * authentication system. */ +int +init_ext_or_cookie_authentication(int is_enabled) +{ + char *fname = NULL; + int retval; + + if (!is_enabled) { + ext_or_auth_cookie_is_set = 0; + return 0; + } + + fname = get_ext_or_auth_cookie_file_name(); + retval = init_cookie_authentication(fname, EXT_OR_PORT_AUTH_COOKIE_HEADER, + EXT_OR_PORT_AUTH_COOKIE_HEADER_LEN, + get_options()->ExtORPortCookieAuthFileGroupReadable, + &ext_or_auth_cookie, + &ext_or_auth_cookie_is_set); + tor_free(fname); + return retval; +} + +/** Read data from <b>conn</b> and see if the client sent us the + * authentication type that they prefer to use in this session. + * + * Return -1 if we received corrupted data or if we don't support the + * authentication type. Return 0 if we need more data in + * <b>conn</b>. Return 1 if the authentication type negotiation was + * successful. */ +static int +connection_ext_or_auth_neg_auth_type(connection_t *conn) +{ + char authtype[1] = {0}; + + if (connection_get_inbuf_len(conn) < 1) + return 0; + + if (connection_fetch_from_buf(authtype, 1, conn) < 0) + return -1; + + log_debug(LD_GENERAL, "Client wants us to use %d auth type", authtype[0]); + if (authtype[0] != EXT_OR_AUTHTYPE_SAFECOOKIE) { + /* '1' is the only auth type supported atm */ + return -1; + } + + conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE; + return 1; +} + +/* DOCDOC */ +STATIC int +handle_client_auth_nonce(const char *client_nonce, size_t client_nonce_len, + char **client_hash_out, + char **reply_out, size_t *reply_len_out) +{ + char server_hash[EXT_OR_PORT_AUTH_HASH_LEN] = {0}; + char server_nonce[EXT_OR_PORT_AUTH_NONCE_LEN] = {0}; + char *reply; + size_t reply_len; + + if (client_nonce_len != EXT_OR_PORT_AUTH_NONCE_LEN) + return -1; + + /* Get our nonce */ + crypto_rand(server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + + { /* set up macs */ + size_t hmac_s_msg_len = strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST) + + 2*EXT_OR_PORT_AUTH_NONCE_LEN; + size_t hmac_c_msg_len = strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST) + + 2*EXT_OR_PORT_AUTH_NONCE_LEN; + + char *hmac_s_msg = tor_malloc_zero(hmac_s_msg_len); + char *hmac_c_msg = tor_malloc_zero(hmac_c_msg_len); + char *correct_client_hash = tor_malloc_zero(EXT_OR_PORT_AUTH_HASH_LEN); + + memcpy(hmac_s_msg, + EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST, + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST)); + memcpy(hmac_s_msg + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST), + client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + memcpy(hmac_s_msg + strlen(EXT_OR_PORT_AUTH_SERVER_TO_CLIENT_CONST) + + EXT_OR_PORT_AUTH_NONCE_LEN, + server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + + memcpy(hmac_c_msg, + EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST, + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST)); + memcpy(hmac_c_msg + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST), + client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + memcpy(hmac_c_msg + strlen(EXT_OR_PORT_AUTH_CLIENT_TO_SERVER_CONST) + + EXT_OR_PORT_AUTH_NONCE_LEN, + server_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + + crypto_hmac_sha256(server_hash, + (char*)ext_or_auth_cookie, + EXT_OR_PORT_AUTH_COOKIE_LEN, + hmac_s_msg, + hmac_s_msg_len); + + crypto_hmac_sha256(correct_client_hash, + (char*)ext_or_auth_cookie, + EXT_OR_PORT_AUTH_COOKIE_LEN, + hmac_c_msg, + hmac_c_msg_len); + + /* Store the client hash we generated. We will need to compare it + with the hash sent by the client. */ + *client_hash_out = correct_client_hash; + + memwipe(hmac_s_msg, 0, hmac_s_msg_len); + memwipe(hmac_c_msg, 0, hmac_c_msg_len); + + tor_free(hmac_s_msg); + tor_free(hmac_c_msg); + } + + { /* debug logging */ /* XXX disable this codepath if not logging on debug?*/ + char server_hash_encoded[(2*EXT_OR_PORT_AUTH_HASH_LEN) + 1]; + char server_nonce_encoded[(2*EXT_OR_PORT_AUTH_NONCE_LEN) + 1]; + char client_nonce_encoded[(2*EXT_OR_PORT_AUTH_NONCE_LEN) + 1]; + + base16_encode(server_hash_encoded, sizeof(server_hash_encoded), + server_hash, sizeof(server_hash)); + base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded), + server_nonce, sizeof(server_nonce)); + base16_encode(client_nonce_encoded, sizeof(client_nonce_encoded), + client_nonce, EXT_OR_PORT_AUTH_NONCE_LEN); + + log_debug(LD_GENERAL, + "server_hash: '%s'\nserver_nonce: '%s'\nclient_nonce: '%s'", + server_hash_encoded, server_nonce_encoded, client_nonce_encoded); + + memwipe(server_hash_encoded, 0, sizeof(server_hash_encoded)); + memwipe(server_nonce_encoded, 0, sizeof(server_nonce_encoded)); + memwipe(client_nonce_encoded, 0, sizeof(client_nonce_encoded)); + } + + { /* write reply: (server_hash, server_nonce) */ + + reply_len = EXT_OR_PORT_AUTH_COOKIE_LEN+EXT_OR_PORT_AUTH_NONCE_LEN; + reply = tor_malloc_zero(reply_len); + memcpy(reply, server_hash, EXT_OR_PORT_AUTH_HASH_LEN); + memcpy(reply + EXT_OR_PORT_AUTH_HASH_LEN, server_nonce, + EXT_OR_PORT_AUTH_NONCE_LEN); + } + + *reply_out = reply; + *reply_len_out = reply_len; + + return 0; +} + +/** Read the client's nonce out of <b>conn</b>, setup the safe-cookie + * crypto, and then send our own hash and nonce to the client + * + * Return -1 if there was an error; return 0 if we need more data in + * <b>conn</b>, and return 1 if we successfully retrieved the + * client's nonce and sent our own. */ +static int +connection_ext_or_auth_handle_client_nonce(connection_t *conn) +{ + char client_nonce[EXT_OR_PORT_AUTH_NONCE_LEN]; + char *reply=NULL; + size_t reply_len=0; + + if (!ext_or_auth_cookie_is_set) { /* this should not happen */ + log_warn(LD_BUG, "Extended ORPort authentication cookie was not set. " + "That's weird since we should have done that on startup. " + "This might be a Tor bug, please file a bug report. "); + return -1; + } + + if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_NONCE_LEN) + return 0; + + if (connection_fetch_from_buf(client_nonce, + EXT_OR_PORT_AUTH_NONCE_LEN, conn) < 0) + return -1; + + /* We extract the ClientNonce from the received data, and use it to + calculate ServerHash and ServerNonce according to proposal 217. + + We also calculate our own ClientHash value and save it in the + connection state. We validate it later against the ClientHash + sent by the client. */ + if (handle_client_auth_nonce(client_nonce, sizeof(client_nonce), + &TO_OR_CONN(conn)->ext_or_auth_correct_client_hash, + &reply, &reply_len) < 0) + return -1; + + connection_write_to_buf(reply, reply_len, conn); + + memwipe(reply, 0, reply_len); + tor_free(reply); + + log_debug(LD_GENERAL, "Got client nonce, and sent our own nonce and hash."); + + conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH; + return 1; +} + +#define connection_ext_or_auth_send_result_success(c) \ + connection_ext_or_auth_send_result(c, 1) +#define connection_ext_or_auth_send_result_fail(c) \ + connection_ext_or_auth_send_result(c, 0) + +/** Send authentication results to <b>conn</b>. Successful results if + * <b>success</b> is set; failure results otherwise. */ +static void +connection_ext_or_auth_send_result(connection_t *conn, int success) +{ + if (success) + connection_write_to_buf("\x01", 1, conn); + else + connection_write_to_buf("\x00", 1, conn); +} + +/** Receive the client's hash from <b>conn</b>, validate that it's + * correct, and then send the authentication results to the client. + * + * Return -1 if there was an error during validation; return 0 if we + * need more data in <b>conn</b>, and return 1 if we successfully + * validated the client's hash and sent a happy authentication + * result. */ +static int +connection_ext_or_auth_handle_client_hash(connection_t *conn) +{ + char provided_client_hash[EXT_OR_PORT_AUTH_HASH_LEN] = {0}; + + if (connection_get_inbuf_len(conn) < EXT_OR_PORT_AUTH_HASH_LEN) + return 0; + + if (connection_fetch_from_buf(provided_client_hash, + EXT_OR_PORT_AUTH_HASH_LEN, conn) < 0) + return -1; + + if (tor_memneq(TO_OR_CONN(conn)->ext_or_auth_correct_client_hash, + provided_client_hash, EXT_OR_PORT_AUTH_HASH_LEN)) { + log_warn(LD_GENERAL, "Incorrect client hash. Authentication failed."); + connection_ext_or_auth_send_result_fail(conn); + return -1; + } + + log_debug(LD_GENERAL, "Got client's hash and it was legit."); + + /* send positive auth result */ + connection_ext_or_auth_send_result_success(conn); + conn->state = EXT_OR_CONN_STATE_OPEN; + return 1; +} + +/** Handle data from <b>or_conn</b> received on Extended ORPort. + * Return -1 on error. 0 on unsufficient data. 1 on correct. */ +static int +connection_ext_or_auth_process_inbuf(or_connection_t *or_conn) +{ + connection_t *conn = TO_CONN(or_conn); + + /* State transitions of the Extended ORPort authentication protocol: + + EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE (start state) -> + EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE -> + EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH -> + EXT_OR_CONN_STATE_OPEN + + During EXT_OR_CONN_STATE_OPEN, data is handled by + connection_ext_or_process_inbuf(). + */ + + switch (conn->state) { /* Functionify */ + case EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE: + return connection_ext_or_auth_neg_auth_type(conn); + + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE: + return connection_ext_or_auth_handle_client_nonce(conn); + + case EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH: + return connection_ext_or_auth_handle_client_hash(conn); + + default: + log_warn(LD_BUG, "Encountered unexpected connection state %d while trying " + "to process Extended ORPort authentication data.", conn->state); + return -1; + } +} + +/** Extended ORPort commands (Transport-to-Bridge) */ +#define EXT_OR_CMD_TB_DONE 0x0000 +#define EXT_OR_CMD_TB_USERADDR 0x0001 +#define EXT_OR_CMD_TB_TRANSPORT 0x0002 + +/** Extended ORPort commands (Bridge-to-Transport) */ +#define EXT_OR_CMD_BT_OKAY 0x1000 +#define EXT_OR_CMD_BT_DENY 0x1001 +#define EXT_OR_CMD_BT_CONTROL 0x1002 + +/** Process a USERADDR command from the Extended + * ORPort. <b>payload</b> is a payload of size <b>len</b>. + * + * If the USERADDR command was well formed, change the address of + * <b>conn</b> to the address on the USERADDR command. + * + * Return 0 on success and -1 on error. */ +static int +connection_ext_or_handle_cmd_useraddr(connection_t *conn, + const char *payload, uint16_t len) +{ + /* Copy address string. */ + tor_addr_t addr; + uint16_t port; + char *addr_str; + char *address_part=NULL; + int res; + if (memchr(payload, '\0', len)) { + log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unexpected NUL in ExtORPort UserAddr"); + return -1; + } + + addr_str = tor_memdup_nulterm(payload, len); + + res = tor_addr_port_split(LOG_INFO, addr_str, &address_part, &port); + tor_free(addr_str); + if (res<0) + return -1; + + res = tor_addr_parse(&addr, address_part); + tor_free(address_part); + if (res<0) + return -1; + + { /* do some logging */ + char *old_address = tor_dup_addr(&conn->addr); + char *new_address = tor_dup_addr(&addr); + + log_debug(LD_NET, "Received USERADDR." + "We rewrite our address from '%s:%u' to '%s:%u'.", + safe_str(old_address), conn->port, safe_str(new_address), port); + + tor_free(old_address); + tor_free(new_address); + } + + /* record the address */ + tor_addr_copy(&conn->addr, &addr); + conn->port = port; + if (conn->address) { + tor_free(conn->address); + } + conn->address = tor_dup_addr(&addr); + + return 0; +} + +/** Process a TRANSPORT command from the Extended + * ORPort. <b>payload</b> is a payload of size <b>len</b>. + * + * If the TRANSPORT command was well formed, register the name of the + * transport on <b>conn</b>. + * + * Return 0 on success and -1 on error. */ +static int +connection_ext_or_handle_cmd_transport(or_connection_t *conn, + const char *payload, uint16_t len) +{ + char *transport_str; + if (memchr(payload, '\0', len)) { + log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unexpected NUL in ExtORPort Transport"); + return -1; + } + + transport_str = tor_memdup_nulterm(payload, len); + + /* Transport names MUST be C-identifiers. */ + if (!string_is_C_identifier(transport_str)) { + tor_free(transport_str); + return -1; + } + + /* If ext_or_transport is already occupied (because the PT sent two + * TRANSPORT commands), deallocate the old name and keep the new + * one */ + if (conn->ext_or_transport) + tor_free(conn->ext_or_transport); + + conn->ext_or_transport = transport_str; + return 0; +} + +#define EXT_OR_CONN_STATE_IS_AUTHENTICATING(st) \ + ((st) <= EXT_OR_CONN_STATE_AUTH_MAX) + +/** Process Extended ORPort messages from <b>or_conn</b>. */ +int +connection_ext_or_process_inbuf(or_connection_t *or_conn) +{ + connection_t *conn = TO_CONN(or_conn); + ext_or_cmd_t *command; + int r; + + /* DOCDOC Document the state machine and transitions in this function */ + + /* If we are still in the authentication stage, process traffic as + authentication data: */ + while (EXT_OR_CONN_STATE_IS_AUTHENTICATING(conn->state)) { + log_debug(LD_GENERAL, "Got Extended ORPort authentication data (%u).", + (unsigned int) connection_get_inbuf_len(conn)); + r = connection_ext_or_auth_process_inbuf(or_conn); + if (r < 0) { + connection_mark_for_close(conn); + return -1; + } else if (r == 0) { + return 0; + } + /* if r > 0, loop and process more data (if any). */ + } + + while (1) { + log_debug(LD_GENERAL, "Got Extended ORPort data."); + command = NULL; + r = connection_fetch_ext_or_cmd_from_buf(conn, &command); + if (r < 0) + goto err; + else if (r == 0) + return 0; /* need to wait for more data */ + + /* Got a command! */ + tor_assert(command); + + if (command->cmd == EXT_OR_CMD_TB_DONE) { + if (connection_get_inbuf_len(conn)) { + /* The inbuf isn't empty; the client is misbehaving. */ + goto err; + } + + log_debug(LD_NET, "Received DONE."); + + /* If the transport proxy did not use the TRANSPORT command to + * specify the transport name, mark this as unknown transport. */ + if (!or_conn->ext_or_transport) { + /* We write this string this way to avoid ??>, which is a C + * trigraph. */ + or_conn->ext_or_transport = tor_strdup("<?" "?>"); + } + + connection_write_ext_or_command(conn, EXT_OR_CMD_BT_OKAY, NULL, 0); + + /* can't transition immediately; need to flush first. */ + conn->state = EXT_OR_CONN_STATE_FLUSHING; + connection_stop_reading(conn); + } else if (command->cmd == EXT_OR_CMD_TB_USERADDR) { + if (connection_ext_or_handle_cmd_useraddr(conn, + command->body, command->len) < 0) + goto err; + } else if (command->cmd == EXT_OR_CMD_TB_TRANSPORT) { + if (connection_ext_or_handle_cmd_transport(or_conn, + command->body, command->len) < 0) + goto err; + } else { + log_notice(LD_NET,"Got Extended ORPort command we don't regognize (%u).", + command->cmd); + } + + ext_or_cmd_free(command); + } + + return 0; + + err: + ext_or_cmd_free(command); + connection_mark_for_close(conn); + return -1; +} + +/** <b>conn</b> finished flushing Extended ORPort messages to the + * network, and is now ready to accept OR traffic. This function + * does the transition. */ +int +connection_ext_or_finished_flushing(or_connection_t *conn) +{ + if (conn->base_.state == EXT_OR_CONN_STATE_FLUSHING) { + connection_start_reading(TO_CONN(conn)); + connection_ext_or_transition(conn); + } + return 0; +} + +/** Initiate Extended ORPort authentication, by sending the list of + * supported authentication types to the client. */ +int +connection_ext_or_start_auth(or_connection_t *or_conn) +{ + connection_t *conn = TO_CONN(or_conn); + const uint8_t authtypes[] = { + /* We only support authtype '1' for now. */ + EXT_OR_AUTHTYPE_SAFECOOKIE, + /* Marks the end of the list. */ + 0 + }; + + log_debug(LD_GENERAL, + "ExtORPort authentication: Sending supported authentication types"); + + connection_write_to_buf((const char *)authtypes, sizeof(authtypes), conn); + conn->state = EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE; + + return 0; +} + +/** Free any leftover allocated memory of the ext_orport.c subsystem. */ +void +ext_orport_free_all(void) +{ + if (ext_or_auth_cookie) /* Free the auth cookie */ + tor_free(ext_or_auth_cookie); +} + diff --git a/src/or/ext_orport.h b/src/or/ext_orport.h new file mode 100644 index 0000000000..33d954e8d0 --- /dev/null +++ b/src/or/ext_orport.h @@ -0,0 +1,42 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef EXT_ORPORT_H +#define EXT_ORPORT_H + +int connection_ext_or_start_auth(or_connection_t *or_conn); + +ext_or_cmd_t *ext_or_cmd_new(uint16_t len); +void ext_or_cmd_free(ext_or_cmd_t *cmd); +void connection_or_set_ext_or_identifier(or_connection_t *conn); +void connection_or_remove_from_ext_or_id_map(or_connection_t *conn); +void connection_or_clear_ext_or_id_map(void); +or_connection_t *connection_or_get_by_ext_or_id(const char *id); + +int connection_ext_or_finished_flushing(or_connection_t *conn); +int connection_ext_or_process_inbuf(or_connection_t *or_conn); + +int init_ext_or_cookie_authentication(int is_enabled); +char *get_ext_or_auth_cookie_file_name(void); +void ext_orport_free_all(void); + +#ifdef EXT_ORPORT_PRIVATE +STATIC int connection_write_ext_or_command(connection_t *conn, + uint16_t command, + const char *body, + size_t bodylen); +STATIC int handle_client_auth_nonce(const char *client_nonce, + size_t client_nonce_len, + char **client_hash_out, + char **reply_out, size_t *reply_len_out); +#ifdef TOR_UNIT_TESTS +extern uint8_t *ext_or_auth_cookie; +extern int ext_or_auth_cookie_is_set; +#endif +#endif + +#endif + diff --git a/src/or/fallback_dirs.inc b/src/or/fallback_dirs.inc new file mode 100644 index 0000000000..be94ff50fc --- /dev/null +++ b/src/or/fallback_dirs.inc @@ -0,0 +1,423 @@ +/* Whitelist & blacklist excluded 1177 of 1389 candidates. */ +/* To comment-out entries in this file, use C comments, and add * to the start of each line. (stem finds fallback entries using " at the start of a line.) */ +/* Checked IPv4 DirPorts served a consensus within 15.0s. */ +/* +Final Count: 177 (Eligible 212, Target 392 (1963 * 0.20), Max 200) +Excluded: 35 (Same Operator 35, Failed/Skipped Download 0, Excess 0) +Bandwidth Range: 1.2 - 107.3 MByte/s +*/ +/* +Onionoo Source: details Date: 2016-12-19 03:00:00 Version: 3.1 +URL: https:onionoo.torproject.orgdetails?fields=fingerprint%2Cnickname%2Ccontact%2Clast_changed_address_or_port%2Cconsensus_weight%2Cadvertised_bandwidth%2Cor_addresses%2Cdir_address%2Crecommended_version%2Cflags%2Ceffective_family%2Cplatform&flag=V2Dir&type=relay&last_seen_days=-0&first_seen_days=7- +*/ +/* +Onionoo Source: uptime Date: 2016-12-19 03:00:00 Version: 3.1 +URL: https:onionoo.torproject.orguptime?first_seen_days=7-&flag=V2Dir&type=relay&last_seen_days=-0 +*/ +"185.13.39.197:80 orport=443 id=001524DD403D729F08F7E5D77813EF12756CFA8D" +" weight=10", +"185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E" +" weight=10", +"62.210.92.11:9030 orport=9001 id=0266B0660F3F20A7D1F3D8335931C95EF50F6C6B" +" ipv6=[2001:bc8:338c::1]:9001" +" weight=10", +"185.97.32.18:9030 orport=9001 id=04250C3835019B26AA6764E85D836088BE441088" +" weight=10", +"92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C" +" weight=10", +"163.172.149.155:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F" +" weight=10", +"5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6" +" ipv6=[2001:41d0:8:b1c7::1]:443" +" weight=10", +"163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F" +" weight=10", +"178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163" +" weight=10", +"185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0" +" weight=10", +"5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797" +" weight=10", +"193.11.114.43:9030 orport=9001 id=12AD30E5D25AA67F519780E2111E611A455FDC89" +" ipv6=[2001:6b0:30:1000::99]:9050" +" weight=10", +"37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2" +" weight=10", +"178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794" +" weight=10", +"204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70" +" weight=10", +"92.222.4.102:9030 orport=9001 id=1A6B8B8272632D8AD38442027F822A367128405C" +" weight=10", +"5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA" +" ipv6=[2a01:4f8:190:514a::2]:443" +" weight=10", +"46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8" +" weight=10", +"91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7" +" weight=10", +"5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11" +" weight=10", +"212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68" +" ipv6=[2001:bc8:4400:2100::f03]:9001" +" weight=10", +"91.219.236.222:80 orport=443 id=20704E7DD51501DC303FA51B738D7B7E61397CF6" +" weight=10", +"144.76.163.93:9030 orport=9001 id=22F08CF09764C4E8982640D77F71ED72FF26A9AC" +" weight=10", +"163.172.176.167:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207" +" weight=10", +"212.47.240.10:82 orport=443 id=2A4C448784F5A83AFE6C78DA357D5E31F7989DEB" +" weight=10", +"144.76.26.175:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510" +" weight=10", +"178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2" +" ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443" +" weight=10", +"62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E" +" ipv6=[2001:bc8:3f23:100::1]:9101" +" weight=10", +"97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94" +" weight=10", +"213.61.66.118:9031 orport=9001 id=30648BC64CEDB3020F4A405E4AB2A6347FB8FA22" +" weight=10", +"107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA" +" weight=10", +"64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2" +" weight=10", +"212.83.154.33:8080 orport=8443 id=322C6E3A973BC10FC36DE3037AD27BC89F14723B" +" weight=10", +"109.105.109.162:52860 orport=60784 id=32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F" +" ipv6=[2001:948:7:2::163]:5001" +" weight=10", +"185.100.84.212:80 orport=443 id=330CD3DB6AD266DC70CDB512B036957D03D9BC59" +" ipv6=[2a06:1700:0:7::1]:443" +" weight=10", +"163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB" +" ipv6=[2001:bc8:38cb:201::8]:9001" +" weight=10", +"45.62.255.25:80 orport=443 id=3473ED788D9E63361D1572B7E82EC54338953D2A" +" weight=10", +"217.79.190.25:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B" +" weight=10", +"37.187.22.87:9030 orport=9001 id=36B9E7AC1E36B62A9D6F330ABEB6012BA7F0D400" +" ipv6=[2001:41d0:a:1657::1]:9001" +" weight=10", +"176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442" +" ipv6=[2a02:59e0:0:7::12]:81" +" weight=10", +"198.50.191.95:80 orport=443 id=39F096961ED2576975C866D450373A9913AFDC92" +" weight=10", +"164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B" +" weight=10", +"212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97" +" weight=10", +"217.79.179.177:9030 orport=9001 id=3E53D3979DB07EFD736661C934A1DED14127B684" +" ipv6=[2001:4ba0:fff9:131:6c4f::90d3]:9001" +" weight=10", +"212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6" +" weight=10", +"185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270" +" weight=10", +"178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1" +" ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050" +" weight=10", +"163.172.157.213:8080 orport=443 id=4623A9EC53BFD83155929E56D6F7B55B5E718C24" +" weight=10", +"31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798" +" weight=10", +"69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6" +" ipv6=[2607:f128:40:1212::45a2:8b09]:9001" +" weight=10", +"51.254.246.203:9030 orport=9001 id=47B596B81C9E6277B98623A84B7629798A16E8D5" +" weight=10", +"37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053" +" ipv6=[2001:41d0:a:26ba::1]:9001" +" weight=10", +"188.165.194.195:9030 orport=9001 id=49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF" +" weight=10", +"62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF" +" weight=10", +"51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF" +" weight=10", +"81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82" +" ipv6=[2a02:180:1:1::517:10b6]:993" +" weight=10", +"138.201.130.32:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257" +" weight=10", +"94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69" +" weight=10", +"95.130.12.119:80 orport=443 id=587E0A9552E4274B251F29B5B2673D38442EE4BF" +" weight=10", +"185.21.100.50:9030 orport=9001 id=58ED9C9C35E433EE58764D62892B4FFD518A3CD0" +" ipv6=[2a00:1158:2:cd00:0:74:6f:72]:443" +" weight=10", +"78.142.142.246:80 orport=443 id=5A5E03355C1908EBF424CAF1F3ED70782C0D2F74" +" weight=10", +"46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321" +" weight=10", +"120.29.217.46:80 orport=443 id=5E853C94AB1F655E9C908924370A0A6707508C62" +" weight=10", +"95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90" +" ipv6=[2a02:ec0:209:10::4]:443" +" weight=10", +"195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6" +" weight=10", +"163.172.139.104:8080 orport=443 id=68F175CCABE727AA2D2309BCD8789499CEE36ED7" +" weight=10", +"85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911" +" weight=10", +"95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A" +" weight=10", +"91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F" +" ipv6=[2001:41d0:1:8989::1]:4051" +" weight=10", +"213.61.66.117:9032 orport=9002 id=6E44A52E3D1FF7683FE5C399C3FB5E912DE1C6B4" +" weight=10", +"80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621" +" ipv6=[2001:981:47c1:1::6]:443" +" weight=10", +"95.183.48.12:80 orport=443 id=7187CED1A3871F837D0E60AC98F374AC541CB0DA" +" weight=10", +"163.172.35.247:80 orport=443 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4" +" weight=10", +"85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE" +" weight=10", +"46.101.237.246:9030 orport=9001 id=75F1992FD3F403E9C082A5815EB5D12934CDF46C" +" ipv6=[2a03:b0c0:3:d0::208:5001]:9050" +" weight=10", +"188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7" +" ipv6=[2a03:b0c0:2:d0::5:f001]:9001" +" weight=10", +"81.30.158.213:9030 orport=9001 id=789EA6C9AE9ADDD8760903171CFA9AC5741B0C70" +" ipv6=[2001:4ba0:cafe:e84::1]:9001" +" weight=10", +"185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778" +" weight=10", +"171.25.193.131:80 orport=443 id=79861CF8522FC637EF046F7688F5289E49D94576" +" weight=10", +"62.210.129.246:80 orport=443 id=79E169B25E4C7CE99584F6ED06F379478F23E2B8" +" weight=10", +"82.223.21.74:9030 orport=9001 id=7A32C9519D80CA458FC8B034A28F5F6815649A98" +" ipv6=[2001:470:53e0::cafe]:9050" +" weight=10", +"51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63" +" weight=10", +"193.11.114.45:9031 orport=9002 id=80AAF8D5956A43C197104CEF2550CD42D165C6FB" +" weight=10", +"192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F" +" ipv6=[2605:e200:d00c:c01d::1111]:9002" +" weight=10", +"192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7" +" weight=10", +"163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D" +" ipv6=[2001:bc8:4400:2100::1:3]:443" +" weight=10", +"188.166.23.127:80 orport=443 id=8672E8A01B4D3FA4C0BBE21C740D4506302EA487" +" ipv6=[2a03:b0c0:2:d0::27b:7001]:9050" +" weight=10", +"93.180.156.84:9030 orport=9001 id=8844D87E9B038BE3270938F05AF797E1D3C74C0F" +" weight=10", +"212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C" +" weight=10", +"163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451" +" weight=10", +"178.254.44.135:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422" +" weight=10", +"151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605" +" ipv6=[2001:41d0:e:f67::114]:9001" +" weight=10", +"173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7" +" weight=10", +"51.255.41.65:9030 orport=9001 id=9231DF741915AA1630031A93026D88726877E93A" +" weight=10", +"178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A" +" ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443" +" weight=10", +"91.219.237.244:80 orport=443 id=92ECC9E0E2AF81BB954719B189AC362E254AD4A5" +" weight=10", +"204.8.156.142:80 orport=443 id=94C4B7B8C50C86A92B6A20107539EE2678CF9A28" +" weight=10", +"176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F" +" weight=10", +"85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A" +" weight=10", +"85.214.206.219:9030 orport=9001 id=98F8D5F359949E41DE8DF3DBB1975A86E96A84A0" +" weight=10", +"163.172.223.200:80 orport=443 id=998BF3ED7F70E33D1C307247B9626D9E7573C438" +" weight=10", +"81.7.10.93:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7" +" weight=10", +"91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F" +" weight=10", +"66.111.2.20:9030 orport=9001 id=9A68B85A02318F4E7E87F2828039FBD5D75B0142" +" weight=10", +"5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE" +" weight=10", +"5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A" +" weight=10", +"86.105.212.130:9030 orport=443 id=9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022" +" weight=10", +"146.185.177.103:80 orport=9030 id=9EC5E097663862DF861A18C32B37C5F82284B27D" +" weight=10", +"178.254.20.134:80 orport=443 id=9F5068310818ED7C70B0BC4087AB55CB12CB4377" +" weight=10", +"46.28.110.244:80 orport=443 id=9F7D6E6420183C2B76D3CE99624EBC98A21A967E" +" weight=10", +"178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1" +" ipv6=[2a03:b0c0:1:d0::174:1]:9050" +" weight=10", +"171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A" +" ipv6=[2001:67c:289c:3::77]:443" +" weight=10", +"171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE" +" ipv6=[2001:67c:289c:3::78]:443" +" weight=10", +"178.16.208.58:80 orport=443 id=A4C98CEA3F34E05299417E9F885A642C88EF6029" +" ipv6=[2a00:1c20:4089:1234:cdae:1b3e:cc38:3d45]:443" +" weight=10", +"163.172.149.122:80 orport=443 id=A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2" +" weight=10", +"213.61.66.116:9033 orport=9003 id=A9DEB920B42B4EC1DE6249034039B06D61F38690" +" weight=10", +"192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8" +" weight=10", +"195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C" +" weight=10", +"86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0" +" weight=10", +"185.129.62.62:9030 orport=9001 id=ACDD9E85A05B127BA010466C13C8C47212E8A38F" +" ipv6=[2a06:d380:0:3700::62]:9001" +" weight=10", +"188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601" +" weight=10", +"163.172.131.88:80 orport=443 id=AD253B49E303C6AB1E048B014392AC569E8A7DAE" +" ipv6=[2001:bc8:4400:2100::2:1009]:443" +" weight=10", +"176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88" +" weight=10", +"31.185.104.20:80 orport=443 id=ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D" +" weight=10", +"37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05" +" ipv6=[2001:41d0:a:74a::1]:443" +" weight=10", +"176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A" +" ipv6=[2a02:59e0:0:7::11]:9003" +" weight=10", +"212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7" +" weight=10", +"185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529" +" weight=10", +"198.199.64.217:80 orport=443 id=B1D81825CFD7209BD1B4520B040EF5653C204A23" +" ipv6=[2604:a880:400:d0::1a9:b001]:9050" +" weight=10", +"136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664" +" weight=10", +"212.47.233.86:9030 orport=9001 id=B4CAFD9CBFB34EC5DAAC146920DC7DFAFE91EA20" +" weight=10", +"93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D" +" weight=10", +"81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243" +" ipv6=[2001:15e8:201:1::d10a]:80" +" weight=10", +"193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E" +" weight=10", +"85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82" +" ipv6=[2a00:1298:8011:212::164]:9004" +" weight=10", +"197.231.221.211:9030 orport=9001 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626" +" weight=10", +"89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410" +" weight=10", +"198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E" +" weight=10", +"128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5" +" ipv6=[2a03:b0c0:2:d0::158:3001]:9001" +" weight=10", +"148.251.190.229:9030 orport=9010 id=BF0FB582E37F738CD33C3651125F2772705BB8E8" +" ipv6=[2a01:4f8:211:c68::2]:9010" +" weight=10", +"163.172.35.249:80 orport=443 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A" +" weight=10", +"185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08" +" ipv6=[2a02:ed06::221]:9001" +" weight=10", +"213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2" +" ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337" +" weight=10", +"188.138.112.60:1433 orport=1521 id=C414F28FD2BEC1553024299B31D4E726BEB8E788" +" weight=10", +"37.59.46.159:9030 orport=9001 id=CBD0D1BD110EC52963082D839AC6A89D0AE243E7" +" weight=10", +"178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE" +" ipv6=[2a03:b0c0:2:d0::b7:5001]:443" +" weight=10", +"134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE" +" weight=10", +"185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361" +" weight=10", +"37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811" +" weight=10", +"46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C" +" ipv6=[2a03:b0c0:3:d0::74f:a001]:9001" +" weight=10", +"46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204" +" weight=10", +"193.35.52.53:9030 orport=9001 id=DAA39FC00B196B353C2A271459C305C429AF09E4" +" weight=10", +"178.33.183.251:80 orport=443 id=DD823AFB415380A802DCAEB9461AE637604107FB" +" ipv6=[2001:41d0:2:a683::251]:443" +" weight=10", +"178.62.173.203:9030 orport=9001 id=DD85503F2D1F52EF9EAD621E942298F46CD2FC10" +" ipv6=[2a03:b0c0:0:1010::a4:b001]:9001" +" weight=10", +"83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0" +" ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443" +" weight=10", +"5.34.183.205:80 orport=443 id=DDD7871C1B7FA32CB55061E08869A236E61BDDF8" +" weight=10", +"167.114.66.61:9696 orport=443 id=DE6CD5F09DF26076F26321B0BDFBE78ACD935C65" +" ipv6=[2607:5300:100::78d]:443" +" weight=10", +"78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123" +" weight=10", +"92.222.38.67:80 orport=443 id=DED6892FF89DBD737BA689698A171B2392EB3E82" +" weight=10", +"217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05" +" ipv6=[2a02:27a8:0:2::7e]:443" +" weight=10", +"167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48" +" weight=10", +"212.47.244.38:8080 orport=443 id=E81EF60A73B3809F8964F73766B01BAA0A171E20" +" weight=10", +"131.188.40.188:443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8" +" weight=10", +"89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7" +" weight=10", +"192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024" +" weight=10", +"212.83.40.238:9030 orport=9001 id=F409FA7902FD89270E8DE0D7977EA23BC38E5887" +" weight=10", +"5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265" +" weight=10", +"46.28.207.141:80 orport=443 id=F69BED36177ED727706512BA6A97755025EEA0FB" +" weight=10", +"78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE" +" weight=10", +"178.254.13.126:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE" +" weight=10", +"185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216" +" weight=10", +"104.243.35.196:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD" +" weight=10", +"86.59.119.83:80 orport=443 id=FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B" +" weight=10", +"192.187.124.98:9030 orport=9001 id=FD1871854BFC06D7B02F10742073069F0528B5CC" +" weight=10", +"212.129.38.254:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4" +" weight=10", +"149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76" +" weight=10", +"193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07" +" ipv6=[2001:6b0:7:125::243]:9001" +" weight=10", diff --git a/src/or/fp_pair.c b/src/or/fp_pair.c index 4d8a835c83..53b311e580 100644 --- a/src/or/fp_pair.c +++ b/src/or/fp_pair.c @@ -1,6 +1,14 @@ -/* Copyright (c) 2013, The Tor Project, Inc. */ +/* Copyright (c) 2013-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * \file fp_pair.c + * + * \brief Manages data structures for associating pairs of fingerprints. Used + * to handle combinations of identity/signing-key fingerprints for + * authorities. + **/ + #include "or.h" #include "fp_pair.h" @@ -21,7 +29,7 @@ struct fp_pair_map_s { */ /** Compare fp_pair_entry_t objects by key value. */ -static INLINE int +static inline int fp_pair_map_entries_eq(const fp_pair_map_entry_t *a, const fp_pair_map_entry_t *b) { @@ -29,20 +37,11 @@ fp_pair_map_entries_eq(const fp_pair_map_entry_t *a, } /** Return a hash value for an fp_pair_entry_t. */ -static INLINE unsigned int +static inline unsigned int fp_pair_map_entry_hash(const fp_pair_map_entry_t *a) { - const uint32_t *p; - unsigned int hash; - - p = (const uint32_t *)(a->key.first); - /* Hashes are 20 bytes long, so 5 times uint32_t */ - hash = p[0] ^ p[1] ^ p[2] ^ p[3] ^ p[4]; - /* Now XOR in the second fingerprint */ - p = (const uint32_t *)(a->key.second); - hash ^= p[0] ^ p[1] ^ p[2] ^ p[3] ^ p[4]; - - return hash; + tor_assert(sizeof(a->key) == DIGEST_LEN*2); + return (unsigned) siphash24g(&a->key, DIGEST_LEN*2); } /* @@ -51,9 +50,9 @@ fp_pair_map_entry_hash(const fp_pair_map_entry_t *a) HT_PROTOTYPE(fp_pair_map_impl, fp_pair_map_entry_s, node, fp_pair_map_entry_hash, fp_pair_map_entries_eq) -HT_GENERATE(fp_pair_map_impl, fp_pair_map_entry_s, node, - fp_pair_map_entry_hash, fp_pair_map_entries_eq, - 0.6, tor_malloc, tor_realloc, tor_free) +HT_GENERATE2(fp_pair_map_impl, fp_pair_map_entry_s, node, + fp_pair_map_entry_hash, fp_pair_map_entries_eq, + 0.6, tor_reallocarray_, tor_free_) /** Constructor to create a new empty map from fp_pair_t to void * */ diff --git a/src/or/fp_pair.h b/src/or/fp_pair.h index 89f664a813..b1466581d2 100644 --- a/src/or/fp_pair.h +++ b/src/or/fp_pair.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013, The Tor Project, Inc. */ +/* Copyright (c) 2013-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/geoip.c b/src/or/geoip.c index e2e98e8ec4..b563db0418 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -18,7 +18,6 @@ #include "geoip.h" #include "routerlist.h" -static void clear_geoip_db(void); static void init_geoip_countries(void); /** An entry from the GeoIP IPv4 file: maps an IPv4 range to a country. */ @@ -58,8 +57,8 @@ static char geoip6_digest[DIGEST_LEN]; /** Return the index of the <b>country</b>'s entry in the GeoIP * country list if it is a valid 2-letter country code, otherwise * return -1. */ -country_t -geoip_get_country(const char *country) +MOCK_IMPL(country_t, +geoip_get_country,(const char *country)) { void *idxplus1_; intptr_t idx; @@ -120,7 +119,7 @@ geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, /** Add an entry to the GeoIP table indicated by <b>family</b>, * parsing it from <b>line</b>. The format is as for geoip_load_file(). */ -/*private*/ int +STATIC int geoip_parse_entry(const char *line, sa_family_t family) { tor_addr_t low_addr, high_addr; @@ -363,7 +362,7 @@ geoip_load_file(sa_family_t family, const char *filename) * be less than geoip_get_n_countries(). To decode it, call * geoip_get_country_name(). */ -int +STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr) { geoip_ipv4_entry_t *ent; @@ -379,7 +378,7 @@ geoip_get_country_by_ipv4(uint32_t ipaddr) * 0 for the 'unknown country'. The return value will always be less than * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). */ -int +STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr) { geoip_ipv6_entry_t *ent; @@ -396,8 +395,8 @@ geoip_get_country_by_ipv6(const struct in6_addr *addr) * the 'unknown country'. The return value will always be less than * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). */ -int -geoip_get_country_by_addr(const tor_addr_t *addr) +MOCK_IMPL(int, +geoip_get_country_by_addr,(const tor_addr_t *addr)) { if (tor_addr_family(addr) == AF_INET) { return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr)); @@ -409,8 +408,8 @@ geoip_get_country_by_addr(const tor_addr_t *addr) } /** Return the number of countries recognized by the GeoIP country list. */ -int -geoip_get_n_countries(void) +MOCK_IMPL(int, +geoip_get_n_countries,(void)) { if (!geoip_countries) init_geoip_countries(); @@ -430,8 +429,8 @@ geoip_get_country_name(country_t num) } /** Return true iff we have loaded a GeoIP database.*/ -int -geoip_is_loaded(sa_family_t family) +MOCK_IMPL(int, +geoip_is_loaded,(sa_family_t family)) { tor_assert(family == AF_INET || family == AF_INET6); if (geoip_countries == NULL) @@ -461,6 +460,10 @@ geoip_db_digest(sa_family_t family) typedef struct clientmap_entry_t { HT_ENTRY(clientmap_entry_t) node; tor_addr_t addr; + /* Name of pluggable transport used by this client. NULL if no + pluggable transport was used. */ + char *transport_name; + /** Time when we last saw this IP address, in MINUTES since the epoch. * * (This will run out of space around 4011 CE. If Tor is still in use around @@ -479,23 +482,42 @@ static HT_HEAD(clientmap, clientmap_entry_t) client_history = HT_INITIALIZER(); /** Hashtable helper: compute a hash of a clientmap_entry_t. */ -static INLINE unsigned +static inline unsigned clientmap_entry_hash(const clientmap_entry_t *a) { - return ht_improve_hash(tor_addr_hash(&a->addr)); + unsigned h = (unsigned) tor_addr_hash(&a->addr); + + if (a->transport_name) + h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name)); + + return h; } /** Hashtable helper: compare two clientmap_entry_t values for equality. */ -static INLINE int +static inline int clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) { + if (strcmp_opt(a->transport_name, b->transport_name)) + return 0; + return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) && a->action == b->action; } HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq); -HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, - clientmap_entries_eq, 0.6, malloc, realloc, free); +HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash, + clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_) + +/** Free all storage held by <b>ent</b>. */ +static void +clientmap_entry_free(clientmap_entry_t *ent) +{ + if (!ent) + return; + + tor_free(ent->transport_name); + tor_free(ent); +} /** Clear history of connecting clients used by entry and bridge stats. */ static void @@ -507,7 +529,7 @@ client_history_clear(void) if ((*ent)->action == GEOIP_CLIENT_CONNECT) { this = *ent; next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + clientmap_entry_free(this); } else { next = HT_NEXT(clientmap, &client_history, ent); } @@ -519,27 +541,40 @@ client_history_clear(void) * configured accordingly. */ void geoip_note_client_seen(geoip_client_action_t action, - const tor_addr_t *addr, time_t now) + const tor_addr_t *addr, + const char *transport_name, + time_t now) { const or_options_t *options = get_options(); clientmap_entry_t lookup, *ent; + memset(&lookup, 0, sizeof(clientmap_entry_t)); + if (action == GEOIP_CLIENT_CONNECT) { /* Only remember statistics as entry guard or as bridge. */ if (!options->EntryStatistics && (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) return; } else { - if (options->BridgeRelay || options->BridgeAuthoritativeDir || - !options->DirReqStatistics) + /* Only gather directory-request statistics if configured, and + * forcibly disable them on bridge authorities. */ + if (!options->DirReqStatistics || options->BridgeAuthoritativeDir) return; } + log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.", + safe_str_client(fmt_addr((addr))), + transport_name ? transport_name : "<no transport>"); + tor_addr_copy(&lookup.addr, addr); lookup.action = (int)action; + lookup.transport_name = (char*) transport_name; ent = HT_FIND(clientmap, &client_history, &lookup); + if (! ent) { ent = tor_malloc_zero(sizeof(clientmap_entry_t)); tor_addr_copy(&ent->addr, addr); + if (transport_name) + ent->transport_name = tor_strdup(transport_name); ent->action = (int)action; HT_INSERT(clientmap, &client_history, ent); } @@ -566,7 +601,7 @@ remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff) { time_t cutoff = *(time_t*)_cutoff / 60; if (ent->last_seen_in_minutes < cutoff) { - tor_free(ent); + clientmap_entry_free(ent); return 1; } else { return 0; @@ -684,8 +719,8 @@ dirreq_map_ent_hash(const dirreq_map_entry_t *entry) HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, dirreq_map_ent_eq); -HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, - dirreq_map_ent_eq, 0.6, malloc, realloc, free); +HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) /** Helper: Put <b>entry</b> into map of directory requests using * <b>type</b> and <b>dirreq_id</b> as key parts. If there is @@ -769,6 +804,106 @@ geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type, } } +/** Return the bridge-ip-transports string that should be inserted in + * our extra-info descriptor. Return NULL if the bridge-ip-transports + * line should be empty. */ +char * +geoip_get_transport_history(void) +{ + unsigned granularity = IP_GRANULARITY; + /** String hash table (name of transport) -> (number of users). */ + strmap_t *transport_counts = strmap_new(); + + /** Smartlist that contains copies of the names of the transports + that have been used. */ + smartlist_t *transports_used = smartlist_new(); + + /* Special string to signify that no transport was used for this + connection. Pluggable transport names can't have symbols in their + names, so this string will never collide with a real transport. */ + static const char* no_transport_str = "<OR>"; + + clientmap_entry_t **ent; + const char *transport_name = NULL; + smartlist_t *string_chunks = smartlist_new(); + char *the_string = NULL; + + /* If we haven't seen any clients yet, return NULL. */ + if (HT_EMPTY(&client_history)) + goto done; + + /** We do the following steps to form the transport history string: + * a) Foreach client that uses a pluggable transport, we increase the + * times that transport was used by one. If the client did not use + * a transport, we increase the number of times someone connected + * without obfuscation. + * b) Foreach transport we observed, we write its transport history + * string and push it to string_chunks. So, for example, if we've + * seen 665 obfs2 clients, we write "obfs2=665". + * c) We concatenate string_chunks to form the final string. + */ + + log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.", + HT_SIZE(&client_history)); + + /* Loop through all clients. */ + HT_FOREACH(ent, clientmap, &client_history) { + uintptr_t val; + void *ptr; + transport_name = (*ent)->transport_name; + if (!transport_name) + transport_name = no_transport_str; + + /* Increase the count for this transport name. */ + ptr = strmap_get(transport_counts, transport_name); + val = (uintptr_t)ptr; + val++; + ptr = (void*)val; + strmap_set(transport_counts, transport_name, ptr); + + /* If it's the first time we see this transport, note it. */ + if (val == 1) + smartlist_add(transports_used, tor_strdup(transport_name)); + + log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. " + "I've now seen %d clients.", + safe_str_client(fmt_addr(&(*ent)->addr)), + transport_name ? transport_name : "<no transport>", + (int)val); + } + + /* Sort the transport names (helps with unit testing). */ + smartlist_sort_strings(transports_used); + + /* Loop through all seen transports. */ + SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) { + void *transport_count_ptr = strmap_get(transport_counts, transport_name); + uintptr_t transport_count = (uintptr_t) transport_count_ptr; + + log_debug(LD_GENERAL, "We got "U64_FORMAT" clients with transport '%s'.", + U64_PRINTF_ARG((uint64_t)transport_count), transport_name); + + smartlist_add_asprintf(string_chunks, "%s="U64_FORMAT, + transport_name, + U64_PRINTF_ARG(round_uint64_to_next_multiple_of( + (uint64_t)transport_count, + granularity))); + } SMARTLIST_FOREACH_END(transport_name); + + the_string = smartlist_join_strings(string_chunks, ",", 0, NULL); + + log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string); + + done: + strmap_free(transport_counts, NULL); + SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s)); + smartlist_free(transports_used); + SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s)); + smartlist_free(string_chunks); + + return the_string; +} + /** Return a newly allocated comma-separated string containing statistics * on network status downloads. The string contains the number of completed * requests, timeouts, and still running requests as well as the download @@ -827,14 +962,14 @@ geoip_get_dirreq_history(dirreq_type_t type) /* We may have rounded 'completed' up. Here we want to use the * real value. */ complete = smartlist_len(dirreq_completed); - dltimes = tor_malloc_zero(sizeof(uint32_t) * complete); + dltimes = tor_calloc(complete, sizeof(uint32_t)); SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { uint32_t bytes_per_second; uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, &ent->completion_time); if (time_diff == 0) time_diff = 1; /* Avoid DIV/0; "instant" answers are impossible - * by law of nature or something, but a milisecond + * by law of nature or something, but a millisecond * is a bit greater than "instantly" */ bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff); dltimes[ent_sl_idx] = bytes_per_second; @@ -897,7 +1032,7 @@ geoip_get_client_history(geoip_client_action_t action, if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6)) return -1; - counts = tor_malloc_zero(sizeof(unsigned)*n_countries); + counts = tor_calloc(n_countries, sizeof(unsigned)); HT_FOREACH(ent, clientmap, &client_history) { int country; if ((*ent)->action != (int)action) @@ -1037,7 +1172,7 @@ geoip_reset_dirreq_stats(time_t now) if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) { this = *ent; next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + clientmap_entry_free(this); } else { next = HT_NEXT(clientmap, &client_history, ent); } @@ -1071,9 +1206,9 @@ geoip_format_dirreq_stats(time_t now) { char t[ISO_TIME_LEN+1]; int i; - char *v3_ips_string, *v3_reqs_string, *v3_direct_dl_string, - *v3_tunneled_dl_string; - char *result; + char *v3_ips_string = NULL, *v3_reqs_string = NULL, + *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL; + char *result = NULL; if (!start_of_dirreq_stats_interval) return NULL; /* Not initialized. */ @@ -1132,7 +1267,7 @@ geoip_format_dirreq_stats(time_t now) time_t geoip_dirreq_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_dirreq_stats_interval) return 0; /* Not initialized. */ @@ -1144,23 +1279,17 @@ geoip_dirreq_stats_write(time_t now) /* Generate history string .*/ str = geoip_format_dirreq_stats(now); + if (! str) + goto done; /* Write dirreq-stats string to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics"); + /* Reset measurement interval start. */ + geoip_reset_dirreq_stats(now); } - filename = get_datadir_fname2("stats", "dirreq-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write dirreq statistics to disk!"); - - /* Reset measurement interval start. */ - geoip_reset_dirreq_stats(now); done: - tor_free(statsdir); - tor_free(filename); tor_free(str); return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL; } @@ -1197,6 +1326,8 @@ validate_bridge_stats(const char *stats_str, time_t now) const char *BRIDGE_STATS_END = "bridge-stats-end "; const char *BRIDGE_IPS = "bridge-ips "; const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n"; + const char *BRIDGE_TRANSPORTS = "bridge-ip-transports "; + const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n"; const char *tmp; time_t stats_end_time; int seconds; @@ -1231,6 +1362,15 @@ validate_bridge_stats(const char *stats_str, time_t now) return 0; } + /* Parse: "bridge-ip-transports PT=N,PT=N,..." */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS); + if (!tmp) { + /* Look if there is an empty "bridge-ip-transports" line */ + tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE); + if (!tmp) + return 0; + } + return 1; } @@ -1244,7 +1384,8 @@ static char *bridge_stats_extrainfo = NULL; char * geoip_format_bridge_stats(time_t now) { - char *out = NULL, *country_data = NULL, *ipver_data = NULL; + char *out = NULL; + char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL; long duration = now - start_of_bridge_stats_interval; char written[ISO_TIME_LEN+1]; @@ -1255,16 +1396,20 @@ geoip_format_bridge_stats(time_t now) format_iso_time(written, now); geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data); + transport_data = geoip_get_transport_history(); tor_asprintf(&out, "bridge-stats-end %s (%ld s)\n" "bridge-ips %s\n" - "bridge-ip-versions %s\n", + "bridge-ip-versions %s\n" + "bridge-ip-transports %s\n", written, duration, country_data ? country_data : "", - ipver_data ? ipver_data : ""); + ipver_data ? ipver_data : "", + transport_data ? transport_data : ""); tor_free(country_data); tor_free(ipver_data); + tor_free(transport_data); return out; } @@ -1292,12 +1437,45 @@ format_bridge_stats_controller(time_t now) return out; } +/** Return a newly allocated string holding our bridge usage stats by + * country in a format suitable for inclusion in our heartbeat + * message. Return NULL on failure. */ +char * +format_client_stats_heartbeat(time_t now) +{ + const int n_hours = 6; + char *out = NULL; + int n_clients = 0; + clientmap_entry_t **ent; + unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 ); + + if (!start_of_bridge_stats_interval) + return NULL; /* Not initialized. */ + + /* count unique IPs */ + HT_FOREACH(ent, clientmap, &client_history) { + /* only count directly connecting clients */ + if ((*ent)->action != GEOIP_CLIENT_CONNECT) + continue; + if ((*ent)->last_seen_in_minutes < cutoff) + continue; + n_clients++; + } + + tor_asprintf(&out, "Heartbeat: " + "In the last %d hours, I have seen %d unique clients.", + n_hours, + n_clients); + + return out; +} + /** Write bridge statistics to $DATADIR/stats/bridge-stats and return * when we should next try to write statistics. */ time_t geoip_bridge_stats_write(time_t now) { - char *filename = NULL, *val = NULL, *statsdir = NULL; + char *val = NULL; /* Check if 24 hours have passed since starting measurements. */ if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL) @@ -1317,24 +1495,20 @@ geoip_bridge_stats_write(time_t now) start_of_bridge_stats_interval = now; /* Write it to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) - goto done; - filename = get_datadir_fname2("stats", "bridge-stats"); - - write_str_to_file(filename, bridge_stats_extrainfo, 0); - - /* Tell the controller, "hey, there are clients!" */ - { - char *controller_str = format_bridge_stats_controller(now); - if (controller_str) - control_event_clients_seen(controller_str); - tor_free(controller_str); + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "bridge-stats", + bridge_stats_extrainfo, "bridge statistics"); + + /* Tell the controller, "hey, there are clients!" */ + { + char *controller_str = format_bridge_stats_controller(now); + if (controller_str) + control_event_clients_seen(controller_str); + tor_free(controller_str); + } } - done: - tor_free(filename); - tor_free(statsdir); + done: return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL; } @@ -1436,7 +1610,7 @@ geoip_format_entry_stats(time_t now) time_t geoip_entry_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_entry_stats_interval) return 0; /* Not initialized. */ @@ -1450,21 +1624,14 @@ geoip_entry_stats_write(time_t now) str = geoip_format_entry_stats(now); /* Write entry-stats string to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; - } - filename = get_datadir_fname2("stats", "entry-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write entry statistics to disk!"); + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "entry-stats", str, "entry statistics"); - /* Reset measurement interval start. */ - geoip_reset_entry_stats(now); + /* Reset measurement interval start. */ + geoip_reset_entry_stats(now); + } done: - tor_free(statsdir); - tor_free(filename); tor_free(str); return start_of_entry_stats_interval + WRITE_STATS_INTERVAL; } @@ -1500,7 +1667,7 @@ getinfo_helper_geoip(control_connection_t *control_conn, } /** Release all storage held by the GeoIP databases and country list. */ -static void +STATIC void clear_geoip_db(void) { if (geoip_countries) { @@ -1534,7 +1701,7 @@ geoip_free_all(void) for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) { this = *ent; next = HT_NEXT_RMV(clientmap, &client_history, ent); - tor_free(this); + clientmap_entry_free(this); } HT_CLEAR(clientmap, &client_history); } @@ -1549,5 +1716,6 @@ geoip_free_all(void) } clear_geoip_db(); + tor_free(bridge_stats_extrainfo); } diff --git a/src/or/geoip.h b/src/or/geoip.h index ebefee5f4e..070296dd07 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,25 +12,30 @@ #ifndef TOR_GEOIP_H #define TOR_GEOIP_H +#include "testsupport.h" + #ifdef GEOIP_PRIVATE -int geoip_parse_entry(const char *line, sa_family_t family); -int geoip_get_country_by_ipv4(uint32_t ipaddr); -int geoip_get_country_by_ipv6(const struct in6_addr *addr); +STATIC int geoip_parse_entry(const char *line, sa_family_t family); +STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr); +STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr); +STATIC void clear_geoip_db(void); #endif int should_record_bridge_info(const or_options_t *options); int geoip_load_file(sa_family_t family, const char *filename); -int geoip_get_country_by_addr(const tor_addr_t *addr); -int geoip_get_n_countries(void); +MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr)); +MOCK_DECL(int, geoip_get_n_countries, (void)); const char *geoip_get_country_name(country_t num); -int geoip_is_loaded(sa_family_t family); +MOCK_DECL(int, geoip_is_loaded, (sa_family_t family)); const char *geoip_db_digest(sa_family_t family); -country_t geoip_get_country(const char *countrycode); +MOCK_DECL(country_t, geoip_get_country, (const char *countrycode)); void geoip_note_client_seen(geoip_client_action_t action, - const tor_addr_t *addr, time_t now); + const tor_addr_t *addr, const char *transport_name, + time_t now); void geoip_remove_old_clients(time_t cutoff); void geoip_note_ns_response(geoip_ns_response_t response); +char *geoip_get_transport_history(void); int geoip_get_client_history(geoip_client_action_t action, char **country_str, char **ipver_str); char *geoip_get_request_history(void); @@ -60,6 +65,7 @@ time_t geoip_bridge_stats_write(time_t now); void geoip_bridge_stats_term(void); const char *geoip_get_bridge_stats_extrainfo(time_t); char *geoip_get_bridge_stats_controller(time_t); +char *format_client_stats_heartbeat(time_t now); #endif diff --git a/src/or/hibernate.c b/src/or/hibernate.c index a412571331..9408925d96 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -239,8 +239,8 @@ accounting_parse_options(const or_options_t *options, int validate_only) /** If we want to manage the accounting system and potentially * hibernate, return 1, else return 0. */ -int -accounting_is_enabled(const or_options_t *options) +MOCK_IMPL(int, +accounting_is_enabled,(const or_options_t *options)) { if (options->AccountingMax) return 1; @@ -255,6 +255,13 @@ accounting_get_interval_length(void) return (int)(interval_end_time - interval_start_time); } +/** Return the time at which the current accounting interval will end. */ +MOCK_IMPL(time_t, +accounting_get_end_time,(void)) +{ + return interval_end_time; +} + /** Called from main.c to tell us that <b>seconds</b> seconds have * passed, <b>n_read</b> bytes have been read, and <b>n_written</b> * bytes have been written. */ @@ -403,6 +410,21 @@ configure_accounting(time_t now) accounting_set_wakeup_time(); } +/** Return the relevant number of bytes sent/received this interval + * based on the set AccountingRule */ +uint64_t +get_accounting_bytes(void) +{ + if (get_options()->AccountingRule == ACCT_SUM) + return n_bytes_read_in_interval+n_bytes_written_in_interval; + else if (get_options()->AccountingRule == ACCT_IN) + return n_bytes_read_in_interval; + else if (get_options()->AccountingRule == ACCT_OUT) + return n_bytes_written_in_interval; + else + return MAX(n_bytes_read_in_interval, n_bytes_written_in_interval); +} + /** Set expected_bandwidth_usage based on how much we sent/received * per minute last interval (if we were up for at least 30 minutes), * or based on our declared bandwidth otherwise. */ @@ -414,6 +436,11 @@ update_expected_bandwidth(void) uint64_t max_configured = (options->RelayBandwidthRate > 0 ? options->RelayBandwidthRate : options->BandwidthRate) * 60; + /* max_configured is the larger of bytes read and bytes written + * If we are accounting based on sum, worst case is both are + * at max, doubling the expected sum of bandwidth */ + if (get_options()->AccountingRule == ACCT_SUM) + max_configured *= 2; #define MIN_TIME_FOR_MEASUREMENT (1800) @@ -432,8 +459,7 @@ update_expected_bandwidth(void) * doesn't know to store soft-limit info. Just take rate at which * we were reading/writing in the last interval as our expected rate. */ - uint64_t used = MAX(n_bytes_written_in_interval, - n_bytes_read_in_interval); + uint64_t used = get_accounting_bytes(); expected = used / (n_seconds_active_in_interval / 60); } else { /* If we haven't gotten enough data last interval, set 'expected' @@ -468,7 +494,7 @@ reset_accounting(time_t now) } /** Return true iff we should save our bandwidth usage to disk. */ -static INLINE int +static inline int time_to_record_bandwidth_usage(time_t now) { /* Note every 600 sec */ @@ -641,7 +667,15 @@ read_bandwidth_usage(void) { char *fname = get_datadir_fname("bw_accounting"); - unlink(fname); + int res; + + res = unlink(fname); + if (res != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + fname, strerror(errno)); + } + tor_free(fname); } @@ -700,8 +734,7 @@ hibernate_hard_limit_reached(void) uint64_t hard_limit = get_options()->AccountingMax; if (!hard_limit) return 0; - return n_bytes_read_in_interval >= hard_limit - || n_bytes_written_in_interval >= hard_limit; + return get_accounting_bytes() >= hard_limit; } /** Return true iff we have sent/received almost all the bytes we are willing @@ -732,8 +765,7 @@ hibernate_soft_limit_reached(void) if (!soft_limit) return 0; - return n_bytes_read_in_interval >= soft_limit - || n_bytes_written_in_interval >= soft_limit; + return get_accounting_bytes() >= soft_limit; } /** Called when we get a SIGINT, or when bandwidth soft limit is @@ -757,8 +789,7 @@ hibernate_begin(hibernate_state_t new_state, time_t now) hibernate_state == HIBERNATE_STATE_LIVE) { soft_limit_hit_at = now; n_seconds_to_hit_soft_limit = n_seconds_active_in_interval; - n_bytes_at_soft_limit = MAX(n_bytes_read_in_interval, - n_bytes_written_in_interval); + n_bytes_at_soft_limit = get_accounting_bytes(); } /* close listeners. leave control listener(s). */ @@ -808,8 +839,8 @@ hibernate_begin_shutdown(void) } /** Return true iff we are currently hibernating. */ -int -we_are_hibernating(void) +MOCK_IMPL(int, +we_are_hibernating,(void)) { return hibernate_state != HIBERNATE_STATE_LIVE; } @@ -983,18 +1014,39 @@ getinfo_helper_accounting(control_connection_t *conn, else *answer = tor_strdup("awake"); } else if (!strcmp(question, "accounting/bytes")) { - tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, + tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, U64_PRINTF_ARG(n_bytes_read_in_interval), U64_PRINTF_ARG(n_bytes_written_in_interval)); } else if (!strcmp(question, "accounting/bytes-left")) { uint64_t limit = get_options()->AccountingMax; - uint64_t read_left = 0, write_left = 0; - if (n_bytes_read_in_interval < limit) - read_left = limit - n_bytes_read_in_interval; - if (n_bytes_written_in_interval < limit) - write_left = limit - n_bytes_written_in_interval; - tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, - U64_PRINTF_ARG(read_left), U64_PRINTF_ARG(write_left)); + if (get_options()->AccountingRule == ACCT_SUM) { + uint64_t total_left = 0; + uint64_t total_bytes = get_accounting_bytes(); + if (total_bytes < limit) + total_left = limit - total_bytes; + tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, + U64_PRINTF_ARG(total_left), U64_PRINTF_ARG(total_left)); + } else if (get_options()->AccountingRule == ACCT_IN) { + uint64_t read_left = 0; + if (n_bytes_read_in_interval < limit) + read_left = limit - n_bytes_read_in_interval; + tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, + U64_PRINTF_ARG(read_left), U64_PRINTF_ARG(limit)); + } else if (get_options()->AccountingRule == ACCT_OUT) { + uint64_t write_left = 0; + if (n_bytes_written_in_interval < limit) + write_left = limit - n_bytes_written_in_interval; + tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, + U64_PRINTF_ARG(limit), U64_PRINTF_ARG(write_left)); + } else { + uint64_t read_left = 0, write_left = 0; + if (n_bytes_read_in_interval < limit) + read_left = limit - n_bytes_read_in_interval; + if (n_bytes_written_in_interval < limit) + write_left = limit - n_bytes_written_in_interval; + tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, + U64_PRINTF_ARG(read_left), U64_PRINTF_ARG(write_left)); + } } else if (!strcmp(question, "accounting/interval-start")) { *answer = tor_malloc(ISO_TIME_LEN+1); format_iso_time(*answer, interval_start_time); @@ -1010,6 +1062,7 @@ getinfo_helper_accounting(control_connection_t *conn, return 0; } +#ifdef TOR_UNIT_TESTS /** * Manually change the hibernation state. Private; used only by the unit * tests. @@ -1019,4 +1072,5 @@ hibernate_set_state_for_testing_(hibernate_state_t newstate) { hibernate_state = newstate; } +#endif diff --git a/src/or/hibernate.h b/src/or/hibernate.h index d2d6989e10..fa9da6de39 100644 --- a/src/or/hibernate.h +++ b/src/or/hibernate.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,19 +12,24 @@ #ifndef TOR_HIBERNATE_H #define TOR_HIBERNATE_H +#include "testsupport.h" + int accounting_parse_options(const or_options_t *options, int validate_only); -int accounting_is_enabled(const or_options_t *options); +MOCK_DECL(int, accounting_is_enabled, (const or_options_t *options)); int accounting_get_interval_length(void); +MOCK_DECL(time_t, accounting_get_end_time, (void)); void configure_accounting(time_t now); +uint64_t get_accounting_bytes(void); void accounting_run_housekeeping(time_t now); void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); int accounting_record_bandwidth_usage(time_t now, or_state_t *state); void hibernate_begin_shutdown(void); -int we_are_hibernating(void); +MOCK_DECL(int, we_are_hibernating, (void)); void consider_hibernation(time_t now); int getinfo_helper_accounting(control_connection_t *conn, const char *question, char **answer, const char **errmsg); +uint64_t get_accounting_max_total(void); #ifdef HIBERNATE_PRIVATE /** Possible values of hibernate_state */ @@ -45,8 +50,10 @@ typedef enum { HIBERNATE_STATE_INITIAL=5 } hibernate_state_t; +#ifdef TOR_UNIT_TESTS void hibernate_set_state_for_testing_(hibernate_state_t newstate); #endif +#endif #endif diff --git a/src/or/include.am b/src/or/include.am index 65dbeff53e..712ae18406 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -1,5 +1,13 @@ bin_PROGRAMS+= src/or/tor -noinst_LIBRARIES+= src/or/libtor.a +noinst_LIBRARIES += \ + src/or/libtor.a +if UNITTESTS_ENABLED +noinst_LIBRARIES += \ + src/or/libtor-testing.a +endif +if COVERAGE_ENABLED +noinst_PROGRAMS+= src/or/tor-cov +endif if BUILD_NT_SERVICES tor_platform_source=src/or/ntmain.c @@ -7,7 +15,7 @@ else tor_platform_source= endif -EXTRA_DIST+= src/or/ntmain.c src/or/or_sha1.i src/or/Makefile.nmake +EXTRA_DIST+= src/or/ntmain.c src/or/Makefile.nmake if USE_EXTERNAL_EVDNS evdns_source= @@ -15,17 +23,12 @@ else evdns_source=src/ext/eventdns.c endif -if CURVE25519_ENABLED -onion_ntor_source=src/or/onion_ntor.c -else -onion_ntor_source= -endif - -src_or_libtor_a_SOURCES = \ +LIBTOR_A_SOURCES = \ src/or/addressmap.c \ src/or/buffers.c \ src/or/channel.c \ src/or/channeltls.c \ + src/or/circpathbias.c \ src/or/circuitbuild.c \ src/or/circuitlist.c \ src/or/circuitmux.c \ @@ -40,15 +43,18 @@ src_or_libtor_a_SOURCES = \ src/or/connection_or.c \ src/or/control.c \ src/or/cpuworker.c \ + src/or/dircollate.c \ src/or/directory.c \ src/or/dirserv.c \ src/or/dirvote.c \ src/or/dns.c \ src/or/dnsserv.c \ - src/or/fp_pair.c \ + src/or/fp_pair.c \ src/or/geoip.c \ src/or/entrynodes.c \ + src/or/ext_orport.c \ src/or/hibernate.c \ + src/or/keypin.c \ src/or/main.c \ src/or/microdesc.c \ src/or/networkstatus.c \ @@ -57,9 +63,11 @@ src_or_libtor_a_SOURCES = \ src/or/onion_fast.c \ src/or/onion_tap.c \ src/or/transports.c \ + src/or/periodic.c \ src/or/policies.c \ src/or/reasons.c \ src/or/relay.c \ + src/or/rendcache.c \ src/or/rendclient.c \ src/or/rendcommon.c \ src/or/rendmid.c \ @@ -67,28 +75,33 @@ src_or_libtor_a_SOURCES = \ src/or/rephist.c \ src/or/replaycache.c \ src/or/router.c \ + src/or/routerkeys.c \ src/or/routerlist.c \ src/or/routerparse.c \ src/or/routerset.c \ + src/or/scheduler.c \ src/or/statefile.c \ src/or/status.c \ + src/or/torcert.c \ + src/or/onion_ntor.c \ $(evdns_source) \ - $(tor_platform_source) \ - $(onion_ntor_source) \ - src/or/config_codedigest.c - -#libtor_a_LIBADD = ../common/libor.a ../common/libor-crypto.a \ -# ../common/libor-event.a + $(tor_platform_source) +src_or_libtor_a_SOURCES = $(LIBTOR_A_SOURCES) +src_or_libtor_testing_a_SOURCES = $(LIBTOR_A_SOURCES) src_or_tor_SOURCES = src/or/tor_main.c AM_CPPFLAGS += -I$(srcdir)/src/or -Isrc/or -src/or/tor_main.o: micro-revision.i +src/or/tor_main.$(OBJEXT) \ + src/or/src_or_tor_cov-tor_main.$(OBJEXT): micro-revision.i + +AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\"" \ + -DLOCALSTATEDIR="\"$(localstatedir)\"" \ + -DBINDIR="\"$(bindir)\"" -AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\"" \ - -DLOCALSTATEDIR="\"$(localstatedir)\"" \ - -DBINDIR="\"$(bindir)\"" +src_or_libtor_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_or_libtor_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) # -L flags need to go in LDFLAGS. -l flags need to go in LDADD. # This seems to matter nowhere but on windows, but I assure you that it @@ -97,16 +110,29 @@ AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\"" \ src_or_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ src_or_tor_LDADD = src/or/libtor.a src/common/libor.a \ - src/common/libor-crypto.a $(LIBDONNA) \ - src/common/libor-event.a \ + src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ + src/common/libor-event.a src/trunnel/libor-trunnel.a \ + @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + +if COVERAGE_ENABLED +src_or_tor_cov_SOURCES = src/or/tor_main.c +src_or_tor_cov_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_or_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_or_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ +src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ + src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ + src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ +endif ORHEADERS = \ src/or/addressmap.h \ src/or/buffers.h \ src/or/channel.h \ src/or/channeltls.h \ + src/or/circpathbias.h \ src/or/circuitbuild.h \ src/or/circuitlist.h \ src/or/circuitmux.h \ @@ -121,16 +147,21 @@ ORHEADERS = \ src/or/connection_or.h \ src/or/control.h \ src/or/cpuworker.h \ + src/or/dircollate.h \ src/or/directory.h \ src/or/dirserv.h \ src/or/dirvote.h \ src/or/dns.h \ + src/or/dns_structs.h \ src/or/dnsserv.h \ src/or/eventdns_tor.h \ + src/or/ext_orport.h \ + src/or/fallback_dirs.inc \ src/or/fp_pair.h \ src/or/geoip.h \ src/or/entrynodes.h \ src/or/hibernate.h \ + src/or/keypin.h \ src/or/main.h \ src/or/microdesc.h \ src/or/networkstatus.h \ @@ -142,9 +173,11 @@ ORHEADERS = \ src/or/onion_tap.h \ src/or/or.h \ src/or/transports.h \ + src/or/periodic.h \ src/or/policies.h \ src/or/reasons.h \ src/or/relay.h \ + src/or/rendcache.h \ src/or/rendclient.h \ src/or/rendcommon.h \ src/or/rendmid.h \ @@ -152,44 +185,36 @@ ORHEADERS = \ src/or/rephist.h \ src/or/replaycache.h \ src/or/router.h \ + src/or/routerkeys.h \ src/or/routerlist.h \ + src/or/routerkeys.h \ src/or/routerset.h \ src/or/routerparse.h \ + src/or/scheduler.h \ src/or/statefile.h \ - src/or/status.h + src/or/status.h \ + src/or/torcert.h noinst_HEADERS+= $(ORHEADERS) micro-revision.i -src/or/config_codedigest.o: src/or/or_sha1.i - micro-revision.i: FORCE - @rm -f micro-revision.tmp; \ - if test -d "$(top_srcdir)/.git" && \ - test -x "`which git 2>&1;true`"; then \ - HASH="`cd "$(top_srcdir)" && git rev-parse --short=16 HEAD`"; \ - echo \"$$HASH\" > micro-revision.tmp; \ - fi; \ - if test ! -f micro-revision.tmp ; then \ - if test ! -f micro-revision.i ; then \ - echo '""' > micro-revision.i; \ - fi; \ - elif test ! -f micro-revision.i || \ - test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ - mv micro-revision.tmp micro-revision.i; \ - fi; true - -src/or/or_sha1.i: $(src_or_tor_SOURCES) $(src_or_libtor_a_SOURCES) $(ORHEADERS) - $(AM_V_GEN)if test "@SHA1SUM@" != none; then \ - (cd "$(srcdir)" && "@SHA1SUM@" $(src_or_tor_SOURCES) $(src_or_libtor_a_SOURCES) $(ORHEADERS) ) | \ - "@SED@" -n 's/^\(.*\)$$/"\1\\n"/p' > src/or/or_sha1.i; \ - elif test "@OPENSSL@" != none; then \ - (cd "$(srcdir)" && "@OPENSSL@" sha1 $(src_or_tor_SOURCES) $(src_or_libtor_a_SOURCES) $(ORHEADERS)) | \ - "@SED@" -n 's/SHA1(\(.*\))= \(.*\)/"\2 \1\\n"/p' > src/or/or_sha1.i; \ - else \ - rm src/or/or_sha1.i; \ - touch src/or/or_sha1.i; \ - fi - -CLEANFILES+= micro-revision.i src/or/micro-revision.i + $(AM_V_at)rm -f micro-revision.tmp; \ + if test -d "$(top_srcdir)/.git" && \ + test -x "`which git 2>&1;true`"; then \ + HASH="`cd "$(top_srcdir)" && git rev-parse --short=16 HEAD`"; \ + echo \"$$HASH\" > micro-revision.tmp; \ + fi; \ + if test ! -f micro-revision.tmp; then \ + if test ! -f micro-revision.i; then \ + echo '""' > micro-revision.i; \ + fi; \ + elif test ! -f micro-revision.i || \ + test x"`cat micro-revision.tmp`" != x"`cat micro-revision.i`"; then \ + mv micro-revision.tmp micro-revision.i; \ + fi; \ + rm -f micro-revision.tmp; \ + true + +CLEANFILES+= micro-revision.i src/or/micro-revision.i micro-revision.tmp FORCE: diff --git a/src/or/keypin.c b/src/or/keypin.c new file mode 100644 index 0000000000..1f82eccf86 --- /dev/null +++ b/src/or/keypin.c @@ -0,0 +1,486 @@ +/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file keypin.c + * + * \brief Functions and structures for associating routers' RSA key + * fingerprints with their ED25519 keys. + */ + +#define KEYPIN_PRIVATE + +#include "orconfig.h" +#include "compat.h" +#include "crypto.h" +#include "crypto_format.h" +#include "di_ops.h" +#include "ht.h" +#include "keypin.h" +#include "siphash.h" +#include "torint.h" +#include "torlog.h" +#include "util.h" +#include "util_format.h" + +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#ifdef _WIN32 +#include <io.h> +#endif + +/** + * @file keypin.c + * @brief Key-pinning for RSA and Ed25519 identity keys at directory + * authorities. + * + * This module implements a key-pinning mechanism to ensure that it's safe + * to use RSA keys as identitifers even as we migrate to Ed25519 keys. It + * remembers, for every Ed25519 key we've seen, what the associated Ed25519 + * key is. This way, if we see a different Ed25519 key with that RSA key, + * we'll know that there's a mismatch. + * + * We persist these entries to disk using a simple format, where each line + * has a base64-encoded RSA SHA1 hash, then a base64-endoded Ed25519 key. + * Empty lines, misformed lines, and lines beginning with # are + * ignored. Lines beginning with @ are reserved for future extensions. + */ + +static int keypin_journal_append_entry(const uint8_t *rsa_id_digest, + const uint8_t *ed25519_id_key); +static int keypin_check_and_add_impl(const uint8_t *rsa_id_digest, + const uint8_t *ed25519_id_key, + const int do_not_add, + const int replace); +static int keypin_add_or_replace_entry_in_map(keypin_ent_t *ent); + +static HT_HEAD(rsamap, keypin_ent_st) the_rsa_map = HT_INITIALIZER(); +static HT_HEAD(edmap, keypin_ent_st) the_ed_map = HT_INITIALIZER(); + +/** Hashtable helper: compare two keypin table entries and return true iff + * they have the same RSA key IDs. */ +static inline int +keypin_ents_eq_rsa(const keypin_ent_t *a, const keypin_ent_t *b) +{ + return tor_memeq(a->rsa_id, b->rsa_id, sizeof(a->rsa_id)); +} + +/** Hashtable helper: hash a keypin table entries based on its RSA key ID */ +static inline unsigned +keypin_ent_hash_rsa(const keypin_ent_t *a) +{ +return (unsigned) siphash24g(a->rsa_id, sizeof(a->rsa_id)); +} + +/** Hashtable helper: compare two keypin table entries and return true iff + * they have the same ed25519 keys */ +static inline int +keypin_ents_eq_ed(const keypin_ent_t *a, const keypin_ent_t *b) +{ + return tor_memeq(a->ed25519_key, b->ed25519_key, sizeof(a->ed25519_key)); +} + +/** Hashtable helper: hash a keypin table entries based on its ed25519 key */ +static inline unsigned +keypin_ent_hash_ed(const keypin_ent_t *a) +{ +return (unsigned) siphash24g(a->ed25519_key, sizeof(a->ed25519_key)); +} + +HT_PROTOTYPE(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa, + keypin_ents_eq_rsa); +HT_GENERATE2(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa, + keypin_ents_eq_rsa, 0.6, tor_reallocarray, tor_free_); + +HT_PROTOTYPE(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed, + keypin_ents_eq_ed); +HT_GENERATE2(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed, + keypin_ents_eq_ed, 0.6, tor_reallocarray, tor_free_); + +/** + * Check whether we already have an entry in the key pinning table for a + * router with RSA ID digest <b>rsa_id_digest</b> or for ed25519 key + * <b>ed25519_id_key</b>. If we have an entry that matches both keys, + * return KEYPIN_FOUND. If we find an entry that matches one key but + * not the other, return KEYPIN_MISMATCH. If we have no entry for either + * key, add such an entry to the table and return KEYPIN_ADDED. + * + * If <b>replace_existing_entry</b> is true, then any time we would have said + * KEYPIN_FOUND, we instead add this entry anyway and return KEYPIN_ADDED. + */ +int +keypin_check_and_add(const uint8_t *rsa_id_digest, + const uint8_t *ed25519_id_key, + const int replace_existing_entry) +{ + return keypin_check_and_add_impl(rsa_id_digest, ed25519_id_key, 0, + replace_existing_entry); +} + +/** + * As keypin_check_and_add, but do not add. Return KEYPIN_NOT_FOUND if + * we would add. + */ +int +keypin_check(const uint8_t *rsa_id_digest, + const uint8_t *ed25519_id_key) +{ + return keypin_check_and_add_impl(rsa_id_digest, ed25519_id_key, 1, 0); +} + +/** + * Helper: implements keypin_check and keypin_check_and_add. + */ +static int +keypin_check_and_add_impl(const uint8_t *rsa_id_digest, + const uint8_t *ed25519_id_key, + const int do_not_add, + const int replace) +{ + keypin_ent_t search, *ent; + memset(&search, 0, sizeof(search)); + memcpy(search.rsa_id, rsa_id_digest, sizeof(search.rsa_id)); + memcpy(search.ed25519_key, ed25519_id_key, sizeof(search.ed25519_key)); + + /* Search by RSA key digest first */ + ent = HT_FIND(rsamap, &the_rsa_map, &search); + if (ent) { + tor_assert(fast_memeq(ent->rsa_id, rsa_id_digest, sizeof(ent->rsa_id))); + if (tor_memeq(ent->ed25519_key, ed25519_id_key,sizeof(ent->ed25519_key))) { + return KEYPIN_FOUND; /* Match on both keys. Great. */ + } else { + if (!replace) + return KEYPIN_MISMATCH; /* Found RSA with different Ed key */ + } + } + + /* See if we know a different RSA key for this ed key */ + if (! replace) { + ent = HT_FIND(edmap, &the_ed_map, &search); + if (ent) { + /* If we got here, then the ed key matches and the RSA doesn't */ + tor_assert(fast_memeq(ent->ed25519_key, ed25519_id_key, + sizeof(ent->ed25519_key))); + tor_assert(fast_memneq(ent->rsa_id, rsa_id_digest, sizeof(ent->rsa_id))); + return KEYPIN_MISMATCH; + } + } + + /* Okay, this one is new to us. */ + if (do_not_add) + return KEYPIN_NOT_FOUND; + + ent = tor_memdup(&search, sizeof(search)); + int r = keypin_add_or_replace_entry_in_map(ent); + if (! replace) { + tor_assert(r == 1); + } else { + tor_assert(r != 0); + } + keypin_journal_append_entry(rsa_id_digest, ed25519_id_key); + return KEYPIN_ADDED; +} + +/** + * Helper: add <b>ent</b> to the hash tables. + */ +MOCK_IMPL(STATIC void, +keypin_add_entry_to_map, (keypin_ent_t *ent)) +{ + HT_INSERT(rsamap, &the_rsa_map, ent); + HT_INSERT(edmap, &the_ed_map, ent); +} + +/** + * Helper: add 'ent' to the maps, replacing any entries that contradict it. + * Take ownership of 'ent', freeing it if needed. + * + * Return 0 if the entry was a duplicate, -1 if there was a conflict, + * and 1 if there was no conflict. + */ +static int +keypin_add_or_replace_entry_in_map(keypin_ent_t *ent) +{ + int r = 1; + keypin_ent_t *ent2 = HT_FIND(rsamap, &the_rsa_map, ent); + keypin_ent_t *ent3 = HT_FIND(edmap, &the_ed_map, ent); + if (ent2 && + fast_memeq(ent2->ed25519_key, ent->ed25519_key, DIGEST256_LEN)) { + /* We already have this mapping stored. Ignore it. */ + tor_free(ent); + return 0; + } else if (ent2 || ent3) { + /* We have a conflict. (If we had no entry, we would have ent2 == ent3 + * == NULL. If we had a non-conflicting duplicate, we would have found + * it above.) + * + * We respond by having this entry (ent) supersede all entries that it + * contradicts (ent2 and/or ent3). In other words, if we receive + * <rsa,ed>, we remove all <rsa,ed'> and all <rsa',ed>, for rsa'!=rsa + * and ed'!= ed. + */ + const keypin_ent_t *t; + if (ent2) { + t = HT_REMOVE(rsamap, &the_rsa_map, ent2); + tor_assert(ent2 == t); + t = HT_REMOVE(edmap, &the_ed_map, ent2); + tor_assert(ent2 == t); + } + if (ent3 && ent2 != ent3) { + t = HT_REMOVE(rsamap, &the_rsa_map, ent3); + tor_assert(ent3 == t); + t = HT_REMOVE(edmap, &the_ed_map, ent3); + tor_assert(ent3 == t); + tor_free(ent3); + } + tor_free(ent2); + r = -1; + /* Fall through */ + } + + keypin_add_entry_to_map(ent); + return r; +} + +/** + * Check whether we already have an entry in the key pinning table for a + * router with RSA ID digest <b>rsa_id_digest</b>. If we have no such entry, + * return KEYPIN_NOT_FOUND. If we find an entry that matches the RSA key but + * which has an ed25519 key, return KEYPIN_MISMATCH. + */ +int +keypin_check_lone_rsa(const uint8_t *rsa_id_digest) +{ + keypin_ent_t search, *ent; + memset(&search, 0, sizeof(search)); + memcpy(search.rsa_id, rsa_id_digest, sizeof(search.rsa_id)); + + /* Search by RSA key digest first */ + ent = HT_FIND(rsamap, &the_rsa_map, &search); + if (ent) { + return KEYPIN_MISMATCH; + } else { + return KEYPIN_NOT_FOUND; + } +} + +/** Open fd to the keypinning journal file. */ +static int keypin_journal_fd = -1; + +/** Open the key-pinning journal to append to <b>fname</b>. Return 0 on + * success, -1 on failure. */ +int +keypin_open_journal(const char *fname) +{ + /* O_SYNC ??*/ + int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY, 0600); + if (fd < 0) + goto err; + + if (tor_fd_seekend(fd) < 0) + goto err; + + /* Add a newline in case the last line was only partially written */ + if (write(fd, "\n", 1) < 1) + goto err; + + /* Add something about when we opened this file. */ + char buf[80]; + char tbuf[ISO_TIME_LEN+1]; + format_iso_time(tbuf, approx_time()); + tor_snprintf(buf, sizeof(buf), "@opened-at %s\n", tbuf); + if (write_all(fd, buf, strlen(buf), 0) < 0) + goto err; + + keypin_journal_fd = fd; + return 0; + err: + if (fd >= 0) + close(fd); + return -1; +} + +/** Close the keypinning journal file. */ +int +keypin_close_journal(void) +{ + if (keypin_journal_fd >= 0) + close(keypin_journal_fd); + keypin_journal_fd = -1; + return 0; +} + +/** Length of a keypinning journal line, including terminating newline. */ +#define JOURNAL_LINE_LEN (BASE64_DIGEST_LEN + BASE64_DIGEST256_LEN + 2) + +/** Add an entry to the keypinning journal to map <b>rsa_id_digest</b> and + * <b>ed25519_id_key</b>. */ +static int +keypin_journal_append_entry(const uint8_t *rsa_id_digest, + const uint8_t *ed25519_id_key) +{ + if (keypin_journal_fd == -1) + return -1; + char line[JOURNAL_LINE_LEN]; + digest_to_base64(line, (const char*)rsa_id_digest); + line[BASE64_DIGEST_LEN] = ' '; + digest256_to_base64(line + BASE64_DIGEST_LEN + 1, + (const char*)ed25519_id_key); + line[BASE64_DIGEST_LEN+1+BASE64_DIGEST256_LEN] = '\n'; + + if (write_all(keypin_journal_fd, line, JOURNAL_LINE_LEN, 0)<0) { + log_warn(LD_DIRSERV, "Error while adding a line to the key-pinning " + "journal: %s", strerror(errno)); + keypin_close_journal(); + return -1; + } + + return 0; +} + +/** Load a journal from the <b>size</b>-byte region at <b>data</b>. Return 0 + * on success, -1 on failure. */ +STATIC int +keypin_load_journal_impl(const char *data, size_t size) +{ + const char *start = data, *end = data + size, *next; + + int n_corrupt_lines = 0; + int n_entries = 0; + int n_duplicates = 0; + int n_conflicts = 0; + + for (const char *cp = start; cp < end; cp = next) { + const char *eol = memchr(cp, '\n', end-cp); + const char *eos = eol ? eol : end; + const size_t len = eos - cp; + + next = eol ? eol + 1 : end; + + if (len == 0) { + continue; + } + + if (*cp == '@') { + /* Lines that start with @ are reserved. Ignore for now. */ + continue; + } + if (*cp == '#') { + /* Lines that start with # are comments. */ + continue; + } + + /* Is it the right length? (The -1 here is for the newline.) */ + if (len != JOURNAL_LINE_LEN - 1) { + /* Lines with a bad length are corrupt unless they are empty. + * Ignore them either way */ + for (const char *s = cp; s < eos; ++s) { + if (! TOR_ISSPACE(*s)) { + ++n_corrupt_lines; + break; + } + } + continue; + } + + keypin_ent_t *ent = keypin_parse_journal_line(cp); + + if (ent == NULL) { + ++n_corrupt_lines; + continue; + } + + const int r = keypin_add_or_replace_entry_in_map(ent); + if (r == 0) { + ++n_duplicates; + } else if (r == -1) { + ++n_conflicts; + } + + ++n_entries; + } + + int severity = (n_corrupt_lines || n_duplicates) ? LOG_WARN : LOG_INFO; + tor_log(severity, LD_DIRSERV, + "Loaded %d entries from keypin journal. " + "Found %d corrupt lines, %d duplicates, and %d conflicts.", + n_entries, n_corrupt_lines, n_duplicates, n_conflicts); + + return 0; +} + +/** + * Load a journal from the file called <b>fname</b>. Return 0 on success, + * -1 on failure. + */ +int +keypin_load_journal(const char *fname) +{ + tor_mmap_t *map = tor_mmap_file(fname); + if (!map) { + if (errno == ENOENT) + return 0; + else + return -1; + } + int r = keypin_load_journal_impl(map->data, map->size); + tor_munmap_file(map); + return r; +} + +/** Parse a single keypinning journal line entry from <b>cp</b>. The input + * does not need to be NUL-terminated, but it <em>does</em> need to have + * KEYPIN_JOURNAL_LINE_LEN -1 bytes available to read. Return a new entry + * on success, and NULL on failure. + */ +STATIC keypin_ent_t * +keypin_parse_journal_line(const char *cp) +{ + /* XXXX assumes !USE_OPENSSL_BASE64 */ + keypin_ent_t *ent = tor_malloc_zero(sizeof(keypin_ent_t)); + + if (base64_decode((char*)ent->rsa_id, sizeof(ent->rsa_id), + cp, BASE64_DIGEST_LEN) != DIGEST_LEN || + cp[BASE64_DIGEST_LEN] != ' ' || + base64_decode((char*)ent->ed25519_key, sizeof(ent->ed25519_key), + cp+BASE64_DIGEST_LEN+1, BASE64_DIGEST256_LEN) != DIGEST256_LEN) { + tor_free(ent); + return NULL; + } else { + return ent; + } +} + +/** Remove all entries from the keypinning table.*/ +void +keypin_clear(void) +{ + int bad_entries = 0; + { + keypin_ent_t **ent, **next, *this; + for (ent = HT_START(rsamap, &the_rsa_map); ent != NULL; ent = next) { + this = *ent; + next = HT_NEXT_RMV(rsamap, &the_rsa_map, ent); + + keypin_ent_t *other_ent = HT_REMOVE(edmap, &the_ed_map, this); + bad_entries += (other_ent != this); + + tor_free(this); + } + } + bad_entries += HT_SIZE(&the_ed_map); + + HT_CLEAR(edmap,&the_ed_map); + HT_CLEAR(rsamap,&the_rsa_map); + + if (bad_entries) { + log_warn(LD_BUG, "Found %d discrepencies in the the keypin database.", + bad_entries); + } +} + diff --git a/src/or/keypin.h b/src/or/keypin.h new file mode 100644 index 0000000000..673f24d9e3 --- /dev/null +++ b/src/or/keypin.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_KEYPIN_H +#define TOR_KEYPIN_H + +#include "testsupport.h" + +int keypin_check_and_add(const uint8_t *rsa_id_digest, + const uint8_t *ed25519_id_key, + const int replace_existing_entry); +int keypin_check(const uint8_t *rsa_id_digest, + const uint8_t *ed25519_id_key); + +int keypin_open_journal(const char *fname); +int keypin_close_journal(void); +int keypin_load_journal(const char *fname); +void keypin_clear(void); +int keypin_check_lone_rsa(const uint8_t *rsa_id_digest); + +#define KEYPIN_FOUND 0 +#define KEYPIN_ADDED 1 +#define KEYPIN_MISMATCH -1 +#define KEYPIN_NOT_FOUND -2 + +#ifdef KEYPIN_PRIVATE + +/** + * In-memory representation of a key-pinning table entry. + */ +typedef struct keypin_ent_st { + HT_ENTRY(keypin_ent_st) rsamap_node; + HT_ENTRY(keypin_ent_st) edmap_node; + /** SHA1 hash of the RSA key */ + uint8_t rsa_id[DIGEST_LEN]; + /** Ed2219 key. */ + uint8_t ed25519_key[DIGEST256_LEN]; +} keypin_ent_t; + +STATIC keypin_ent_t * keypin_parse_journal_line(const char *cp); +STATIC int keypin_load_journal_impl(const char *data, size_t size); + +MOCK_DECL(STATIC void, keypin_add_entry_to_map, (keypin_ent_t *ent)); +#endif + +#endif + diff --git a/src/or/main.c b/src/or/main.c index bd23141b97..d4d98ee317 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,6 +13,7 @@ #define MAIN_PRIVATE #include "or.h" #include "addressmap.h" +#include "backtrace.h" #include "buffers.h" #include "channel.h" #include "channeltls.h" @@ -21,11 +22,13 @@ #include "circuituse.h" #include "command.h" #include "config.h" +#include "confparse.h" #include "connection.h" #include "connection_edge.h" #include "connection_or.h" #include "control.h" #include "cpuworker.h" +#include "crypto_s2k.h" #include "directory.h" #include "dirserv.h" #include "dirvote.h" @@ -34,12 +37,14 @@ #include "entrynodes.h" #include "geoip.h" #include "hibernate.h" +#include "keypin.h" #include "main.h" #include "microdesc.h" #include "networkstatus.h" #include "nodelist.h" #include "ntmain.h" #include "onion.h" +#include "periodic.h" #include "policies.h" #include "transports.h" #include "relay.h" @@ -48,15 +53,20 @@ #include "rendservice.h" #include "rephist.h" #include "router.h" +#include "routerkeys.h" #include "routerlist.h" #include "routerparse.h" +#include "scheduler.h" #include "statefile.h" #include "status.h" +#include "util_process.h" +#include "ext_orport.h" #ifdef USE_DMALLOC #include <dmalloc.h> #include <openssl/crypto.h> #endif #include "memarea.h" +#include "sandbox.h" #ifdef HAVE_EVENT2_EVENT_H #include <event2/event.h> @@ -68,6 +78,16 @@ #include <event2/bufferevent.h> #endif +#ifdef HAVE_SYSTEMD +# if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) +/* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse + * Coverity. Here's a kludge to unconfuse it. + */ +# define __INCLUDE_LEVEL__ 2 +# endif +#include <systemd/sd-daemon.h> +#endif + void evdns_shutdown(int); /********* PROTOTYPES **********/ @@ -80,6 +100,8 @@ static void second_elapsed_callback(periodic_timer_t *timer, void *args); static int conn_close_if_marked(int i); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); +static int run_main_loop_until_done(void); +static void process_signal(int sig); /********* START VARIABLES **********/ @@ -113,8 +135,6 @@ static uint64_t stats_n_bytes_written = 0; time_t time_of_process_start = 0; /** How many seconds have we been running? */ long stats_n_seconds_working = 0; -/** When do we next launch DNS wildcarding checks? */ -static time_t time_to_check_for_correct_dns = 0; /** How often will we honor SIGNEWNYM requests? */ #define MAX_SIGNEWNYM_RATE 10 @@ -144,7 +164,7 @@ static int called_loop_once = 0; * any longer (a big time jump happened, when we notice our directory is * heinously out-of-date, etc. */ -int can_complete_circuit=0; +static int can_complete_circuits = 0; /** How often do we check for router descriptors that we should download * when we have too little directory info? */ @@ -155,8 +175,6 @@ int can_complete_circuit=0; /** How often do we 'forgive' undownloadable router descriptors and attempt * to download them again? */ #define DESCRIPTOR_FAILURE_RESET_INTERVAL (60*60) -/** How long do we let a directory connection stall before expiring it? */ -#define DIR_CONN_MAX_STALL (5*60) /** Decides our behavior when no logs are configured/before any * logs have been configured. For 0, we log notice to stdout as normal. @@ -167,37 +185,11 @@ int quiet_level = 0; /********* END VARIABLES ************/ /**************************************************************************** -* -* This section contains accessors and other methods on the connection_array -* variables (which are global within this file and unavailable outside it). -* -****************************************************************************/ - -#if 0 && defined(USE_BUFFEREVENTS) -static void -free_old_inbuf(connection_t *conn) -{ - if (! conn->inbuf) - return; - - tor_assert(conn->outbuf); - tor_assert(buf_datalen(conn->inbuf) == 0); - tor_assert(buf_datalen(conn->outbuf) == 0); - buf_free(conn->inbuf); - buf_free(conn->outbuf); - conn->inbuf = conn->outbuf = NULL; - - if (conn->read_event) { - event_del(conn->read_event); - tor_event_free(conn->read_event); - } - if (conn->write_event) { - event_del(conn->read_event); - tor_event_free(conn->write_event); - } - conn->read_event = conn->write_event = NULL; -} -#endif + * + * This section contains accessors and other methods on the connection_array + * variables (which are global within this file and unavailable outside it). + * + ****************************************************************************/ #if defined(_WIN32) && defined(USE_BUFFEREVENTS) /** Remove the kernel-space send and receive buffers for <b>s</b>. For use @@ -207,11 +199,13 @@ set_buffer_lengths_to_zero(tor_socket_t s) { int zero = 0; int r = 0; - if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, (void*)&zero, sizeof(zero))) { + if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, (void*)&zero, + (socklen_t)sizeof(zero))) { log_warn(LD_NET, "Unable to clear SO_SNDBUF"); r = -1; } - if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, (void*)&zero, sizeof(zero))) { + if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, (void*)&zero, + (socklen_t)sizeof(zero))) { log_warn(LD_NET, "Unable to clear SO_RCVBUF"); r = -1; } @@ -219,6 +213,31 @@ set_buffer_lengths_to_zero(tor_socket_t s) } #endif +/** Return 1 if we have successfully built a circuit, and nothing has changed + * to make us think that maybe we can't. + */ +int +have_completed_a_circuit(void) +{ + return can_complete_circuits; +} + +/** Note that we have successfully built a circuit, so that reachability + * testing and introduction points and so on may be attempted. */ +void +note_that_we_completed_a_circuit(void) +{ + can_complete_circuits = 1; +} + +/** Note that something has happened (like a clock jump, or DisableNetwork) to + * make us think that maybe we can't complete circuits. */ +void +note_that_we_maybe_cant_complete_circuits(void) +{ + can_complete_circuits = 0; +} + /** Add <b>conn</b> to the array of connections that we can poll on. The * connection's socket must be set; the connection starts out * non-reading and non-writing. @@ -351,6 +370,12 @@ connection_remove(connection_t *conn) (int)conn->s, conn_type_to_string(conn->type), smartlist_len(connection_array)); + if (conn->type == CONN_TYPE_AP && conn->socket_family == AF_UNIX) { + log_info(LD_NET, "Closing SOCKS SocksSocket connection"); + } + + control_event_conn_bandwidth(conn); + tor_assert(conn->conn_array_index >= 0); current_index = conn->conn_array_index; connection_unregister_events(conn); /* This is redundant, but cheap. */ @@ -414,6 +439,19 @@ connection_unlink(connection_t *conn) connection_free(conn); } +/** Initialize the global connection list, closeable connection list, + * and active connection list. */ +STATIC void +init_connection_lists(void) +{ + if (!connection_array) + connection_array = smartlist_new(); + if (!closeable_connection_lst) + closeable_connection_lst = smartlist_new(); + if (!active_linked_connection_lst) + active_linked_connection_lst = smartlist_new(); +} + /** Schedule <b>conn</b> to be closed. **/ void add_connection_to_closeable_list(connection_t *conn) @@ -438,8 +476,7 @@ connection_in_array(connection_t *conn) return smartlist_contains(connection_array, conn); } -/** Set <b>*array</b> to an array of all connections, and <b>*n</b> - * to the length of the array. <b>*array</b> and <b>*n</b> must not +/** Set <b>*array</b> to an array of all connections. <b>*array</b> must not * be modified. */ smartlist_t * @@ -452,15 +489,15 @@ get_connection_array(void) /** Provides the traffic read and written over the life of the process. */ -uint64_t -get_bytes_read(void) +MOCK_IMPL(uint64_t, +get_bytes_read,(void)) { return stats_n_bytes_read; } /* DOCDOC get_bytes_written */ -uint64_t -get_bytes_written(void) +MOCK_IMPL(uint64_t, +get_bytes_written,(void)) { return stats_n_bytes_written; } @@ -506,9 +543,49 @@ connection_is_reading(connection_t *conn) (conn->read_event && event_pending(conn->read_event, EV_READ, NULL)); } +/** Check whether <b>conn</b> is correct in having (or not having) a + * read/write event (passed in <b>ev</b>). On success, return 0. On failure, + * log a warning and return -1. */ +static int +connection_check_event(connection_t *conn, struct event *ev) +{ + int bad; + + if (conn->type == CONN_TYPE_AP && TO_EDGE_CONN(conn)->is_dns_request) { + /* DNS requests which we launch through the dnsserv.c module do not have + * any underlying socket or any underlying linked connection, so they + * shouldn't have any attached events either. + */ + bad = ev != NULL; + } else { + /* Everytyhing else should have an underlying socket, or a linked + * connection (which is also tracked with a read_event/write_event pair). + */ + bad = ev == NULL; + } + + if (bad) { + log_warn(LD_BUG, "Event missing on connection %p [%s;%s]. " + "socket=%d. linked=%d. " + "is_dns_request=%d. Marked_for_close=%s:%d", + conn, + conn_type_to_string(conn->type), + conn_state_to_string(conn->type, conn->state), + (int)conn->s, (int)conn->linked, + (conn->type == CONN_TYPE_AP && + TO_EDGE_CONN(conn)->is_dns_request), + conn->marked_for_close_file ? conn->marked_for_close_file : "-", + conn->marked_for_close + ); + log_backtrace(LOG_WARN, LD_BUG, "Backtrace attached."); + return -1; + } + return 0; +} + /** Tell the main loop to stop notifying <b>conn</b> of any read events. */ -void -connection_stop_reading(connection_t *conn) +MOCK_IMPL(void, +connection_stop_reading,(connection_t *conn)) { tor_assert(conn); @@ -517,7 +594,9 @@ connection_stop_reading(connection_t *conn) return; }); - tor_assert(conn->read_event); + if (connection_check_event(conn, conn->read_event) < 0) { + return; + } if (conn->linked) { conn->reading_from_linked_conn = 0; @@ -532,8 +611,8 @@ connection_stop_reading(connection_t *conn) } /** Tell the main loop to start notifying <b>conn</b> of any read events. */ -void -connection_start_reading(connection_t *conn) +MOCK_IMPL(void, +connection_start_reading,(connection_t *conn)) { tor_assert(conn); @@ -542,7 +621,9 @@ connection_start_reading(connection_t *conn) return; }); - tor_assert(conn->read_event); + if (connection_check_event(conn, conn->read_event) < 0) { + return; + } if (conn->linked) { conn->reading_from_linked_conn = 1; @@ -572,8 +653,8 @@ connection_is_writing(connection_t *conn) } /** Tell the main loop to stop notifying <b>conn</b> of any write events. */ -void -connection_stop_writing(connection_t *conn) +MOCK_IMPL(void, +connection_stop_writing,(connection_t *conn)) { tor_assert(conn); @@ -582,7 +663,9 @@ connection_stop_writing(connection_t *conn) return; }); - tor_assert(conn->write_event); + if (connection_check_event(conn, conn->write_event) < 0) { + return; + } if (conn->linked) { conn->writing_to_linked_conn = 0; @@ -598,8 +681,8 @@ connection_stop_writing(connection_t *conn) } /** Tell the main loop to start notifying <b>conn</b> of any write events. */ -void -connection_start_writing(connection_t *conn) +MOCK_IMPL(void, +connection_start_writing,(connection_t *conn)) { tor_assert(conn); @@ -608,7 +691,9 @@ connection_start_writing(connection_t *conn) return; }); - tor_assert(conn->write_event); + if (connection_check_event(conn, conn->write_event) < 0) { + return; + } if (conn->linked) { conn->writing_to_linked_conn = 1; @@ -641,6 +726,19 @@ connection_should_read_from_linked_conn(connection_t *conn) return 0; } +/** If we called event_base_loop() and told it to never stop until it + * runs out of events, now we've changed our mind: tell it we want it to + * finish. */ +void +tell_event_loop_to_finish(void) +{ + if (!called_loop_once) { + struct timeval tv = { 0, 0 }; + tor_event_base_loopexit(tor_libevent_get_base(), &tv); + called_loop_once = 1; /* hack to avoid adding more exit events */ + } +} + /** Helper: Tell the main loop to begin reading bytes into <b>conn</b> from * its linked connection, if it is not doing so already. Called by * connection_start_reading and connection_start_writing as appropriate. */ @@ -653,14 +751,10 @@ connection_start_reading_from_linked_conn(connection_t *conn) if (!conn->active_on_link) { conn->active_on_link = 1; smartlist_add(active_linked_connection_lst, conn); - if (!called_loop_once) { - /* This is the first event on the list; we won't be in LOOP_ONCE mode, - * so we need to make sure that the event_base_loop() actually exits at - * the end of its run through the current connections and lets us - * activate read events for linked connections. */ - struct timeval tv = { 0, 0 }; - tor_event_base_loopexit(tor_libevent_get_base(), &tv); - } + /* make sure that the event_base_loop() function exits at + * the end of its run through the current connections, so we can + * activate read events for linked connections. */ + tell_event_loop_to_finish(); } else { tor_assert(smartlist_contains(active_linked_connection_lst, conn)); } @@ -687,7 +781,7 @@ connection_stop_reading_from_linked_conn(connection_t *conn) } /** Close all connections that have been scheduled to get closed. */ -static void +STATIC void close_closeable_connections(void) { int i; @@ -883,18 +977,6 @@ conn_close_if_marked(int i) * would make much more sense to react in * connection_handle_read_impl, or to just stop reading in * mark_and_flush */ -#if 0 -#define MARKED_READING_RATE 180 - static ratelim_t marked_read_lim = RATELIM_INIT(MARKED_READING_RATE); - char *m; - if ((m = rate_limit_log(&marked_read_lim, now))) { - log_warn(LD_BUG, "Marked connection (fd %d, type %s, state %s) " - "is still reading; that shouldn't happen.%s", - (int)conn->s, conn_type_to_string(conn->type), - conn_state_to_string(conn->type, conn->state), m); - tor_free(m); - } -#endif conn->read_blocked_on_bw = 1; connection_stop_reading(conn); } @@ -902,16 +984,7 @@ conn_close_if_marked(int i) return 0; } if (connection_wants_to_flush(conn)) { - int severity; - if (conn->type == CONN_TYPE_EXIT || - (conn->type == CONN_TYPE_OR && server_mode(get_options())) || - (conn->type == CONN_TYPE_DIR && conn->purpose == DIR_PURPOSE_SERVER)) - severity = LOG_INFO; - else - severity = LOG_NOTICE; - /* XXXX Maybe allow this to happen a certain amount per hour; it usually - * is meaningless. */ - log_fn(severity, LD_NET, "We stalled too much while trying to write %d " + log_fn(LOG_INFO, LD_NET, "We stalled too much while trying to write %d " "bytes to address %s. If this happens a lot, either " "something is wrong with your network connection, or " "something is wrong with theirs. " @@ -931,19 +1004,18 @@ conn_close_if_marked(int i) return 1; } -/** We've just tried every dirserver we know about, and none of - * them were reachable. Assume the network is down. Change state - * so next time an application connection arrives we'll delay it - * and try another directory fetch. Kill off all the circuit_wait - * streams that are waiting now, since they will all timeout anyway. +/** Implementation for directory_all_unreachable. This is done in a callback, + * since otherwise it would complicate Tor's control-flow graph beyond all + * reason. */ -void -directory_all_unreachable(time_t now) +static void +directory_all_unreachable_cb(evutil_socket_t fd, short event, void *arg) { - connection_t *conn; - (void)now; + (void)fd; + (void)event; + (void)arg; - stats_n_seconds_working=0; /* reset it */ + connection_t *conn; while ((conn = connection_get_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT))) { @@ -956,18 +1028,43 @@ directory_all_unreachable(time_t now) connection_mark_unattached_ap(entry_conn, END_STREAM_REASON_NET_UNREACHABLE); } - control_event_general_status(LOG_ERR, "DIR_ALL_UNREACHABLE"); + control_event_general_error("DIR_ALL_UNREACHABLE"); +} + +static struct event *directory_all_unreachable_cb_event = NULL; + +/** We've just tried every dirserver we know about, and none of + * them were reachable. Assume the network is down. Change state + * so next time an application connection arrives we'll delay it + * and try another directory fetch. Kill off all the circuit_wait + * streams that are waiting now, since they will all timeout anyway. + */ +void +directory_all_unreachable(time_t now) +{ + (void)now; + + stats_n_seconds_working=0; /* reset it */ + + if (!directory_all_unreachable_cb_event) { + directory_all_unreachable_cb_event = + tor_event_new(tor_libevent_get_base(), + -1, EV_READ, directory_all_unreachable_cb, NULL); + tor_assert(directory_all_unreachable_cb_event); + } + + event_active(directory_all_unreachable_cb_event, EV_READ, 1); } /** This function is called whenever we successfully pull down some new * network statuses or server descriptors. */ void -directory_info_has_arrived(time_t now, int from_cache) +directory_info_has_arrived(time_t now, int from_cache, int suppress_logs) { const or_options_t *options = get_options(); if (!router_have_minimum_dir_info()) { - int quiet = from_cache || + int quiet = suppress_logs || from_cache || directory_too_idle_to_fetch_descriptors(options, now); tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR, "I learned some more directory information, but not enough to " @@ -989,19 +1086,10 @@ directory_info_has_arrived(time_t now, int from_cache) } if (server_mode(options) && !net_is_disabled() && !from_cache && - (can_complete_circuit || !any_predicted_circuits(now))) + (have_completed_a_circuit() || !any_predicted_circuits(now))) consider_testing_reachability(1, 1); } -/** How long do we wait before killing OR connections with no circuits? - * In Tor versions up to 0.2.1.25 and 0.2.2.12-alpha, we waited 15 minutes - * before cancelling these connections, which caused fast relays to accrue - * many many idle connections. Hopefully 3 minutes is low enough that - * it kills most idle connections, without being so low that we cause - * clients to bounce on and off. - */ -#define IDLE_OR_CONN_TIMEOUT 180 - /** Perform regular maintenance tasks for a single connection. This * function gets run once per second per connection by run_scheduled_events. */ @@ -1012,6 +1100,8 @@ run_connection_housekeeping(int i, time_t now) connection_t *conn = smartlist_get(connection_array, i); const or_options_t *options = get_options(); or_connection_t *or_conn; + channel_t *chan = NULL; + int have_any_circuits; int past_keepalive = now >= conn->timestamp_lastwritten + options->KeepalivePeriod; @@ -1028,9 +1118,11 @@ run_connection_housekeeping(int i, time_t now) * if a server or received if a client) for 5 min */ if (conn->type == CONN_TYPE_DIR && ((DIR_CONN_IS_SERVER(conn) && - conn->timestamp_lastwritten + DIR_CONN_MAX_STALL < now) || + conn->timestamp_lastwritten + + options->TestingDirConnectionMaxStall < now) || (!DIR_CONN_IS_SERVER(conn) && - conn->timestamp_lastread + DIR_CONN_MAX_STALL < now))) { + conn->timestamp_lastread + + options->TestingDirConnectionMaxStall < now))) { log_info(LD_DIR,"Expiring wedged directory conn (fd %d, purpose %d)", (int)conn->s, conn->purpose); /* This check is temporary; it's to let us know whether we should consider @@ -1059,8 +1151,18 @@ run_connection_housekeeping(int i, time_t now) tor_assert(conn->outbuf); #endif + chan = TLS_CHAN_TO_BASE(or_conn->chan); + tor_assert(chan); + + if (channel_num_circuits(chan) != 0) { + have_any_circuits = 1; + chan->timestamp_last_had_circuits = now; + } else { + have_any_circuits = 0; + } + if (channel_is_bad_for_new_circs(TLS_CHAN_TO_BASE(or_conn->chan)) && - !connection_or_get_num_circuits(or_conn)) { + ! have_any_circuits) { /* It's bad for new circuits, and has no unmarked circuits on it: * mark it now. */ log_info(LD_OR, @@ -1079,19 +1181,22 @@ run_connection_housekeeping(int i, time_t now) connection_or_close_normally(TO_OR_CONN(conn), 0); } } else if (we_are_hibernating() && - !connection_or_get_num_circuits(or_conn) && + ! have_any_circuits && !connection_get_outbuf_len(conn)) { /* We're hibernating, there's no circuits, and nothing to flush.*/ log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) " "[Hibernating or exiting].", (int)conn->s,conn->address, conn->port); connection_or_close_normally(TO_OR_CONN(conn), 1); - } else if (!connection_or_get_num_circuits(or_conn) && - now >= or_conn->timestamp_last_added_nonpadding + - IDLE_OR_CONN_TIMEOUT) { + } else if (!have_any_circuits && + now - or_conn->idle_timeout >= + chan->timestamp_last_had_circuits) { log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) " - "[idle %d].", (int)conn->s,conn->address, conn->port, - (int)(now - or_conn->timestamp_last_added_nonpadding)); + "[no circuits for %d; timeout %d; %scanonical].", + (int)conn->s, conn->address, conn->port, + (int)(now - chan->timestamp_last_had_circuits), + or_conn->idle_timeout, + or_conn->is_canonical ? "" : "non"); connection_or_close_normally(TO_OR_CONN(conn), 0); } else if ( now >= or_conn->timestamp_lastempty + options->KeepalivePeriod*10 && @@ -1143,55 +1248,228 @@ get_signewnym_epoch(void) return newnym_epoch; } +/** True iff we have initialized all the members of <b>periodic_events</b>. + * Used to prevent double-initialization. */ +static int periodic_events_initialized = 0; + +/* Declare all the timer callback functions... */ +#undef CALLBACK +#define CALLBACK(name) \ + static int name ## _callback(time_t, const or_options_t *) +CALLBACK(rotate_onion_key); +CALLBACK(check_ed_keys); +CALLBACK(launch_descriptor_fetches); +CALLBACK(reset_descriptor_failures); +CALLBACK(rotate_x509_certificate); +CALLBACK(add_entropy); +CALLBACK(launch_reachability_tests); +CALLBACK(downrate_stability); +CALLBACK(save_stability); +CALLBACK(check_authority_cert); +CALLBACK(check_expired_networkstatus); +CALLBACK(write_stats_file); +CALLBACK(record_bridge_stats); +CALLBACK(clean_caches); +CALLBACK(rend_cache_failure_clean); +CALLBACK(retry_dns); +CALLBACK(check_descriptor); +CALLBACK(check_for_reachability_bw); +CALLBACK(fetch_networkstatus); +CALLBACK(retry_listeners); +CALLBACK(expire_old_ciruits_serverside); +CALLBACK(check_dns_honesty); +CALLBACK(write_bridge_ns); +CALLBACK(check_fw_helper_app); +CALLBACK(heartbeat); + +#undef CALLBACK + +/* Now we declare an array of periodic_event_item_t for each periodic event */ +#define CALLBACK(name) PERIODIC_EVENT(name) + +static periodic_event_item_t periodic_events[] = { + CALLBACK(rotate_onion_key), + CALLBACK(check_ed_keys), + CALLBACK(launch_descriptor_fetches), + CALLBACK(reset_descriptor_failures), + CALLBACK(rotate_x509_certificate), + CALLBACK(add_entropy), + CALLBACK(launch_reachability_tests), + CALLBACK(downrate_stability), + CALLBACK(save_stability), + CALLBACK(check_authority_cert), + CALLBACK(check_expired_networkstatus), + CALLBACK(write_stats_file), + CALLBACK(record_bridge_stats), + CALLBACK(clean_caches), + CALLBACK(rend_cache_failure_clean), + CALLBACK(retry_dns), + CALLBACK(check_descriptor), + CALLBACK(check_for_reachability_bw), + CALLBACK(fetch_networkstatus), + CALLBACK(retry_listeners), + CALLBACK(expire_old_ciruits_serverside), + CALLBACK(check_dns_honesty), + CALLBACK(write_bridge_ns), + CALLBACK(check_fw_helper_app), + CALLBACK(heartbeat), + END_OF_PERIODIC_EVENTS +}; +#undef CALLBACK + +/* These are pointers to members of periodic_events[] that are used to + * implement particular callbacks. We keep them separate here so that we + * can access them by name. We also keep them inside periodic_events[] + * so that we can implement "reset all timers" in a reasonable way. */ +static periodic_event_item_t *check_descriptor_event=NULL; +static periodic_event_item_t *fetch_networkstatus_event=NULL; +static periodic_event_item_t *launch_descriptor_fetches_event=NULL; +static periodic_event_item_t *check_dns_honesty_event=NULL; + +/** Reset all the periodic events so we'll do all our actions again as if we + * just started up. + * Useful if our clock just moved back a long time from the future, + * so we don't wait until that future arrives again before acting. + */ +void +reset_all_main_loop_timers(void) +{ + int i; + for (i = 0; periodic_events[i].name; ++i) { + periodic_event_reschedule(&periodic_events[i]); + } +} + +/** Return the member of periodic_events[] whose name is <b>name</b>. + * Return NULL if no such event is found. + */ +static periodic_event_item_t * +find_periodic_event(const char *name) +{ + int i; + for (i = 0; periodic_events[i].name; ++i) { + if (strcmp(name, periodic_events[i].name) == 0) + return &periodic_events[i]; + } + return NULL; +} + +/** Helper, run one second after setup: + * Initializes all members of periodic_events and starts them running. + * + * (We do this one second after setup for backward-compatibility reasons; + * it might not actually be necessary.) */ +static void +initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data) +{ + (void) fd; + (void) events; + (void) data; + int i; + for (i = 0; periodic_events[i].name; ++i) { + periodic_event_launch(&periodic_events[i]); + } +} + +/** Set up all the members of periodic_events[], and configure them all to be + * launched from a callback. */ +STATIC void +initialize_periodic_events(void) +{ + tor_assert(periodic_events_initialized == 0); + periodic_events_initialized = 1; + + int i; + for (i = 0; periodic_events[i].name; ++i) { + periodic_event_setup(&periodic_events[i]); + } + +#define NAMED_CALLBACK(name) \ + STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END + + NAMED_CALLBACK(check_descriptor); + NAMED_CALLBACK(fetch_networkstatus); + NAMED_CALLBACK(launch_descriptor_fetches); + NAMED_CALLBACK(check_dns_honesty); + + struct timeval one_second = { 1, 0 }; + event_base_once(tor_libevent_get_base(), -1, EV_TIMEOUT, + initialize_periodic_events_cb, NULL, + &one_second); +} + +STATIC void +teardown_periodic_events(void) +{ + int i; + for (i = 0; periodic_events[i].name; ++i) { + periodic_event_destroy(&periodic_events[i]); + } +} + +/** + * Update our schedule so that we'll check whether we need to update our + * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL + * seconds. + */ +void +reschedule_descriptor_update_check(void) +{ + tor_assert(check_descriptor_event); + periodic_event_reschedule(check_descriptor_event); +} + +/** + * Update our schedule so that we'll check whether we need to fetch directory + * info immediately. + */ +void +reschedule_directory_downloads(void) +{ + tor_assert(fetch_networkstatus_event); + tor_assert(launch_descriptor_fetches_event); + + periodic_event_reschedule(fetch_networkstatus_event); + periodic_event_reschedule(launch_descriptor_fetches_event); +} + +#define LONGEST_TIMER_PERIOD (30 * 86400) +/** Helper: Return the number of seconds between <b>now</b> and <b>next</b>, + * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */ +static inline int +safe_timer_diff(time_t now, time_t next) +{ + if (next > now) { + /* There were no computers at signed TIME_MIN (1902 on 32-bit systems), + * and nothing that could run Tor. It's a bug if 'next' is around then. + * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big + * Bang. We cannot extrapolate past a singularity, but there was probably + * nothing that could run Tor then, either. + **/ + tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD); + + if (next - LONGEST_TIMER_PERIOD > now) + return LONGEST_TIMER_PERIOD; + return (int)(next - now); + } else { + return 1; + } +} + /** Perform regular maintenance tasks. This function gets run once per * second by second_elapsed_callback(). */ static void run_scheduled_events(time_t now) { - static time_t last_rotated_x509_certificate = 0; - static time_t time_to_check_v3_certificate = 0; - static time_t time_to_check_listeners = 0; - static time_t time_to_check_descriptor = 0; - static time_t time_to_shrink_memory = 0; - static time_t time_to_try_getting_descriptors = 0; - static time_t time_to_reset_descriptor_failures = 0; - static time_t time_to_add_entropy = 0; - static time_t time_to_write_bridge_status_file = 0; - static time_t time_to_downrate_stability = 0; - static time_t time_to_save_stability = 0; - static time_t time_to_clean_caches = 0; - static time_t time_to_recheck_bandwidth = 0; - static time_t time_to_check_for_expired_networkstatus = 0; - static time_t time_to_write_stats_files = 0; - static time_t time_to_write_bridge_stats = 0; - static time_t time_to_check_port_forwarding = 0; - static time_t time_to_launch_reachability_tests = 0; - static int should_init_bridge_stats = 1; - static time_t time_to_retry_dns_init = 0; - static time_t time_to_next_heartbeat = 0; const or_options_t *options = get_options(); - int is_server = server_mode(options); - int i; - int have_dir_info; - - /** 0. See if we've been asked to shut down and our timeout has + /* 0. See if we've been asked to shut down and our timeout has * expired; or if our bandwidth limits are exhausted and we * should hibernate; or if it's time to wake up from hibernation. */ consider_hibernation(now); -#if 0 - { - static time_t nl_check_time = 0; - if (nl_check_time <= now) { - nodelist_assert_ok(); - nl_check_time = now + 30; - } - } -#endif - /* 0b. If we've deferred a signewnym, make sure it gets handled * eventually. */ if (signewnym_is_pending && @@ -1203,201 +1481,378 @@ run_scheduled_events(time_t now) /* 0c. If we've deferred log messages for the controller, handle them now */ flush_pending_log_callbacks(); - /** 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys, + if (options->UseBridges && !options->DisableNetwork) { + fetch_bridge_descriptors(options, now); + } + + if (accounting_is_enabled(options)) { + accounting_run_housekeeping(now); + } + + if (authdir_mode_v3(options)) { + dirvote_act(options, now); + } + + /* 3a. Every second, we examine pending circuits and prune the + * ones which have been pending for more than a few seconds. + * We do this before step 4, so it can try building more if + * it's not comfortable with the number of available circuits. + */ + /* (If our circuit build timeout can ever become lower than a second (which + * it can't, currently), we should do this more often.) */ + circuit_expire_building(); + + /* 3b. Also look at pending streams and prune the ones that 'began' + * a long time ago but haven't gotten a 'connected' yet. + * Do this before step 4, so we can put them back into pending + * state to be picked up by the new circuit. + */ + connection_ap_expire_beginning(); + + /* 3c. And expire connections that we've held open for too long. + */ + connection_expire_held_open(); + + /* 4. Every second, we try a new circuit if there are no valid + * circuits. Every NewCircuitPeriod seconds, we expire circuits + * that became dirty more than MaxCircuitDirtiness seconds ago, + * and we make a new circ if there are no clean circuits. + */ + const int have_dir_info = router_have_minimum_dir_info(); + if (have_dir_info && !net_is_disabled()) { + circuit_build_needed_circs(now); + } else { + circuit_expire_old_circs_as_needed(now); + } + + if (!net_is_disabled()) { + /* This is usually redundant with circuit_build_needed_circs() above, + * but it is very fast when there is no work to do. */ + connection_ap_attach_pending(0); + } + + /* 5. We do housekeeping for each connection... */ + connection_or_set_bad_connections(NULL, 0); + int i; + for (i=0;i<smartlist_len(connection_array);i++) { + run_connection_housekeeping(i, now); + } + + /* 6. And remove any marked circuits... */ + circuit_close_all_marked(); + + /* 7. And upload service descriptors if necessary. */ + if (have_completed_a_circuit() && !net_is_disabled()) { + rend_consider_services_upload(now); + rend_consider_descriptor_republication(); + } + + /* 8. and blow away any connections that need to die. have to do this now, + * because if we marked a conn for close and left its socket -1, then + * we'll pass it to poll/select and bad things will happen. + */ + close_closeable_connections(); + + /* 8b. And if anything in our state is ready to get flushed to disk, we + * flush it. */ + or_state_save(now); + + /* 8c. Do channel cleanup just like for connections */ + channel_run_cleanup(); + channel_listener_run_cleanup(); + + /* 11b. check pending unconfigured managed proxies */ + if (!net_is_disabled() && pt_proxies_configuration_pending()) + pt_configure_remaining_proxies(); +} + +static int +rotate_onion_key_callback(time_t now, const or_options_t *options) +{ + /* 1a. Every MIN_ONION_KEY_LIFETIME seconds, rotate the onion keys, * shut down and restart all cpuworkers, and update the directory if * necessary. */ - if (is_server && - get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME < now) { + if (server_mode(options)) { + time_t rotation_time = get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME; + if (rotation_time > now) { + return safe_timer_diff(now, rotation_time); + } + log_info(LD_GENERAL,"Rotating onion key."); rotate_onion_key(); - cpuworkers_rotate(); + cpuworkers_rotate_keyinfo(); if (router_rebuild_descriptor(1)<0) { log_info(LD_CONFIG, "Couldn't rebuild router descriptor"); } if (advertised_server_mode() && !options->DisableNetwork) router_upload_dir_desc_to_dirservers(0); + return MIN_ONION_KEY_LIFETIME; } + return PERIODIC_EVENT_NO_UPDATE; +} - if (!options->DisableNetwork && time_to_try_getting_descriptors < now) { - update_all_descriptor_downloads(now); - update_extrainfo_downloads(now); - if (router_have_minimum_dir_info()) - time_to_try_getting_descriptors = now + LAZY_DESCRIPTOR_RETRY_INTERVAL; - else - time_to_try_getting_descriptors = now + GREEDY_DESCRIPTOR_RETRY_INTERVAL; +static int +check_ed_keys_callback(time_t now, const or_options_t *options) +{ + if (server_mode(options)) { + if (should_make_new_ed_keys(options, now)) { + if (load_ed_keys(options, now) < 0 || + generate_ed_link_cert(options, now)) { + log_err(LD_OR, "Unable to update Ed25519 keys! Exiting."); + tor_cleanup(); + exit(0); + } + } + return 30; } + return PERIODIC_EVENT_NO_UPDATE; +} - if (time_to_reset_descriptor_failures < now) { - router_reset_descriptor_download_failures(); - time_to_reset_descriptor_failures = - now + DESCRIPTOR_FAILURE_RESET_INTERVAL; - } +static int +launch_descriptor_fetches_callback(time_t now, const or_options_t *options) +{ + if (should_delay_dir_fetches(options, NULL)) + return PERIODIC_EVENT_NO_UPDATE; - if (options->UseBridges) - fetch_bridge_descriptors(options, now); + update_all_descriptor_downloads(now); + update_extrainfo_downloads(now); + if (router_have_minimum_dir_info()) + return LAZY_DESCRIPTOR_RETRY_INTERVAL; + else + return GREEDY_DESCRIPTOR_RETRY_INTERVAL; +} - /** 1b. Every MAX_SSL_KEY_LIFETIME_INTERNAL seconds, we change our +static int +reset_descriptor_failures_callback(time_t now, const or_options_t *options) +{ + (void)now; + (void)options; + router_reset_descriptor_download_failures(); + return DESCRIPTOR_FAILURE_RESET_INTERVAL; +} + +static int +rotate_x509_certificate_callback(time_t now, const or_options_t *options) +{ + static int first = 1; + (void)now; + (void)options; + if (first) { + first = 0; + return MAX_SSL_KEY_LIFETIME_INTERNAL; + } + + /* 1b. Every MAX_SSL_KEY_LIFETIME_INTERNAL seconds, we change our * TLS context. */ - if (!last_rotated_x509_certificate) - last_rotated_x509_certificate = now; - if (last_rotated_x509_certificate+MAX_SSL_KEY_LIFETIME_INTERNAL < now) { - log_info(LD_GENERAL,"Rotating tls context."); - if (router_initialize_tls_context() < 0) { - log_warn(LD_BUG, "Error reinitializing TLS context"); - /* XXX is it a bug here, that we just keep going? -RD */ - } - last_rotated_x509_certificate = now; - /* We also make sure to rotate the TLS connections themselves if they've - * been up for too long -- but that's done via is_bad_for_new_circs in - * connection_run_housekeeping() above. */ + log_info(LD_GENERAL,"Rotating tls context."); + if (router_initialize_tls_context() < 0) { + log_warn(LD_BUG, "Error reinitializing TLS context"); + tor_assert(0); } - if (time_to_add_entropy < now) { - if (time_to_add_entropy) { - /* We already seeded once, so don't die on failure. */ - crypto_seed_rng(0); - } -/** How often do we add more entropy to OpenSSL's RNG pool? */ -#define ENTROPY_INTERVAL (60*60) - time_to_add_entropy = now + ENTROPY_INTERVAL; + /* We also make sure to rotate the TLS connections themselves if they've + * been up for too long -- but that's done via is_bad_for_new_circs in + * run_connection_housekeeping() above. */ + return MAX_SSL_KEY_LIFETIME_INTERNAL; +} + +static int +add_entropy_callback(time_t now, const or_options_t *options) +{ + (void)now; + (void)options; + /* We already seeded once, so don't die on failure. */ + if (crypto_seed_rng() < 0) { + log_warn(LD_GENERAL, "Tried to re-seed RNG, but failed. We already " + "seeded once, though, so we won't exit here."); } - /** 1c. If we have to change the accounting interval or record - * bandwidth used in this accounting interval, do so. */ - if (accounting_is_enabled(options)) - accounting_run_housekeeping(now); + /** How often do we add more entropy to OpenSSL's RNG pool? */ +#define ENTROPY_INTERVAL (60*60) + return ENTROPY_INTERVAL; +} - if (time_to_launch_reachability_tests < now && - (authdir_mode_tests_reachability(options)) && - !net_is_disabled()) { - time_to_launch_reachability_tests = now + REACHABILITY_TEST_INTERVAL; +static int +launch_reachability_tests_callback(time_t now, const or_options_t *options) +{ + if (authdir_mode_tests_reachability(options) && + !net_is_disabled()) { /* try to determine reachability of the other Tor relays */ dirserv_test_reachability(now); } + return REACHABILITY_TEST_INTERVAL; +} - /** 1d. Periodically, we discount older stability information so that new +static int +downrate_stability_callback(time_t now, const or_options_t *options) +{ + (void)options; + /* 1d. Periodically, we discount older stability information so that new * stability info counts more, and save the stability information to disk as * appropriate. */ - if (time_to_downrate_stability < now) - time_to_downrate_stability = rep_hist_downrate_old_runs(now); + time_t next = rep_hist_downrate_old_runs(now); + return safe_timer_diff(now, next); +} + +static int +save_stability_callback(time_t now, const or_options_t *options) +{ if (authdir_mode_tests_reachability(options)) { - if (time_to_save_stability < now) { - if (time_to_save_stability && rep_hist_record_mtbf_data(now, 1)<0) { - log_warn(LD_GENERAL, "Couldn't store mtbf data."); - } -#define SAVE_STABILITY_INTERVAL (30*60) - time_to_save_stability = now + SAVE_STABILITY_INTERVAL; + if (rep_hist_record_mtbf_data(now, 1)<0) { + log_warn(LD_GENERAL, "Couldn't store mtbf data."); } } +#define SAVE_STABILITY_INTERVAL (30*60) + return SAVE_STABILITY_INTERVAL; +} +static int +check_authority_cert_callback(time_t now, const or_options_t *options) +{ + (void)now; + (void)options; /* 1e. Periodically, if we're a v3 authority, we check whether our cert is * close to expiring and warn the admin if it is. */ - if (time_to_check_v3_certificate < now) { - v3_authority_check_key_expiry(); + v3_authority_check_key_expiry(); #define CHECK_V3_CERTIFICATE_INTERVAL (5*60) - time_to_check_v3_certificate = now + CHECK_V3_CERTIFICATE_INTERVAL; - } + return CHECK_V3_CERTIFICATE_INTERVAL; +} +static int +check_expired_networkstatus_callback(time_t now, const or_options_t *options) +{ + (void)options; /* 1f. Check whether our networkstatus has expired. */ - if (time_to_check_for_expired_networkstatus < now) { - networkstatus_t *ns = networkstatus_get_latest_consensus(); - /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in - * networkstatus_get_reasonably_live_consensus(), but that value is way - * way too high. Arma: is the bridge issue there resolved yet? -NM */ + networkstatus_t *ns = networkstatus_get_latest_consensus(); + /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in + * networkstatus_get_reasonably_live_consensus(), but that value is way + * way too high. Arma: is the bridge issue there resolved yet? -NM */ #define NS_EXPIRY_SLOP (24*60*60) - if (ns && ns->valid_until < now+NS_EXPIRY_SLOP && - router_have_minimum_dir_info()) { - router_dir_info_changed(); - } -#define CHECK_EXPIRED_NS_INTERVAL (2*60) - time_to_check_for_expired_networkstatus = now + CHECK_EXPIRED_NS_INTERVAL; + if (ns && ns->valid_until < now+NS_EXPIRY_SLOP && + router_have_minimum_dir_info()) { + router_dir_info_changed(); } +#define CHECK_EXPIRED_NS_INTERVAL (2*60) + return CHECK_EXPIRED_NS_INTERVAL; +} +static int +write_stats_file_callback(time_t now, const or_options_t *options) +{ /* 1g. Check whether we should write statistics to disk. */ - if (time_to_write_stats_files < now) { #define CHECK_WRITE_STATS_INTERVAL (60*60) - time_t next_time_to_write_stats_files = (time_to_write_stats_files > 0 ? - time_to_write_stats_files : now) + CHECK_WRITE_STATS_INTERVAL; - if (options->CellStatistics) { - time_t next_write = - rep_hist_buffer_stats_write(time_to_write_stats_files); - if (next_write && next_write < next_time_to_write_stats_files) - next_time_to_write_stats_files = next_write; - } - if (options->DirReqStatistics) { - time_t next_write = geoip_dirreq_stats_write(time_to_write_stats_files); - if (next_write && next_write < next_time_to_write_stats_files) - next_time_to_write_stats_files = next_write; - } - if (options->EntryStatistics) { - time_t next_write = geoip_entry_stats_write(time_to_write_stats_files); - if (next_write && next_write < next_time_to_write_stats_files) - next_time_to_write_stats_files = next_write; - } - if (options->ExitPortStatistics) { - time_t next_write = rep_hist_exit_stats_write(time_to_write_stats_files); - if (next_write && next_write < next_time_to_write_stats_files) - next_time_to_write_stats_files = next_write; - } - if (options->ConnDirectionStatistics) { - time_t next_write = rep_hist_conn_stats_write(time_to_write_stats_files); - if (next_write && next_write < next_time_to_write_stats_files) - next_time_to_write_stats_files = next_write; - } - if (options->BridgeAuthoritativeDir) { - time_t next_write = rep_hist_desc_stats_write(time_to_write_stats_files); - if (next_write && next_write < next_time_to_write_stats_files) - next_time_to_write_stats_files = next_write; - } - time_to_write_stats_files = next_time_to_write_stats_files; - } + time_t next_time_to_write_stats_files = now + CHECK_WRITE_STATS_INTERVAL; + if (options->CellStatistics) { + time_t next_write = + rep_hist_buffer_stats_write(now); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->DirReqStatistics) { + time_t next_write = geoip_dirreq_stats_write(now); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->EntryStatistics) { + time_t next_write = geoip_entry_stats_write(now); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->HiddenServiceStatistics) { + time_t next_write = rep_hist_hs_stats_write(now); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->ExitPortStatistics) { + time_t next_write = rep_hist_exit_stats_write(now); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->ConnDirectionStatistics) { + time_t next_write = rep_hist_conn_stats_write(now); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + if (options->BridgeAuthoritativeDir) { + time_t next_write = rep_hist_desc_stats_write(now); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } + + return safe_timer_diff(now, next_time_to_write_stats_files); +} + +static int +record_bridge_stats_callback(time_t now, const or_options_t *options) +{ + static int should_init_bridge_stats = 1; /* 1h. Check whether we should write bridge statistics to disk. */ if (should_record_bridge_info(options)) { - if (time_to_write_bridge_stats < now) { - if (should_init_bridge_stats) { - /* (Re-)initialize bridge statistics. */ + if (should_init_bridge_stats) { + /* (Re-)initialize bridge statistics. */ geoip_bridge_stats_init(now); - time_to_write_bridge_stats = now + WRITE_STATS_INTERVAL; should_init_bridge_stats = 0; - } else { - /* Possibly write bridge statistics to disk and ask when to write - * them next time. */ - time_to_write_bridge_stats = geoip_bridge_stats_write( - time_to_write_bridge_stats); - } + return WRITE_STATS_INTERVAL; + } else { + /* Possibly write bridge statistics to disk and ask when to write + * them next time. */ + time_t next = geoip_bridge_stats_write(now); + return safe_timer_diff(now, next); } } else if (!should_init_bridge_stats) { /* Bridge mode was turned off. Ensure that stats are re-initialized * next time bridge mode is turned on. */ should_init_bridge_stats = 1; } + return PERIODIC_EVENT_NO_UPDATE; +} +static int +clean_caches_callback(time_t now, const or_options_t *options) +{ /* Remove old information from rephist and the rend cache. */ - if (time_to_clean_caches < now) { - rep_history_clean(now - options->RephistTrackTime); - rend_cache_clean(now); - rend_cache_clean_v2_descs_as_dir(now); - microdesc_cache_rebuild(NULL, 0); + rep_history_clean(now - options->RephistTrackTime); + rend_cache_clean(now, REND_CACHE_TYPE_CLIENT); + rend_cache_clean(now, REND_CACHE_TYPE_SERVICE); + rend_cache_clean_v2_descs_as_dir(now, 0); + microdesc_cache_rebuild(NULL, 0); #define CLEAN_CACHES_INTERVAL (30*60) - time_to_clean_caches = now + CLEAN_CACHES_INTERVAL; - } + return CLEAN_CACHES_INTERVAL; +} + +static int +rend_cache_failure_clean_callback(time_t now, const or_options_t *options) +{ + (void)options; + /* We don't keep entries that are more than five minutes old so we try to + * clean it as soon as we can since we want to make sure the client waits + * as little as possible for reachability reasons. */ + rend_cache_failure_clean(now); + return 30; +} +static int +retry_dns_callback(time_t now, const or_options_t *options) +{ + (void)now; #define RETRY_DNS_INTERVAL (10*60) /* If we're a server and initializing dns failed, retry periodically. */ - if (time_to_retry_dns_init < now) { - time_to_retry_dns_init = now + RETRY_DNS_INTERVAL; - if (is_server && has_dns_init_failed()) - dns_init(); - } + if (server_mode(options) && has_dns_init_failed()) + dns_init(); + return RETRY_DNS_INTERVAL; +} - /** 2. Periodically, we consider force-uploading our descriptor + /* 2. Periodically, we consider force-uploading our descriptor * (if we've passed our internal checks). */ +static int +check_descriptor_callback(time_t now, const or_options_t *options) +{ /** How often do we check whether part of our router info has changed in a * way that would require an upload? That includes checking whether our IP * address has changed. */ @@ -1405,186 +1860,184 @@ run_scheduled_events(time_t now) /* 2b. Once per minute, regenerate and upload the descriptor if the old * one is inaccurate. */ - if (time_to_check_descriptor < now && !options->DisableNetwork) { - static int dirport_reachability_count = 0; - time_to_check_descriptor = now + CHECK_DESCRIPTOR_INTERVAL; + if (!options->DisableNetwork) { check_descriptor_bandwidth_changed(now); check_descriptor_ipaddress_changed(now); mark_my_descriptor_dirty_if_too_old(now); consider_publishable_server(0); - /* also, check religiously for reachability, if it's within the first - * 20 minutes of our uptime. */ - if (is_server && - (can_complete_circuit || !any_predicted_circuits(now)) && - !we_are_hibernating()) { - if (stats_n_seconds_working < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { - consider_testing_reachability(1, dirport_reachability_count==0); - if (++dirport_reachability_count > 5) - dirport_reachability_count = 0; - } else if (time_to_recheck_bandwidth < now) { - /* If we haven't checked for 12 hours and our bandwidth estimate is - * low, do another bandwidth test. This is especially important for - * bridges, since they might go long periods without much use. */ - const routerinfo_t *me = router_get_my_routerinfo(); - if (time_to_recheck_bandwidth && me && - me->bandwidthcapacity < me->bandwidthrate && - me->bandwidthcapacity < 51200) { - reset_bandwidth_test(); - } -#define BANDWIDTH_RECHECK_INTERVAL (12*60*60) - time_to_recheck_bandwidth = now + BANDWIDTH_RECHECK_INTERVAL; - } - } - /* If any networkstatus documents are no longer recent, we need to * update all the descriptors' running status. */ - /* purge obsolete entries */ - networkstatus_v2_list_clean(now); /* Remove dead routers. */ + /* XXXX This doesn't belong here, but it was here in the pre- + * XXXX refactoring code. */ routerlist_remove_old_routers(); - - /* Also, once per minute, check whether we want to download any - * networkstatus documents. - */ - update_networkstatus_downloads(now); } - /** 2c. Let directory voting happen. */ - if (authdir_mode_v3(options)) - dirvote_act(options, now); + return CHECK_DESCRIPTOR_INTERVAL; +} - /** 3a. Every second, we examine pending circuits and prune the - * ones which have been pending for more than a few seconds. - * We do this before step 4, so it can try building more if - * it's not comfortable with the number of available circuits. - */ - /* (If our circuit build timeout can ever become lower than a second (which - * it can't, currently), we should do this more often.) */ - circuit_expire_building(); +static int +check_for_reachability_bw_callback(time_t now, const or_options_t *options) +{ + /* XXXX This whole thing was stuck in the middle of what is now + * XXXX check_descriptor_callback. I'm not sure it's right. */ - /** 3b. Also look at pending streams and prune the ones that 'began' - * a long time ago but haven't gotten a 'connected' yet. - * Do this before step 4, so we can put them back into pending - * state to be picked up by the new circuit. - */ - connection_ap_expire_beginning(); + static int dirport_reachability_count = 0; + /* also, check religiously for reachability, if it's within the first + * 20 minutes of our uptime. */ + if (server_mode(options) && + (have_completed_a_circuit() || !any_predicted_circuits(now)) && + !we_are_hibernating()) { + if (stats_n_seconds_working < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { + consider_testing_reachability(1, dirport_reachability_count==0); + if (++dirport_reachability_count > 5) + dirport_reachability_count = 0; + return 1; + } else { + /* If we haven't checked for 12 hours and our bandwidth estimate is + * low, do another bandwidth test. This is especially important for + * bridges, since they might go long periods without much use. */ + const routerinfo_t *me = router_get_my_routerinfo(); + static int first_time = 1; + if (!first_time && me && + me->bandwidthcapacity < me->bandwidthrate && + me->bandwidthcapacity < 51200) { + reset_bandwidth_test(); + } + first_time = 0; +#define BANDWIDTH_RECHECK_INTERVAL (12*60*60) + return BANDWIDTH_RECHECK_INTERVAL; + } + } + return CHECK_DESCRIPTOR_INTERVAL; +} - /** 3c. And expire connections that we've held open for too long. - */ - connection_expire_held_open(); +static int +fetch_networkstatus_callback(time_t now, const or_options_t *options) +{ + /* 2c. Every minute (or every second if TestingTorNetwork, or during + * client bootstrap), check whether we want to download any networkstatus + * documents. */ + + /* How often do we check whether we should download network status + * documents? */ + const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping( + now); + const int prefer_mirrors = !directory_fetches_from_authorities( + get_options()); + int networkstatus_dl_check_interval = 60; + /* check more often when testing, or when bootstrapping from mirrors + * (connection limits prevent too many connections being made) */ + if (options->TestingTorNetwork + || (we_are_bootstrapping && prefer_mirrors)) { + networkstatus_dl_check_interval = 1; + } + + if (should_delay_dir_fetches(options, NULL)) + return PERIODIC_EVENT_NO_UPDATE; + + update_networkstatus_downloads(now); + return networkstatus_dl_check_interval; +} - /** 3d. And every 60 seconds, we relaunch listeners if any died. */ - if (!net_is_disabled() && time_to_check_listeners < now) { +static int +retry_listeners_callback(time_t now, const or_options_t *options) +{ + (void)now; + (void)options; + /* 3d. And every 60 seconds, we relaunch listeners if any died. */ + if (!net_is_disabled()) { retry_all_listeners(NULL, NULL, 0); - time_to_check_listeners = now+60; + return 60; } + return PERIODIC_EVENT_NO_UPDATE; +} - /** 4. Every second, we try a new circuit if there are no valid - * circuits. Every NewCircuitPeriod seconds, we expire circuits - * that became dirty more than MaxCircuitDirtiness seconds ago, - * and we make a new circ if there are no clean circuits. - */ - have_dir_info = router_have_minimum_dir_info(); - if (have_dir_info && !net_is_disabled()) - circuit_build_needed_circs(now); +static int +expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options) +{ + (void)options; + /* every 11 seconds, so not usually the same second as other such events */ + circuit_expire_old_circuits_serverside(now); + return 11; +} - /* every 10 seconds, but not at the same second as other such events */ - if (now % 10 == 5) - circuit_expire_old_circuits_serverside(now); +static int +check_dns_honesty_callback(time_t now, const or_options_t *options) +{ + (void)now; + /* 9. and if we're an exit node, check whether our DNS is telling stories + * to us. */ + if (net_is_disabled() || + ! public_server_mode(options) || + router_my_exit_policy_is_reject_star()) + return PERIODIC_EVENT_NO_UPDATE; + + static int first_time = 1; + if (first_time) { + /* Don't launch right when we start */ + first_time = 0; + return crypto_rand_int_range(60, 180); + } + + dns_launch_correctness_checks(); + return 12*3600 + crypto_rand_int(12*3600); +} - /** 5. We do housekeeping for each connection... */ - connection_or_set_bad_connections(NULL, 0); - for (i=0;i<smartlist_len(connection_array);i++) { - run_connection_housekeeping(i, now); +static int +write_bridge_ns_callback(time_t now, const or_options_t *options) +{ + /* 10. write bridge networkstatus file to disk */ + if (options->BridgeAuthoritativeDir) { + networkstatus_dump_bridge_status_to_file(now); +#define BRIDGE_STATUSFILE_INTERVAL (30*60) + return BRIDGE_STATUSFILE_INTERVAL; } - if (time_to_shrink_memory < now) { - SMARTLIST_FOREACH(connection_array, connection_t *, conn, { - if (conn->outbuf) - buf_shrink(conn->outbuf); - if (conn->inbuf) - buf_shrink(conn->inbuf); - }); - clean_cell_pool(); - buf_shrink_freelists(0); -/** How often do we check buffers and pools for empty space that can be - * deallocated? */ -#define MEM_SHRINK_INTERVAL (60) - time_to_shrink_memory = now + MEM_SHRINK_INTERVAL; - } - - /** 6. And remove any marked circuits... */ - circuit_close_all_marked(); + return PERIODIC_EVENT_NO_UPDATE; +} - /** 7. And upload service descriptors if necessary. */ - if (can_complete_circuit && !net_is_disabled()) { - rend_consider_services_upload(now); - rend_consider_descriptor_republication(); +static int +check_fw_helper_app_callback(time_t now, const or_options_t *options) +{ + if (net_is_disabled() || + ! server_mode(options) || + ! options->PortForwarding) { + return PERIODIC_EVENT_NO_UPDATE; } + /* 11. check the port forwarding app */ - /** 8. and blow away any connections that need to die. have to do this now, - * because if we marked a conn for close and left its socket -1, then - * we'll pass it to poll/select and bad things will happen. - */ - close_closeable_connections(); - - /** 8b. And if anything in our state is ready to get flushed to disk, we - * flush it. */ - or_state_save(now); - - /** 8c. Do channel cleanup just like for connections */ - channel_run_cleanup(); - channel_listener_run_cleanup(); +#define PORT_FORWARDING_CHECK_INTERVAL 5 + smartlist_t *ports_to_forward = get_list_of_ports_to_forward(); + if (ports_to_forward) { + tor_check_port_forwarding(options->PortForwardingHelper, + ports_to_forward, + now); - /** 9. and if we're a server, check whether our DNS is telling stories to - * us. */ - if (!net_is_disabled() && - public_server_mode(options) && time_to_check_for_correct_dns < now) { - if (!time_to_check_for_correct_dns) { - time_to_check_for_correct_dns = now + 60 + crypto_rand_int(120); - } else { - dns_launch_correctness_checks(); - time_to_check_for_correct_dns = now + 12*3600 + - crypto_rand_int(12*3600); - } + SMARTLIST_FOREACH(ports_to_forward, char *, cp, tor_free(cp)); + smartlist_free(ports_to_forward); } + return PORT_FORWARDING_CHECK_INTERVAL; +} - /** 10. write bridge networkstatus file to disk */ - if (options->BridgeAuthoritativeDir && - time_to_write_bridge_status_file < now) { - networkstatus_dump_bridge_status_to_file(now); -#define BRIDGE_STATUSFILE_INTERVAL (30*60) - time_to_write_bridge_status_file = now+BRIDGE_STATUSFILE_INTERVAL; - } +/** Callback to write heartbeat message in the logs. */ +static int +heartbeat_callback(time_t now, const or_options_t *options) +{ + static int first = 1; - /** 11. check the port forwarding app */ - if (!net_is_disabled() && - time_to_check_port_forwarding < now && - options->PortForwarding && - is_server) { -#define PORT_FORWARDING_CHECK_INTERVAL 5 - smartlist_t *ports_to_forward = get_list_of_ports_to_forward(); - if (ports_to_forward) { - tor_check_port_forwarding(options->PortForwardingHelper, - ports_to_forward, - now); - - SMARTLIST_FOREACH(ports_to_forward, char *, cp, tor_free(cp)); - smartlist_free(ports_to_forward); - } - time_to_check_port_forwarding = now+PORT_FORWARDING_CHECK_INTERVAL; + /* Check if heartbeat is disabled */ + if (!options->HeartbeatPeriod) { + return PERIODIC_EVENT_NO_UPDATE; } - /** 11b. check pending unconfigured managed proxies */ - if (!net_is_disabled() && pt_proxies_configuration_pending()) - pt_configure_remaining_proxies(); - - /** 12. write the heartbeat message */ - if (options->HeartbeatPeriod && - time_to_next_heartbeat <= now) { - if (time_to_next_heartbeat) /* don't log the first heartbeat */ - log_heartbeat(now); - time_to_next_heartbeat = now+options->HeartbeatPeriod; + /* Write the heartbeat message */ + if (first) { + first = 0; /* Skip the first one. */ + } else { + log_heartbeat(now); } + + return options->HeartbeatPeriod; } /** Timer: used to invoke second_elapsed_callback() once per second. */ @@ -1638,35 +2091,44 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written); control_event_stream_bandwidth_used(); + control_event_conn_bandwidth_used(); + control_event_circ_bandwidth_used(); + control_event_circuit_cell_stats(); if (server_mode(options) && !net_is_disabled() && seconds_elapsed > 0 && - can_complete_circuit && + have_completed_a_circuit() && stats_n_seconds_working / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT != (stats_n_seconds_working+seconds_elapsed) / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { /* every 20 minutes, check and complain if necessary */ const routerinfo_t *me = router_get_my_routerinfo(); - if (me && !check_whether_orport_reachable()) { + if (me && !check_whether_orport_reachable(options)) { + char *address = tor_dup_ip(me->addr); log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " - "its ORPort is reachable. Please check your firewalls, ports, " - "address, /etc/hosts file, etc.", - me->address, me->or_port); + "its ORPort is reachable. Relays do not publish descriptors " + "until their ORPort and DirPort are reachable. Please check " + "your firewalls, ports, address, /etc/hosts file, etc.", + address, me->or_port); control_event_server_status(LOG_WARN, "REACHABILITY_FAILED ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); + tor_free(address); } - if (me && !check_whether_dirport_reachable()) { + if (me && !check_whether_dirport_reachable(options)) { + char *address = tor_dup_ip(me->addr); log_warn(LD_CONFIG, "Your server (%s:%d) has not managed to confirm that its " - "DirPort is reachable. Please check your firewalls, ports, " - "address, /etc/hosts file, etc.", - me->address, me->dir_port); + "DirPort is reachable. Relays do not publish descriptors " + "until their ORPort and DirPort are reachable. Please check " + "your firewalls, ports, address, /etc/hosts file, etc.", + address, me->dir_port); control_event_server_status(LOG_WARN, "REACHABILITY_FAILED DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); + tor_free(address); } } @@ -1676,8 +2138,6 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN || seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { circuit_note_clock_jumped(seconds_elapsed); - /* XXX if the time jumps *back* many months, do our events in - * run_scheduled_events() recover? I don't think they do. -RD */ } else if (seconds_elapsed > 0) stats_n_seconds_working += seconds_elapsed; @@ -1686,6 +2146,19 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) current_second = now; /* remember which second it is, for next time */ } +#ifdef HAVE_SYSTEMD_209 +static periodic_timer_t *systemd_watchdog_timer = NULL; + +/** Libevent callback: invoked to reset systemd watchdog. */ +static void +systemd_watchdog_callback(periodic_timer_t *timer, void *arg) +{ + (void)timer; + (void)arg; + sd_notify(0, "WATCHDOG=1"); +} +#endif + #ifndef USE_BUFFEREVENTS /** Timer: used to invoke refill_callback(). */ static periodic_timer_t *refill_timer = NULL; @@ -1728,7 +2201,7 @@ refill_callback(periodic_timer_t *timer, void *arg) accounting_add_bytes(bytes_read, bytes_written, seconds_rolled_over); if (milliseconds_elapsed > 0) - connection_bucket_refill(milliseconds_elapsed, now.tv_sec); + connection_bucket_refill(milliseconds_elapsed, (time_t)now.tv_sec); stats_prev_global_read_bucket = global_read_bucket; stats_prev_global_write_bucket = global_write_bucket; @@ -1760,12 +2233,15 @@ got_libevent_error(void) void ip_address_changed(int at_interface) { - int server = server_mode(get_options()); + const or_options_t *options = get_options(); + int server = server_mode(options); + int exit_reject_private = (server && options->ExitRelay + && options->ExitPolicyRejectPrivate); if (at_interface) { if (! server) { /* Okay, change our keys. */ - if (init_keys()<0) + if (init_keys_client() < 0) log_warn(LD_GENERAL, "Unable to rotate keys after IP change!"); } } else { @@ -1774,10 +2250,15 @@ ip_address_changed(int at_interface) reset_bandwidth_test(); stats_n_seconds_working = 0; router_reset_reachability(); - mark_my_descriptor_dirty("IP address changed"); } } + /* Exit relays incorporate interface addresses in their exit policies when + * ExitPolicyRejectPrivate is set */ + if (exit_reject_private || (server && !at_interface)) { + mark_my_descriptor_dirty("IP address changed"); + } + dns_servers_relaunch_checks(); } @@ -1788,7 +2269,10 @@ dns_servers_relaunch_checks(void) { if (server_mode(get_options())) { dns_reset_correctness_checks(); - time_to_check_for_correct_dns = 0; + if (periodic_events_initialized) { + tor_assert(check_dns_honesty_event); + periodic_event_reschedule(check_dns_honesty_event); + } } } @@ -1820,6 +2304,10 @@ do_hup(void) return -1; } options = get_options(); /* they have changed now */ + /* Logs are only truncated the first time they are opened, but were + probably intended to be cleaned up on signal. */ + if (options->TruncateLogFile) + truncate_logs(); } else { char *msg = NULL; log_notice(LD_GENERAL, "Not reloading config file: the controller told " @@ -1856,21 +2344,35 @@ do_hup(void) * force a retry there. */ if (server_mode(options)) { - /* Restart cpuworker and dnsworker processes, so they get up-to-date + /* Maybe we've been given a new ed25519 key or certificate? + */ + time_t now = approx_time(); + if (load_ed_keys(options, now) < 0 || + generate_ed_link_cert(options, now)) { + log_warn(LD_OR, "Problem reloading Ed25519 keys; still using old keys."); + } + + /* Update cpuworker and dnsworker processes, so they get up-to-date * configuration options. */ - cpuworkers_rotate(); + cpuworkers_rotate_keyinfo(); dns_reset(); } return 0; } /** Tor main loop. */ -/* static */ int +int do_main_loop(void) { - int loop_result; time_t now; + /* initialize the periodic events first, so that code that depends on the + * events being present does not assert. + */ + if (! periodic_events_initialized) { + initialize_periodic_events(); + } + /* initialize dns resolve map, spawn workers if needed */ if (dns_init() < 0) { if (get_options()->ServerDNSAllowBrokenConfig) @@ -1895,14 +2397,11 @@ do_main_loop(void) * TLS context. */ if (! client_identity_key_is_set()) { if (init_keys() < 0) { - log_err(LD_BUG,"Error initializing keys; exiting"); + log_err(LD_OR, "Error initializing keys; exiting"); return -1; } } - /* Set up the packed_cell_t memory pool. */ - init_cell_pool(); - /* Set up our buckets */ connection_bucket_init(); #ifndef USE_BUFFEREVENTS @@ -1913,13 +2412,38 @@ do_main_loop(void) /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); + /* Initialize the keypinning log. */ + if (authdir_mode_v3(get_options())) { + char *fname = get_datadir_fname("key-pinning-journal"); + int r = 0; + if (keypin_load_journal(fname)<0) { + log_err(LD_DIR, "Error loading key-pinning journal: %s",strerror(errno)); + r = -1; + } + if (keypin_open_journal(fname)<0) { + log_err(LD_DIR, "Error opening key-pinning journal: %s",strerror(errno)); + r = -1; + } + tor_free(fname); + if (r) + return r; + } + { + /* This is the old name for key-pinning-journal. These got corrupted + * in a couple of cases by #16530, so we started over. See #16580 for + * the rationale and for other options we didn't take. We can remove + * this code once all the authorities that ran 0.2.7.1-alpha-dev are + * upgraded. + */ + char *fname = get_datadir_fname("key-pinning-entries"); + unlink(fname); + tor_free(fname); + } + if (trusted_dirs_reload_certs()) { log_warn(LD_DIR, "Couldn't load all cached v3 certificates. Starting anyway."); } - if (router_reload_v2_networkstatus()) { - return -1; - } if (router_reload_consensus_networkstatus()) { return -1; } @@ -1931,7 +2455,7 @@ do_main_loop(void) * appropriate.) */ now = time(NULL); - directory_info_has_arrived(now, 1); + directory_info_has_arrived(now, 1, 0); if (server_mode(get_options())) { /* launch cpuworkers. Need to do this *after* we've read the onion key. */ @@ -1951,6 +2475,28 @@ do_main_loop(void) tor_assert(second_timer); } +#ifdef HAVE_SYSTEMD_209 + uint64_t watchdog_delay; + /* set up systemd watchdog notification. */ + if (sd_watchdog_enabled(1, &watchdog_delay) > 0) { + if (! systemd_watchdog_timer) { + struct timeval watchdog; + /* The manager will "act on" us if we don't send them a notification + * every 'watchdog_delay' microseconds. So, send notifications twice + * that often. */ + watchdog_delay /= 2; + watchdog.tv_sec = watchdog_delay / 1000000; + watchdog.tv_usec = watchdog_delay % 1000000; + + systemd_watchdog_timer = periodic_timer_new(tor_libevent_get_base(), + &watchdog, + systemd_watchdog_callback, + NULL); + tor_assert(systemd_watchdog_timer); + } + } +#endif + #ifndef USE_BUFFEREVENTS if (!refill_timer) { struct timeval refill_interval; @@ -1967,70 +2513,115 @@ do_main_loop(void) } #endif - for (;;) { - if (nt_service_is_stopping()) - return 0; +#ifdef HAVE_SYSTEMD + { + const int r = sd_notify(0, "READY=1"); + if (r < 0) { + log_warn(LD_GENERAL, "Unable to send readiness to systemd: %s", + strerror(r)); + } else if (r > 0) { + log_notice(LD_GENERAL, "Signaled readiness to systemd"); + } else { + log_info(LD_GENERAL, "Systemd NOTIFY_SOCKET not present."); + } + } +#endif + + return run_main_loop_until_done(); +} + +/** + * Run the main loop a single time. Return 0 for "exit"; -1 for "exit with + * error", and 1 for "run this again." + */ +static int +run_main_loop_once(void) +{ + int loop_result; + + if (nt_service_is_stopping()) + return 0; #ifndef _WIN32 - /* Make it easier to tell whether libevent failure is our fault or not. */ - errno = 0; + /* Make it easier to tell whether libevent failure is our fault or not. */ + errno = 0; #endif - /* All active linked conns should get their read events activated. */ - SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn, - event_active(conn->read_event, EV_READ, 1)); - called_loop_once = smartlist_len(active_linked_connection_lst) ? 1 : 0; - - update_approx_time(time(NULL)); - - /* poll until we have an event, or the second ends, or until we have - * some active linked connections to trigger events for. */ - loop_result = event_base_loop(tor_libevent_get_base(), - called_loop_once ? EVLOOP_ONCE : 0); - - /* let catch() handle things like ^c, and otherwise don't worry about it */ - if (loop_result < 0) { - int e = tor_socket_errno(-1); - /* let the program survive things like ^z */ - if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { - log_err(LD_NET,"libevent call with %s failed: %s [%d]", - tor_libevent_get_method(), tor_socket_strerror(e), e); - return -1; + /* All active linked conns should get their read events activated. */ + SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn, + event_active(conn->read_event, EV_READ, 1)); + called_loop_once = smartlist_len(active_linked_connection_lst) ? 1 : 0; + + update_approx_time(time(NULL)); + + /* poll until we have an event, or the second ends, or until we have + * some active linked connections to trigger events for. */ + loop_result = event_base_loop(tor_libevent_get_base(), + called_loop_once ? EVLOOP_ONCE : 0); + + /* let catch() handle things like ^c, and otherwise don't worry about it */ + if (loop_result < 0) { + int e = tor_socket_errno(-1); + /* let the program survive things like ^z */ + if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { + log_err(LD_NET,"libevent call with %s failed: %s [%d]", + tor_libevent_get_method(), tor_socket_strerror(e), e); + return -1; #ifndef _WIN32 - } else if (e == EINVAL) { - log_warn(LD_NET, "EINVAL from libevent: should you upgrade libevent?"); - if (got_libevent_error()) - return -1; + } else if (e == EINVAL) { + log_warn(LD_NET, "EINVAL from libevent: should you upgrade libevent?"); + if (got_libevent_error()) + return -1; #endif - } else { - if (ERRNO_IS_EINPROGRESS(e)) - log_warn(LD_BUG, - "libevent call returned EINPROGRESS? Please report."); - log_debug(LD_NET,"libevent call interrupted."); - /* You can't trust the results of this poll(). Go back to the - * top of the big for loop. */ - continue; - } + } else { + if (ERRNO_IS_EINPROGRESS(e)) + log_warn(LD_BUG, + "libevent call returned EINPROGRESS? Please report."); + log_debug(LD_NET,"libevent call interrupted."); + /* You can't trust the results of this poll(). Go back to the + * top of the big for loop. */ + return 1; } } + + /* This will be pretty fast if nothing new is pending. Note that this gets + * called once per libevent loop, which will make it happen once per group + * of events that fire, or once per second. */ + connection_ap_attach_pending(0); + + return 1; +} + +/** Run the run_main_loop_once() function until it declares itself done, + * and return its final return value. + * + * Shadow won't invoke this function, so don't fill it up with things. + */ +static int +run_main_loop_until_done(void) +{ + int loop_result = 1; + do { + loop_result = run_main_loop_once(); + } while (loop_result == 1); + return loop_result; } -#ifndef _WIN32 /* Only called when we're willing to use signals */ /** Libevent callback: invoked when we get a signal. */ static void -signal_callback(int fd, short events, void *arg) +signal_callback(evutil_socket_t fd, short events, void *arg) { - uintptr_t sig = (uintptr_t)arg; + const int *sigptr = arg; + const int sig = *sigptr; (void)fd; (void)events; process_signal(sig); } -#endif /** Do the work of acting on a signal received in <b>sig</b> */ -void -process_signal(uintptr_t sig) +static void +process_signal(int sig) { switch (sig) { @@ -2045,6 +2636,9 @@ process_signal(uintptr_t sig) tor_cleanup(); exit(0); } +#ifdef HAVE_SYSTEMD + sd_notify(0, "STOPPING=1"); +#endif hibernate_begin_shutdown(); break; #ifdef SIGPIPE @@ -2064,17 +2658,22 @@ process_signal(uintptr_t sig) control_event_signal(sig); break; case SIGHUP: +#ifdef HAVE_SYSTEMD + sd_notify(0, "RELOADING=1"); +#endif if (do_hup() < 0) { log_warn(LD_CONFIG,"Restart failed (config error?). Exiting."); tor_cleanup(); exit(1); } +#ifdef HAVE_SYSTEMD + sd_notify(0, "READY=1"); +#endif control_event_signal(sig); break; #ifdef SIGCHLD case SIGCHLD: - while (waitpid(-1,NULL,WNOHANG) > 0) ; /* keep reaping until no more - zombies */ + notify_pending_waitpid_callbacks(); break; #endif case SIGNEWNYM: { @@ -2093,12 +2692,16 @@ process_signal(uintptr_t sig) addressmap_clear_transient(); control_event_signal(sig); break; + case SIGHEARTBEAT: + log_heartbeat(time(NULL)); + control_event_signal(sig); + break; } } /** Returns Tor's uptime. */ -long -get_uptime(void) +MOCK_IMPL(long, +get_uptime,(void)) { return stats_n_seconds_working; } @@ -2118,7 +2721,6 @@ dumpmemusage(int severity) dump_routerlist_mem_usage(severity); dump_cell_pool_usage(severity); dump_dns_mem_usage(severity); - buf_dump_freelist_sizes(severity); tor_log_mallinfo(severity); } @@ -2160,12 +2762,13 @@ dumpstats(int severity) if (conn->type == CONN_TYPE_OR) { or_connection_t *or_conn = TO_OR_CONN(conn); if (or_conn->tls) { - tor_tls_get_buffer_sizes(or_conn->tls, &rbuf_cap, &rbuf_len, - &wbuf_cap, &wbuf_len); - tor_log(severity, LD_GENERAL, - "Conn %d: %d/%d bytes used on OpenSSL read buffer; " - "%d/%d bytes used on write buffer.", - i, (int)rbuf_len, (int)rbuf_cap, (int)wbuf_len, (int)wbuf_cap); + if (tor_tls_get_buffer_sizes(or_conn->tls, &rbuf_cap, &rbuf_len, + &wbuf_cap, &wbuf_len) == 0) { + tor_log(severity, LD_GENERAL, + "Conn %d: %d/%d bytes used on OpenSSL read buffer; " + "%d/%d bytes used on write buffer.", + i, (int)rbuf_len, (int)rbuf_cap, (int)wbuf_len, (int)wbuf_cap); + } } } } @@ -2242,35 +2845,73 @@ exit_function(void) #endif } -/** Set up the signal handlers for either parent or child. */ +#ifdef _WIN32 +#define UNIX_ONLY 0 +#else +#define UNIX_ONLY 1 +#endif +static struct { + int signal_value; + int try_to_register; + struct event *signal_event; +} signal_handlers[] = { +#ifdef SIGINT + { SIGINT, UNIX_ONLY, NULL }, /* do a controlled slow shutdown */ +#endif +#ifdef SIGTERM + { SIGTERM, UNIX_ONLY, NULL }, /* to terminate now */ +#endif +#ifdef SIGPIPE + { SIGPIPE, UNIX_ONLY, NULL }, /* otherwise SIGPIPE kills us */ +#endif +#ifdef SIGUSR1 + { SIGUSR1, UNIX_ONLY, NULL }, /* dump stats */ +#endif +#ifdef SIGUSR2 + { SIGUSR2, UNIX_ONLY, NULL }, /* go to loglevel debug */ +#endif +#ifdef SIGHUP + { SIGHUP, UNIX_ONLY, NULL }, /* to reload config, retry conns, etc */ +#endif +#ifdef SIGXFSZ + { SIGXFSZ, UNIX_ONLY, NULL }, /* handle file-too-big resource exhaustion */ +#endif +#ifdef SIGCHLD + { SIGCHLD, UNIX_ONLY, NULL }, /* handle dns/cpu workers that exit */ +#endif + /* These are controller-only */ + { SIGNEWNYM, 0, NULL }, + { SIGCLEARDNSCACHE, 0, NULL }, + { SIGHEARTBEAT, 0, NULL }, + { -1, -1, NULL } +}; + +/** Set up the signal handlers for either parent or child process */ void handle_signals(int is_parent) { -#ifndef _WIN32 /* do signal stuff only on Unix */ int i; - static const int signals[] = { - SIGINT, /* do a controlled slow shutdown */ - SIGTERM, /* to terminate now */ - SIGPIPE, /* otherwise SIGPIPE kills us */ - SIGUSR1, /* dump stats */ - SIGUSR2, /* go to loglevel debug */ - SIGHUP, /* to reload config, retry conns, etc */ -#ifdef SIGXFSZ - SIGXFSZ, /* handle file-too-big resource exhaustion */ -#endif - SIGCHLD, /* handle dns/cpu workers that exit */ - -1 }; - static struct event *signal_events[16]; /* bigger than it has to be. */ if (is_parent) { - for (i = 0; signals[i] >= 0; ++i) { - signal_events[i] = tor_evsignal_new( - tor_libevent_get_base(), signals[i], signal_callback, - (void*)(uintptr_t)signals[i]); - if (event_add(signal_events[i], NULL)) - log_warn(LD_BUG, "Error from libevent when adding event for signal %d", - signals[i]); + for (i = 0; signal_handlers[i].signal_value >= 0; ++i) { + if (signal_handlers[i].try_to_register) { + signal_handlers[i].signal_event = + tor_evsignal_new(tor_libevent_get_base(), + signal_handlers[i].signal_value, + signal_callback, + &signal_handlers[i].signal_value); + if (event_add(signal_handlers[i].signal_event, NULL)) + log_warn(LD_BUG, "Error from libevent when adding " + "event for signal %d", + signal_handlers[i].signal_value); + } else { + signal_handlers[i].signal_event = + tor_event_new(tor_libevent_get_base(), -1, + EV_SIGNAL, signal_callback, + &signal_handlers[i].signal_value); + } } } else { +#ifndef _WIN32 struct sigaction action; action.sa_flags = 0; sigemptyset(&action.sa_mask); @@ -2284,29 +2925,43 @@ handle_signals(int is_parent) #ifdef SIGXFSZ sigaction(SIGXFSZ, &action, NULL); #endif +#endif + } +} + +/* Make sure the signal handler for signal_num will be called. */ +void +activate_signal(int signal_num) +{ + int i; + for (i = 0; signal_handlers[i].signal_value >= 0; ++i) { + if (signal_handlers[i].signal_value == signal_num) { + event_active(signal_handlers[i].signal_event, EV_SIGNAL, 1); + return; + } } -#else /* MS windows */ - (void)is_parent; -#endif /* signal stuff */ } /** Main entry point for the Tor command-line client. */ -/* static */ int +int tor_init(int argc, char *argv[]) { - char buf[256]; - int i, quiet = 0; + char progname[256]; + int quiet = 0; + time_of_process_start = time(NULL); - if (!connection_array) - connection_array = smartlist_new(); - if (!closeable_connection_lst) - closeable_connection_lst = smartlist_new(); - if (!active_linked_connection_lst) - active_linked_connection_lst = smartlist_new(); + init_connection_lists(); /* Have the log set up with our application name. */ - tor_snprintf(buf, sizeof(buf), "Tor %s", get_version()); - log_set_application_name(buf); + tor_snprintf(progname, sizeof(progname), "Tor %s", get_version()); + log_set_application_name(progname); + + /* Set up the crypto nice and early */ + if (crypto_early_init() < 0) { + log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!"); + return -1; + } + /* Initialize the history structures. */ rep_hist_init(); /* Initialize the service cache. */ @@ -2314,17 +2969,32 @@ tor_init(int argc, char *argv[]) addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ + { /* We search for the "quiet" option first, since it decides whether we * will log anything at all to the command line. */ - for (i=1;i<argc;++i) { - if (!strcmp(argv[i], "--hush")) - quiet = 1; - if (!strcmp(argv[i], "--quiet")) - quiet = 2; - /* --version implies --quiet */ - if (!strcmp(argv[i], "--version")) - quiet = 2; + config_line_t *opts = NULL, *cmdline_opts = NULL; + const config_line_t *cl; + (void) config_parse_commandline(argc, argv, 1, &opts, &cmdline_opts); + for (cl = cmdline_opts; cl; cl = cl->next) { + if (!strcmp(cl->key, "--hush")) + quiet = 1; + if (!strcmp(cl->key, "--quiet") || + !strcmp(cl->key, "--dump-config")) + quiet = 2; + /* The following options imply --hush */ + if (!strcmp(cl->key, "--version") || !strcmp(cl->key, "--digests") || + !strcmp(cl->key, "--list-torrc-options") || + !strcmp(cl->key, "--library-versions") || + !strcmp(cl->key, "--hash-password") || + !strcmp(cl->key, "-h") || !strcmp(cl->key, "--help")) { + if (quiet < 1) + quiet = 1; + } + } + config_free_lines(opts); + config_free_lines(cmdline_opts); } + /* give it somewhere to log to initially */ switch (quiet) { case 2: @@ -2346,11 +3016,12 @@ tor_init(int argc, char *argv[]) #else ""; #endif - log_notice(LD_GENERAL, "Tor v%s %srunning on %s with Libevent %s " - "and OpenSSL %s.", version, bev_str, + log_notice(LD_GENERAL, "Tor v%s %srunning on %s with Libevent %s, " + "OpenSSL %s and Zlib %s.", version, bev_str, get_uname(), tor_libevent_get_version_str(), - crypto_openssl_get_version_str()); + crypto_openssl_get_version_str(), + tor_zlib_get_version_str()); log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! " "Learn how to be safe at " @@ -2390,6 +3061,9 @@ tor_init(int argc, char *argv[]) return -1; } stream_choice_seed_weak_rng(); + if (tor_init_libevent_rng() < 0) { + log_warn(LD_NET, "Problem initializing libevent RNG."); + } return 0; } @@ -2488,19 +3162,25 @@ tor_free_all(int postfork) channel_tls_free_all(); channel_free_all(); connection_free_all(); - buf_shrink_freelists(1); - memarea_clear_freelist(); + connection_edge_free_all(); + scheduler_free_all(); nodelist_free_all(); microdesc_free_all(); + ext_orport_free_all(); + control_free_all(); + sandbox_free_getaddrinfo_cache(); if (!postfork) { config_free_all(); or_state_free_all(); router_free_all(); + routerkeys_free_all(); policies_free_all(); } - free_cell_pool(); if (!postfork) { tor_tls_free_all(); +#ifndef _WIN32 + tor_getpwnam(NULL); +#endif } /* stuff in main.c */ @@ -2508,6 +3188,7 @@ tor_free_all(int postfork) smartlist_free(closeable_connection_lst); smartlist_free(active_linked_connection_lst); periodic_timer_free(second_timer); + teardown_periodic_events(); #ifndef USE_BUFFEREVENTS periodic_timer_free(refill_timer); #endif @@ -2532,16 +3213,26 @@ tor_cleanup(void) time_t now = time(NULL); /* Remove our pid file. We don't care if there was an error when we * unlink, nothing we could do about it anyways. */ - if (options->PidFile) - unlink(options->PidFile); - if (options->ControlPortWriteToFile) - unlink(options->ControlPortWriteToFile); + if (options->PidFile) { + if (unlink(options->PidFile) != 0) { + log_warn(LD_FS, "Couldn't unlink pid file %s: %s", + options->PidFile, strerror(errno)); + } + } + if (options->ControlPortWriteToFile) { + if (unlink(options->ControlPortWriteToFile) != 0) { + log_warn(LD_FS, "Couldn't unlink control port file %s: %s", + options->ControlPortWriteToFile, + strerror(errno)); + } + } if (accounting_is_enabled(options)) accounting_record_bandwidth_usage(now, get_or_state()); or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */ or_state_save(now); if (authdir_mode_tests_reachability(options)) rep_hist_record_mtbf_data(now, 0); + keypin_close_journal(); } #ifdef USE_DMALLOC dmalloc_log_stats(); @@ -2558,20 +3249,21 @@ tor_cleanup(void) } /** Read/create keys as needed, and echo our fingerprint to stdout. */ -/* static */ int +static int do_list_fingerprint(void) { char buf[FINGERPRINT_LEN+1]; crypto_pk_t *k; const char *nickname = get_options()->Nickname; + sandbox_disable_getaddrinfo_cache(); if (!server_mode(get_options())) { log_err(LD_GENERAL, - "Clients don't have long-term identity keys. Exiting.\n"); + "Clients don't have long-term identity keys. Exiting."); return -1; } tor_assert(nickname); if (init_keys() < 0) { - log_err(LD_BUG,"Error initializing keys; can't display fingerprint"); + log_err(LD_GENERAL,"Error initializing keys; exiting."); return -1; } if (!(k = get_server_identity_key())) { @@ -2588,46 +3280,319 @@ do_list_fingerprint(void) /** Entry point for password hashing: take the desired password from * the command line, and print its salted hash to stdout. **/ -/* static */ void +static void do_hash_password(void) { char output[256]; - char key[S2K_SPECIFIER_LEN+DIGEST_LEN]; + char key[S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN]; - crypto_rand(key, S2K_SPECIFIER_LEN-1); - key[S2K_SPECIFIER_LEN-1] = (uint8_t)96; /* Hash 64 K of data. */ - secret_to_key(key+S2K_SPECIFIER_LEN, DIGEST_LEN, + crypto_rand(key, S2K_RFC2440_SPECIFIER_LEN-1); + key[S2K_RFC2440_SPECIFIER_LEN-1] = (uint8_t)96; /* Hash 64 K of data. */ + secret_to_key_rfc2440(key+S2K_RFC2440_SPECIFIER_LEN, DIGEST_LEN, get_options()->command_arg, strlen(get_options()->command_arg), key); base16_encode(output, sizeof(output), key, sizeof(key)); printf("16:%s\n",output); } -#if defined (WINCE) -int -find_flashcard_path(PWCHAR path, size_t size) +/** Entry point for configuration dumping: write the configuration to + * stdout. */ +static int +do_dump_config(void) { - WIN32_FIND_DATA d = {0}; - HANDLE h = NULL; + const or_options_t *options = get_options(); + const char *arg = options->command_arg; + int how; + char *opts; + + if (!strcmp(arg, "short")) { + how = OPTIONS_DUMP_MINIMAL; + } else if (!strcmp(arg, "non-builtin")) { + how = OPTIONS_DUMP_DEFAULTS; + } else if (!strcmp(arg, "full")) { + how = OPTIONS_DUMP_ALL; + } else { + fprintf(stderr, "No valid argument to --dump-config found!\n"); + fprintf(stderr, "Please select 'short', 'non-builtin', or 'full'.\n"); - if (!path) return -1; + } - h = FindFirstFlashCard(&d); - if (h == INVALID_HANDLE_VALUE) - return -1; + opts = options_dump(options, how); + printf("%s", opts); + tor_free(opts); - if (wcslen(d.cFileName) == 0) { - FindClose(h); - return -1; + return 0; +} + +static void +init_addrinfo(void) +{ + if (! server_mode(get_options()) || + (get_options()->Address && strlen(get_options()->Address) > 0)) { + /* We don't need to seed our own hostname, because we won't be calling + * resolve_my_address on it. + */ + return; } + char hname[256]; - wcsncpy(path,d.cFileName,size); - FindClose(h); - return 0; + // host name to sandbox + gethostname(hname, sizeof(hname)); + sandbox_add_addrinfo(hname); +} + +static sandbox_cfg_t* +sandbox_init_filter(void) +{ + const or_options_t *options = get_options(); + sandbox_cfg_t *cfg = sandbox_cfg_new(); + int i; + + sandbox_cfg_allow_openat_filename(&cfg, + get_datadir_fname("cached-status")); + +#define OPEN(name) \ + sandbox_cfg_allow_open_filename(&cfg, tor_strdup(name)) + +#define OPEN_DATADIR(name) \ + sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname(name)) + +#define OPEN_DATADIR2(name, name2) \ + sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname2((name), (name2))) + +#define OPEN_DATADIR_SUFFIX(name, suffix) do { \ + OPEN_DATADIR(name); \ + OPEN_DATADIR(name suffix); \ + } while (0) + +#define OPEN_DATADIR2_SUFFIX(name, name2, suffix) do { \ + OPEN_DATADIR2(name, name2); \ + OPEN_DATADIR2(name, name2 suffix); \ + } while (0) + + OPEN(options->DataDirectory); + OPEN_DATADIR("keys"); + OPEN_DATADIR_SUFFIX("cached-certs", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("unverified-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("unverified-microdesc-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-microdesc-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-microdescs", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-microdescs.new", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-descriptors", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-descriptors.new", ".tmp"); + OPEN_DATADIR("cached-descriptors.tmp.tmp"); + OPEN_DATADIR_SUFFIX("cached-extrainfo", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-extrainfo.new", ".tmp"); + OPEN_DATADIR("cached-extrainfo.tmp.tmp"); + OPEN_DATADIR_SUFFIX("state", ".tmp"); + OPEN_DATADIR_SUFFIX("unparseable-desc", ".tmp"); + OPEN_DATADIR_SUFFIX("v3-status-votes", ".tmp"); + OPEN_DATADIR("key-pinning-journal"); + OPEN("/dev/srandom"); + OPEN("/dev/urandom"); + OPEN("/dev/random"); + OPEN("/etc/hosts"); + OPEN("/proc/meminfo"); + + if (options->BridgeAuthoritativeDir) + OPEN_DATADIR_SUFFIX("networkstatus-bridges", ".tmp"); + + if (authdir_mode_handles_descs(options, -1)) + OPEN_DATADIR("approved-routers"); + + if (options->ServerDNSResolvConfFile) + sandbox_cfg_allow_open_filename(&cfg, + tor_strdup(options->ServerDNSResolvConfFile)); + else + sandbox_cfg_allow_open_filename(&cfg, tor_strdup("/etc/resolv.conf")); + + for (i = 0; i < 2; ++i) { + if (get_torrc_fname(i)) { + sandbox_cfg_allow_open_filename(&cfg, tor_strdup(get_torrc_fname(i))); + } + } + +#define RENAME_SUFFIX(name, suffix) \ + sandbox_cfg_allow_rename(&cfg, \ + get_datadir_fname(name suffix), \ + get_datadir_fname(name)) + +#define RENAME_SUFFIX2(prefix, name, suffix) \ + sandbox_cfg_allow_rename(&cfg, \ + get_datadir_fname2(prefix, name suffix), \ + get_datadir_fname2(prefix, name)) + + RENAME_SUFFIX("cached-certs", ".tmp"); + RENAME_SUFFIX("cached-consensus", ".tmp"); + RENAME_SUFFIX("unverified-consensus", ".tmp"); + RENAME_SUFFIX("unverified-microdesc-consensus", ".tmp"); + RENAME_SUFFIX("cached-microdesc-consensus", ".tmp"); + RENAME_SUFFIX("cached-microdescs", ".tmp"); + RENAME_SUFFIX("cached-microdescs", ".new"); + RENAME_SUFFIX("cached-microdescs.new", ".tmp"); + RENAME_SUFFIX("cached-descriptors", ".tmp"); + RENAME_SUFFIX("cached-descriptors", ".new"); + RENAME_SUFFIX("cached-descriptors.new", ".tmp"); + RENAME_SUFFIX("cached-extrainfo", ".tmp"); + RENAME_SUFFIX("cached-extrainfo", ".new"); + RENAME_SUFFIX("cached-extrainfo.new", ".tmp"); + RENAME_SUFFIX("state", ".tmp"); + RENAME_SUFFIX("unparseable-desc", ".tmp"); + RENAME_SUFFIX("v3-status-votes", ".tmp"); + + if (options->BridgeAuthoritativeDir) + RENAME_SUFFIX("networkstatus-bridges", ".tmp"); + +#define STAT_DATADIR(name) \ + sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname(name)) + +#define STAT_DATADIR2(name, name2) \ + sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname2((name), (name2))) + + STAT_DATADIR(NULL); + STAT_DATADIR("lock"); + STAT_DATADIR("state"); + STAT_DATADIR("router-stability"); + STAT_DATADIR("cached-extrainfo.new"); + + { + smartlist_t *files = smartlist_new(); + tor_log_get_logfile_names(files); + SMARTLIST_FOREACH(files, char *, file_name, { + /* steals reference */ + sandbox_cfg_allow_open_filename(&cfg, file_name); + }); + smartlist_free(files); + } + + { + smartlist_t *files = smartlist_new(); + smartlist_t *dirs = smartlist_new(); + rend_services_add_filenames_to_lists(files, dirs); + SMARTLIST_FOREACH(files, char *, file_name, { + char *tmp_name = NULL; + tor_asprintf(&tmp_name, "%s.tmp", file_name); + sandbox_cfg_allow_rename(&cfg, + tor_strdup(tmp_name), tor_strdup(file_name)); + /* steals references */ + sandbox_cfg_allow_open_filename(&cfg, file_name); + sandbox_cfg_allow_open_filename(&cfg, tmp_name); + }); + SMARTLIST_FOREACH(dirs, char *, dir, { + /* steals reference */ + sandbox_cfg_allow_stat_filename(&cfg, dir); + }); + smartlist_free(files); + smartlist_free(dirs); + } + + { + char *fname; + if ((fname = get_controller_cookie_file_name())) { + sandbox_cfg_allow_open_filename(&cfg, fname); + } + if ((fname = get_ext_or_auth_cookie_file_name())) { + sandbox_cfg_allow_open_filename(&cfg, fname); + } + } + + SMARTLIST_FOREACH_BEGIN(get_configured_ports(), port_cfg_t *, port) { + if (!port->is_unix_addr) + continue; + /* When we open an AF_UNIX address, we want permission to open the + * directory that holds it. */ + char *dirname = tor_strdup(port->unix_addr); + if (get_parent_directory(dirname) == 0) { + OPEN(dirname); + } + tor_free(dirname); + sandbox_cfg_allow_chmod_filename(&cfg, tor_strdup(port->unix_addr)); + sandbox_cfg_allow_chown_filename(&cfg, tor_strdup(port->unix_addr)); + } SMARTLIST_FOREACH_END(port); + + if (options->DirPortFrontPage) { + sandbox_cfg_allow_open_filename(&cfg, + tor_strdup(options->DirPortFrontPage)); + } + + // orport + if (server_mode(get_options())) { + + OPEN_DATADIR2_SUFFIX("keys", "secret_id_key", ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "secret_onion_key", ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "secret_onion_key_ntor", ".tmp"); + OPEN_DATADIR2("keys", "secret_id_key.old"); + OPEN_DATADIR2("keys", "secret_onion_key.old"); + OPEN_DATADIR2("keys", "secret_onion_key_ntor.old"); + + OPEN_DATADIR2_SUFFIX("keys", "ed25519_master_id_secret_key", ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "ed25519_master_id_secret_key_encrypted", + ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "ed25519_master_id_public_key", ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_secret_key", ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_secret_key_encrypted", + ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_public_key", ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "ed25519_signing_cert", ".tmp"); + + OPEN_DATADIR2_SUFFIX("stats", "bridge-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "dirreq-stats", ".tmp"); + + OPEN_DATADIR2_SUFFIX("stats", "entry-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "exit-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "buffer-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "conn-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "hidserv-stats", ".tmp"); + + OPEN_DATADIR("approved-routers"); + OPEN_DATADIR_SUFFIX("fingerprint", ".tmp"); + OPEN_DATADIR_SUFFIX("hashed-fingerprint", ".tmp"); + OPEN_DATADIR_SUFFIX("router-stability", ".tmp"); + + OPEN("/etc/resolv.conf"); + + RENAME_SUFFIX("fingerprint", ".tmp"); + RENAME_SUFFIX2("keys", "secret_onion_key_ntor", ".tmp"); + RENAME_SUFFIX2("keys", "secret_id_key", ".tmp"); + RENAME_SUFFIX2("keys", "secret_id_key.old", ".tmp"); + RENAME_SUFFIX2("keys", "secret_onion_key", ".tmp"); + RENAME_SUFFIX2("keys", "secret_onion_key.old", ".tmp"); + RENAME_SUFFIX2("stats", "bridge-stats", ".tmp"); + RENAME_SUFFIX2("stats", "dirreq-stats", ".tmp"); + RENAME_SUFFIX2("stats", "entry-stats", ".tmp"); + RENAME_SUFFIX2("stats", "exit-stats", ".tmp"); + RENAME_SUFFIX2("stats", "buffer-stats", ".tmp"); + RENAME_SUFFIX2("stats", "conn-stats", ".tmp"); + RENAME_SUFFIX2("stats", "hidserv-stats", ".tmp"); + RENAME_SUFFIX("hashed-fingerprint", ".tmp"); + RENAME_SUFFIX("router-stability", ".tmp"); + + RENAME_SUFFIX2("keys", "ed25519_master_id_secret_key", ".tmp"); + RENAME_SUFFIX2("keys", "ed25519_master_id_secret_key_encrypted", ".tmp"); + RENAME_SUFFIX2("keys", "ed25519_master_id_public_key", ".tmp"); + RENAME_SUFFIX2("keys", "ed25519_signing_secret_key", ".tmp"); + RENAME_SUFFIX2("keys", "ed25519_signing_cert", ".tmp"); + + sandbox_cfg_allow_rename(&cfg, + get_datadir_fname2("keys", "secret_onion_key"), + get_datadir_fname2("keys", "secret_onion_key.old")); + sandbox_cfg_allow_rename(&cfg, + get_datadir_fname2("keys", "secret_onion_key_ntor"), + get_datadir_fname2("keys", "secret_onion_key_ntor.old")); + + STAT_DATADIR("keys"); + OPEN_DATADIR("stats"); + STAT_DATADIR("stats"); + STAT_DATADIR2("stats", "dirreq-stats"); + } + + init_addrinfo(); + + return cfg; } -#endif /** Main entry point for the Tor process. Called from main(). */ /* This function is distinct from main() only so we can link main.c into @@ -2636,31 +3601,6 @@ int tor_main(int argc, char *argv[]) { int result = 0; -#if defined (WINCE) - WCHAR path [MAX_PATH] = {0}; - WCHAR fullpath [MAX_PATH] = {0}; - PWCHAR p = NULL; - FILE* redir = NULL; - FILE* redirdbg = NULL; - - // this is to facilitate debugging by opening - // a file on a folder shared by the wm emulator. - // if no flashcard (real or emulated) is present, - // log files will be written in the root folder - if (find_flashcard_path(path,MAX_PATH) == -1) { - redir = _wfreopen( L"\\stdout.log", L"w", stdout ); - redirdbg = _wfreopen( L"\\stderr.log", L"w", stderr ); - } else { - swprintf(fullpath,L"\\%s\\tor",path); - CreateDirectory(fullpath,NULL); - - swprintf(fullpath,L"\\%s\\tor\\stdout.log",path); - redir = _wfreopen( fullpath, L"w", stdout ); - - swprintf(fullpath,L"\\%s\\tor\\stderr.log",path); - redirdbg = _wfreopen( fullpath, L"w", stderr ); - } -#endif #ifdef _WIN32 /* Call SetProcessDEPPolicy to permanently enable DEP. @@ -2675,9 +3615,11 @@ tor_main(int argc, char *argv[]) } #endif + configure_backtrace_handler(get_version()); + update_approx_time(time(NULL)); tor_threads_init(); - init_logging(); + init_logging(0); #ifdef USE_DMALLOC { /* Instruct OpenSSL to use our internal wrappers for malloc, @@ -2695,6 +3637,22 @@ tor_main(int argc, char *argv[]) #endif if (tor_init(argc, argv)<0) return -1; + + if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) { + sandbox_cfg_t* cfg = sandbox_init_filter(); + + if (sandbox_init(cfg)) { + log_err(LD_BUG,"Failed to create syscall sandbox filter"); + return -1; + } + + // registering libevent rng +#ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE + evutil_secure_rng_set_urandom_device_file( + (char*) sandbox_intern_string("/dev/urandom")); +#endif + } + switch (get_options()->command) { case CMD_RUN_TOR: #ifdef NT_SERVICE @@ -2702,6 +3660,9 @@ tor_main(int argc, char *argv[]) #endif result = do_main_loop(); break; + case CMD_KEYGEN: + result = load_ed_keys(get_options(), time(NULL)); + break; case CMD_LIST_FINGERPRINT: result = do_list_fingerprint(); break; @@ -2710,9 +3671,13 @@ tor_main(int argc, char *argv[]) result = 0; break; case CMD_VERIFY_CONFIG: - printf("Configuration was valid\n"); + if (quiet_level == 0) + printf("Configuration was valid\n"); result = 0; break; + case CMD_DUMP_CONFIG: + result = do_dump_config(); + break; case CMD_RUN_UNITTESTS: /* only set by test.c */ default: log_warn(LD_BUG,"Illegal command number %d: internal error.", diff --git a/src/or/main.h b/src/or/main.h index 338449b6a6..6949376f3e 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,7 +12,9 @@ #ifndef TOR_MAIN_H #define TOR_MAIN_H -extern int can_complete_circuit; +int have_completed_a_circuit(void); +void note_that_we_completed_a_circuit(void); +void note_that_we_maybe_cant_complete_circuits(void); int connection_add_impl(connection_t *conn, int is_connecting); #define connection_add(conn) connection_add_impl((conn), 0) @@ -24,8 +26,8 @@ void add_connection_to_closeable_list(connection_t *conn); int connection_is_on_closeable_list(connection_t *conn); smartlist_t *get_connection_array(void); -uint64_t get_bytes_read(void); -uint64_t get_bytes_written(void); +MOCK_DECL(uint64_t,get_bytes_read,(void)); +MOCK_DECL(uint64_t,get_bytes_written,(void)); /** Bitmask for events that we can turn on and off with * connection_watch_events. */ @@ -36,26 +38,32 @@ typedef enum watchable_events { } watchable_events_t; void connection_watch_events(connection_t *conn, watchable_events_t events); int connection_is_reading(connection_t *conn); -void connection_stop_reading(connection_t *conn); -void connection_start_reading(connection_t *conn); +MOCK_DECL(void,connection_stop_reading,(connection_t *conn)); +MOCK_DECL(void,connection_start_reading,(connection_t *conn)); int connection_is_writing(connection_t *conn); -void connection_stop_writing(connection_t *conn); -void connection_start_writing(connection_t *conn); +MOCK_DECL(void,connection_stop_writing,(connection_t *conn)); +MOCK_DECL(void,connection_start_writing,(connection_t *conn)); + +void tell_event_loop_to_finish(void); void connection_stop_reading_from_linked_conn(connection_t *conn); void directory_all_unreachable(time_t now); -void directory_info_has_arrived(time_t now, int from_cache); +void directory_info_has_arrived(time_t now, int from_cache, int suppress_logs); void ip_address_changed(int at_interface); void dns_servers_relaunch_checks(void); +void reset_all_main_loop_timers(void); +void reschedule_descriptor_update_check(void); +void reschedule_directory_downloads(void); + +MOCK_DECL(long,get_uptime,(void)); -long get_uptime(void); unsigned get_signewnym_epoch(void); void handle_signals(int is_parent); -void process_signal(uintptr_t sig); +void activate_signal(int signal_num); int try_locking(const or_options_t *options, int err_if_locked); int have_lockfile(void); @@ -66,11 +74,14 @@ void tor_free_all(int postfork); int tor_main(int argc, char *argv[]); -#ifdef MAIN_PRIVATE int do_main_loop(void); -int do_list_fingerprint(void); -void do_hash_password(void); int tor_init(int argc, char **argv); + +#ifdef MAIN_PRIVATE +STATIC void init_connection_lists(void); +STATIC void close_closeable_connections(void); +STATIC void initialize_periodic_events(void); +STATIC void teardown_periodic_events(void); #endif #endif diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0e72c0b89b..5b5c29a6d2 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -1,6 +1,13 @@ -/* Copyright (c) 2009-2013, The Tor Project, Inc. */ +/* Copyright (c) 2009-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * \file microdesc.c + * + * \brief Implements microdescriptors -- an abbreviated description of + * less-frequently-changing router information. + */ + #include "or.h" #include "circuitbuild.h" #include "config.h" @@ -39,22 +46,23 @@ struct microdesc_cache_t { uint64_t total_len_seen; /** Total number of microdescriptors we have added to this cache */ unsigned n_seen; + + /** True iff we have loaded this cache from disk ever. */ + int is_loaded; }; +static microdesc_cache_t *get_microdesc_cache_noload(void); + /** Helper: computes a hash of <b>md</b> to place it in a hash table. */ -static INLINE unsigned int +static inline unsigned int microdesc_hash_(microdesc_t *md) { - unsigned *d = (unsigned*)md->digest; -#if SIZEOF_INT == 4 - return d[0] ^ d[1] ^ d[2] ^ d[3] ^ d[4] ^ d[5] ^ d[6] ^ d[7]; -#else - return d[0] ^ d[1] ^ d[2] ^ d[3]; -#endif + return (unsigned) siphash24g(md->digest, sizeof(md->digest)); } -/** Helper: compares <b>a</b> and </b> for equality for hash-table purposes. */ -static INLINE int +/** Helper: compares <b>a</b> and <b>b</b> for equality for hash-table + * purposes. */ +static inline int microdesc_eq_(microdesc_t *a, microdesc_t *b) { return tor_memeq(a->digest, b->digest, DIGEST256_LEN); @@ -62,9 +70,9 @@ microdesc_eq_(microdesc_t *a, microdesc_t *b) HT_PROTOTYPE(microdesc_map, microdesc_t, node, microdesc_hash_, microdesc_eq_); -HT_GENERATE(microdesc_map, microdesc_t, node, +HT_GENERATE2(microdesc_map, microdesc_t, node, microdesc_hash_, microdesc_eq_, 0.6, - malloc, realloc, free); + tor_reallocarray_, tor_free_) /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. * On success, return the total number of bytes written, and set @@ -118,12 +126,23 @@ static microdesc_cache_t *the_microdesc_cache = NULL; microdesc_cache_t * get_microdesc_cache(void) { + microdesc_cache_t *cache = get_microdesc_cache_noload(); + if (PREDICT_UNLIKELY(cache->is_loaded == 0)) { + microdesc_cache_reload(cache); + } + return cache; +} + +/** Return a pointer to the microdescriptor cache, creating (but not loading) + * it if necessary. */ +static microdesc_cache_t * +get_microdesc_cache_noload(void) +{ if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) { - microdesc_cache_t *cache = tor_malloc_zero(sizeof(microdesc_cache_t)); + microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache)); HT_INIT(microdesc_map, &cache->map); cache->cache_fname = get_datadir_fname("cached-microdescs"); cache->journal_fname = get_datadir_fname("cached-microdescs.new"); - microdesc_cache_reload(cache); the_microdesc_cache = cache; } return the_microdesc_cache; @@ -139,7 +158,7 @@ get_microdesc_cache(void) * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>, * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE, * leave their bodies as pointers to the mmap'd cache. If where is - * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is positive, + * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1, * set the last_listed field of every microdesc to listed_at. If * requested_digests is non-null, then it contains a list of digests we mean * to allow, so we should reject any non-requested microdesc with a different @@ -152,39 +171,81 @@ microdescs_add_to_cache(microdesc_cache_t *cache, int no_save, time_t listed_at, smartlist_t *requested_digests256) { + void * const DIGEST_REQUESTED = (void*)1; + void * const DIGEST_RECEIVED = (void*)2; + void * const DIGEST_INVALID = (void*)3; + smartlist_t *descriptors, *added; const int allow_annotations = (where != SAVED_NOWHERE); + smartlist_t *invalid_digests = smartlist_new(); descriptors = microdescs_parse_from_string(s, eos, allow_annotations, - where); - if (listed_at > 0) { + where, invalid_digests); + if (listed_at != (time_t)-1) { SMARTLIST_FOREACH(descriptors, microdesc_t *, md, md->last_listed = listed_at); } if (requested_digests256) { - digestmap_t *requested; /* XXXX actually we should just use a - digest256map */ - requested = digestmap_new(); - SMARTLIST_FOREACH(requested_digests256, const char *, cp, - digestmap_set(requested, cp, (void*)1)); + digest256map_t *requested; + requested = digest256map_new(); + /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */ + SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp, + digest256map_set(requested, cp, DIGEST_REQUESTED)); + /* Set requested[d] to DIGEST_INVALID for every md we requested which we + * will never be able to parse. Remove the ones we didn't request from + * invalid_digests. + */ + SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) { + if (digest256map_get(requested, cp)) { + digest256map_set(requested, cp, DIGEST_INVALID); + } else { + tor_free(cp); + SMARTLIST_DEL_CURRENT(invalid_digests, cp); + } + } SMARTLIST_FOREACH_END(cp); + /* Update requested[d] to 2 for the mds we asked for and got. Delete the + * ones we never requested from the 'descriptors' smartlist. + */ SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { - if (digestmap_get(requested, md->digest)) { - digestmap_set(requested, md->digest, (void*)2); + if (digest256map_get(requested, (const uint8_t*)md->digest)) { + digest256map_set(requested, (const uint8_t*)md->digest, + DIGEST_RECEIVED); } else { log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc"); microdesc_free(md); SMARTLIST_DEL_CURRENT(descriptors, md); } } SMARTLIST_FOREACH_END(md); - SMARTLIST_FOREACH_BEGIN(requested_digests256, char *, cp) { - if (digestmap_get(requested, cp) == (void*)2) { + /* Remove the ones we got or the invalid ones from requested_digests256. + */ + SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) { + void *status = digest256map_get(requested, cp); + if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) { tor_free(cp); SMARTLIST_DEL_CURRENT(requested_digests256, cp); } } SMARTLIST_FOREACH_END(cp); - digestmap_free(requested, NULL); + digest256map_free(requested, NULL); + } + + /* For every requested microdescriptor that was unparseable, mark it + * as not to be retried. */ + if (smartlist_len(invalid_digests)) { + networkstatus_t *ns = + networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC); + if (ns) { + SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) { + routerstatus_t *rs = + router_get_mutable_consensus_status_by_descriptor_digest(ns, d); + if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) { + download_status_mark_impossible(&rs->dl_status); + } + } SMARTLIST_FOREACH_END(d); + } } + SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d)); + smartlist_free(invalid_digests); added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); smartlist_free(descriptors); @@ -280,6 +341,7 @@ void microdesc_cache_clear(microdesc_cache_t *cache) { microdesc_t **entry, **next; + for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) { microdesc_t *md = *entry; next = HT_NEXT_RMV(microdesc_map, &cache->map, entry); @@ -288,7 +350,13 @@ microdesc_cache_clear(microdesc_cache_t *cache) } HT_CLEAR(microdesc_map, &cache->map); if (cache->cache_content) { - tor_munmap_file(cache->cache_content); + int res = tor_munmap_file(cache->cache_content); + if (res != 0) { + log_warn(LD_FS, + "tor_munmap_file() failed clearing microdesc cache; " + "we are probably about to leak memory."); + /* TODO something smarter? */ + } cache->cache_content = NULL; } cache->total_len_seen = 0; @@ -309,6 +377,8 @@ microdesc_cache_reload(microdesc_cache_t *cache) microdesc_cache_clear(cache); + cache->is_loaded = 1; + mm = cache->cache_content = tor_mmap_file(cache->cache_fname); if (mm) { added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, @@ -368,7 +438,9 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) cutoff = now - TOLERATE_MICRODESC_AGE; for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) { - if ((*mdp)->last_listed < cutoff) { + const int is_old = (*mdp)->last_listed < cutoff; + const unsigned held_by_nodes = (*mdp)->held_by_nodes; + if (is_old && !held_by_nodes) { ++dropped; victim = *mdp; mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp); @@ -376,6 +448,57 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) bytes_dropped += victim->bodylen; microdesc_free(victim); } else { + if (is_old) { + /* It's old, but it has held_by_nodes set. That's not okay. */ + /* Let's try to diagnose and fix #7164 . */ + smartlist_t *nodes = nodelist_find_nodes_with_microdesc(*mdp); + const networkstatus_t *ns = networkstatus_get_latest_consensus(); + long networkstatus_age = -1; + const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map); + if (ns) { + networkstatus_age = now - ns->valid_after; + } + log_warn(LD_BUG, "Microdescriptor seemed very old " + "(last listed %d hours ago vs %d hour cutoff), but is still " + "marked as being held by %d node(s). I found %d node(s) " + "holding it. Current networkstatus is %ld hours old. " + "Hashtable badness is %d.", + (int)((now - (*mdp)->last_listed) / 3600), + (int)((now - cutoff) / 3600), + held_by_nodes, + smartlist_len(nodes), + networkstatus_age / 3600, + ht_badness); + + SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) { + const char *rs_match = "No RS"; + const char *rs_present = ""; + if (node->rs) { + if (tor_memeq(node->rs->descriptor_digest, + (*mdp)->digest, DIGEST256_LEN)) { + rs_match = "Microdesc digest in RS matches"; + } else { + rs_match = "Microdesc digest in RS does match"; + } + if (ns) { + /* This should be impossible, but let's see! */ + rs_present = " RS not present in networkstatus."; + SMARTLIST_FOREACH(ns->routerstatus_list, routerstatus_t *,rs, { + if (rs == node->rs) { + rs_present = " RS okay in networkstatus."; + } + }); + } + } + log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s", + node_sl_idx, + hex_str(node->identity, DIGEST_LEN), + node->md, node->rs, node->ri, rs_match, rs_present); + } SMARTLIST_FOREACH_END(node); + smartlist_free(nodes); + (*mdp)->last_listed = now; + } + ++kept; mdp = HT_NEXT(microdesc_map, &cache->map, mdp); } @@ -434,7 +557,7 @@ int microdesc_cache_rebuild(microdesc_cache_t *cache, int force) { open_file_t *open_file; - int fd = -1; + int fd = -1, res; microdesc_t **mdp; smartlist_t *wrote; ssize_t size; @@ -489,7 +612,8 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) "By my count, I'm at "I64_FORMAT ", but I should be at "I64_FORMAT, I64_PRINTF_ARG(off), I64_PRINTF_ARG(off_real)); - off = off_real; + if (off_real >= 0) + off = off_real; } if (md->saved_location != SAVED_IN_CACHE) { tor_free(md->body); @@ -500,8 +624,14 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) /* We must do this unmap _before_ we call finish_writing_to_file(), or * windows will not actually replace the file. */ - if (cache->cache_content) - tor_munmap_file(cache->cache_content); + if (cache->cache_content) { + res = tor_munmap_file(cache->cache_content); + if (res != 0) { + log_warn(LD_FS, + "Failed to unmap old microdescriptor cache while rebuilding"); + } + cache->cache_content = NULL; + } if (finish_writing_to_file(open_file) < 0) { log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s", @@ -514,6 +644,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) microdesc_wipe_body(md); } } + smartlist_free(wrote); return -1; } @@ -592,7 +723,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) /* Make sure that the microdesc was really removed from the appropriate data structures. */ if (md->held_in_map) { - microdesc_cache_t *cache = get_microdesc_cache(); + microdesc_cache_t *cache = get_microdesc_cache_noload(); microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md); if (md2 == md) { log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still " @@ -605,8 +736,10 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) tor_fragile_assert(); } if (md->held_by_nodes) { + microdesc_cache_t *cache = get_microdesc_cache_noload(); int found=0; const smartlist_t *nodes = nodelist_get_list(); + const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map); SMARTLIST_FOREACH(nodes, node_t *, node, { if (node->md == md) { ++found; @@ -614,13 +747,14 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) } }); if (found) { - log_info(LD_BUG, "microdesc_free() called from %s:%d, but md was still " - "referenced %d node(s); held_by_nodes == %u", - fname, lineno, found, md->held_by_nodes); + log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still " + "referenced %d node(s); held_by_nodes == %u, ht_badness == %d", + fname, lineno, found, md->held_by_nodes, ht_badness); } else { log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes " - "set to %u, but md was not referenced by any nodes", - fname, lineno, md->held_by_nodes); + "set to %u, but md was not referenced by any nodes. " + "ht_badness == %d", + fname, lineno, md->held_by_nodes, ht_badness); } tor_fragile_assert(); } @@ -630,6 +764,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) if (md->onion_pkey) crypto_pk_free(md->onion_pkey); tor_free(md->onion_curve25519_pkey); + tor_free(md->ed25519_identity_pkey); if (md->body && md->saved_location != SAVED_IN_CACHE) tor_free(md->body); @@ -686,7 +821,7 @@ microdesc_average_size(microdesc_cache_t *cache) * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */ smartlist_t * microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, - int downloadable_only, digestmap_t *skip) + int downloadable_only, digest256map_t *skip) { smartlist_t *result = smartlist_new(); time_t now = time(NULL); @@ -696,9 +831,9 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, continue; if (downloadable_only && !download_status_is_ready(&rs->dl_status, now, - MAX_MICRODESC_DOWNLOAD_FAILURES)) + get_options()->TestingMicrodescMaxDownloadTries)) continue; - if (skip && digestmap_get(skip, rs->descriptor_digest)) + if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest)) continue; if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN)) continue; @@ -713,7 +848,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, /** Launch download requests for microdescriptors as appropriate. * * Specifically, we should launch download requests if we are configured to - * download mirodescriptors, and there are some microdescriptors listed the + * download mirodescriptors, and there are some microdescriptors listed in the * current microdesc consensus that we don't have, and either we never asked * for them, or we failed to download them but we're willing to retry. */ @@ -723,9 +858,9 @@ update_microdesc_downloads(time_t now) const or_options_t *options = get_options(); networkstatus_t *consensus; smartlist_t *missing; - digestmap_t *pending; + digest256map_t *pending; - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (directory_too_idle_to_fetch_descriptors(options, now)) return; @@ -737,14 +872,14 @@ update_microdesc_downloads(time_t now) if (!we_fetch_microdescriptors(options)) return; - pending = digestmap_new(); + pending = digest256map_new(); list_pending_microdesc_downloads(pending); missing = microdesc_list_missing_digest256(consensus, get_microdesc_cache(), 1, pending); - digestmap_free(pending, NULL); + digest256map_free(pending, NULL); launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC, missing, NULL, now); @@ -820,8 +955,8 @@ we_fetch_router_descriptors(const or_options_t *options) } /** Return the consensus flavor we actually want to use to build circuits. */ -int -usable_consensus_flavor(void) +MOCK_IMPL(int, +usable_consensus_flavor,(void)) { if (we_use_microdescriptors_for_circuits(get_options())) { return FLAV_MICRODESC; diff --git a/src/or/microdesc.h b/src/or/microdesc.h index 7adb8c68af..40c83139e9 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -37,7 +37,7 @@ size_t microdesc_average_size(microdesc_cache_t *cache); smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, int downloadable_only, - digestmap_t *skip); + digest256map_t *skip); void microdesc_free_(microdesc_t *md, const char *fname, int line); #define microdesc_free(md) \ @@ -47,7 +47,7 @@ void microdesc_free_all(void); void update_microdesc_downloads(time_t now); void update_microdescs_from_networkstatus(time_t now); -int usable_consensus_flavor(void); +MOCK_DECL(int, usable_consensus_flavor,(void)); int we_fetch_microdescriptors(const or_options_t *options); int we_fetch_router_descriptors(const or_options_t *options); int we_use_microdescriptors_for_circuits(const or_options_t *options); diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 23b7304b39..1cedfef9b7 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -10,6 +10,7 @@ * client or cache. */ +#define NETWORKSTATUS_PRIVATE #include "or.h" #include "channel.h" #include "circuitmux.h" @@ -31,18 +32,7 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" - -/* For tracking v2 networkstatus documents. Only caches do this now. */ - -/** Map from descriptor digest of routers listed in the v2 networkstatus - * documents to download_status_t* */ -static digestmap_t *v2_download_status_map = NULL; -/** Global list of all of the current v2 network_status documents that we know - * about. This list is kept sorted by published_on. */ -static smartlist_t *networkstatus_v2_list = NULL; -/** True iff any member of networkstatus_v2_list has changed since the last - * time we called download_status_map_update_from_v2_networkstatus() */ -static int networkstatus_v2_list_has_changed = 0; +#include "transports.h" /** Map from lowercase nickname to identity digest of named server, if any. */ static strmap_t *named_server_map = NULL; @@ -88,17 +78,38 @@ typedef struct consensus_waiting_for_certs_t { static consensus_waiting_for_certs_t consensus_waiting_for_certs[N_CONSENSUS_FLAVORS]; -/** The last time we tried to download a networkstatus, or 0 for "never". We - * use this to rate-limit download attempts for directory caches (including - * mirrors). Clients don't use this now. */ -static time_t last_networkstatus_download_attempted = 0; - /** A time before which we shouldn't try to replace the current consensus: * this will be at some point after the next consensus becomes valid, but * before the current consensus becomes invalid. */ static time_t time_to_download_next_consensus[N_CONSENSUS_FLAVORS]; /** Download status for the current consensus networkstatus. */ -static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS]; +static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS] = + { + { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, + DL_SCHED_INCREMENT_FAILURE }, + { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, + DL_SCHED_INCREMENT_FAILURE }, + }; + +#define N_CONSENSUS_BOOTSTRAP_SCHEDULES 2 +#define CONSENSUS_BOOTSTRAP_SOURCE_AUTHORITY 0 +#define CONSENSUS_BOOTSTRAP_SOURCE_ANY_DIRSERVER 1 + +/* Using DL_SCHED_INCREMENT_ATTEMPT on these schedules means that + * download_status_increment_failure won't increment these entries. + * However, any bootstrap connection failures that occur after we have + * a valid consensus will count against the failure counts on the non-bootstrap + * schedules. There should only be one of these, as all the others will have + * been cancelled. (This doesn't seem to be a significant issue.) */ +static download_status_t + consensus_bootstrap_dl_status[N_CONSENSUS_BOOTSTRAP_SCHEDULES] = + { + { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY, + DL_SCHED_INCREMENT_ATTEMPT }, + /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */ + { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, + DL_SCHED_INCREMENT_ATTEMPT }, + }; /** True iff we have logged a warning about this OR's version being older than * listed by the authorities. */ @@ -107,8 +118,10 @@ static int have_warned_about_old_version = 0; * listed by the authorities. */ static int have_warned_about_new_version = 0; -static void download_status_map_update_from_v2_networkstatus(void); static void routerstatus_list_update_named_server_map(void); +static void update_consensus_bootstrap_multiple_downloads( + time_t now, + const or_options_t *options); /** Forget that we've warned about anything networkstatus-related, so we will * give fresh warnings if the same behavior happens again. */ @@ -131,86 +144,12 @@ void networkstatus_reset_download_failures(void) { int i; - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { - if (!router_get_by_descriptor_digest(rs->descriptor_digest)) - rs->need_to_mirror = 1; - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); for (i=0; i < N_CONSENSUS_FLAVORS; ++i) download_status_reset(&consensus_dl_status[i]); - if (v2_download_status_map) { - digestmap_iter_t *iter; - digestmap_t *map = v2_download_status_map; - const char *key; - void *val; - download_status_t *dls; - for (iter = digestmap_iter_init(map); !digestmap_iter_done(iter); - iter = digestmap_iter_next(map, iter) ) { - digestmap_iter_get(iter, &key, &val); - dls = val; - download_status_reset(dls); - } - } -} - -/** Repopulate our list of network_status_t objects from the list cached on - * disk. Return 0 on success, -1 on failure. */ -int -router_reload_v2_networkstatus(void) -{ - smartlist_t *entries; - struct stat st; - char *s; - char *filename = get_datadir_fname("cached-status"); - int maybe_delete = !directory_caches_v2_dir_info(get_options()); - time_t now = time(NULL); - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); - entries = tor_listdir(filename); - if (!entries) { /* dir doesn't exist */ - tor_free(filename); - return 0; - } else if (!smartlist_len(entries) && maybe_delete) { - rmdir(filename); - tor_free(filename); - smartlist_free(entries); - return 0; - } - tor_free(filename); - SMARTLIST_FOREACH_BEGIN(entries, const char *, fn) { - char buf[DIGEST_LEN]; - if (maybe_delete) { - filename = get_datadir_fname2("cached-status", fn); - remove_file_if_very_old(filename, now); - tor_free(filename); - continue; - } - if (strlen(fn) != HEX_DIGEST_LEN || - base16_decode(buf, sizeof(buf), fn, strlen(fn))) { - log_info(LD_DIR, - "Skipping cached-status file with unexpected name \"%s\"",fn); - continue; - } - filename = get_datadir_fname2("cached-status", fn); - s = read_file_to_str(filename, 0, &st); - if (s) { - if (router_set_networkstatus_v2(s, st.st_mtime, NS_FROM_CACHE, - NULL)<0) { - log_warn(LD_FS, "Couldn't load networkstatus from \"%s\"",filename); - } - tor_free(s); - } - tor_free(filename); - } SMARTLIST_FOREACH_END(fn); - SMARTLIST_FOREACH(entries, char *, fn, tor_free(fn)); - smartlist_free(entries); - networkstatus_v2_list_clean(time(NULL)); - routers_update_all_from_networkstatus(time(NULL), 2); - return 0; + for (i=0; i < N_CONSENSUS_BOOTSTRAP_SCHEDULES; ++i) + download_status_reset(&consensus_bootstrap_dl_status[i]); } /** Read every cached v3 consensus networkstatus from the disk. */ @@ -277,7 +216,7 @@ router_reload_consensus_networkstatus(void) } /** Free all storage held by the vote_routerstatus object <b>rs</b>. */ -static void +STATIC void vote_routerstatus_free(vote_routerstatus_t *rs) { vote_microdesc_hash_t *h, *next; @@ -303,26 +242,6 @@ routerstatus_free(routerstatus_t *rs) tor_free(rs); } -/** Free all storage held by the networkstatus object <b>ns</b>. */ -void -networkstatus_v2_free(networkstatus_v2_t *ns) -{ - if (!ns) - return; - tor_free(ns->source_address); - tor_free(ns->contact); - if (ns->signing_key) - crypto_pk_free(ns->signing_key); - tor_free(ns->client_versions); - tor_free(ns->server_versions); - if (ns->entries) { - SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs, - routerstatus_free(rs)); - smartlist_free(ns->entries); - } - tor_free(ns); -} - /** Free all storage held in <b>sig</b> */ void document_signature_free(document_signature_t *sig) @@ -366,6 +285,10 @@ networkstatus_vote_free(networkstatus_t *ns) SMARTLIST_FOREACH(ns->supported_methods, char *, c, tor_free(c)); smartlist_free(ns->supported_methods); } + if (ns->package_lines) { + SMARTLIST_FOREACH(ns->package_lines, char *, c, tor_free(c)); + smartlist_free(ns->package_lines); + } if (ns->voters) { SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) { tor_free(voter->nickname); @@ -648,296 +571,10 @@ networkstatus_check_consensus_signature(networkstatus_t *consensus, return -2; } -/** Helper: return a newly allocated string containing the name of the filename - * where we plan to cache the network status with the given identity digest. */ -char * -networkstatus_get_cache_filename(const char *identity_digest) -{ - char fp[HEX_DIGEST_LEN+1]; - base16_encode(fp, HEX_DIGEST_LEN+1, identity_digest, DIGEST_LEN); - return get_datadir_fname2("cached-status", fp); -} - -/** Helper for smartlist_sort: Compare two networkstatus objects by - * publication date. */ -static int -compare_networkstatus_v2_published_on_(const void **_a, const void **_b) -{ - const networkstatus_v2_t *a = *_a, *b = *_b; - if (a->published_on < b->published_on) - return -1; - else if (a->published_on > b->published_on) - return 1; - else - return 0; -} - -/** Add the parsed v2 networkstatus in <b>ns</b> (with original document in - * <b>s</b>) to the disk cache (and the in-memory directory server cache) as - * appropriate. */ -static int -add_networkstatus_to_cache(const char *s, - v2_networkstatus_source_t source, - networkstatus_v2_t *ns) -{ - if (source != NS_FROM_CACHE) { - char *fn = networkstatus_get_cache_filename(ns->identity_digest); - if (write_str_to_file(fn, s, 0)<0) { - log_notice(LD_FS, "Couldn't write cached network status to \"%s\"", fn); - } - tor_free(fn); - } - - if (directory_caches_v2_dir_info(get_options())) - dirserv_set_cached_networkstatus_v2(s, - ns->identity_digest, - ns->published_on); - - return 0; -} - /** How far in the future do we allow a network-status to get before removing * it? (seconds) */ #define NETWORKSTATUS_ALLOW_SKEW (24*60*60) -/** Given a string <b>s</b> containing a network status that we received at - * <b>arrived_at</b> from <b>source</b>, try to parse it, see if we want to - * store it, and put it into our cache as necessary. - * - * If <b>source</b> is NS_FROM_DIR or NS_FROM_CACHE, do not replace our - * own networkstatus_t (if we're an authoritative directory server). - * - * If <b>source</b> is NS_FROM_CACHE, do not write our networkstatus_t to the - * cache. - * - * If <b>requested_fingerprints</b> is provided, it must contain a list of - * uppercased identity fingerprints. Do not update any networkstatus whose - * fingerprint is not on the list; after updating a networkstatus, remove its - * fingerprint from the list. - * - * Return 0 on success, -1 on failure. - * - * Callers should make sure that routers_update_all_from_networkstatus() is - * invoked after this function succeeds. - */ -int -router_set_networkstatus_v2(const char *s, time_t arrived_at, - v2_networkstatus_source_t source, - smartlist_t *requested_fingerprints) -{ - networkstatus_v2_t *ns; - int i, found; - time_t now; - int skewed = 0; - dir_server_t *trusted_dir = NULL; - const char *source_desc = NULL; - char fp[HEX_DIGEST_LEN+1]; - char published[ISO_TIME_LEN+1]; - - if (!directory_caches_v2_dir_info(get_options())) - return 0; /* Don't bother storing it. */ - - ns = networkstatus_v2_parse_from_string(s); - if (!ns) { - log_warn(LD_DIR, "Couldn't parse network status."); - return -1; - } - base16_encode(fp, HEX_DIGEST_LEN+1, ns->identity_digest, DIGEST_LEN); - if (!(trusted_dir = - router_get_trusteddirserver_by_digest(ns->identity_digest)) || - !(trusted_dir->type & V2_DIRINFO)) { - log_info(LD_DIR, "Network status was signed, but not by an authoritative " - "directory we recognize."); - source_desc = fp; - } else { - source_desc = trusted_dir->description; - } - now = time(NULL); - if (arrived_at > now) - arrived_at = now; - - ns->received_on = arrived_at; - - format_iso_time(published, ns->published_on); - - if (ns->published_on > now + NETWORKSTATUS_ALLOW_SKEW) { - char dbuf[64]; - long delta = now - ns->published_on; - format_time_interval(dbuf, sizeof(dbuf), delta); - log_warn(LD_GENERAL, "Network status from %s was published %s in the " - "future (%s UTC). Check your time and date settings! " - "Not caching.", - source_desc, dbuf, published); - control_event_general_status(LOG_WARN, - "CLOCK_SKEW MIN_SKEW=%ld SOURCE=NETWORKSTATUS:%s:%d", - delta, ns->source_address, ns->source_dirport); - skewed = 1; - } - - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); - - if ( (source == NS_FROM_DIR_BY_FP || source == NS_FROM_DIR_ALL) && - router_digest_is_me(ns->identity_digest)) { - /* Don't replace our own networkstatus when we get it from somebody else.*/ - networkstatus_v2_free(ns); - return 0; - } - - if (requested_fingerprints) { - if (smartlist_contains_string(requested_fingerprints, fp)) { - smartlist_string_remove(requested_fingerprints, fp); - } else { - if (source != NS_FROM_DIR_ALL) { - char *requested = - smartlist_join_strings(requested_fingerprints," ",0,NULL); - log_warn(LD_DIR, - "We received a network status with a fingerprint (%s) that we " - "never requested. (We asked for: %s.) Dropping.", - fp, requested); - tor_free(requested); - return 0; - } - } - } - - if (!trusted_dir) { - if (!skewed) { - /* We got a non-trusted networkstatus, and we're a directory cache. - * This means that we asked an authority, and it told us about another - * authority we didn't recognize. */ - log_info(LD_DIR, - "We do not recognize authority (%s) but we are willing " - "to cache it.", fp); - add_networkstatus_to_cache(s, source, ns); - networkstatus_v2_free(ns); - } - return 0; - } - - found = 0; - for (i=0; i < smartlist_len(networkstatus_v2_list); ++i) { - networkstatus_v2_t *old_ns = smartlist_get(networkstatus_v2_list, i); - - if (tor_memeq(old_ns->identity_digest, ns->identity_digest, DIGEST_LEN)) { - if (tor_memeq(old_ns->networkstatus_digest, - ns->networkstatus_digest, DIGEST_LEN)) { - /* Same one we had before. */ - networkstatus_v2_free(ns); - tor_assert(trusted_dir); - log_info(LD_DIR, - "Not replacing network-status from %s (published %s); " - "we already have it.", - trusted_dir->description, published); - if (old_ns->received_on < arrived_at) { - if (source != NS_FROM_CACHE) { - char *fn; - fn = networkstatus_get_cache_filename(old_ns->identity_digest); - /* We use mtime to tell when it arrived, so update that. */ - touch_file(fn); - tor_free(fn); - } - old_ns->received_on = arrived_at; - } - download_status_failed(&trusted_dir->v2_ns_dl_status, 0); - return 0; - } else if (old_ns->published_on >= ns->published_on) { - char old_published[ISO_TIME_LEN+1]; - format_iso_time(old_published, old_ns->published_on); - tor_assert(trusted_dir); - log_info(LD_DIR, - "Not replacing network-status from %s (published %s);" - " we have a newer one (published %s) for this authority.", - trusted_dir->description, published, - old_published); - networkstatus_v2_free(ns); - download_status_failed(&trusted_dir->v2_ns_dl_status, 0); - return 0; - } else { - networkstatus_v2_free(old_ns); - smartlist_set(networkstatus_v2_list, i, ns); - found = 1; - break; - } - } - } - - if (source != NS_FROM_CACHE && trusted_dir) { - download_status_reset(&trusted_dir->v2_ns_dl_status); - } - - if (!found) - smartlist_add(networkstatus_v2_list, ns); - -/** Retain any routerinfo mentioned in a V2 networkstatus for at least this - * long. */ -#define V2_NETWORKSTATUS_ROUTER_LIFETIME (3*60*60) - - { - time_t live_until = ns->published_on + V2_NETWORKSTATUS_ROUTER_LIFETIME; - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { - signed_descriptor_t *sd = - router_get_by_descriptor_digest(rs->descriptor_digest); - if (sd) { - if (sd->last_listed_as_valid_until < live_until) - sd->last_listed_as_valid_until = live_until; - } else { - rs->need_to_mirror = 1; - } - } SMARTLIST_FOREACH_END(rs); - } - - log_info(LD_DIR, "Setting networkstatus %s %s (published %s)", - source == NS_FROM_CACHE?"cached from": - ((source == NS_FROM_DIR_BY_FP || source == NS_FROM_DIR_ALL) ? - "downloaded from":"generated for"), - trusted_dir->description, published); - networkstatus_v2_list_has_changed = 1; - - smartlist_sort(networkstatus_v2_list, - compare_networkstatus_v2_published_on_); - - if (!skewed) - add_networkstatus_to_cache(s, source, ns); - - return 0; -} - -/** Remove all very-old network_status_t objects from memory and from the - * disk cache. */ -void -networkstatus_v2_list_clean(time_t now) -{ - int i; - if (!networkstatus_v2_list) - return; - - for (i = 0; i < smartlist_len(networkstatus_v2_list); ++i) { - networkstatus_v2_t *ns = smartlist_get(networkstatus_v2_list, i); - char *fname = NULL; - if (ns->published_on + MAX_NETWORKSTATUS_AGE > now) - continue; - /* Okay, this one is too old. Remove it from the list, and delete it - * from the cache. */ - smartlist_del(networkstatus_v2_list, i--); - fname = networkstatus_get_cache_filename(ns->identity_digest); - if (file_status(fname) == FN_FILE) { - log_info(LD_DIR, "Removing too-old networkstatus in %s", fname); - unlink(fname); - } - tor_free(fname); - if (directory_caches_v2_dir_info(get_options())) { - dirserv_set_cached_networkstatus_v2(NULL, ns->identity_digest, 0); - } - networkstatus_v2_free(ns); - } - - /* And now go through the directory cache for any cached untrusted - * networkstatuses and other network info. */ - dirserv_clear_old_networkstatuses(now - MAX_NETWORKSTATUS_AGE); - dirserv_clear_old_v1_info(now); -} - /** Helper for bsearching a list of routerstatus_t pointers: compare a * digest in the key to the identity digest of a routerstatus_t. */ int @@ -959,22 +596,6 @@ compare_digest_to_vote_routerstatus_entry(const void *_key, return tor_memcmp(key, vrs->status.identity_digest, DIGEST_LEN); } -/** As networkstatus_v2_find_entry, but do not return a const pointer */ -routerstatus_t * -networkstatus_v2_find_mutable_entry(networkstatus_v2_t *ns, const char *digest) -{ - return smartlist_bsearch(ns->entries, digest, - compare_digest_to_routerstatus_entry); -} - -/** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or - * NULL if none was found. */ -const routerstatus_t * -networkstatus_v2_find_entry(networkstatus_v2_t *ns, const char *digest) -{ - return networkstatus_v2_find_mutable_entry(ns, digest); -} - /** As networkstatus_find_entry, but do not return a const pointer */ routerstatus_t * networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest) @@ -1004,21 +625,12 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, found_out); } -/** Return a list of the v2 networkstatus documents. */ -const smartlist_t * -networkstatus_get_v2_list(void) -{ - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); - return networkstatus_v2_list; -} - /** As router_get_consensus_status_by_descriptor_digest, but does not return * a const pointer. */ -routerstatus_t * -router_get_mutable_consensus_status_by_descriptor_digest( +MOCK_IMPL(routerstatus_t *, +router_get_mutable_consensus_status_by_descriptor_digest,( networkstatus_t *consensus, - const char *digest) + const char *digest)) { if (!consensus) consensus = current_consensus; @@ -1048,8 +660,8 @@ router_get_consensus_status_by_descriptor_digest(networkstatus_t *consensus, /** Given the digest of a router descriptor, return its current download * status, or NULL if the digest is unrecognized. */ -download_status_t * -router_get_dl_status_by_descriptor_digest(const char *d) +MOCK_IMPL(download_status_t *, +router_get_dl_status_by_descriptor_digest,(const char *d)) { routerstatus_t *rs; if (!current_ns_consensus) @@ -1057,8 +669,6 @@ router_get_dl_status_by_descriptor_digest(const char *d) if ((rs = router_get_mutable_consensus_status_by_descriptor_digest( current_ns_consensus, d))) return &rs->dl_status; - if (v2_download_status_map) - return digestmap_get(v2_download_status_map, d); return NULL; } @@ -1124,72 +734,6 @@ networkstatus_nickname_is_unnamed(const char *nickname) * networkstatus documents? */ #define NONAUTHORITY_NS_CACHE_INTERVAL (60*60) -/** We are a directory server, and so cache network_status documents. - * Initiate downloads as needed to update them. For v2 authorities, - * this means asking each trusted directory for its network-status. - * For caches, this means asking a random v2 authority for all - * network-statuses. - */ -static void -update_v2_networkstatus_cache_downloads(time_t now) -{ - int authority = authdir_mode_v2(get_options()); - int interval = - authority ? AUTHORITY_NS_CACHE_INTERVAL : NONAUTHORITY_NS_CACHE_INTERVAL; - const smartlist_t *trusted_dir_servers = router_get_trusted_dir_servers(); - - if (last_networkstatus_download_attempted + interval >= now) - return; - - last_networkstatus_download_attempted = now; - - if (authority) { - /* An authority launches a separate connection for everybody. */ - SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, dir_server_t *, ds) - { - char resource[HEX_DIGEST_LEN+6]; /* fp/hexdigit.z\0 */ - tor_addr_t addr; - if (!(ds->type & V2_DIRINFO)) - continue; - if (router_digest_is_me(ds->digest)) - continue; - tor_addr_from_ipv4h(&addr, ds->addr); - /* Is this quite sensible with IPv6 or multiple addresses? */ - if (connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &addr, ds->dir_port, - DIR_PURPOSE_FETCH_V2_NETWORKSTATUS)) { - /* XXX the above dir_port won't be accurate if we're - * doing a tunneled conn. In that case it should be or_port. - * How to guess from here? Maybe make the function less general - * and have it know that it's looking for dir conns. -RD */ - /* Only directory caches download v2 networkstatuses, and they - * don't use tunneled connections. I think it's okay to ignore - * this. */ - continue; - } - strlcpy(resource, "fp/", sizeof(resource)); - base16_encode(resource+3, sizeof(resource)-3, ds->digest, DIGEST_LEN); - strlcat(resource, ".z", sizeof(resource)); - directory_initiate_command_routerstatus( - &ds->fake_status, DIR_PURPOSE_FETCH_V2_NETWORKSTATUS, - ROUTER_PURPOSE_GENERAL, - DIRIND_ONEHOP, - resource, - NULL, 0 /* No payload. */, - 0 /* No I-M-S. */); - } - SMARTLIST_FOREACH_END(ds); - } else { - /* A non-authority cache launches one connection to a random authority. */ - /* (Check whether we're currently fetching network-status objects.) */ - if (!connection_get_by_type_purpose(CONN_TYPE_DIR, - DIR_PURPOSE_FETCH_V2_NETWORKSTATUS)) - directory_get_from_dirserver(DIR_PURPOSE_FETCH_V2_NETWORKSTATUS, - ROUTER_PURPOSE_GENERAL, "all.z", - PDS_RETRY_IF_NO_SERVERS); - } -} - /** Return true iff, given the options listed in <b>options</b>, <b>flavor</b> * is the flavor of a consensus networkstatus that we would like to fetch. */ static int @@ -1214,12 +758,39 @@ we_want_to_fetch_flavor(const or_options_t *options, int flavor) return flavor == usable_consensus_flavor(); } -/** How many times will we try to fetch a consensus before we give up? */ -#define CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES 8 /** How long will we hang onto a possibly live consensus for which we're * fetching certs before we check whether there is a better one? */ #define DELAY_WHILE_FETCHING_CERTS (20*60) +/* Check if a downloaded consensus flavor should still wait for certificates + * to download now. + * If so, return 1. If not, fail dls and return 0. */ +static int +check_consensus_waiting_for_certs(int flavor, time_t now, + download_status_t *dls) +{ + consensus_waiting_for_certs_t *waiting; + + /* We should always have a known flavor, because we_want_to_fetch_flavor() + * filters out unknown flavors. */ + tor_assert(flavor >= 0 && flavor < N_CONSENSUS_FLAVORS); + + waiting = &consensus_waiting_for_certs[flavor]; + if (waiting->consensus) { + /* XXXX make sure this doesn't delay sane downloads. */ + if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) { + return 1; + } else { + if (!waiting->dl_failed) { + download_status_failed(dls, 0); + waiting->dl_failed=1; + } + } + } + + return 0; +} + /** If we want to download a fresh consensus, launch a new download as * appropriate. */ static void @@ -1227,12 +798,19 @@ update_consensus_networkstatus_downloads(time_t now) { int i; const or_options_t *options = get_options(); + const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping( + now); + const int use_multi_conn = + networkstatus_consensus_can_use_multiple_directories(options); + + if (should_delay_dir_fetches(options, NULL)) + return; for (i=0; i < N_CONSENSUS_FLAVORS; ++i) { /* XXXX need some way to download unknown flavors if we are caching. */ const char *resource; - consensus_waiting_for_certs_t *waiting; networkstatus_t *c; + int max_in_progress_conns = 1; if (! we_want_to_fetch_flavor(options, i)) continue; @@ -1248,32 +826,147 @@ update_consensus_networkstatus_downloads(time_t now) resource = networkstatus_get_flavor_name(i); - if (!download_status_is_ready(&consensus_dl_status[i], now, - CONSENSUS_NETWORKSTATUS_MAX_DL_TRIES)) - continue; /* We failed downloading a consensus too recently. */ - if (connection_dir_get_by_purpose_and_resource( - DIR_PURPOSE_FETCH_CONSENSUS, resource)) - continue; /* There's an in-progress download.*/ + /* Check if we already have enough connections in progress */ + if (we_are_bootstrapping) { + max_in_progress_conns = + options->ClientBootstrapConsensusMaxInProgressTries; + } + if (connection_dir_count_by_purpose_and_resource( + DIR_PURPOSE_FETCH_CONSENSUS, + resource) + >= max_in_progress_conns) { + continue; + } - waiting = &consensus_waiting_for_certs[i]; - if (waiting->consensus) { - /* XXXX make sure this doesn't delay sane downloads. */ - if (waiting->set_at + DELAY_WHILE_FETCHING_CERTS > now) { - continue; /* We're still getting certs for this one. */ - } else { - if (!waiting->dl_failed) { - download_status_failed(&consensus_dl_status[i], 0); - waiting->dl_failed=1; - } + /* Check if we want to launch another download for a usable consensus. + * Only used during bootstrap. */ + if (we_are_bootstrapping && use_multi_conn + && i == usable_consensus_flavor()) { + + /* Check if we're already downloading a usable consensus */ + if (networkstatus_consensus_is_already_downloading(resource)) + continue; + + /* Make multiple connections for a bootstrap consensus download. */ + update_consensus_bootstrap_multiple_downloads(now, options); + } else { + /* Check if we failed downloading a consensus too recently */ + int max_dl_tries = options->TestingConsensusMaxDownloadTries; + + /* Let's make sure we remembered to update consensus_dl_status */ + tor_assert(consensus_dl_status[i].schedule == DL_SCHED_CONSENSUS); + + if (!download_status_is_ready(&consensus_dl_status[i], + now, + max_dl_tries)) { + continue; } + + /* Check if we're waiting for certificates to download */ + if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i])) + continue; + + /* Try the requested attempt */ + log_info(LD_DIR, "Launching %s standard networkstatus consensus " + "download.", networkstatus_get_flavor_name(i)); + directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS, + ROUTER_PURPOSE_GENERAL, resource, + PDS_RETRY_IF_NO_SERVERS, + consensus_dl_status[i].want_authority); } + } +} + +/** When we're bootstrapping, launch one or more consensus download + * connections, if schedule indicates connection(s) should be made after now. + * If is_authority, connect to an authority, otherwise, use a fallback + * directory mirror. + */ +static void +update_consensus_bootstrap_attempt_downloads( + time_t now, + const or_options_t *options, + download_status_t *dls, + download_want_authority_t want_authority) +{ + int use_fallbacks = networkstatus_consensus_can_use_extra_fallbacks(options); + int max_dl_tries = options->ClientBootstrapConsensusMaxDownloadTries; + if (!use_fallbacks) { + max_dl_tries = + options->ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries; + } + + const char *resource = networkstatus_get_flavor_name( + usable_consensus_flavor()); + + /* Let's make sure we remembered to update schedule */ + tor_assert(dls->schedule == DL_SCHED_CONSENSUS); - log_info(LD_DIR, "Launching %s networkstatus consensus download.", - networkstatus_get_flavor_name(i)); + /* Allow for multiple connections in the same second, if the schedule value + * is 0. */ + while (download_status_is_ready(dls, now, max_dl_tries)) { + log_info(LD_DIR, "Launching %s bootstrap %s networkstatus consensus " + "download.", resource, (want_authority == DL_WANT_AUTHORITY + ? "authority" + : "mirror")); directory_get_from_dirserver(DIR_PURPOSE_FETCH_CONSENSUS, ROUTER_PURPOSE_GENERAL, resource, - PDS_RETRY_IF_NO_SERVERS); + PDS_RETRY_IF_NO_SERVERS, want_authority); + /* schedule the next attempt */ + download_status_increment_attempt(dls, resource, now); + } +} + +/** If we're bootstrapping, check the connection schedules and see if we want + * to make additional, potentially concurrent, consensus download + * connections. + * Only call when bootstrapping, and when we want to make additional + * connections. Only nodes that satisfy + * networkstatus_consensus_can_use_multiple_directories make additional + * connections. + */ +static void +update_consensus_bootstrap_multiple_downloads(time_t now, + const or_options_t *options) +{ + const int usable_flavor = usable_consensus_flavor(); + + /* make sure we can use multiple connections */ + if (!networkstatus_consensus_can_use_multiple_directories(options)) { + return; + } + + /* Launch concurrent consensus download attempt(s) based on the mirror and + * authority schedules. Try the mirror first - this makes it slightly more + * likely that we'll connect to the fallback first, and then end the + * authority connection attempt. */ + + /* If a consensus download fails because it's waiting for certificates, + * we'll fail both the authority and fallback schedules. This is better than + * failing only one of the schedules, and having the other continue + * unchecked. + */ + + /* If we don't have or can't use extra fallbacks, don't try them. */ + if (networkstatus_consensus_can_use_extra_fallbacks(options)) { + download_status_t *dls_f = + &consensus_bootstrap_dl_status[CONSENSUS_BOOTSTRAP_SOURCE_ANY_DIRSERVER]; + + if (!check_consensus_waiting_for_certs(usable_flavor, now, dls_f)) { + /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */ + update_consensus_bootstrap_attempt_downloads(now, options, dls_f, + DL_WANT_ANY_DIRSERVER); + } + } + + /* Now try an authority. */ + download_status_t *dls_a = + &consensus_bootstrap_dl_status[CONSENSUS_BOOTSTRAP_SOURCE_AUTHORITY]; + + if (!check_consensus_waiting_for_certs(usable_flavor, now, dls_a)) { + update_consensus_bootstrap_attempt_downloads(now, options, dls_a, + DL_WANT_AUTHORITY); } } @@ -1319,12 +1012,16 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) a crazy-fast voting interval, though, 2 minutes may be too much. */ min_sec_before_caching = interval/16; + /* make sure we always delay by at least a second before caching */ + if (min_sec_before_caching == 0) { + min_sec_before_caching = 1; + } } if (directory_fetches_dir_info_early(options)) { /* We want to cache the next one at some point after this one * is no longer fresh... */ - start = c->fresh_until + min_sec_before_caching; + start = (time_t)(c->fresh_until + min_sec_before_caching); /* Some clients may need the consensus sooner than others. */ if (options->FetchDirInfoExtraEarly || authdir_mode_v3(options)) { dl_interval = 60; @@ -1335,9 +1032,9 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) dl_interval = interval/2; } } else { - /* We're an ordinary client or a bridge. Give all the caches enough - * time to download the consensus. */ - start = c->fresh_until + (interval*3)/4; + /* We're an ordinary client, a bridge, or a hidden service. + * Give all the caches enough time to download the consensus. */ + start = (time_t)(c->fresh_until + (interval*3)/4); /* But download the next one well before this one is expired. */ dl_interval = ((c->valid_until - start) * 7 )/ 8; @@ -1345,13 +1042,22 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) * to choose the rest of the interval *after* them. */ if (directory_fetches_dir_info_later(options)) { /* Give all the *clients* enough time to download the consensus. */ - start = start + dl_interval + min_sec_before_caching; + start = (time_t)(start + dl_interval + min_sec_before_caching); /* But try to get it before ours actually expires. */ dl_interval = (c->valid_until - start) - min_sec_before_caching; } } + /* catch low dl_interval in crazy-fast networks */ if (dl_interval < 1) dl_interval = 1; + /* catch late start in crazy-fast networks */ + if (start+dl_interval >= c->valid_until) + start = c->valid_until - dl_interval - 1; + log_debug(LD_DIR, + "fresh_until: %ld start: %ld " + "dl_interval: %ld valid_until: %ld ", + (long)c->fresh_until, (long)start, dl_interval, + (long)c->valid_until); /* We must not try to replace c while it's still fresh: */ tor_assert(c->fresh_until < start); /* We must download the next one before c is invalid: */ @@ -1391,14 +1097,45 @@ update_consensus_networkstatus_fetch_time(time_t now) /** Return 1 if there's a reason we shouldn't try any directory * fetches yet (e.g. we demand bridges and none are yet known). - * Else return 0. */ + * Else return 0. + + * If we return 1 and <b>msg_out</b> is provided, set <b>msg_out</b> + * to an explanation of why directory fetches are delayed. (If we + * return 0, we set msg_out to NULL.) + */ int -should_delay_dir_fetches(const or_options_t *options) +should_delay_dir_fetches(const or_options_t *options, const char **msg_out) { - if (options->UseBridges && !any_bridge_descriptors_known()) { - log_info(LD_DIR, "delaying dir fetches (no running bridges known)"); + if (msg_out) { + *msg_out = NULL; + } + + if (options->DisableNetwork) { + if (msg_out) { + *msg_out = "DisableNetwork is set."; + } + log_info(LD_DIR, "Delaying dir fetches (DisableNetwork is set)"); return 1; } + + if (options->UseBridges) { + if (!any_bridge_descriptors_known()) { + if (msg_out) { + *msg_out = "No running bridges"; + } + log_info(LD_DIR, "Delaying dir fetches (no running bridges known)"); + return 1; + } + + if (pt_proxies_configuration_pending()) { + if (msg_out) { + *msg_out = "Pluggable transport proxies still configuring"; + } + log_info(LD_DIR, "Delaying dir fetches (pt proxies still configuring)"); + return 1; + } + } + return 0; } @@ -1408,10 +1145,8 @@ void update_networkstatus_downloads(time_t now) { const or_options_t *options = get_options(); - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; - if (authdir_mode_any_main(options) || options->FetchV2Networkstatus) - update_v2_networkstatus_cache_downloads(now); update_consensus_networkstatus_downloads(now); update_certificate_downloads(now); } @@ -1453,8 +1188,8 @@ networkstatus_get_latest_consensus(void) /** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL * if we don't have one. */ -networkstatus_t * -networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f) +MOCK_IMPL(networkstatus_t *, +networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f)) { if (f == FLAV_NS) return current_ns_consensus; @@ -1498,6 +1233,100 @@ networkstatus_get_reasonably_live_consensus(time_t now, int flavor) return NULL; } +/** Check if we need to download a consensus during tor's bootstrap phase. + * If we have no consensus, or our consensus is unusably old, return 1. + * As soon as we have received a consensus, return 0, even if we don't have + * enough certificates to validate it. + * If a fallback directory gives us a consensus we can never get certs for, + * check_consensus_waiting_for_certs() will wait 20 minutes before failing + * the cert downloads. After that, a new consensus will be fetched from a + * randomly chosen fallback. */ +MOCK_IMPL(int, +networkstatus_consensus_is_bootstrapping,(time_t now)) +{ + /* If we have a validated, reasonably live consensus, we're not + * bootstrapping a consensus at all. */ + if (networkstatus_get_reasonably_live_consensus( + now, + usable_consensus_flavor())) { + return 0; + } + + /* If we have a consensus, but we're waiting for certificates, + * we're not waiting for a consensus download while bootstrapping. */ + if (consensus_is_waiting_for_certs()) { + return 0; + } + + /* If we have no consensus, or our consensus is very old, we are + * bootstrapping, and we need to download a consensus. */ + return 1; +} + +/** Check if we can use multiple directories for a consensus download. + * Only clients (including bridge relays, which act like clients) benefit + * from multiple simultaneous consensus downloads. */ +int +networkstatus_consensus_can_use_multiple_directories( + const or_options_t *options) +{ + /* If we are a client, bridge, bridge client, or hidden service */ + return !public_server_mode(options); +} + +/** Check if we can use fallback directory mirrors for a consensus download. + * If we have fallbacks and don't want to fetch from the authorities, + * we can use them. */ +MOCK_IMPL(int, +networkstatus_consensus_can_use_extra_fallbacks,(const or_options_t *options)) +{ + /* The list length comparisons are a quick way to check if we have any + * non-authority fallback directories. If we ever have any authorities that + * aren't fallback directories, we will need to change this code. */ + tor_assert(smartlist_len(router_get_fallback_dir_servers()) + >= smartlist_len(router_get_trusted_dir_servers())); + /* If we don't fetch from the authorities, and we have additional mirrors, + * we can use them. */ + return (!directory_fetches_from_authorities(options) + && (smartlist_len(router_get_fallback_dir_servers()) + > smartlist_len(router_get_trusted_dir_servers()))); +} + +/* Is there a consensus fetch for flavor <b>resource</b> that's far + * enough along to be attached to a circuit? */ +int +networkstatus_consensus_is_already_downloading(const char *resource) +{ + int answer = 0; + + /* First, get a list of all the dir conns that are fetching a consensus, + * fetching *this* consensus, and are in state "reading" (meaning they + * have already flushed their request onto the socks connection). */ + smartlist_t *fetching_conns = + connection_dir_list_by_purpose_resource_and_state( + DIR_PURPOSE_FETCH_CONSENSUS, resource, DIR_CONN_STATE_CLIENT_READING); + + /* Then, walk through each conn, to see if its linked socks connection + * is in an attached state. We have to check this separately, since with + * the optimistic data feature, fetches can send their request to the + * socks connection and go into state 'reading', even before they're + * attached to any circuit. */ + SMARTLIST_FOREACH_BEGIN(fetching_conns, dir_connection_t *, dirconn) { + /* Do any of these other dir conns have a linked socks conn that is + * attached to a circuit already? */ + connection_t *base = TO_CONN(dirconn); + if (base->linked_conn && + base->linked_conn->type == CONN_TYPE_AP && + !AP_CONN_STATE_IS_UNATTACHED(base->linked_conn->state)) { + answer = 1; + break; /* stop looping, because we know the answer will be yes */ + } + } SMARTLIST_FOREACH_END(dirconn); + smartlist_free(fetching_conns); + + return answer; +} + /** Given two router status entries for the same router identity, return 1 if * if the contents have changed between them. Otherwise, return 0. */ static int @@ -1518,10 +1347,8 @@ routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b) a->is_named != b->is_named || a->is_unnamed != b->is_unnamed || a->is_valid != b->is_valid || - a->is_v2_dir != b->is_v2_dir || a->is_possible_guard != b->is_possible_guard || a->is_bad_exit != b->is_bad_exit || - a->is_bad_directory != b->is_bad_directory || a->is_hs_dir != b->is_hs_dir || a->version_known != b->version_known; } @@ -1583,13 +1410,45 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c, rs_new->last_dir_503_at = rs_old->last_dir_503_at; if (tor_memeq(rs_old->descriptor_digest, rs_new->descriptor_digest, - DIGEST_LEN)) { + DIGEST256_LEN)) { /* And the same descriptor too! */ memcpy(&rs_new->dl_status, &rs_old->dl_status,sizeof(download_status_t)); } } SMARTLIST_FOREACH_JOIN_END(rs_old, rs_new); } +#ifdef TOR_UNIT_TESTS +/**Accept a <b>flavor</b> consensus <b>c</b> without any additional + * validation. This is exclusively for unit tests. + * We copy any ancillary information from a pre-existing consensus + * and then free the current one and replace it with the newly + * provided instance. Returns -1 on unrecognized flavor, 0 otherwise. + */ +int +networkstatus_set_current_consensus_from_ns(networkstatus_t *c, + const char *flavor) +{ + int flav = networkstatus_parse_flavor_name(flavor); + switch (flav) { + case FLAV_NS: + if (current_ns_consensus) { + networkstatus_copy_old_consensus_info(c, current_ns_consensus); + networkstatus_vote_free(current_ns_consensus); + } + current_ns_consensus = c; + break; + case FLAV_MICRODESC: + if (current_md_consensus) { + networkstatus_copy_old_consensus_info(c, current_md_consensus); + networkstatus_vote_free(current_md_consensus); + } + current_md_consensus = c; + break; + } + return current_md_consensus ? 0 : -1; +} +#endif //TOR_UNIT_TESTS + /** Try to replace the current cached v3 networkstatus with the one in * <b>consensus</b>. If we don't have enough certificates to validate it, * store it in consensus_waiting_for_certs and launch a certificate fetch. @@ -1623,7 +1482,7 @@ networkstatus_set_current_consensus(const char *consensus, const unsigned dl_certs = !(flags & NSSET_DONT_DOWNLOAD_CERTS); const unsigned accept_obsolete = flags & NSSET_ACCEPT_OBSOLETE; const unsigned require_flavor = flags & NSSET_REQUIRE_FLAVOR; - const digests_t *current_digests = NULL; + const common_digests_t *current_digests = NULL; consensus_waiting_for_certs_t *waiting = NULL; time_t current_valid_after = 0; int free_consensus = 1; /* Free 'c' at the end of the function */ @@ -1740,7 +1599,11 @@ networkstatus_set_current_consensus(const char *consensus, /* Even if we had enough signatures, we'd never use this as the * latest consensus. */ if (was_waiting_for_certs && from_cache) - unlink(unverified_fname); + if (unlink(unverified_fname) != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); + } } goto done; } else { @@ -1750,8 +1613,13 @@ networkstatus_set_current_consensus(const char *consensus, "consensus"); result = -2; } - if (was_waiting_for_certs && (r < -1) && from_cache) - unlink(unverified_fname); + if (was_waiting_for_certs && (r < -1) && from_cache) { + if (unlink(unverified_fname) != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); + } + } goto done; } } @@ -1763,7 +1631,9 @@ networkstatus_set_current_consensus(const char *consensus, if (r != 1 && dl_certs) authority_certs_fetch_missing(c, now); - if (flav == usable_consensus_flavor()) { + const int is_usable_flavor = flav == usable_consensus_flavor(); + + if (is_usable_flavor) { notify_control_networkstatus_changed(current_consensus, c); } if (flav == FLAV_NS) { @@ -1799,23 +1669,19 @@ networkstatus_set_current_consensus(const char *consensus, waiting->body = NULL; waiting->set_at = 0; waiting->dl_failed = 0; - unlink(unverified_fname); + if (unlink(unverified_fname) != 0) { + log_warn(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); + } } - /* Reset the failure count only if this consensus is actually valid. */ - if (c->valid_after <= now && now <= c->valid_until) { - download_status_reset(&consensus_dl_status[flav]); - } else { - if (!from_cache) - download_status_failed(&consensus_dl_status[flav], 0); - } + if (is_usable_flavor) { + nodelist_set_consensus(c); - if (flav == usable_consensus_flavor()) { /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/ update_consensus_networkstatus_fetch_time(now); - nodelist_set_consensus(current_consensus); - dirvote_recalculate_timing(options, now); routerstatus_list_update_named_server_map(); @@ -1835,7 +1701,16 @@ networkstatus_set_current_consensus(const char *consensus, * current consensus really alter our view of any OR's rate limits? */ connection_or_update_token_buckets(get_connection_array(), options); - circuit_build_times_new_consensus_params(&circ_times, current_consensus); + circuit_build_times_new_consensus_params(get_circuit_build_times_mutable(), + current_consensus); + } + + /* Reset the failure count only if this consensus is actually valid. */ + if (c->valid_after <= now && now <= c->valid_until) { + download_status_reset(&consensus_dl_status[flav]); + } else { + if (!from_cache) + download_status_failed(&consensus_dl_status[flav], 0); } if (directory_caches_dir_info(options)) { @@ -1912,9 +1787,6 @@ routers_update_all_from_networkstatus(time_t now, int dir_version) networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_NS); - if (networkstatus_v2_list_has_changed) - download_status_map_update_from_v2_networkstatus(); - if (!consensus || dir_version < 3) /* nothing more we should do */ return; @@ -1969,35 +1841,6 @@ routers_update_all_from_networkstatus(time_t now, int dir_version) } } -/** Update v2_download_status_map to contain an entry for every router - * descriptor listed in the v2 networkstatuses. */ -static void -download_status_map_update_from_v2_networkstatus(void) -{ - digestmap_t *dl_status; - if (!networkstatus_v2_list) - return; - if (!v2_download_status_map) - v2_download_status_map = digestmap_new(); - - dl_status = digestmap_new(); - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - SMARTLIST_FOREACH_BEGIN(ns->entries, const routerstatus_t *, rs) { - const char *d = rs->descriptor_digest; - download_status_t *s; - if (digestmap_get(dl_status, d)) - continue; - if (!(s = digestmap_remove(v2_download_status_map, d))) { - s = tor_malloc_zero(sizeof(download_status_t)); - } - digestmap_set(dl_status, d, s); - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); - digestmap_free(v2_download_status_map, tor_free_); - v2_download_status_map = dl_status; - networkstatus_v2_list_has_changed = 0; -} - /** Update our view of the list of named servers from the most recently * retrieved networkstatus consensus. */ static void @@ -2029,14 +1872,11 @@ void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, int reset_failures) { - dir_server_t *ds; const or_options_t *options = get_options(); - int authdir = authdir_mode_v2(options) || authdir_mode_v3(options); + int authdir = authdir_mode_v3(options); networkstatus_t *ns = current_consensus; if (!ns || !smartlist_len(ns->routerstatus_list)) return; - if (!networkstatus_v2_list) - networkstatus_v2_list = smartlist_new(); routers_sort_by_identity(routers); @@ -2046,11 +1886,6 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, router->cache_info.identity_digest, DIGEST_LEN), { }) { - /* We have a routerstatus for this router. */ - const char *digest = router->cache_info.identity_digest; - - ds = router_get_fallback_dirserver_by_digest(digest); - /* Is it the same descriptor, or only the same identity? */ if (tor_memeq(router->cache_info.signed_descriptor_digest, rs->descriptor_digest, DIGEST_LEN)) { @@ -2068,30 +1903,11 @@ routers_update_status_from_consensus_networkstatus(smartlist_t *routers, dirserv_should_launch_reachability_test(router, old_router); } } - if (rs->is_flagged_running && ds) { - download_status_reset(&ds->v2_ns_dl_status); - } if (reset_failures) { download_status_reset(&rs->dl_status); } } SMARTLIST_FOREACH_JOIN_END(rs, router); - /* Now update last_listed_as_valid_until from v2 networkstatuses. */ - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - time_t live_until = ns->published_on + V2_NETWORKSTATUS_ROUTER_LIFETIME; - SMARTLIST_FOREACH_JOIN(ns->entries, const routerstatus_t *, rs, - routers, routerinfo_t *, ri, - tor_memcmp(rs->identity_digest, - ri->cache_info.identity_digest, DIGEST_LEN), - STMT_NIL) { - if (tor_memeq(ri->cache_info.signed_descriptor_digest, - rs->descriptor_digest, DIGEST_LEN)) { - if (live_until > ri->cache_info.last_listed_as_valid_until) - ri->cache_info.last_listed_as_valid_until = live_until; - } - } SMARTLIST_FOREACH_JOIN_END(rs, ri); - } SMARTLIST_FOREACH_END(ns); - router_dir_info_changed(); } @@ -2166,7 +1982,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE) dirserv_set_router_is_running(ri, now); /* then generate and write out status lines for each of them */ - set_routerstatus_from_routerinfo(&rs, node, ri, now, 0, 0, 0, 0); + set_routerstatus_from_routerinfo(&rs, node, ri, now, 0); smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs)); } SMARTLIST_FOREACH_END(ri); @@ -2183,9 +1999,21 @@ networkstatus_dump_bridge_status_to_file(time_t now) char *status = networkstatus_getinfo_by_purpose("bridge", now); const or_options_t *options = get_options(); char *fname = NULL; + char *thresholds = NULL; + char *published_thresholds_and_status = NULL; + char published[ISO_TIME_LEN+1]; + + format_iso_time(published, now); + dirserv_compute_bridge_flag_thresholds(); + thresholds = dirserv_get_flag_thresholds_line(); + tor_asprintf(&published_thresholds_and_status, + "published %s\nflag-thresholds %s\n%s", + published, thresholds, status); tor_asprintf(&fname, "%s"PATH_SEPARATOR"networkstatus-bridges", options->DataDirectory); - write_str_to_file(fname,status,0); + write_str_to_file(fname,published_thresholds_and_status,0); + tor_free(thresholds); + tor_free(published_thresholds_and_status); tor_free(fname); tor_free(status); } @@ -2388,6 +2216,33 @@ getinfo_helper_networkstatus(control_connection_t *conn, } else if (!strcmpstart(question, "ns/purpose/")) { *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL)); return *answer ? 0 : -1; + } else if (!strcmp(question, "consensus/packages")) { + const networkstatus_t *ns = networkstatus_get_latest_consensus(); + if (ns && ns->package_lines) + *answer = smartlist_join_strings(ns->package_lines, "\n", 0, NULL); + else + *errmsg = "No consensus available"; + return *answer ? 0 : -1; + } else if (!strcmp(question, "consensus/valid-after") || + !strcmp(question, "consensus/fresh-until") || + !strcmp(question, "consensus/valid-until")) { + const networkstatus_t *ns = networkstatus_get_latest_consensus(); + if (ns) { + time_t t; + if (!strcmp(question, "consensus/valid-after")) + t = ns->valid_after; + else if (!strcmp(question, "consensus/fresh-until")) + t = ns->fresh_until; + else + t = ns->valid_until; + + char tbuf[ISO_TIME_LEN+1]; + format_iso_time(tbuf, t); + *answer = tor_strdup(tbuf); + } else { + *errmsg = "No consensus available"; + } + return *answer ? 0 : -1; } else { return 0; } @@ -2402,15 +2257,6 @@ void networkstatus_free_all(void) { int i; - if (networkstatus_v2_list) { - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - networkstatus_v2_free(ns)); - smartlist_free(networkstatus_v2_list); - networkstatus_v2_list = NULL; - } - - digestmap_free(v2_download_status_map, tor_free_); - v2_download_status_map = NULL; networkstatus_vote_free(current_ns_consensus); networkstatus_vote_free(current_md_consensus); current_md_consensus = current_ns_consensus = NULL; diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 761f8e7f0e..ac93e5de91 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,16 +12,12 @@ #ifndef TOR_NETWORKSTATUS_H #define TOR_NETWORKSTATUS_H -/** How old do we allow a v2 network-status to get before removing it - * completely? */ -#define MAX_NETWORKSTATUS_AGE (10*24*60*60) +#include "testsupport.h" void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); -int router_reload_v2_networkstatus(void); int router_reload_consensus_networkstatus(void); void routerstatus_free(routerstatus_t *rs); -void networkstatus_v2_free(networkstatus_v2_t *ns); void networkstatus_vote_free(networkstatus_t *ns); networkstatus_voter_info_t *networkstatus_get_voter_by_id( networkstatus_t *vote, @@ -31,36 +27,29 @@ int networkstatus_check_consensus_signature(networkstatus_t *consensus, int networkstatus_check_document_signature(const networkstatus_t *consensus, document_signature_t *sig, const authority_cert_t *cert); -char *networkstatus_get_cache_filename(const char *identity_digest); -int router_set_networkstatus_v2(const char *s, time_t arrived_at, - v2_networkstatus_source_t source, - smartlist_t *requested_fingerprints); -void networkstatus_v2_list_clean(time_t now); int compare_digest_to_routerstatus_entry(const void *_key, const void **_member); int compare_digest_to_vote_routerstatus_entry(const void *_key, const void **_member); -const routerstatus_t *networkstatus_v2_find_entry(networkstatus_v2_t *ns, - const char *digest); const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest); -routerstatus_t *networkstatus_v2_find_mutable_entry(networkstatus_v2_t *ns, - const char *digest); routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest); int networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out); -const smartlist_t *networkstatus_get_v2_list(void); -download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); + +MOCK_DECL(download_status_t *,router_get_dl_status_by_descriptor_digest, + (const char *d)); + const routerstatus_t *router_get_consensus_status_by_id(const char *digest); routerstatus_t *router_get_mutable_consensus_status_by_id( const char *digest); const routerstatus_t *router_get_consensus_status_by_descriptor_digest( networkstatus_t *consensus, const char *digest); -routerstatus_t *router_get_mutable_consensus_status_by_descriptor_digest( - networkstatus_t *consensus, - const char *digest); +MOCK_DECL(routerstatus_t *, + router_get_mutable_consensus_status_by_descriptor_digest, + (networkstatus_t *consensus, const char *digest)); const routerstatus_t *router_get_consensus_status_by_nickname( const char *nickname, int warn_if_unnamed); @@ -69,18 +58,25 @@ int networkstatus_nickname_is_unnamed(const char *nickname); void networkstatus_consensus_download_failed(int status_code, const char *flavname); void update_consensus_networkstatus_fetch_time(time_t now); -int should_delay_dir_fetches(const or_options_t *options); +int should_delay_dir_fetches(const or_options_t *options,const char **msg_out); void update_networkstatus_downloads(time_t now); void update_certificate_downloads(time_t now); int consensus_is_waiting_for_certs(void); int client_would_use_router(const routerstatus_t *rs, time_t now, const or_options_t *options); networkstatus_t *networkstatus_get_latest_consensus(void); -networkstatus_t *networkstatus_get_latest_consensus_by_flavor( - consensus_flavor_t f); +MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor, + (consensus_flavor_t f)); networkstatus_t *networkstatus_get_live_consensus(time_t now); networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now, int flavor); +MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now)); +int networkstatus_consensus_can_use_multiple_directories( + const or_options_t *options); +MOCK_DECL(int, networkstatus_consensus_can_use_extra_fallbacks,( + const or_options_t *options)); +int networkstatus_consensus_is_already_downloading(const char *resource); + #define NSSET_FROM_CACHE 1 #define NSSET_WAS_WAITING_FOR_CERTS 2 #define NSSET_DONT_DOWNLOAD_CERTS 4 @@ -115,5 +111,13 @@ document_signature_t *document_signature_dup(const document_signature_t *sig); void networkstatus_free_all(void); int networkstatus_get_weight_scale_param(networkstatus_t *ns); +#ifdef NETWORKSTATUS_PRIVATE +STATIC void vote_routerstatus_free(vote_routerstatus_t *rs); +#ifdef TOR_UNIT_TESTS +STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c, + const char *flavor); +#endif // TOR_UNIT_TESTS +#endif + #endif diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 178f084b69..89b5355c8d 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -1,9 +1,17 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * \file nodelist.c + * + * \brief Structures and functions for tracking what we know about the routers + * on the Tor network, and correlating information from networkstatus, + * routerinfo, and microdescs. + */ + #include "or.h" #include "address.h" #include "config.h" @@ -24,7 +32,26 @@ static void nodelist_drop_node(node_t *node, int remove_from_ht); static void node_free(node_t *node); + +/** count_usable_descriptors counts descriptors with these flag(s) + */ +typedef enum { + /* All descriptors regardless of flags */ + USABLE_DESCRIPTOR_ALL = 0, + /* Only descriptors with the Exit flag */ + USABLE_DESCRIPTOR_EXIT_ONLY = 1 +} usable_descriptor_t; +static void count_usable_descriptors(int *num_present, + int *num_usable, + smartlist_t *descs_out, + const networkstatus_t *consensus, + const or_options_t *options, + time_t now, + routerset_t *in_set, + usable_descriptor_t exit_only); static void update_router_have_minimum_dir_info(void); +static double get_frac_paths_needed_for_circs(const or_options_t *options, + const networkstatus_t *ns); /** A nodelist_t holds a node_t object for every router we're "willing to use * for something". Specifically, it should hold a node_t for every node that @@ -38,28 +65,21 @@ typedef struct nodelist_t { } nodelist_t; -static INLINE unsigned int +static inline unsigned int node_id_hash(const node_t *node) { -#if SIZEOF_INT == 4 - const uint32_t *p = (const uint32_t*)node->identity; - return p[0] ^ p[1] ^ p[2] ^ p[3] ^ p[4]; -#elif SIZEOF_INT == 8 - const uint64_t *p = (const uint32_t*)node->identity; - const uint32_t *p32 = (const uint32_t*)node->identity; - return p[0] ^ p[1] ^ p32[4]; -#endif + return (unsigned) siphash24g(node->identity, DIGEST_LEN); } -static INLINE unsigned int +static inline unsigned int node_id_eq(const node_t *node1, const node_t *node2) { return tor_memeq(node1->identity, node2->identity, DIGEST_LEN); } HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq); -HT_GENERATE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq, - 0.6, malloc, realloc, free); +HT_GENERATE2(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq, + 0.6, tor_reallocarray_, tor_free_) /** The global nodelist. */ static nodelist_t *the_nodelist=NULL; @@ -90,8 +110,8 @@ node_get_mutable_by_id(const char *identity_digest) /** Return the node_t whose identity is <b>identity_digest</b>, or NULL * if no such node exists. */ -const node_t * -node_get_by_id(const char *identity_digest) +MOCK_IMPL(const node_t *, +node_get_by_id,(const char *identity_digest)) { return node_get_mutable_by_id(identity_digest); } @@ -164,7 +184,7 @@ nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out) if (authdir_mode(get_options()) && !had_router) { const char *discard=NULL; - uint32_t status = dirserv_router_get_status(ri, &discard); + uint32_t status = dirserv_router_get_status(ri, &discard, LOG_INFO); dirserv_set_node_flags_from_authoritative_status(node, status); } @@ -211,8 +231,7 @@ void nodelist_set_consensus(networkstatus_t *ns) { const or_options_t *options = get_options(); - int authdir = authdir_mode_v2(options) || authdir_mode_v3(options); - int client = !server_mode(options); + int authdir = authdir_mode_v3(options); init_nodelist(); if (ns->flavor == FLAV_MICRODESC) @@ -246,11 +265,10 @@ nodelist_set_consensus(networkstatus_t *ns) node->is_stable = rs->is_stable; node->is_possible_guard = rs->is_possible_guard; node->is_exit = rs->is_exit; - node->is_bad_directory = rs->is_bad_directory; node->is_bad_exit = rs->is_bad_exit; node->is_hs_dir = rs->is_hs_dir; node->ipv6_preferred = 0; - if (client && options->ClientPreferIPv6ORPort == 1 && + if (fascist_firewall_prefer_ipv6_orport(options) && (tor_addr_is_null(&rs->ipv6_addr) == 0 || (node->md && tor_addr_is_null(&node->md->ipv6_addr) == 0))) node->ipv6_preferred = 1; @@ -272,8 +290,7 @@ nodelist_set_consensus(networkstatus_t *ns) node->is_valid = node->is_running = node->is_hs_dir = node->is_fast = node->is_stable = node->is_possible_guard = node->is_exit = - node->is_bad_exit = node->is_bad_directory = - node->ipv6_preferred = 0; + node->is_bad_exit = node->ipv6_preferred = 0; } } } SMARTLIST_FOREACH_END(node); @@ -281,7 +298,7 @@ nodelist_set_consensus(networkstatus_t *ns) } /** Helper: return true iff a node has a usable amount of information*/ -static INLINE int +static inline int node_is_usable(const node_t *node) { return (node->rs) || (node->ri); @@ -337,6 +354,25 @@ nodelist_drop_node(node_t *node, int remove_from_ht) node->nodelist_idx = -1; } +/** Return a newly allocated smartlist of the nodes that have <b>md</b> as + * their microdescriptor. */ +smartlist_t * +nodelist_find_nodes_with_microdesc(const microdesc_t *md) +{ + smartlist_t *result = smartlist_new(); + + if (the_nodelist == NULL) + return result; + + SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) { + if (node->md == md) { + smartlist_add(result, node); + } + } SMARTLIST_FOREACH_END(node); + + return result; +} + /** Release storage held by <b>node</b> */ static void node_free(node_t *node) @@ -460,8 +496,8 @@ nodelist_assert_ok(void) /** Return a list of a node_t * for every node we know about. The caller * MUST NOT modify the list. (You can set and clear flags in the nodes if * you must, but you must not add or remove nodes.) */ -smartlist_t * -nodelist_get_list(void) +MOCK_IMPL(smartlist_t *, +nodelist_get_list,(void)) { init_nodelist(); return the_nodelist->nodes; @@ -503,8 +539,8 @@ node_get_by_hex_id(const char *hex_id) * the corresponding node_t, or NULL if none exists. Warn the user if * <b>warn_if_unnamed</b> is set, and they have specified a router by * nickname, but the Named flag isn't set for that router. */ -const node_t * -node_get_by_nickname(const char *nickname, int warn_if_unnamed) +MOCK_IMPL(const node_t *, +node_get_by_nickname,(const char *nickname, int warn_if_unnamed)) { const node_t *node; if (!the_nodelist) @@ -558,10 +594,10 @@ node_get_by_nickname(const char *nickname, int warn_if_unnamed) "but none is listed as Named in the directory consensus. " "Choosing one arbitrarily.", nickname); } - } else if (smartlist_len(matches)>1 && warn_if_unnamed) { + } else if (smartlist_len(matches)==1 && warn_if_unnamed) { char fp[HEX_DIGEST_LEN+1]; node_t *node = smartlist_get(matches, 0); - if (node->name_lookup_warned) { + if (! node->name_lookup_warned) { base16_encode(fp, sizeof(fp), node->identity, DIGEST_LEN); log_warn(LD_CONFIG, "You specified a server \"%s\" by name, but the directory " @@ -615,12 +651,19 @@ node_is_named(const node_t *node) int node_is_dir(const node_t *node) { - if (node->rs) - return node->rs->dir_port != 0; - else if (node->ri) - return node->ri->dir_port != 0; - else + if (node->rs) { + routerstatus_t * rs = node->rs; + /* This is true if supports_tunnelled_dir_requests is true which + * indicates that we support directory request tunnelled or through the + * DirPort. */ + return rs->is_v2_dir; + } else if (node->ri) { + routerinfo_t * ri = node->ri; + /* Both tunnelled request is supported or DirPort is set. */ + return ri->supports_tunnelled_dir_requests; + } else { return 0; + } } /** Return true iff <b>node</b> has either kind of usable descriptor -- that @@ -644,7 +687,7 @@ node_get_purpose(const node_t *node) /** Compute the verbose ("extended") nickname of <b>node</b> and store it * into the MAX_VERBOSE_NICKNAME_LEN+1 character buffer at - * <b>verbose_nickname_out</b> */ + * <b>verbose_name_out</b> */ void node_get_verbose_nickname(const node_t *node, char *verbose_name_out) @@ -660,6 +703,25 @@ node_get_verbose_nickname(const node_t *node, strlcpy(verbose_name_out+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1); } +/** Compute the verbose ("extended") nickname of node with + * given <b>id_digest</b> and store it into the MAX_VERBOSE_NICKNAME_LEN+1 + * character buffer at <b>verbose_name_out</b> + * + * If node_get_by_id() returns NULL, base 16 encoding of + * <b>id_digest</b> is returned instead. */ +void +node_get_verbose_nickname_by_id(const char *id_digest, + char *verbose_name_out) +{ + const node_t *node = node_get_by_id(id_digest); + if (!node) { + verbose_name_out[0] = '$'; + base16_encode(verbose_name_out+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN); + } else { + node_get_verbose_nickname(node, verbose_name_out); + } +} + /** Return true iff it seems that <b>node</b> allows circuits to exit * through it directlry from the client. */ int @@ -706,6 +768,40 @@ node_exit_policy_is_exact(const node_t *node, sa_family_t family) return 1; } +/* Check if the "addr" and port_field fields from r are a valid non-listening + * address/port. If so, set valid to true and add a newly allocated + * tor_addr_port_t containing "addr" and port_field to sl. + * "addr" is an IPv4 host-order address and port_field is a uint16_t. + * r is typically a routerinfo_t or routerstatus_t. + */ +#define SL_ADD_NEW_IPV4_AP(r, port_field, sl, valid) \ + STMT_BEGIN \ + if (tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \ + valid = 1; \ + tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); \ + tor_addr_from_ipv4h(&ap->addr, (r)->addr); \ + ap->port = (r)->port_field; \ + smartlist_add((sl), ap); \ + } \ + STMT_END + +/* Check if the "addr" and port_field fields from r are a valid non-listening + * address/port. If so, set valid to true and add a newly allocated + * tor_addr_port_t containing "addr" and port_field to sl. + * "addr" is a tor_addr_t and port_field is a uint16_t. + * r is typically a routerinfo_t or routerstatus_t. + */ +#define SL_ADD_NEW_IPV6_AP(r, port_field, sl, valid) \ + STMT_BEGIN \ + if (tor_addr_port_is_valid(&(r)->ipv6_addr, (r)->port_field, 0)) { \ + valid = 1; \ + tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); \ + tor_addr_copy(&ap->addr, &(r)->ipv6_addr); \ + ap->port = (r)->port_field; \ + smartlist_add((sl), ap); \ + } \ + STMT_END + /** Return list of tor_addr_port_t with all OR ports (in the sense IP * addr + TCP port) for <b>node</b>. Caller must free all elements * using tor_free() and free the list using smartlist_free(). @@ -718,30 +814,38 @@ smartlist_t * node_get_all_orports(const node_t *node) { smartlist_t *sl = smartlist_new(); + int valid = 0; + /* Find a valid IPv4 address and port */ if (node->ri != NULL) { - if (node->ri->addr != 0) { - tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); - tor_addr_from_ipv4h(&ap->addr, node->ri->addr); - ap->port = node->ri->or_port; - smartlist_add(sl, ap); - } - if (!tor_addr_is_null(&node->ri->ipv6_addr)) { - tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); - tor_addr_copy(&ap->addr, &node->ri->ipv6_addr); - ap->port = node->ri->or_port; - smartlist_add(sl, ap); - } - } else if (node->rs != NULL) { - tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); - tor_addr_from_ipv4h(&ap->addr, node->rs->addr); - ap->port = node->rs->or_port; - smartlist_add(sl, ap); + SL_ADD_NEW_IPV4_AP(node->ri, or_port, sl, valid); + } + + /* If we didn't find a valid address/port in the ri, try the rs */ + if (!valid && node->rs != NULL) { + SL_ADD_NEW_IPV4_AP(node->rs, or_port, sl, valid); + } + + /* Find a valid IPv6 address and port */ + valid = 0; + if (node->ri != NULL) { + SL_ADD_NEW_IPV6_AP(node->ri, ipv6_orport, sl, valid); + } + + if (!valid && node->rs != NULL) { + SL_ADD_NEW_IPV6_AP(node->rs, ipv6_orport, sl, valid); + } + + if (!valid && node->md != NULL) { + SL_ADD_NEW_IPV6_AP(node->md, ipv6_orport, sl, valid); } return sl; } +#undef SL_ADD_NEW_IPV4_AP +#undef SL_ADD_NEW_IPV6_AP + /** Wrapper around node_get_prim_orport for backward compatibility. */ void @@ -757,9 +861,13 @@ node_get_addr(const node_t *node, tor_addr_t *addr_out) uint32_t node_get_prim_addr_ipv4h(const node_t *node) { - if (node->ri) { + /* Don't check the ORPort or DirPort, as this function isn't port-specific, + * and the node might have a valid IPv4 address, yet have a zero + * ORPort or DirPort. + */ + if (node->ri && tor_addr_is_valid_ipv4h(node->ri->addr, 0)) { return node->ri->addr; - } else if (node->rs) { + } else if (node->rs && tor_addr_is_valid_ipv4h(node->rs->addr, 0)) { return node->rs->addr; } return 0; @@ -770,13 +878,13 @@ node_get_prim_addr_ipv4h(const node_t *node) void node_get_address_string(const node_t *node, char *buf, size_t len) { - if (node->ri) { - strlcpy(buf, node->ri->address, len); - } else if (node->rs) { + uint32_t ipv4_addr = node_get_prim_addr_ipv4h(node); + + if (tor_addr_is_valid_ipv4h(ipv4_addr, 0)) { tor_addr_t addr; - tor_addr_from_ipv4h(&addr, node->rs->addr); + tor_addr_from_ipv4h(&addr, ipv4_addr); tor_addr_to_str(buf, &addr, len, 0); - } else { + } else if (len > 0) { buf[0] = '\0'; } } @@ -835,30 +943,83 @@ node_get_declared_family(const node_t *node) return NULL; } +/* Does this node have a valid IPv6 address? + * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for + * checking specific ports. */ +int +node_has_ipv6_addr(const node_t *node) +{ + /* Don't check the ORPort or DirPort, as this function isn't port-specific, + * and the node might have a valid IPv6 address, yet have a zero + * ORPort or DirPort. + */ + if (node->ri && tor_addr_is_valid(&node->ri->ipv6_addr, 0)) + return 1; + if (node->rs && tor_addr_is_valid(&node->rs->ipv6_addr, 0)) + return 1; + if (node->md && tor_addr_is_valid(&node->md->ipv6_addr, 0)) + return 1; + + return 0; +} + +/* Does this node have a valid IPv6 ORPort? */ +int +node_has_ipv6_orport(const node_t *node) +{ + tor_addr_port_t ipv6_orport; + node_get_pref_ipv6_orport(node, &ipv6_orport); + return tor_addr_port_is_valid_ap(&ipv6_orport, 0); +} + +/* Does this node have a valid IPv6 DirPort? */ +int +node_has_ipv6_dirport(const node_t *node) +{ + tor_addr_port_t ipv6_dirport; + node_get_pref_ipv6_dirport(node, &ipv6_dirport); + return tor_addr_port_is_valid_ap(&ipv6_dirport, 0); +} + /** Return 1 if we prefer the IPv6 address and OR TCP port of * <b>node</b>, else 0. * - * We prefer the IPv6 address if the router has an IPv6 address and + * We prefer the IPv6 address if the router has an IPv6 address, + * and we can use IPv6 addresses, and: * i) the node_t says that it prefers IPv6 * or - * ii) the router has no IPv4 address. */ + * ii) the router has no IPv4 OR address. + * + * If you don't have a node, consider looking it up. + * If there is no node, use fascist_firewall_prefer_ipv6_orport(). + */ int -node_ipv6_preferred(const node_t *node) +node_ipv6_or_preferred(const node_t *node) { + const or_options_t *options = get_options(); tor_addr_port_t ipv4_addr; node_assert_ok(node); - if (node->ipv6_preferred || node_get_prim_orport(node, &ipv4_addr)) { - if (node->ri) - return !tor_addr_is_null(&node->ri->ipv6_addr); - if (node->md) - return !tor_addr_is_null(&node->md->ipv6_addr); - if (node->rs) - return !tor_addr_is_null(&node->rs->ipv6_addr); + /* XX/teor - node->ipv6_preferred is set from + * fascist_firewall_prefer_ipv6_orport() each time the consensus is loaded. + */ + if (!fascist_firewall_use_ipv6(options)) { + return 0; + } else if (node->ipv6_preferred || node_get_prim_orport(node, &ipv4_addr)) { + return node_has_ipv6_orport(node); } return 0; } +#define RETURN_IPV4_AP(r, port_field, ap_out) \ + STMT_BEGIN \ + if (r && tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \ + tor_addr_from_ipv4h(&(ap_out)->addr, (r)->addr); \ + (ap_out)->port = (r)->port_field; \ + return 0; \ + } \ + STMT_END + /** Copy the primary (IPv4) OR port (IP address and TCP port) for * <b>node</b> into *<b>ap_out</b>. Return 0 if a valid address and * port was copied, else return non-zero.*/ @@ -868,20 +1029,10 @@ node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out) node_assert_ok(node); tor_assert(ap_out); - if (node->ri) { - if (node->ri->addr == 0 || node->ri->or_port == 0) - return -1; - tor_addr_from_ipv4h(&ap_out->addr, node->ri->addr); - ap_out->port = node->ri->or_port; - return 0; - } - if (node->rs) { - if (node->rs->addr == 0 || node->rs->or_port == 0) - return -1; - tor_addr_from_ipv4h(&ap_out->addr, node->rs->addr); - ap_out->port = node->rs->or_port; - return 0; - } + RETURN_IPV4_AP(node->ri, or_port, ap_out); + RETURN_IPV4_AP(node->rs, or_port, ap_out); + /* Microdescriptors only have an IPv6 address */ + return -1; } @@ -890,21 +1041,12 @@ node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out) void node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out) { - const or_options_t *options = get_options(); tor_assert(ap_out); - /* Cheap implementation of config option ClientUseIPv6 -- simply - don't prefer IPv6 when ClientUseIPv6 is not set and we're not a - client running with bridges. See #4455 for more on this subject. - - Note that this filter is too strict since we're hindering not - only clients! Erring on the safe side shouldn't be a problem - though. XXX move this check to where outgoing connections are - made? -LN */ - if ((options->ClientUseIPv6 || options->UseBridges) && - node_ipv6_preferred(node)) { + if (node_ipv6_or_preferred(node)) { node_get_pref_ipv6_orport(node, ap_out); } else { + /* the primary ORPort is always on IPv4 */ node_get_prim_orport(node, ap_out); } } @@ -917,20 +1059,115 @@ node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out) node_assert_ok(node); tor_assert(ap_out); - /* We prefer the microdesc over a potential routerstatus here. They - are not being synchronised atm so there might be a chance that - they differ at some point, f.ex. when flipping - UseMicrodescriptors? -LN */ + /* Prefer routerstatus over microdesc for consistency with the + * fascist_firewall_* functions. Also check if the address or port are valid, + * and try another alternative if they are not. */ - if (node->ri) { + if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr, + node->ri->ipv6_orport, 0)) { tor_addr_copy(&ap_out->addr, &node->ri->ipv6_addr); ap_out->port = node->ri->ipv6_orport; - } else if (node->md) { + } else if (node->rs && tor_addr_port_is_valid(&node->rs->ipv6_addr, + node->rs->ipv6_orport, 0)) { + tor_addr_copy(&ap_out->addr, &node->rs->ipv6_addr); + ap_out->port = node->rs->ipv6_orport; + } else if (node->md && tor_addr_port_is_valid(&node->md->ipv6_addr, + node->md->ipv6_orport, 0)) { tor_addr_copy(&ap_out->addr, &node->md->ipv6_addr); ap_out->port = node->md->ipv6_orport; - } else if (node->rs) { + } else { + tor_addr_make_null(&ap_out->addr, AF_INET6); + ap_out->port = 0; + } +} + +/** Return 1 if we prefer the IPv6 address and Dir TCP port of + * <b>node</b>, else 0. + * + * We prefer the IPv6 address if the router has an IPv6 address, + * and we can use IPv6 addresses, and: + * i) the router has no IPv4 Dir address. + * or + * ii) our preference is for IPv6 Dir addresses. + * + * If there is no node, use fascist_firewall_prefer_ipv6_dirport(). + */ +int +node_ipv6_dir_preferred(const node_t *node) +{ + const or_options_t *options = get_options(); + tor_addr_port_t ipv4_addr; + node_assert_ok(node); + + /* node->ipv6_preferred is set from fascist_firewall_prefer_ipv6_orport(), + * so we can't use it to determine DirPort IPv6 preference. + * This means that bridge clients will use IPv4 DirPorts by default. + */ + if (!fascist_firewall_use_ipv6(options)) { + return 0; + } else if (node_get_prim_dirport(node, &ipv4_addr) + || fascist_firewall_prefer_ipv6_dirport(get_options())) { + return node_has_ipv6_dirport(node); + } + return 0; +} + +/** Copy the primary (IPv4) Dir port (IP address and TCP port) for + * <b>node</b> into *<b>ap_out</b>. Return 0 if a valid address and + * port was copied, else return non-zero.*/ +int +node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out) +{ + node_assert_ok(node); + tor_assert(ap_out); + + RETURN_IPV4_AP(node->ri, dir_port, ap_out); + RETURN_IPV4_AP(node->rs, dir_port, ap_out); + /* Microdescriptors only have an IPv6 address */ + + return -1; +} + +#undef RETURN_IPV4_AP + +/** Copy the preferred Dir port (IP address and TCP port) for + * <b>node</b> into *<b>ap_out</b>. */ +void +node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out) +{ + tor_assert(ap_out); + + if (node_ipv6_dir_preferred(node)) { + node_get_pref_ipv6_dirport(node, ap_out); + } else { + /* the primary DirPort is always on IPv4 */ + node_get_prim_dirport(node, ap_out); + } +} + +/** Copy the preferred IPv6 Dir port (IP address and TCP port) for + * <b>node</b> into *<b>ap_out</b>. */ +void +node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out) +{ + node_assert_ok(node); + tor_assert(ap_out); + + /* Check if the address or port are valid, and try another alternative if + * they are not. Note that microdescriptors have no dir_port. */ + + /* Assume IPv4 and IPv6 dirports are the same */ + if (node->ri && tor_addr_port_is_valid(&node->ri->ipv6_addr, + node->ri->dir_port, 0)) { + tor_addr_copy(&ap_out->addr, &node->ri->ipv6_addr); + ap_out->port = node->ri->dir_port; + } else if (node->rs && tor_addr_port_is_valid(&node->rs->ipv6_addr, + node->rs->dir_port, 0)) { tor_addr_copy(&ap_out->addr, &node->rs->ipv6_addr); - ap_out->port = node->rs->ipv6_orport; + ap_out->port = node->rs->dir_port; + } else { + tor_addr_make_null(&ap_out->addr, AF_INET6); + ap_out->port = 0; } } @@ -973,7 +1210,7 @@ nodelist_refresh_countries(void) /** Return true iff router1 and router2 have similar enough network addresses * that we should treat them as being in the same family */ -static INLINE int +static inline int addrs_in_same_network_family(const tor_addr_t *a1, const tor_addr_t *a2) { @@ -997,7 +1234,7 @@ node_nickname_matches(const node_t *node, const char *nickname) } /** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */ -static INLINE int +static inline int node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) { if (!lst) return 0; @@ -1216,37 +1453,76 @@ router_set_status(const char *digest, int up) if (!up && node_is_me(node) && !net_is_disabled()) log_warn(LD_NET, "We just marked ourself as down. Are your external " "addresses reachable?"); + + if (bool_neq(node->is_running, up)) + router_dir_info_changed(); + node->is_running = up; } - - router_dir_info_changed(); } /** True iff, the last time we checked whether we had enough directory info - * to build circuits, the answer was "yes". */ + * to build circuits, the answer was "yes". If there are no exits in the + * consensus, we act as if we have 100% of the exit directory info. */ static int have_min_dir_info = 0; + +/** Does the consensus contain nodes that can exit? */ +static consensus_path_type_t have_consensus_path = CONSENSUS_PATH_UNKNOWN; + /** True iff enough has changed since the last time we checked whether we had * enough directory info to build circuits that our old answer can no longer * be trusted. */ static int need_to_update_have_min_dir_info = 1; /** String describing what we're missing before we have enough directory * info. */ -static char dir_info_status[256] = ""; - -/** Return true iff we have enough networkstatus and router information to - * start building circuits. Right now, this means "more than half the - * networkstatus documents, and at least 1/4 of expected routers." */ -//XXX should consider whether we have enough exiting nodes here. +static char dir_info_status[512] = ""; + +/** Return true iff we have enough consensus information to + * start building circuits. Right now, this means "a consensus that's + * less than a day old, and at least 60% of router descriptors (configurable), + * weighted by bandwidth. Treat the exit fraction as 100% if there are + * no exits in the consensus." + * To obtain the final weighted bandwidth, we multiply the + * weighted bandwidth fraction for each position (guard, middle, exit). */ int router_have_minimum_dir_info(void) { + static int logged_delay=0; + const char *delay_fetches_msg = NULL; + if (should_delay_dir_fetches(get_options(), &delay_fetches_msg)) { + if (!logged_delay) + log_notice(LD_DIR, "Delaying directory fetches: %s", delay_fetches_msg); + logged_delay=1; + strlcpy(dir_info_status, delay_fetches_msg, sizeof(dir_info_status)); + return 0; + } + logged_delay = 0; /* reset it if we get this far */ + if (PREDICT_UNLIKELY(need_to_update_have_min_dir_info)) { update_router_have_minimum_dir_info(); - need_to_update_have_min_dir_info = 0; } + return have_min_dir_info; } +/** Set to CONSENSUS_PATH_EXIT if there is at least one exit node + * in the consensus. We update this flag in compute_frac_paths_available if + * there is at least one relay that has an Exit flag in the consensus. + * Used to avoid building exit circuits when they will almost certainly fail. + * Set to CONSENSUS_PATH_INTERNAL if there are no exits in the consensus. + * (This situation typically occurs during bootstrap of a test network.) + * Set to CONSENSUS_PATH_UNKNOWN if we have never checked, or have + * reason to believe our last known value was invalid or has expired. + * If we're in a network with TestingDirAuthVoteExit set, + * this can cause router_have_consensus_path() to be set to + * CONSENSUS_PATH_EXIT, even if there are no nodes with accept exit policies. + */ +consensus_path_type_t +router_have_consensus_path(void) +{ + return have_consensus_path; +} + /** Called when our internal view of the directory has changed. This can be * when the authorities change, networkstatuses change, the list of routerdescs * changes, or number of running routers changes. @@ -1267,22 +1543,26 @@ get_dir_info_status_string(void) } /** Iterate over the servers listed in <b>consensus</b>, and count how many of - * them seem like ones we'd use, and how many of <em>those</em> we have - * descriptors for. Store the former in *<b>num_usable</b> and the latter in - * *<b>num_present</b>. If <b>in_set</b> is non-NULL, only consider those - * routers in <b>in_set</b>. If <b>exit_only</b> is true, only consider nodes - * with the Exit flag. If *descs_out is present, add a node_t for each - * usable descriptor to it. + * them seem like ones we'd use (store this in *<b>num_usable</b>), and how + * many of <em>those</em> we have descriptors for (store this in + * *<b>num_present</b>). + * + * If <b>in_set</b> is non-NULL, only consider those routers in <b>in_set</b>. + * If <b>exit_only</b> is USABLE_DESCRIPTOR_EXIT_ONLY, only consider nodes + * with the Exit flag. + * If *<b>descs_out</b> is present, add a node_t for each usable descriptor + * to it. */ static void count_usable_descriptors(int *num_present, int *num_usable, smartlist_t *descs_out, const networkstatus_t *consensus, const or_options_t *options, time_t now, - routerset_t *in_set, int exit_only) + routerset_t *in_set, + usable_descriptor_t exit_only) { const int md = (consensus->flavor == FLAV_MICRODESC); - *num_present = 0, *num_usable=0; + *num_present = 0, *num_usable = 0; SMARTLIST_FOREACH_BEGIN(consensus->routerstatus_list, routerstatus_t *, rs) { @@ -1290,7 +1570,7 @@ count_usable_descriptors(int *num_present, int *num_usable, if (!node) continue; /* This would be a bug: every entry in the consensus is * supposed to have a node. */ - if (exit_only && ! rs->is_exit) + if (exit_only == USABLE_DESCRIPTOR_EXIT_ONLY && ! rs->is_exit) continue; if (in_set && ! routerset_contains_routerstatus(in_set, rs, -1)) continue; @@ -1314,11 +1594,22 @@ count_usable_descriptors(int *num_present, int *num_usable, log_debug(LD_DIR, "%d usable, %d present (%s%s).", *num_usable, *num_present, - md ? "microdesc" : "desc", exit_only ? " exits" : "s"); + md ? "microdesc" : "desc", + exit_only == USABLE_DESCRIPTOR_EXIT_ONLY ? " exits" : "s"); } -/** Return an extimate of which fraction of usable paths through the Tor - * network we have available for use. */ +/** Return an estimate of which fraction of usable paths through the Tor + * network we have available for use. Count how many routers seem like ones + * we'd use (store this in *<b>num_usable_out</b>), and how many of + * <em>those</em> we have descriptors for (store this in + * *<b>num_present_out</b>.) + * + * If **<b>status_out</b> is present, allocate a new string and print the + * available percentages of guard, middle, and exit nodes to it, noting + * whether there are exits in the consensus. + * If there are no guards in the consensus, + * we treat the exit fraction as 100%. + */ static double compute_frac_paths_available(const networkstatus_t *consensus, const or_options_t *options, time_t now, @@ -1328,16 +1619,20 @@ compute_frac_paths_available(const networkstatus_t *consensus, smartlist_t *guards = smartlist_new(); smartlist_t *mid = smartlist_new(); smartlist_t *exits = smartlist_new(); - smartlist_t *myexits= smartlist_new(); - double f_guard, f_mid, f_exit, f_myexit; - int np, nu; /* Ignored */ - const int authdir = authdir_mode_v2(options) || authdir_mode_v3(options); + double f_guard, f_mid, f_exit; + double f_path = 0.0; + /* Used to determine whether there are any exits in the consensus */ + int np = 0; + /* Used to determine whether there are any exits with descriptors */ + int nu = 0; + const int authdir = authdir_mode_v3(options); count_usable_descriptors(num_present_out, num_usable_out, - mid, consensus, options, now, NULL, 0); + mid, consensus, options, now, NULL, + USABLE_DESCRIPTOR_ALL); if (options->EntryNodes) { count_usable_descriptors(&np, &nu, guards, consensus, options, now, - options->EntryNodes, 0); + options->EntryNodes, USABLE_DESCRIPTOR_ALL); } else { SMARTLIST_FOREACH(mid, const node_t *, node, { if (authdir) { @@ -1350,37 +1645,148 @@ compute_frac_paths_available(const networkstatus_t *consensus, }); } + /* All nodes with exit flag + * If we're in a network with TestingDirAuthVoteExit set, + * this can cause false positives on have_consensus_path, + * incorrectly setting it to CONSENSUS_PATH_EXIT. This is + * an unavoidable feature of forcing authorities to declare + * certain nodes as exits. + */ count_usable_descriptors(&np, &nu, exits, consensus, options, now, - NULL, 1); - count_usable_descriptors(&np, &nu, myexits, consensus, options, now, - options->ExitNodes, 1); + NULL, USABLE_DESCRIPTOR_EXIT_ONLY); + log_debug(LD_NET, + "%s: %d present, %d usable", + "exits", + np, + nu); + + /* We need at least 1 exit present in the consensus to consider + * building exit paths */ + /* Update our understanding of whether the consensus has exits */ + consensus_path_type_t old_have_consensus_path = have_consensus_path; + have_consensus_path = ((nu > 0) ? + CONSENSUS_PATH_EXIT : + CONSENSUS_PATH_INTERNAL); + + if (have_consensus_path == CONSENSUS_PATH_INTERNAL + && old_have_consensus_path != have_consensus_path) { + log_notice(LD_NET, + "The current consensus has no exit nodes. " + "Tor can only build internal paths, " + "such as paths to hidden services."); + + /* However, exit nodes can reachability self-test using this consensus, + * join the network, and appear in a later consensus. This will allow + * the network to build exit paths, such as paths for world wide web + * browsing (as distinct from hidden service web browsing). */ + } f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD); f_mid = frac_nodes_with_descriptors(mid, WEIGHT_FOR_MID); f_exit = frac_nodes_with_descriptors(exits, WEIGHT_FOR_EXIT); - f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT); + + log_debug(LD_NET, + "f_guard: %.2f, f_mid: %.2f, f_exit: %.2f", + f_guard, + f_mid, + f_exit); smartlist_free(guards); smartlist_free(mid); smartlist_free(exits); - smartlist_free(myexits); - /* This is a tricky point here: we don't want to make it easy for a - * directory to trickle exits to us until it learns which exits we have - * configured, so require that we have a threshold both of total exits - * and usable exits. */ - if (f_myexit < f_exit) - f_exit = f_myexit; + if (options->ExitNodes) { + double f_myexit, f_myexit_unflagged; + smartlist_t *myexits= smartlist_new(); + smartlist_t *myexits_unflagged = smartlist_new(); + + /* All nodes with exit flag in ExitNodes option */ + count_usable_descriptors(&np, &nu, myexits, consensus, options, now, + options->ExitNodes, USABLE_DESCRIPTOR_EXIT_ONLY); + log_debug(LD_NET, + "%s: %d present, %d usable", + "myexits", + np, + nu); + + /* Now compute the nodes in the ExitNodes option where which we don't know + * what their exit policy is, or we know it permits something. */ + count_usable_descriptors(&np, &nu, myexits_unflagged, + consensus, options, now, + options->ExitNodes, USABLE_DESCRIPTOR_ALL); + log_debug(LD_NET, + "%s: %d present, %d usable", + "myexits_unflagged (initial)", + np, + nu); + + SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) { + if (node_has_descriptor(node) && node_exit_policy_rejects_all(node)) { + SMARTLIST_DEL_CURRENT(myexits_unflagged, node); + /* this node is not actually an exit */ + np--; + /* this node is unusable as an exit */ + nu--; + } + } SMARTLIST_FOREACH_END(node); - tor_asprintf(status_out, - "%d%% of guards bw, " - "%d%% of midpoint bw, and " - "%d%% of exit bw", - (int)(f_guard*100), - (int)(f_mid*100), - (int)(f_exit*100)); + log_debug(LD_NET, + "%s: %d present, %d usable", + "myexits_unflagged (final)", + np, + nu); + + f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT); + f_myexit_unflagged= + frac_nodes_with_descriptors(myexits_unflagged,WEIGHT_FOR_EXIT); + + log_debug(LD_NET, + "f_exit: %.2f, f_myexit: %.2f, f_myexit_unflagged: %.2f", + f_exit, + f_myexit, + f_myexit_unflagged); + + /* If our ExitNodes list has eliminated every possible Exit node, and there + * were some possible Exit nodes, then instead consider nodes that permit + * exiting to some ports. */ + if (smartlist_len(myexits) == 0 && + smartlist_len(myexits_unflagged)) { + f_myexit = f_myexit_unflagged; + } + + smartlist_free(myexits); + smartlist_free(myexits_unflagged); - return f_guard * f_mid * f_exit; + /* This is a tricky point here: we don't want to make it easy for a + * directory to trickle exits to us until it learns which exits we have + * configured, so require that we have a threshold both of total exits + * and usable exits. */ + if (f_myexit < f_exit) + f_exit = f_myexit; + } + + /* if the consensus has no exits, treat the exit fraction as 100% */ + if (router_have_consensus_path() != CONSENSUS_PATH_EXIT) { + f_exit = 1.0; + } + + f_path = f_guard * f_mid * f_exit; + + if (status_out) + tor_asprintf(status_out, + "%d%% of guards bw, " + "%d%% of midpoint bw, and " + "%d%% of exit bw%s = " + "%d%% of path bw", + (int)(f_guard*100), + (int)(f_mid*100), + (int)(f_exit*100), + (router_have_consensus_path() == CONSENSUS_PATH_EXIT ? + "" : + " (no exits in consensus)"), + (int)(f_path*100)); + + return f_path; } /** We just fetched a new set of descriptors. Compute how far through @@ -1391,19 +1797,19 @@ count_loading_descriptors_progress(void) { int num_present = 0, num_usable=0; time_t now = time(NULL); + const or_options_t *options = get_options(); const networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor()); - double fraction; + double paths, fraction; if (!consensus) return 0; /* can't count descriptors if we have no list of them */ - count_usable_descriptors(&num_present, &num_usable, NULL, - consensus, get_options(), now, NULL, 0); + paths = compute_frac_paths_available(consensus, options, now, + &num_present, &num_usable, + NULL); - if (num_usable == 0) - return 0; /* don't div by 0 */ - fraction = num_present / (num_usable/4.); + fraction = paths / get_frac_paths_needed_for_circs(options,consensus); if (fraction > 1.0) return 0; /* it's not the number of descriptors holding us back */ return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int) @@ -1451,16 +1857,9 @@ update_router_have_minimum_dir_info(void) goto done; } - if (should_delay_dir_fetches(get_options())) { - log_notice(LD_DIR, "no known bridge descriptors running yet; stalling"); - strlcpy(dir_info_status, "No live bridge descriptors.", - sizeof(dir_info_status)); - res = 0; - goto done; - } - using_md = consensus->flavor == FLAV_MICRODESC; + /* Check fraction of available paths */ { char *status = NULL; int num_present=0, num_usable=0; @@ -1474,7 +1873,6 @@ update_router_have_minimum_dir_info(void) "can only build %d%% of likely paths. (We have %s.)", using_md?"micro":"", num_present, num_usable, (int)(paths*100), status); - /* log_notice(LD_NET, "%s", dir_info_status); */ tor_free(status); res = 0; control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0); @@ -1486,12 +1884,17 @@ update_router_have_minimum_dir_info(void) } done: + + /* If paths have just become available in this update. */ if (res && !have_min_dir_info) { - log_notice(LD_DIR, - "We now have enough directory information to build circuits."); control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO"); - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_OR, 0); + if (control_event_bootstrap(BOOTSTRAP_STATUS_CONN_OR, 0) == 0) { + log_notice(LD_DIR, + "We now have enough directory information to build circuits."); + } } + + /* If paths have just become unavailable in this update. */ if (!res && have_min_dir_info) { int quiet = directory_too_idle_to_fetch_descriptors(options, now); tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR, @@ -1502,8 +1905,8 @@ update_router_have_minimum_dir_info(void) * is back up and usable, and b) disable some activities that Tor * should only do while circuits are working, like reachability tests * and fetching bridge descriptors only over circuits. */ - can_complete_circuit = 0; - + note_that_we_maybe_cant_complete_circuits(); + have_consensus_path = CONSENSUS_PATH_UNKNOWN; control_event_client_status(LOG_NOTICE, "NOT_ENOUGH_DIR_INFO"); } have_min_dir_info = res; diff --git a/src/or/nodelist.h b/src/or/nodelist.h index 8a4665a8bf..71a91e107f 100644 --- a/src/or/nodelist.h +++ b/src/or/nodelist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,7 +17,7 @@ } STMT_END node_t *node_get_mutable_by_id(const char *identity_digest); -const node_t *node_get_by_id(const char *identity_digest); +MOCK_DECL(const node_t *, node_get_by_id, (const char *identity_digest)); const node_t *node_get_by_hex_id(const char *identity_digest); node_t *nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out); node_t *nodelist_add_microdesc(microdesc_t *md); @@ -26,13 +26,17 @@ void nodelist_set_consensus(networkstatus_t *ns); void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md); void nodelist_remove_routerinfo(routerinfo_t *ri); void nodelist_purge(void); +smartlist_t *nodelist_find_nodes_with_microdesc(const microdesc_t *md); void nodelist_free_all(void); void nodelist_assert_ok(void); -const node_t *node_get_by_nickname(const char *nickname, int warn_if_unnamed); +MOCK_DECL(const node_t *, node_get_by_nickname, + (const char *nickname, int warn_if_unnamed)); void node_get_verbose_nickname(const node_t *node, char *verbose_name_out); +void node_get_verbose_nickname_by_id(const char *id_digest, + char *verbose_name_out); int node_is_named(const node_t *node); int node_is_dir(const node_t *node); int node_has_descriptor(const node_t *node); @@ -51,13 +55,23 @@ void node_get_address_string(const node_t *node, char *cp, size_t len); long node_get_declared_uptime(const node_t *node); time_t node_get_published_on(const node_t *node); const smartlist_t *node_get_declared_family(const node_t *node); -int node_ipv6_preferred(const node_t *node); + +int node_has_ipv6_addr(const node_t *node); +int node_has_ipv6_orport(const node_t *node); +int node_has_ipv6_dirport(const node_t *node); +/* Deprecated - use node_ipv6_or_preferred or node_ipv6_dir_preferred */ +#define node_ipv6_preferred(node) node_ipv6_or_preferred(node) +int node_ipv6_or_preferred(const node_t *node); int node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out); +int node_ipv6_dir_preferred(const node_t *node); +int node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out); +void node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out); +void node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out); int node_has_curve25519_onion_key(const node_t *node); -smartlist_t *nodelist_get_list(void); +MOCK_DECL(smartlist_t *, nodelist_get_list, (void)); /* Temporary during transition to multiple addresses. */ void node_get_addr(const node_t *node, tor_addr_t *addr_out); @@ -75,7 +89,37 @@ int node_is_unreliable(const node_t *router, int need_uptime, int router_exit_policy_all_nodes_reject(const tor_addr_t *addr, uint16_t port, int need_uptime); void router_set_status(const char *digest, int up); + +/** router_have_minimum_dir_info tests to see if we have enough + * descriptor information to create circuits. + * If there are exits in the consensus, we wait until we have enough + * info to create exit paths before creating any circuits. If there are + * no exits in the consensus, we wait for enough info to create internal + * paths, and should avoid creating exit paths, as they will simply fail. + * We make sure we create all available circuit types at the same time. */ int router_have_minimum_dir_info(void); + +/** Set to CONSENSUS_PATH_EXIT if there is at least one exit node + * in the consensus. We update this flag in compute_frac_paths_available if + * there is at least one relay that has an Exit flag in the consensus. + * Used to avoid building exit circuits when they will almost certainly fail. + * Set to CONSENSUS_PATH_INTERNAL if there are no exits in the consensus. + * (This situation typically occurs during bootstrap of a test network.) + * Set to CONSENSUS_PATH_UNKNOWN if we have never checked, or have + * reason to believe our last known value was invalid or has expired. + */ +typedef enum { + /* we haven't checked yet, or we have invalidated our previous check */ + CONSENSUS_PATH_UNKNOWN = -1, + /* The consensus only has internal relays, and we should only + * create internal paths, circuits, streams, ... */ + CONSENSUS_PATH_INTERNAL = 0, + /* The consensus has at least one exit, and can therefore (potentially) + * create exit and internal paths, circuits, streams, ... */ + CONSENSUS_PATH_EXIT = 1 +} consensus_path_type_t; +consensus_path_type_t router_have_consensus_path(void); + void router_dir_info_changed(void); const char *get_dir_info_status_string(void); int count_loading_descriptors_progress(void); diff --git a/src/or/ntmain.c b/src/or/ntmain.c index 8b67b86822..ded0e0d307 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -1,9 +1,16 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ -#define MAIN_PRIVATE +/** + * \file ntmain.c + * + * \brief Entry points for running/configuring Tor as Windows Service. + */ + +#ifdef _WIN32 + #include "or.h" #include "config.h" #include "main.h" @@ -315,8 +322,11 @@ nt_service_main(void) case CMD_LIST_FINGERPRINT: case CMD_HASH_PASSWORD: case CMD_VERIFY_CONFIG: + case CMD_DUMP_CONFIG: + case CMD_KEYGEN: log_err(LD_CONFIG, "Unsupported command (--list-fingerprint, " - "--hash-password, or --verify-config) in NT service."); + "--hash-password, --keygen, --dump-config, or --verify-config) " + "in NT service."); break; case CMD_RUN_UNITTESTS: default: @@ -762,3 +772,5 @@ nt_service_parse_options(int argc, char **argv, int *should_exit) return 0; } +#endif + diff --git a/src/or/ntmain.h b/src/or/ntmain.h index d3027936cd..31bf38c62c 100644 --- a/src/or/ntmain.h +++ b/src/or/ntmain.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,10 +13,8 @@ #define TOR_NTMAIN_H #ifdef _WIN32 -#if !defined (WINCE) #define NT_SERVICE #endif -#endif #ifdef NT_SERVICE int nt_service_parse_options(int argc, char **argv, int *should_exit); diff --git a/src/or/onion.c b/src/or/onion.c index 1a0bcf106e..d6ef3673dd 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,7 +22,6 @@ #include "relay.h" #include "rephist.h" #include "router.h" -#include "tor_queue.h" /** Type for a linked list of circuits that are waiting for a free CPU worker * to process a waiting onion handshake. */ @@ -59,7 +58,7 @@ static void onion_queue_entry_remove(onion_queue_t *victim); * MAX_ONIONSKIN_CHALLENGE/REPLY_LEN." Also, make sure that we can pass * over-large values via EXTEND2/EXTENDED2, for future-compatibility.*/ -/** Return true iff we have room to queue another oninoskin of type +/** Return true iff we have room to queue another onionskin of type * <b>type</b>. */ static int have_room_for_onionskin(uint16_t type) @@ -112,15 +111,11 @@ have_room_for_onionskin(uint16_t type) (uint64_t)options->MaxOnionQueueDelay) return 0; -#ifdef CURVE25519_ENABLED /* If we support the ntor handshake, then don't let TAP handshakes use * more than 2/3 of the space on the queue. */ if (type == ONION_HANDSHAKE_TYPE_TAP && tap_usec / 1000 > (uint64_t)options->MaxOnionQueueDelay * 2 / 3) return 0; -#else - (void) type; -#endif return 1; } @@ -300,6 +295,8 @@ onion_pending_remove(or_circuit_t *circ) victim = circ->onionqueue_entry; if (victim) onion_queue_entry_remove(victim); + + cpuworker_cancel_circ_handshake(circ); } /** Remove a queue entry <b>victim</b> from the queue, unlinking it from @@ -330,50 +327,49 @@ onion_queue_entry_remove(onion_queue_t *victim) void clear_pending_onions(void) { - onion_queue_t *victim; + onion_queue_t *victim, *next; int i; for (i=0; i<=MAX_ONION_HANDSHAKE_TYPE; i++) { - while ((victim = TOR_TAILQ_FIRST(&ol_list[i]))) { + for (victim = TOR_TAILQ_FIRST(&ol_list[i]); victim; victim = next) { + next = TOR_TAILQ_NEXT(victim,next); onion_queue_entry_remove(victim); } + tor_assert(TOR_TAILQ_EMPTY(&ol_list[i])); } memset(ol_entries, 0, sizeof(ol_entries)); } /* ============================================================ */ -/** Fill in a server_onion_keys_t object at <b>keys</b> with all of the keys +/** Return a new server_onion_keys_t object with all of the keys * and other info we might need to do onion handshakes. (We make a copy of * our keys for each cpuworker to avoid race conditions with the main thread, * and to avoid locking) */ -void -setup_server_onion_keys(server_onion_keys_t *keys) +server_onion_keys_t * +server_onion_keys_new(void) { - memset(keys, 0, sizeof(server_onion_keys_t)); + server_onion_keys_t *keys = tor_malloc_zero(sizeof(server_onion_keys_t)); memcpy(keys->my_identity, router_get_my_id_digest(), DIGEST_LEN); dup_onion_keys(&keys->onion_key, &keys->last_onion_key); -#ifdef CURVE25519_ENABLED keys->curve25519_key_map = construct_ntor_key_map(); keys->junk_keypair = tor_malloc_zero(sizeof(curve25519_keypair_t)); curve25519_keypair_generate(keys->junk_keypair, 0); -#endif + return keys; } -/** Release all storage held in <b>keys</b>, but do not free <b>keys</b> - * itself (as it's likely to be stack-allocated.) */ +/** Release all storage held in <b>keys</b>. */ void -release_server_onion_keys(server_onion_keys_t *keys) +server_onion_keys_free(server_onion_keys_t *keys) { if (! keys) return; crypto_pk_free(keys->onion_key); crypto_pk_free(keys->last_onion_key); -#ifdef CURVE25519_ENABLED ntor_key_map_free(keys->curve25519_key_map); tor_free(keys->junk_keypair); -#endif - memset(keys, 0, sizeof(server_onion_keys_t)); + memwipe(keys, 0, sizeof(server_onion_keys_t)); + tor_free(keys); } /** Release whatever storage is held in <b>state</b>, depending on its @@ -390,12 +386,10 @@ onion_handshake_state_release(onion_handshake_state_t *state) fast_handshake_state_free(state->u.fast); state->u.fast = NULL; break; -#ifdef CURVE25519_ENABLED case ONION_HANDSHAKE_TYPE_NTOR: ntor_handshake_state_free(state->u.ntor); state->u.ntor = NULL; break; -#endif default: log_warn(LD_BUG, "called with unknown handshake state type %d", (int)state->tag); @@ -435,7 +429,6 @@ onion_skin_create(int type, r = CREATE_FAST_LEN; break; case ONION_HANDSHAKE_TYPE_NTOR: -#ifdef CURVE25519_ENABLED if (tor_mem_is_zero((const char*)node->curve25519_onion_key.public_key, CURVE25519_PUBKEY_LEN)) return -1; @@ -446,9 +439,6 @@ onion_skin_create(int type, return -1; r = NTOR_ONIONSKIN_LEN; -#else - return -1; -#endif break; default: log_warn(LD_BUG, "called with unknown handshake state type %d", type); @@ -500,7 +490,6 @@ onion_skin_server_handshake(int type, memcpy(rend_nonce_out, reply_out+DIGEST_LEN, DIGEST_LEN); break; case ONION_HANDSHAKE_TYPE_NTOR: -#ifdef CURVE25519_ENABLED if (onionskin_len < NTOR_ONIONSKIN_LEN) return -1; { @@ -521,9 +510,6 @@ onion_skin_server_handshake(int type, tor_free(keys_tmp); r = NTOR_REPLY_LEN; } -#else - return -1; -#endif break; default: log_warn(LD_BUG, "called with unknown handshake state type %d", type); @@ -540,48 +526,59 @@ onion_skin_server_handshake(int type, * bytes worth of key material in <b>keys_out_len</b>, set * <b>rend_authenticator_out</b> to the "KH" field that can be used to * establish introduction points at this hop, and return 0. On failure, - * return -1. */ + * return -1, and set *msg_out to an error message if this is worth + * complaining to the usre about. */ int onion_skin_client_handshake(int type, const onion_handshake_state_t *handshake_state, const uint8_t *reply, size_t reply_len, uint8_t *keys_out, size_t keys_out_len, - uint8_t *rend_authenticator_out) + uint8_t *rend_authenticator_out, + const char **msg_out) { if (handshake_state->tag != type) return -1; switch (type) { case ONION_HANDSHAKE_TYPE_TAP: - if (reply_len != TAP_ONIONSKIN_REPLY_LEN) + if (reply_len != TAP_ONIONSKIN_REPLY_LEN) { + if (msg_out) + *msg_out = "TAP reply was not of the correct length."; return -1; + } if (onion_skin_TAP_client_handshake(handshake_state->u.tap, (const char*)reply, - (char *)keys_out, keys_out_len) < 0) + (char *)keys_out, keys_out_len, + msg_out) < 0) return -1; memcpy(rend_authenticator_out, reply+DH_KEY_LEN, DIGEST_LEN); return 0; case ONION_HANDSHAKE_TYPE_FAST: - if (reply_len != CREATED_FAST_LEN) + if (reply_len != CREATED_FAST_LEN) { + if (msg_out) + *msg_out = "TAP reply was not of the correct length."; return -1; + } if (fast_client_handshake(handshake_state->u.fast, reply, - keys_out, keys_out_len) < 0) + keys_out, keys_out_len, msg_out) < 0) return -1; memcpy(rend_authenticator_out, reply+DIGEST_LEN, DIGEST_LEN); return 0; -#ifdef CURVE25519_ENABLED case ONION_HANDSHAKE_TYPE_NTOR: - if (reply_len < NTOR_REPLY_LEN) + if (reply_len < NTOR_REPLY_LEN) { + if (msg_out) + *msg_out = "ntor reply was not of the correct length."; return -1; + } { size_t keys_tmp_len = keys_out_len + DIGEST_LEN; uint8_t *keys_tmp = tor_malloc(keys_tmp_len); if (onion_skin_ntor_client_handshake(handshake_state->u.ntor, - reply, - keys_tmp, keys_tmp_len) < 0) { + reply, + keys_tmp, keys_tmp_len, msg_out) < 0) { tor_free(keys_tmp); return -1; } @@ -591,7 +588,6 @@ onion_skin_client_handshake(int type, tor_free(keys_tmp); } return 0; -#endif default: log_warn(LD_BUG, "called with unknown handshake state type %d", type); tor_fragile_assert(); @@ -630,12 +626,10 @@ check_create_cell(const create_cell_t *cell, int unknown_ok) if (cell->handshake_len != CREATE_FAST_LEN) return -1; break; -#ifdef CURVE25519_ENABLED case ONION_HANDSHAKE_TYPE_NTOR: if (cell->handshake_len != NTOR_ONIONSKIN_LEN) return -1; break; -#endif default: if (! unknown_ok) return -1; @@ -861,16 +855,19 @@ extend_cell_parse(extend_cell_t *cell_out, const uint8_t command, } case RELAY_COMMAND_EXTEND2: { - uint8_t n_specs = *payload, spectype, speclen; + uint8_t n_specs, spectype, speclen; int i; int found_ipv4 = 0, found_ipv6 = 0, found_id = 0; tor_addr_make_unspec(&cell_out->orport_ipv4.addr); tor_addr_make_unspec(&cell_out->orport_ipv6.addr); + if (payload_length == 0) + return -1; + cell_out->cell_type = RELAY_COMMAND_EXTEND2; - ++payload; + n_specs = *payload++; /* Parse the specifiers. We'll only take the first IPv4 and first IPv6 - * addres, and the node ID, and ignore everything else */ + * address, and the node ID, and ignore everything else */ for (i = 0; i < n_specs; ++i) { if (eop - payload < 2) return -1; diff --git a/src/or/onion.h b/src/or/onion.h index d62f032b87..0275fa00d2 100644 --- a/src/or/onion.h +++ b/src/or/onion.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -23,17 +23,15 @@ typedef struct server_onion_keys_t { uint8_t my_identity[DIGEST_LEN]; crypto_pk_t *onion_key; crypto_pk_t *last_onion_key; -#ifdef CURVE25519_ENABLED di_digest256_map_t *curve25519_key_map; curve25519_keypair_t *junk_keypair; -#endif } server_onion_keys_t; #define MAX_ONIONSKIN_CHALLENGE_LEN 255 #define MAX_ONIONSKIN_REPLY_LEN 255 -void setup_server_onion_keys(server_onion_keys_t *keys); -void release_server_onion_keys(server_onion_keys_t *keys); +server_onion_keys_t *server_onion_keys_new(void); +void server_onion_keys_free(server_onion_keys_t *keys); void onion_handshake_state_release(onion_handshake_state_t *state); @@ -51,7 +49,8 @@ int onion_skin_client_handshake(int type, const onion_handshake_state_t *handshake_state, const uint8_t *reply, size_t reply_len, uint8_t *keys_out, size_t key_out_len, - uint8_t *rend_authenticator_out); + uint8_t *rend_authenticator_out, + const char **msg_out); /** A parsed CREATE, CREATE_FAST, or CREATE2 cell. */ typedef struct create_cell_t { diff --git a/src/or/onion_fast.c b/src/or/onion_fast.c index aa034a8bd6..1f79860596 100644 --- a/src/or/onion_fast.c +++ b/src/or/onion_fast.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,7 +22,7 @@ fast_handshake_state_free(fast_handshake_state_t *victim) tor_free(victim); } -/** Create the state needed to perform a CREATE_FAST hasnshake. Return 0 +/** Create the state needed to perform a CREATE_FAST handshake. Return 0 * on success, -1 on failure. */ int fast_onionskin_create(fast_handshake_state_t **handshake_state_out, @@ -30,10 +30,7 @@ fast_onionskin_create(fast_handshake_state_t **handshake_state_out, { fast_handshake_state_t *s; *handshake_state_out = s = tor_malloc(sizeof(fast_handshake_state_t)); - if (crypto_rand((char*)s->state, sizeof(s->state)) < 0) { - tor_free(s); - return -1; - } + crypto_rand((char*)s->state, sizeof(s->state)); memcpy(handshake_out, s->state, DIGEST_LEN); return 0; } @@ -56,8 +53,7 @@ fast_server_handshake(const uint8_t *key_in, /* DIGEST_LEN bytes */ size_t out_len; int r = -1; - if (crypto_rand((char*)handshake_reply_out, DIGEST_LEN)<0) - return -1; + crypto_rand((char*)handshake_reply_out, DIGEST_LEN); memcpy(tmp, key_in, DIGEST_LEN); memcpy(tmp+DIGEST_LEN, handshake_reply_out, DIGEST_LEN); @@ -92,7 +88,8 @@ int fast_client_handshake(const fast_handshake_state_t *handshake_state, const uint8_t *handshake_reply_out,/*DIGEST_LEN*2 bytes*/ uint8_t *key_out, - size_t key_out_len) + size_t key_out_len, + const char **msg_out) { uint8_t tmp[DIGEST_LEN+DIGEST_LEN]; uint8_t *out; @@ -104,12 +101,14 @@ fast_client_handshake(const fast_handshake_state_t *handshake_state, out_len = key_out_len+DIGEST_LEN; out = tor_malloc(out_len); if (crypto_expand_key_material_TAP(tmp, sizeof(tmp), out, out_len)) { + if (msg_out) + *msg_out = "Failed to expand key material"; goto done; } if (tor_memneq(out, handshake_reply_out+DIGEST_LEN, DIGEST_LEN)) { /* H(K) does *not* match. Something fishy. */ - log_warn(LD_PROTOCOL,"Digest DOES NOT MATCH on fast handshake. " - "Bug or attack."); + if (msg_out) + *msg_out = "Digest DOES NOT MATCH on fast handshake. Bug or attack."; goto done; } memcpy(key_out, out+DIGEST_LEN, key_out_len); diff --git a/src/or/onion_fast.h b/src/or/onion_fast.h index 8c078378d2..b9626002c3 100644 --- a/src/or/onion_fast.h +++ b/src/or/onion_fast.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -32,7 +32,8 @@ int fast_server_handshake(const uint8_t *message_in, int fast_client_handshake(const fast_handshake_state_t *handshake_state, const uint8_t *handshake_reply_out, uint8_t *key_out, - size_t key_out_len); + size_t key_out_len, + const char **msg_out); #endif diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c index 9cf7d5dd6e..9f97a4cfbe 100644 --- a/src/or/onion_ntor.c +++ b/src/or/onion_ntor.c @@ -1,10 +1,16 @@ -/* Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * \file onion_ntor.c + * + * \brief Implementation for the ntor handshake. + */ + #include "orconfig.h" -#include "crypto.h" #define ONION_NTOR_PRIVATE +#include "crypto.h" #include "onion_ntor.h" #include "torlog.h" #include "util.h" @@ -226,7 +232,8 @@ onion_skin_ntor_client_handshake( const ntor_handshake_state_t *handshake_state, const uint8_t *handshake_reply, uint8_t *key_out, - size_t key_out_len) + size_t key_out_len, + const char **msg_out) { const tweakset_t *T = &proto1_tweaks; /* Sensitive stack-allocated material. Kept in an anonymous struct to make @@ -256,7 +263,7 @@ onion_skin_ntor_client_handshake( si += CURVE25519_OUTPUT_LEN; curve25519_handshake(si, &handshake_state->seckey_x, &handshake_state->pubkey_B); - bad |= safe_mem_is_zero(si, CURVE25519_OUTPUT_LEN); + bad |= (safe_mem_is_zero(si, CURVE25519_OUTPUT_LEN) << 1); si += CURVE25519_OUTPUT_LEN; APPEND(si, handshake_state->router_id, DIGEST_LEN); APPEND(si, handshake_state->pubkey_B.public_key, CURVE25519_PUBKEY_LEN); @@ -281,7 +288,7 @@ onion_skin_ntor_client_handshake( /* Compute auth */ h_tweak(s.auth, s.auth_input, sizeof(s.auth_input), T->t_mac); - bad |= tor_memneq(s.auth, auth_candidate, DIGEST256_LEN); + bad |= (tor_memneq(s.auth, auth_candidate, DIGEST256_LEN) << 2); crypto_expand_key_material_rfc5869_sha256( s.secret_input, sizeof(s.secret_input), @@ -290,6 +297,23 @@ onion_skin_ntor_client_handshake( key_out, key_out_len); memwipe(&s, 0, sizeof(s)); + + if (bad) { + if (bad & 4) { + if (msg_out) + *msg_out = NULL; /* Don't report this one; we probably just had the + * wrong onion key.*/ + log_fn(LOG_INFO, LD_PROTOCOL, + "Invalid result from curve25519 handshake: %d", bad); + } + if (bad & 3) { + if (msg_out) + *msg_out = "Zero output from curve25519 handshake"; + log_fn(LOG_WARN, LD_PROTOCOL, + "Invalid result from curve25519 handshake: %d", bad); + } + } + return bad ? -1 : 0; } diff --git a/src/or/onion_ntor.h b/src/or/onion_ntor.h index c942e6e0f0..f637b437fd 100644 --- a/src/or/onion_ntor.h +++ b/src/or/onion_ntor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ONION_NTOR_H @@ -17,7 +17,6 @@ typedef struct ntor_handshake_state_t ntor_handshake_state_t; /** Length of an ntor reply, as sent from server to client. */ #define NTOR_REPLY_LEN 64 -#ifdef CURVE25519_ENABLED void ntor_handshake_state_free(ntor_handshake_state_t *state); int onion_skin_ntor_create(const uint8_t *router_id, @@ -37,7 +36,8 @@ int onion_skin_ntor_client_handshake( const ntor_handshake_state_t *handshake_state, const uint8_t *handshake_reply, uint8_t *key_out, - size_t key_out_len); + size_t key_out_len, + const char **msg_out); #ifdef ONION_NTOR_PRIVATE @@ -59,5 +59,3 @@ struct ntor_handshake_state_t { #endif -#endif - diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index 3782e75abf..bfd472351f 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -122,8 +122,9 @@ onion_skin_TAP_server_handshake( "Couldn't decrypt onionskin: client may be using old onion key"); goto err; } else if (len != DH_KEY_LEN) { - log_warn(LD_PROTOCOL, "Unexpected onionskin length after decryption: %ld", - (long)len); + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Unexpected onionskin length after decryption: %ld", + (long)len); goto err; } @@ -182,7 +183,8 @@ int onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state, const char *handshake_reply, /* TAP_ONIONSKIN_REPLY_LEN bytes */ char *key_out, - size_t key_out_len) + size_t key_out_len, + const char **msg_out) { ssize_t len; char *key_material=NULL; @@ -194,13 +196,16 @@ onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state, len = crypto_dh_compute_secret(LOG_PROTOCOL_WARN, handshake_state, handshake_reply, DH_KEY_LEN, key_material, key_material_len); - if (len < 0) + if (len < 0) { + if (msg_out) + *msg_out = "DH computation failed."; goto err; + } if (tor_memneq(key_material, handshake_reply+DH_KEY_LEN, DIGEST_LEN)) { /* H(K) does *not* match. Something fishy. */ - log_warn(LD_PROTOCOL,"Digest DOES NOT MATCH on onion handshake. " - "Bug or attack."); + if (msg_out) + *msg_out = "Digest DOES NOT MATCH on onion handshake. Bug or attack."; goto err; } diff --git a/src/or/onion_tap.h b/src/or/onion_tap.h index b978b66737..a2880f6e98 100644 --- a/src/or/onion_tap.h +++ b/src/or/onion_tap.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -31,7 +31,8 @@ int onion_skin_TAP_server_handshake(const char *onion_skin, int onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state, const char *handshake_reply, char *key_out, - size_t key_out_len); + size_t key_out_len, + const char **msg_out); #endif diff --git a/src/or/or.h b/src/or/or.h index 4459957a06..da84128530 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,7 +14,7 @@ #include "orconfig.h" -#ifdef __COVERITY__ +#if defined(__clang_analyzer__) || defined(__COVERITY__) /* If we're building for a static analysis, turn on all the off-by-default * features. */ #ifndef INSTRUMENT_DOWNLOADS @@ -22,13 +22,6 @@ #endif #endif -#ifdef _WIN32 -#ifndef _WIN32_WINNT -#define _WIN32_WINNT 0x0501 -#endif -#define WIN32_LEAN_AND_MEAN -#endif - #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -42,9 +35,6 @@ #include <sys/param.h> /* FreeBSD needs this to know what version it is */ #endif #include "torint.h" -#ifdef HAVE_SYS_WAIT_H -#include <sys/wait.h> -#endif #ifdef HAVE_SYS_FCNTL_H #include <sys/fcntl.h> #endif @@ -77,6 +67,7 @@ #endif #ifdef _WIN32 +#include <winsock2.h> #include <io.h> #include <process.h> #include <direct.h> @@ -90,8 +81,9 @@ #endif #include "crypto.h" +#include "crypto_format.h" #include "tortls.h" -#include "../common/torlog.h" +#include "torlog.h" #include "container.h" #include "torgzip.h" #include "address.h" @@ -99,6 +91,9 @@ #include "ht.h" #include "replaycache.h" #include "crypto_curve25519.h" +#include "crypto_ed25519.h" +#include "tor_queue.h" +#include "util_format.h" /* These signals are defined to help handle_control_signal work. */ @@ -121,6 +116,7 @@ * conflict with system-defined signals. */ #define SIGNEWNYM 129 #define SIGCLEARDNSCACHE 130 +#define SIGHEARTBEAT 131 #if (SIZEOF_CELL_T != 0) /* On Irix, stdlib.h defines a cell_t type, so we need to make sure @@ -195,6 +191,7 @@ typedef enum { * and let it use any circuit ID it wants. */ CIRC_ID_TYPE_NEITHER=2 } circ_id_type_t; +#define circ_id_type_bitfield_t ENUM_BF(circ_id_type_t) #define CONN_TYPE_MIN_ 3 /** Type for sockets listening for OR connections. */ @@ -213,8 +210,7 @@ typedef enum { #define CONN_TYPE_DIR_LISTENER 8 /** Type for HTTP connections to the directory server. */ #define CONN_TYPE_DIR 9 -/** Connection from the main process to a CPU worker process. */ -#define CONN_TYPE_CPUWORKER 10 +/* Type 10 is unused. */ /** Type for listening for connections from user interface process. */ #define CONN_TYPE_CONTROL_LISTENER 11 /** Type for connections from user interface process. */ @@ -227,8 +223,14 @@ typedef enum { #define CONN_TYPE_AP_NATD_LISTENER 14 /** Type for sockets listening for DNS requests. */ #define CONN_TYPE_AP_DNS_LISTENER 15 -#define CONN_TYPE_MAX_ 15 -/* !!!! If CONN_TYPE_MAX_ is ever over 15, we must grow the type field in + +/** Type for connections from the Extended ORPort. */ +#define CONN_TYPE_EXT_OR 16 +/** Type for sockets listening for Extended ORPort connections. */ +#define CONN_TYPE_EXT_OR_LISTENER 17 + +#define CONN_TYPE_MAX_ 17 +/* !!!! If _CONN_TYPE_MAX is ever over 31, we must grow the type field in * connection_t. */ /* Proxy client types */ @@ -236,9 +238,11 @@ typedef enum { #define PROXY_CONNECT 1 #define PROXY_SOCKS4 2 #define PROXY_SOCKS5 3 -/* !!!! If there is ever a PROXY_* type over 2, we must grow the proxy_type +/* !!!! If there is ever a PROXY_* type over 3, we must grow the proxy_type * field in or_connection_t */ -/* pluggable transports proxy type */ + +/* Pluggable transport proxy type. Don't use this in or_connection_t, + * instead use the actual underlying proxy type (see above). */ #define PROXY_PLUGGABLE 4 /* Proxy client handshake states */ @@ -268,17 +272,6 @@ typedef enum { /** State for any listener connection. */ #define LISTENER_STATE_READY 0 -#define CPUWORKER_STATE_MIN_ 1 -/** State for a connection to a cpuworker process that's idle. */ -#define CPUWORKER_STATE_IDLE 1 -/** State for a connection to a cpuworker process that's processing a - * handshake. */ -#define CPUWORKER_STATE_BUSY_ONION 2 -#define CPUWORKER_STATE_MAX_ 2 - -#define CPUWORKER_TASK_ONION CPUWORKER_STATE_BUSY_ONION -#define CPUWORKER_TASK_SHUTDOWN 255 - #define OR_CONN_STATE_MIN_ 1 /** State for a connection to an OR: waiting for connect() to finish. */ #define OR_CONN_STATE_CONNECTING 1 @@ -306,6 +299,25 @@ typedef enum { #define OR_CONN_STATE_OPEN 8 #define OR_CONN_STATE_MAX_ 8 +/** States of the Extended ORPort protocol. Be careful before changing + * the numbers: they matter. */ +#define EXT_OR_CONN_STATE_MIN_ 1 +/** Extended ORPort authentication is waiting for the authentication + * type selected by the client. */ +#define EXT_OR_CONN_STATE_AUTH_WAIT_AUTH_TYPE 1 +/** Extended ORPort authentication is waiting for the client nonce. */ +#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_NONCE 2 +/** Extended ORPort authentication is waiting for the client hash. */ +#define EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH 3 +#define EXT_OR_CONN_STATE_AUTH_MAX 3 +/** Authentication finished and the Extended ORPort is now accepting + * traffic. */ +#define EXT_OR_CONN_STATE_OPEN 4 +/** Extended ORPort is flushing its last messages and preparing to + * start accepting OR connections. */ +#define EXT_OR_CONN_STATE_FLUSHING 5 +#define EXT_OR_CONN_STATE_MAX_ 5 + #define EXIT_CONN_STATE_MIN_ 1 /** State for an exit connection: waiting for response from DNS farm. */ #define EXIT_CONN_STATE_RESOLVING 1 @@ -372,16 +384,10 @@ typedef enum { #define CONTROL_CONN_STATE_NEEDAUTH 2 #define CONTROL_CONN_STATE_MAX_ 2 -#define DIR_PURPOSE_MIN_ 3 -/** A connection to a directory server: download a rendezvous - * descriptor. */ -#define DIR_PURPOSE_FETCH_RENDDESC 3 -/** A connection to a directory server: set after a rendezvous +#define DIR_PURPOSE_MIN_ 4 +/** A connection to a directory server: set after a v2 rendezvous * descriptor is downloaded. */ -#define DIR_PURPOSE_HAS_FETCHED_RENDDESC 4 -/** A connection to a directory server: download one or more v2 - * network-status objects */ -#define DIR_PURPOSE_FETCH_V2_NETWORKSTATUS 5 +#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4 /** A connection to a directory server: download one or more server * descriptors. */ #define DIR_PURPOSE_FETCH_SERVERDESC 6 @@ -390,9 +396,6 @@ typedef enum { #define DIR_PURPOSE_FETCH_EXTRAINFO 7 /** A connection to a directory server: upload a server descriptor. */ #define DIR_PURPOSE_UPLOAD_DIR 8 -/** A connection to a directory server: upload a rendezvous - * descriptor. */ -#define DIR_PURPOSE_UPLOAD_RENDDESC 9 /** A connection to a directory server: upload a v3 networkstatus vote. */ #define DIR_PURPOSE_UPLOAD_VOTE 10 /** A connection to a directory server: upload a v3 consensus signature */ @@ -426,7 +429,6 @@ typedef enum { * directory server. */ #define DIR_PURPOSE_IS_UPLOAD(p) \ ((p)==DIR_PURPOSE_UPLOAD_DIR || \ - (p)==DIR_PURPOSE_UPLOAD_RENDDESC || \ (p)==DIR_PURPOSE_UPLOAD_VOTE || \ (p)==DIR_PURPOSE_UPLOAD_SIGNATURES) @@ -456,9 +458,11 @@ typedef enum { #define CIRCUIT_PURPOSE_OR_MIN_ 1 /** OR-side circuit purpose: normal circuit, at OR. */ #define CIRCUIT_PURPOSE_OR 1 -/** OR-side circuit purpose: At OR, from Bob, waiting for intro from Alices. */ +/** OR-side circuit purpose: At OR, from the service, waiting for intro from + * clients. */ #define CIRCUIT_PURPOSE_INTRO_POINT 2 -/** OR-side circuit purpose: At OR, from Alice, waiting for Bob. */ +/** OR-side circuit purpose: At OR, from the client, waiting for the service. + */ #define CIRCUIT_PURPOSE_REND_POINT_WAITING 3 /** OR-side circuit purpose: At OR, both circuits have this purpose. */ #define CIRCUIT_PURPOSE_REND_ESTABLISHED 4 @@ -477,43 +481,47 @@ typedef enum { * to becoming open, or they are open and have sent the * establish_rendezvous cell but haven't received an ack. * circuits that are c_rend_ready are open and have received a - * rend ack, but haven't heard from bob yet. if they have a + * rend ack, but haven't heard from the service yet. if they have a * buildstate->pending_final_cpath then they're expecting a - * cell from bob, else they're not. + * cell from the service, else they're not. * circuits that are c_rend_ready_intro_acked are open, and * some intro circ has sent its intro and received an ack. * circuits that are c_rend_joined are open, have heard from - * bob, and are talking to him. + * the service, and are talking to it. */ /** Client-side circuit purpose: Normal circuit, with cpath. */ #define CIRCUIT_PURPOSE_C_GENERAL 5 -/** Client-side circuit purpose: at Alice, connecting to intro point. */ +/** Client-side circuit purpose: at the client, connecting to intro point. */ #define CIRCUIT_PURPOSE_C_INTRODUCING 6 -/** Client-side circuit purpose: at Alice, sent INTRODUCE1 to intro point, +/** Client-side circuit purpose: at the client, sent INTRODUCE1 to intro point, * waiting for ACK/NAK. */ #define CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT 7 -/** Client-side circuit purpose: at Alice, introduced and acked, closing. */ +/** Client-side circuit purpose: at the client, introduced and acked, closing. + */ #define CIRCUIT_PURPOSE_C_INTRODUCE_ACKED 8 -/** Client-side circuit purpose: at Alice, waiting for ack. */ +/** Client-side circuit purpose: at the client, waiting for ack. */ #define CIRCUIT_PURPOSE_C_ESTABLISH_REND 9 -/** Client-side circuit purpose: at Alice, waiting for Bob. */ +/** Client-side circuit purpose: at the client, waiting for the service. */ #define CIRCUIT_PURPOSE_C_REND_READY 10 -/** Client-side circuit purpose: at Alice, waiting for Bob, INTRODUCE - * has been acknowledged. */ +/** Client-side circuit purpose: at the client, waiting for the service, + * INTRODUCE has been acknowledged. */ #define CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED 11 -/** Client-side circuit purpose: at Alice, rendezvous established. */ +/** Client-side circuit purpose: at the client, rendezvous established. */ #define CIRCUIT_PURPOSE_C_REND_JOINED 12 /** This circuit is used for build time measurement only */ #define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 13 #define CIRCUIT_PURPOSE_C_MAX_ 13 -/** Hidden-service-side circuit purpose: at Bob, waiting for introductions. */ +/** Hidden-service-side circuit purpose: at the service, waiting for + * introductions. */ #define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 14 -/** Hidden-service-side circuit purpose: at Bob, successfully established - * intro. */ +/** Hidden-service-side circuit purpose: at the service, successfully + * established intro. */ #define CIRCUIT_PURPOSE_S_INTRO 15 -/** Hidden-service-side circuit purpose: at Bob, connecting to rend point. */ +/** Hidden-service-side circuit purpose: at the service, connecting to rend + * point. */ #define CIRCUIT_PURPOSE_S_CONNECT_REND 16 -/** Hidden-service-side circuit purpose: at Bob, rendezvous established. */ +/** Hidden-service-side circuit purpose: at the service, rendezvous + * established. */ #define CIRCUIT_PURPOSE_S_REND_JOINED 17 /** A testing circuit; not meant to be used for actual traffic. */ #define CIRCUIT_PURPOSE_TESTING 18 @@ -585,7 +593,8 @@ typedef enum { #define END_OR_CONN_REASON_NO_ROUTE 6 /* no route to host/net */ #define END_OR_CONN_REASON_IO_ERROR 7 /* read/write error */ #define END_OR_CONN_REASON_RESOURCE_LIMIT 8 /* sockets, buffers, etc */ -#define END_OR_CONN_REASON_MISC 9 +#define END_OR_CONN_REASON_PT_MISSING 9 /* PT failed or not available */ +#define END_OR_CONN_REASON_MISC 10 /* Reasons why we (or a remote OR) might close a stream. See tor-spec.txt for * documentation of these. The values must match. */ @@ -659,6 +668,10 @@ typedef enum { /* Negative reasons are internal: we never send them in a DESTROY or TRUNCATE * call; they only go to the controller for tracking */ + +/* Closing introduction point that were opened in parallel. */ +#define END_CIRC_REASON_IP_NOW_REDUNDANT -4 + /** Our post-timeout circuit time measurement period expired. * We must give up now */ #define END_CIRC_REASON_MEASUREMENT_EXPIRED -3 @@ -783,17 +796,34 @@ typedef struct rend_data_t { /** Onion address (without the .onion part) that a client requests. */ char onion_address[REND_SERVICE_ID_LEN_BASE32+1]; + /** Descriptor ID for each replicas computed from the onion address. If + * the onion address is empty, this array MUST be empty. We keep them so + * we know when to purge our entry in the last hsdir request table. */ + char descriptor_id[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS][DIGEST_LEN]; + /** (Optional) descriptor cookie that is used by a client. */ char descriptor_cookie[REND_DESC_COOKIE_LEN]; /** Authorization type for accessing a service used by a client. */ rend_auth_type_t auth_type; + /** Descriptor ID for a client request. The control port command HSFETCH + * uses this. It's set if the descriptor query should only use this + * descriptor ID. */ + char desc_id_fetch[DIGEST_LEN]; + /** Hash of the hidden service's PK used by a service. */ char rend_pk_digest[DIGEST_LEN]; /** Rendezvous cookie used by both, client and service. */ char rend_cookie[REND_COOKIE_LEN]; + + /** List of HSDir fingerprints on which this request has been sent to. + * This contains binary identity digest of the directory. */ + smartlist_t *hsdirs_fp; + + /** Number of streams associated with this rendezvous circuit. */ + int nr_streams; } rend_data_t; /** Time interval for tracking replays of DH public keys received in @@ -823,9 +853,15 @@ typedef enum { /** Maximum number of queued cells on a circuit for which we are the * midpoint before we give up and kill it. This must be >= circwindow * to avoid killing innocent circuits, and >= circwindow*2 to give - * leaky-pipe a chance for being useful someday. + * leaky-pipe a chance of working someday. The ORCIRC_MAX_MIDDLE_KILL_THRESH + * ratio controls the margin of error between emitting a warning and + * killing the circuit. + */ +#define ORCIRC_MAX_MIDDLE_CELLS (CIRCWINDOW_START_MAX*2) +/** Ratio of hard (circuit kill) to soft (warning) thresholds for the + * ORCIRC_MAX_MIDDLE_CELLS tests. */ -#define ORCIRC_MAX_MIDDLE_CELLS (21*(CIRCWINDOW_START_MAX)/10) +#define ORCIRC_MAX_MIDDLE_KILL_THRESH (1.1f) /* Cell commands. These values are defined in tor-spec.txt. */ #define CELL_PADDING 0 @@ -846,6 +882,7 @@ typedef enum { #define CELL_AUTH_CHALLENGE 130 #define CELL_AUTHENTICATE 131 #define CELL_AUTHORIZE 132 +#define CELL_COMMAND_MAX_ 132 /** How long to test reachability before complaining to the user. */ #define TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT (20*60) @@ -884,18 +921,18 @@ typedef enum { #define VAR_CELL_MAX_HEADER_SIZE 7 static int get_cell_network_size(int wide_circ_ids); -static INLINE int get_cell_network_size(int wide_circ_ids) +static inline int get_cell_network_size(int wide_circ_ids) { return wide_circ_ids ? CELL_MAX_NETWORK_SIZE : CELL_MAX_NETWORK_SIZE - 2; } static int get_var_cell_header_size(int wide_circ_ids); -static INLINE int get_var_cell_header_size(int wide_circ_ids) +static inline int get_var_cell_header_size(int wide_circ_ids) { return wide_circ_ids ? VAR_CELL_MAX_HEADER_SIZE : VAR_CELL_MAX_HEADER_SIZE - 2; } static int get_circ_id_size(int wide_circ_ids); -static INLINE int get_circ_id_size(int wide_circ_ids) +static inline int get_circ_id_size(int wide_circ_ids) { return wide_circ_ids ? 4 : 2; } @@ -1073,9 +1110,17 @@ typedef struct var_cell_t { uint8_t payload[FLEXIBLE_ARRAY_MEMBER]; } var_cell_t; +/** A parsed Extended ORPort message. */ +typedef struct ext_or_cmd_t { + uint16_t cmd; /** Command type */ + uint16_t len; /** Body length */ + char body[FLEXIBLE_ARRAY_MEMBER]; /** Message body */ +} ext_or_cmd_t; + /** A cell as packed for writing to the network. */ typedef struct packed_cell_t { - struct packed_cell_t *next; /**< Next cell queued on this circuit. */ + /** Next cell queued on this circuit. */ + TOR_SIMPLEQ_ENTRY(packed_cell_t) next; char body[CELL_MAX_NETWORK_SIZE]; /**< Cell as packed for network. */ uint32_t inserted_time; /**< Time (in milliseconds since epoch, with high * bits truncated) when this cell was inserted. */ @@ -1084,8 +1129,8 @@ typedef struct packed_cell_t { /** A queue of cells on a circuit, waiting to be added to the * or_connection_t's outbuf. */ typedef struct cell_queue_t { - packed_cell_t *head; /**< The first cell, or NULL if the queue is empty. */ - packed_cell_t *tail; /**< The last cell, or NULL if the queue is empty. */ + /** Linked list of packed_cell_t*/ + TOR_SIMPLEQ_HEAD(cell_simpleq, packed_cell_t) head; int n; /**< The number of cells in the queue. */ } cell_queue_t; @@ -1106,6 +1151,53 @@ typedef struct socks_request_t socks_request_t; #define generic_buffer_t buf_t #endif +typedef struct entry_port_cfg_t { + /* Client port types (socks, dns, trans, natd) only: */ + uint8_t isolation_flags; /**< Zero or more isolation flags */ + int session_group; /**< A session group, or -1 if this port is not in a + * session group. */ + + /* Socks only: */ + /** When both no-auth and user/pass are advertised by a SOCKS client, select + * no-auth. */ + unsigned int socks_prefer_no_auth : 1; + /** When ISO_SOCKSAUTH is in use, Keep-Alive circuits indefinitely. */ + unsigned int socks_iso_keep_alive : 1; + + /* Client port types only: */ + unsigned int ipv4_traffic : 1; + unsigned int ipv6_traffic : 1; + unsigned int prefer_ipv6 : 1; + + /** For a socks listener: should we cache IPv4/IPv6 DNS information that + * exit nodes tell us? + * + * @{ */ + unsigned int cache_ipv4_answers : 1; + unsigned int cache_ipv6_answers : 1; + /** @} */ + /** For a socks listeners: if we find an answer in our client-side DNS cache, + * should we use it? + * + * @{ */ + unsigned int use_cached_ipv4_answers : 1; + unsigned int use_cached_ipv6_answers : 1; + /** @} */ + /** For socks listeners: When we can automap an address to IPv4 or IPv6, + * do we prefer IPv6? */ + unsigned int prefer_ipv6_virtaddr : 1; + +} entry_port_cfg_t; + +typedef struct server_port_cfg_t { + /* Server port types (or, dir) only: */ + unsigned int no_advertise : 1; + unsigned int no_listen : 1; + unsigned int all_addrs : 1; + unsigned int bind_ipv4_only : 1; + unsigned int bind_ipv6_only : 1; +} server_port_cfg_t; + /* Values for connection_t.magic: used to make sure that downcasts (casts from * connection_t to foo_connection_t) are safe. */ #define BASE_CONNECTION_MAGIC 0x7C3C304Eu @@ -1139,7 +1231,7 @@ typedef struct connection_t { * *_CONNECTION_MAGIC. */ uint8_t state; /**< Current state of this connection. */ - unsigned int type:4; /**< What kind of connection is this? */ + unsigned int type:5; /**< What kind of connection is this? */ unsigned int purpose:5; /**< Only used for DIR and EXIT types currently. */ /* The next fields are all one-bit booleans. Some are only applicable to @@ -1216,13 +1308,21 @@ typedef struct connection_t { * marked.) */ const char *marked_for_close_file; /**< For debugging: in which file were * we marked for close? */ - char *address; /**< FQDN (or IP) of the guy on the other end. + char *address; /**< FQDN (or IP) of the other end. * strdup into this, because free_connection() frees it. */ /** Another connection that's connected to this one in lieu of a socket. */ struct connection_t *linked_conn; /** Unique identifier for this connection on this Tor instance. */ uint64_t global_identifier; + + /** Bytes read since last call to control_event_conn_bandwidth_used(). + * Only used if we're configured to emit CONN_BW events. */ + uint32_t n_read_conn_bw; + + /** Bytes written since last call to control_event_conn_bandwidth_used(). + * Only used if we're configured to emit CONN_BW events. */ + uint32_t n_written_conn_bw; } connection_t; /** Subtype of connection_t; used for a listener socket. */ @@ -1233,52 +1333,7 @@ typedef struct listener_connection_t { * to the evdns_server_port it uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; - /** @name Isolation parameters - * - * For an AP listener, these fields describe how to isolate streams that - * arrive on the listener. - * - * @{ - */ - /** The session group for this listener. */ - int session_group; - /** One or more ISO_ flags to describe how to isolate streams. */ - uint8_t isolation_flags; - /**@}*/ - /** For SOCKS connections only: If this is set, we will choose "no - * authentication" instead of "username/password" authentication if both - * are offered. Used as input to parse_socks. */ - unsigned int socks_prefer_no_auth : 1; - - /** For a SOCKS listeners, these fields describe whether we should - * allow IPv4 and IPv6 addresses from our exit nodes, respectively. - * - * @{ - */ - unsigned int socks_ipv4_traffic : 1; - unsigned int socks_ipv6_traffic : 1; - /** @} */ - /** For a socks listener: should we tell the exit that we prefer IPv6 - * addresses? */ - unsigned int socks_prefer_ipv6 : 1; - - /** For a socks listener: should we cache IPv4/IPv6 DNS information that - * exit nodes tell us? - * - * @{ */ - unsigned int cache_ipv4_answers : 1; - unsigned int cache_ipv6_answers : 1; - /** @} */ - /** For a socks listeners: if we find an answer in our client-side DNS cache, - * should we use it? - * - * @{ */ - unsigned int use_cached_ipv4_answers : 1; - unsigned int use_cached_ipv6_answers : 1; - /** @} */ - /** For socks listeners: When we can automap an address to IPv4 or IPv6, - * do we prefer IPv6? */ - unsigned int prefer_ipv6_virtaddr : 1; + entry_port_cfg_t entry_cfg; } listener_connection_t; @@ -1303,6 +1358,8 @@ typedef struct listener_connection_t { * in the v3 handshake. The subject key must be a 1024-bit RSA key; it * must be signed by the identity key */ #define OR_CERT_TYPE_AUTH_1024 3 +/* DOCDOC */ +#define OR_CERT_TYPE_RSA_ED_CROSSCERT 7 /**@}*/ /** The one currently supported type of AUTHENTICATE cell. It contains @@ -1378,12 +1435,27 @@ typedef struct or_handshake_state_t { * @{ */ /** The cert for the key that's supposed to sign the AUTHENTICATE cell */ - tor_cert_t *auth_cert; + tor_x509_cert_t *auth_cert; /** A self-signed identity certificate */ - tor_cert_t *id_cert; + tor_x509_cert_t *id_cert; /**@}*/ } or_handshake_state_t; +/** Length of Extended ORPort connection identifier. */ +#define EXT_OR_CONN_ID_LEN DIGEST_LEN /* 20 */ +/* + * OR_CONN_HIGHWATER and OR_CONN_LOWWATER moved from connection_or.c so + * channeltls.c can see them too. + */ + +/** When adding cells to an OR connection's outbuf, keep adding until the + * outbuf is at least this long, or we run out of cells. */ +#define OR_CONN_HIGHWATER (32*1024) + +/** Add cells to an OR connection's outbuf whenever the outbuf's data length + * drops below this size. */ +#define OR_CONN_LOWWATER (16*1024) + /** Subtype of connection_t for an "OR connection" -- that is, one that speaks * cells over TLS. */ typedef struct or_connection_t { @@ -1392,6 +1464,20 @@ typedef struct or_connection_t { /** Hash of the public RSA key for the other side's identity key, or zeroes * if the other side hasn't shown us a valid identity key. */ char identity_digest[DIGEST_LEN]; + + /** Extended ORPort connection identifier. */ + char *ext_or_conn_id; + /** This is the ClientHash value we expect to receive from the + * client during the Extended ORPort authentication protocol. We + * compute it upon receiving the ClientNoce from the client, and we + * compare it with the acual ClientHash value sent by the + * client. */ + char *ext_or_auth_correct_client_hash; + /** String carrying the name of the pluggable transport + * (e.g. "obfs2") that is obfuscating this connection. If no + * pluggable transports are used, it's NULL. */ + char *ext_or_transport; + char *nickname; /**< Nickname of OR on other side (if any). */ tor_tls_t *tls; /**< TLS connection state. */ @@ -1422,15 +1508,20 @@ typedef struct or_connection_t { unsigned int is_outgoing:1; unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */ unsigned int wide_circ_ids:1; + /** True iff this connection has had its bootstrap failure logged with + * control_event_bootstrap_problem. */ + unsigned int have_noted_bootstrap_problem:1; + uint16_t link_proto; /**< What protocol version are we using? 0 for * "none negotiated yet." */ - + uint16_t idle_timeout; /**< How long can this connection sit with no + * circuits on it before we close it? Based on + * IDLE_CIRCUIT_TIMEOUT_{NON,}CANONICAL and + * on is_canonical, randomized. */ or_handshake_state_t *handshake_state; /**< If we are setting this connection * up, state information to do so. */ time_t timestamp_lastempty; /**< When was the outbuf last completely empty?*/ - time_t timestamp_last_added_nonpadding; /** When did we last add a - * non-padding cell to the outbuf? */ /* bandwidth* and *_bucket only used by ORs in OPEN state: */ int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ @@ -1449,6 +1540,18 @@ typedef struct or_connection_t { struct or_connection_t *next_with_same_id; /**< Next connection with same * identity digest as this one. */ + /** Last emptied read token bucket in msec since midnight; only used if + * TB_EMPTY events are enabled. */ + uint32_t read_emptied_time; + /** Last emptied write token bucket in msec since midnight; only used if + * TB_EMPTY events are enabled. */ + uint32_t write_emptied_time; + + /* + * Count the number of bytes flushed out on this orconn, and the number of + * bytes TLS actually sent - used for overhead estimation for scheduling. + */ + uint64_t bytes_xmitted, bytes_xmitted_by_tls; } or_connection_t; /** Subtype of connection_t for an "edge connection" -- that is, an entry (ap) @@ -1520,12 +1623,10 @@ typedef struct entry_connection_t { * only.) */ /* === Isolation related, AP only. === */ - /** AP only: based on which factors do we isolate this stream? */ - uint8_t isolation_flags; - /** AP only: what session group is this stream in? */ - int session_group; + entry_port_cfg_t entry_cfg; /** AP only: The newnym epoch in which we created this connection. */ unsigned nym_epoch; + /** AP only: The original requested address before we rewrote it. */ char *original_dest_address; /* Other fields to isolate on already exist. The ClientAddr is addr. The @@ -1551,6 +1652,13 @@ typedef struct entry_connection_t { * request that we're going to try to answer. */ struct evdns_server_request *dns_server_request; +#define DEBUGGING_17659 + +#ifdef DEBUGGING_17659 + uint16_t marked_pending_circ_line; + const char *marked_pending_circ_file; +#endif + #define NUM_CIRCUITS_LAUNCHED_THRESHOLD 10 /** Number of times we've launched a circuit to handle this stream. If * it gets too high, that could indicate an inconsistency between our @@ -1584,33 +1692,8 @@ typedef struct entry_connection_t { */ unsigned int may_use_optimistic_data : 1; - /** Should we permit IPv4 and IPv6 traffic to use this connection? - * - * @{ */ - unsigned int ipv4_traffic_ok : 1; - unsigned int ipv6_traffic_ok : 1; - /** @} */ - /** Should we say we prefer IPv6 traffic? */ - unsigned int prefer_ipv6_traffic : 1; - - /** For a socks listener: should we cache IPv4/IPv6 DNS information that - * exit nodes tell us? - * - * @{ */ - unsigned int cache_ipv4_answers : 1; - unsigned int cache_ipv6_answers : 1; - /** @} */ - /** For a socks listeners: if we find an answer in our client-side DNS cache, - * should we use it? - * - * @{ */ - unsigned int use_cached_ipv4_answers : 1; - unsigned int use_cached_ipv6_answers : 1; - /** @} */ - /** For socks listeners: When we can automap an address to IPv4 or IPv6, - * do we prefer IPv6? */ - unsigned int prefer_ipv6_virtaddr : 1; - + /** Are we a socks SocksSocket listener? */ + unsigned int is_socks_socket:1; } entry_connection_t; typedef enum { @@ -1619,6 +1702,7 @@ typedef enum { DIR_SPOOL_CACHED_DIR, DIR_SPOOL_NETWORKSTATUS, DIR_SPOOL_MICRODESC, /* NOTE: if we add another entry, add another bit. */ } dir_spool_source_t; +#define dir_spool_source_bitfield_t ENUM_BF(dir_spool_source_t) /** Subtype of connection_t for an "directory connection" -- that is, an HTTP * connection to retrieve or serve directory material. */ @@ -1638,7 +1722,7 @@ typedef struct dir_connection_t { * "spooling" of directory material to the outbuf. Otherwise, we'd have * to append everything to the outbuf in one enormous chunk. */ /** What exactly are we spooling right now? */ - ENUM_BF(dir_spool_source_t) dir_spool_src : 3; + dir_spool_source_bitfield_t dir_spool_src : 3; /** If we're fetching descriptors, what router purpose shall we assign * to them? */ @@ -1668,8 +1752,9 @@ typedef struct dir_connection_t { typedef struct control_connection_t { connection_t base_; - uint32_t event_mask; /**< Bitfield: which events does this controller - * care about? */ + uint64_t event_mask; /**< Bitfield: which events does this controller + * care about? + * EVENT_MAX_ is >31, so we need a 64 bit mask */ /** True if we have sent a protocolinfo reply on this connection. */ unsigned int have_sent_protocolinfo:1; @@ -1677,6 +1762,9 @@ typedef struct control_connection_t { * connection. */ unsigned int is_owning_control_connection:1; + /** List of ephemeral onion services belonging to this connection. */ + smartlist_t *ephemeral_onion_services; + /** If we have sent an AUTHCHALLENGE reply on this connection and * have not received a successful AUTHENTICATE command, points to * the value which the client must send to authenticate itself; @@ -1724,38 +1812,38 @@ static control_connection_t *TO_CONTROL_CONN(connection_t *); * invalid. */ static listener_connection_t *TO_LISTENER_CONN(connection_t *); -static INLINE or_connection_t *TO_OR_CONN(connection_t *c) +static inline or_connection_t *TO_OR_CONN(connection_t *c) { tor_assert(c->magic == OR_CONNECTION_MAGIC); return DOWNCAST(or_connection_t, c); } -static INLINE dir_connection_t *TO_DIR_CONN(connection_t *c) +static inline dir_connection_t *TO_DIR_CONN(connection_t *c) { tor_assert(c->magic == DIR_CONNECTION_MAGIC); return DOWNCAST(dir_connection_t, c); } -static INLINE edge_connection_t *TO_EDGE_CONN(connection_t *c) +static inline edge_connection_t *TO_EDGE_CONN(connection_t *c) { tor_assert(c->magic == EDGE_CONNECTION_MAGIC || c->magic == ENTRY_CONNECTION_MAGIC); return DOWNCAST(edge_connection_t, c); } -static INLINE entry_connection_t *TO_ENTRY_CONN(connection_t *c) +static inline entry_connection_t *TO_ENTRY_CONN(connection_t *c) { tor_assert(c->magic == ENTRY_CONNECTION_MAGIC); return (entry_connection_t*) SUBTYPE_P(c, entry_connection_t, edge_.base_); } -static INLINE entry_connection_t *EDGE_TO_ENTRY_CONN(edge_connection_t *c) +static inline entry_connection_t *EDGE_TO_ENTRY_CONN(edge_connection_t *c) { tor_assert(c->base_.magic == ENTRY_CONNECTION_MAGIC); return (entry_connection_t*) SUBTYPE_P(c, entry_connection_t, edge_); } -static INLINE control_connection_t *TO_CONTROL_CONN(connection_t *c) +static inline control_connection_t *TO_CONTROL_CONN(connection_t *c) { tor_assert(c->magic == CONTROL_CONNECTION_MAGIC); return DOWNCAST(control_connection_t, c); } -static INLINE listener_connection_t *TO_LISTENER_CONN(connection_t *c) +static inline listener_connection_t *TO_LISTENER_CONN(connection_t *c) { tor_assert(c->magic == LISTENER_CONNECTION_MAGIC); return DOWNCAST(listener_connection_t, c); @@ -1811,12 +1899,13 @@ typedef enum { ADDR_POLICY_ACCEPT=1, ADDR_POLICY_REJECT=2, } addr_policy_action_t; +#define addr_policy_action_bitfield_t ENUM_BF(addr_policy_action_t) /** A reference-counted address policy rule. */ typedef struct addr_policy_t { int refcnt; /**< Reference count */ /** What to do when the policy matches.*/ - ENUM_BF(addr_policy_action_t) policy_type:2; + addr_policy_action_bitfield_t policy_type:2; unsigned int is_private:1; /**< True iff this is the pseudo-address, * "private". */ unsigned int is_canonical:1; /**< True iff this policy is the canonical @@ -1846,7 +1935,7 @@ typedef struct cached_dir_t { size_t dir_len; /**< Length of <b>dir</b> (not counting its NUL). */ size_t dir_z_len; /**< Length of <b>dir_z</b>. */ time_t published; /**< When was this object published. */ - digests_t digests; /**< Digests of this object (networkstatus only) */ + common_digests_t digests; /**< Digests of this object (networkstatus only) */ int refcnt; /**< Reference count for this cached_dir_t. */ } cached_dir_t; @@ -1868,23 +1957,85 @@ typedef enum { */ SAVED_IN_JOURNAL } saved_location_t; +#define saved_location_bitfield_t ENUM_BF(saved_location_t) -/** Enumeration: what kind of download schedule are we using for a given - * object? */ +/** Enumeration: what directory object is being downloaded? + * This determines which schedule is selected to perform the download. */ typedef enum { DL_SCHED_GENERIC = 0, DL_SCHED_CONSENSUS = 1, DL_SCHED_BRIDGE = 2, } download_schedule_t; +#define download_schedule_bitfield_t ENUM_BF(download_schedule_t) + +/** Enumeration: is the download schedule for downloading from an authority, + * or from any available directory mirror? + * During bootstrap, "any" means a fallback (or an authority, if there + * are no fallbacks). + * When we have a valid consensus, "any" means any directory server. */ +typedef enum { + DL_WANT_ANY_DIRSERVER = 0, + DL_WANT_AUTHORITY = 1, +} download_want_authority_t; +#define download_want_authority_bitfield_t \ + ENUM_BF(download_want_authority_t) + +/** Enumeration: do we want to increment the schedule position each time a + * connection is attempted (these attempts can be concurrent), or do we want + * to increment the schedule position after a connection fails? */ +typedef enum { + DL_SCHED_INCREMENT_FAILURE = 0, + DL_SCHED_INCREMENT_ATTEMPT = 1, +} download_schedule_increment_t; +#define download_schedule_increment_bitfield_t \ + ENUM_BF(download_schedule_increment_t) /** Information about our plans for retrying downloads for a downloadable - * object. */ + * directory object. + * Each type of downloadable directory object has a corresponding retry + * <b>schedule</b>, which can be different depending on whether the object is + * being downloaded from an authority or a mirror (<b>want_authority</b>). + * <b>next_attempt_at</b> contains the next time we will attempt to download + * the object. + * For schedules that <b>increment_on</b> failure, <b>n_download_failures</b> + * is used to determine the position in the schedule. (Each schedule is a + * smartlist of integer delays, parsed from a CSV option.) Every time a + * connection attempt fails, <b>n_download_failures</b> is incremented, + * the new delay value is looked up from the schedule, and + * <b>next_attempt_at</b> is set delay seconds from the time the previous + * connection failed. Therefore, at most one failure-based connection can be + * in progress for each download_status_t. + * For schedules that <b>increment_on</b> attempt, <b>n_download_attempts</b> + * is used to determine the position in the schedule. Every time a + * connection attempt is made, <b>n_download_attempts</b> is incremented, + * the new delay value is looked up from the schedule, and + * <b>next_attempt_at</b> is set delay seconds from the time the previous + * connection was attempted. Therefore, multiple concurrent attempted-based + * connections can be in progress for each download_status_t. + * After an object is successfully downloaded, any other concurrent connections + * are terminated. A new schedule which starts at position 0 is used for + * subsequent downloads of the same object. + */ typedef struct download_status_t { - time_t next_attempt_at; /**< When should we try downloading this descriptor + time_t next_attempt_at; /**< When should we try downloading this object * again? */ - uint8_t n_download_failures; /**< Number of failures trying to download the - * most recent descriptor. */ - ENUM_BF(download_schedule_t) schedule : 8; + uint8_t n_download_failures; /**< Number of failed downloads of the most + * recent object, since the last success. */ + uint8_t n_download_attempts; /**< Number of (potentially concurrent) attempts + * to download the most recent object, since + * the last success. */ + download_schedule_bitfield_t schedule : 8; /**< What kind of object is being + * downloaded? This determines the + * schedule used for the download. + */ + download_want_authority_bitfield_t want_authority : 1; /**< Is the download + * happening from an authority + * or a mirror? This determines + * the schedule used for the + * download. */ + download_schedule_increment_bitfield_t increment_on : 1; /**< does this + * schedule increment on each attempt, + * or after each failure? */ } download_status_t; /** If n_download_failures is this high, the download can never happen. */ @@ -1914,6 +2065,10 @@ typedef struct signed_descriptor_t { time_t published_on; /** For routerdescs only: digest of the corresponding extrainfo. */ char extra_info_digest[DIGEST_LEN]; + /** For routerdescs only: A SHA256-digest of the extrainfo (if any) */ + char extra_info_digest256[DIGEST256_LEN]; + /** Certificate for ed25519 signing key. */ + struct tor_cert_st *signing_key_cert; /** For routerdescs only: Status of downloading the corresponding * extrainfo. */ download_status_t ei_dl_status; @@ -1926,9 +2081,7 @@ typedef struct signed_descriptor_t { * routerlist->old_routers? -1 for none. */ int routerlist_index; /** The valid-until time of the most recent consensus that listed this - * descriptor, or a bit after the publication time of the most recent v2 - * networkstatus that listed it. 0 for "never listed in a consensus or - * status, so far as we know." */ + * descriptor. 0 for "never listed in a consensus, so far as we know." */ time_t last_listed_as_valid_until; /* If true, we do not ever try to save this object in the cache. */ unsigned int do_not_cache : 1; @@ -1947,7 +2100,6 @@ typedef int16_t country_t; /** Information about another onion router in the network. */ typedef struct { signed_descriptor_t cache_info; - char *address; /**< Location of OR: either a hostname or an IP address. */ char *nickname; /**< Human-readable OR name. */ uint32_t addr; /**< IPv4 address of OR, in host order. */ @@ -1965,6 +2117,9 @@ typedef struct { crypto_pk_t *identity_pkey; /**< Public RSA key for signing. */ /** Public curve25519 key for onions */ curve25519_public_key_t *onion_curve25519_pkey; + /** What's the earliest expiration time on all the certs in this + * routerinfo? */ + time_t cert_expiration_time; char *platform; /**< What software/operating system is this OR using? */ @@ -1998,6 +2153,15 @@ typedef struct { * tests for it. */ unsigned int needs_retest_if_added:1; + /** True iff this router included "tunnelled-dir-server" in its descriptor, + * implying it accepts tunnelled directory requests, or it advertised + * dir_port > 0. */ + unsigned int supports_tunnelled_dir_requests:1; + + /** Used during voting to indicate that we should not include an entry for + * this routerinfo. Used only during voting. */ + unsigned int omit_from_vote:1; + /** Tor can use this router for general positions in circuits; we got it * from a directory server as usual, or we're an authority and a server * uploaded it. */ @@ -2024,6 +2188,8 @@ typedef struct { /** Information needed to keep and cache a signed extra-info document. */ typedef struct extrainfo_t { signed_descriptor_t cache_info; + /** SHA256 digest of this document */ + uint8_t digest256[DIGEST256_LEN]; /** The router's nickname. */ char nickname[MAX_NICKNAME_LEN+1]; /** True iff we found the right key for this extra-info, verified the @@ -2047,7 +2213,7 @@ typedef struct routerstatus_t { /** Digest of the router's most recent descriptor or microdescriptor. * If it's a descriptor, we only use the first DIGEST_LEN bytes. */ char descriptor_digest[DIGEST256_LEN]; - uint32_t addr; /**< IPv4 address for this router. */ + uint32_t addr; /**< IPv4 address for this router, in host order. */ uint16_t or_port; /**< OR port for this router. */ uint16_t dir_port; /**< Directory port for this router. */ tor_addr_t ipv6_addr; /**< IPv6 address for this router. */ @@ -2065,29 +2231,20 @@ typedef struct routerstatus_t { unsigned int is_unnamed:1; /**< True iff "nickname" belongs to another * router. */ unsigned int is_valid:1; /**< True iff this router isn't invalid. */ - unsigned int is_v2_dir:1; /**< True iff this router can serve directory - * information with v2 of the directory - * protocol. (All directory caches cache v1 - * directories.) */ unsigned int is_possible_guard:1; /**< True iff this router would be a good * choice as an entry guard. */ unsigned int is_bad_exit:1; /**< True iff this node is a bad choice for * an exit node. */ - unsigned int is_bad_directory:1; /**< Do we think this directory is junky, - * underpowered, or otherwise useless? */ unsigned int is_hs_dir:1; /**< True iff this router is a v2-or-later hidden * service directory. */ + unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort + * or it claims to accept tunnelled dir requests. + */ /** True iff we know version info for this router. (i.e., a "v" entry was * included.) We'll replace all these with a big tor_version_t or a char[] * if the number of traits we care about ever becomes incredibly big. */ unsigned int version_known:1; - /** True iff this router is a version that, if it caches directory info, - * we can get microdescriptors from. */ - unsigned int version_supports_microdesc_cache:1; - /** True iff this router is a version that allows DATA cells to arrive on - * a stream before it has sent a CONNECTED cell. */ - unsigned int version_supports_optimistic_data:1; /** True iff this router has a version that allows it to accept EXTEND2 * cells */ unsigned int version_supports_extend2_cells:1; @@ -2099,18 +2256,18 @@ typedef struct routerstatus_t { uint32_t bandwidth_kb; /**< Bandwidth (capacity) of the router as reported in * the vote/consensus, in kilobytes/sec. */ + + /** The consensus has guardfraction information for this router. */ + unsigned int has_guardfraction:1; + /** The guardfraction value of this router. */ + uint32_t guardfraction_percentage; + char *exitsummary; /**< exit policy summary - * XXX weasel: this probably should not stay a string. */ /* ---- The fields below aren't derived from the networkstatus; they * hold local information only. */ - /** True if we, as a directory mirror, want to download the corresponding - * routerinfo from the authority who gave us this routerstatus. (That is, - * if we don't have the routerinfo, and if we haven't already tried to get it - * from this authority.) Applies in v2 networkstatus document only. - */ - unsigned int need_to_mirror:1; time_t last_dir_503_at; /**< When did this router last tell us that it * was too busy to serve directory info? */ download_status_t dl_status; @@ -2152,7 +2309,7 @@ typedef struct microdesc_t { */ time_t last_listed; /** Where is this microdescriptor currently stored? */ - ENUM_BF(saved_location_t) saved_location : 3; + saved_location_bitfield_t saved_location : 3; /** If true, do not attempt to cache this microdescriptor on disk. */ unsigned int no_save : 1; /** If true, this microdesc has an entry in the microdesc_map */ @@ -2182,7 +2339,9 @@ typedef struct microdesc_t { crypto_pk_t *onion_pkey; /** As routerinfo_t.onion_curve25519_pkey */ curve25519_public_key_t *onion_curve25519_pkey; - /** As routerinfo_t.ipv6_add */ + /** Ed25519 identity key, if included. */ + ed25519_public_key_t *ed25519_identity_pkey; + /** As routerinfo_t.ipv6_addr */ tor_addr_t ipv6_addr; /** As routerinfo_t.ipv6_orport */ uint16_t ipv6_orport; @@ -2200,7 +2359,7 @@ typedef struct microdesc_t { * Specifically, a node_t is a Tor router as we are using it: a router that * we are considering for circuits, connections, and so on. A node_t is a * thin wrapper around the routerstatus, routerinfo, and microdesc for a - * single wrapper, and provides a consistent interface for all of them. + * single router, and provides a consistent interface for all of them. * * Also, a node_t has mutable state. While a routerinfo, a routerstatus, * and a microdesc have[*] only the information read from a router @@ -2240,8 +2399,6 @@ typedef struct node_t { unsigned int is_exit:1; /**< Do we think this is an OK exit? */ unsigned int is_bad_exit:1; /**< Do we think this exit is censored, borked, * or otherwise nasty? */ - unsigned int is_bad_directory:1; /**< Do we think this directory is junky, - * underpowered, or otherwise useless? */ unsigned int is_hs_dir:1; /**< True iff this router is a hidden service * directory according to the authorities. */ @@ -2259,7 +2416,8 @@ typedef struct node_t { /* Local info: derived. */ - /** True if the IPv6 OR port is preferred over the IPv4 OR port. */ + /** True if the IPv6 OR port is preferred over the IPv4 OR port. + * XX/teor - can this become out of date if the torrc changes? */ unsigned int ipv6_preferred:1; /** According to the geoip db what country is this router in? */ @@ -2275,52 +2433,6 @@ typedef struct node_t { } node_t; -/** How many times will we try to download a router's descriptor before giving - * up? */ -#define MAX_ROUTERDESC_DOWNLOAD_FAILURES 8 - -/** How many times will we try to download a microdescriptor before giving - * up? */ -#define MAX_MICRODESC_DOWNLOAD_FAILURES 8 - -/** Contents of a v2 (non-consensus, non-vote) network status object. */ -typedef struct networkstatus_v2_t { - /** When did we receive the network-status document? */ - time_t received_on; - - /** What was the digest of the document? */ - char networkstatus_digest[DIGEST_LEN]; - - /* These fields come from the actual network-status document.*/ - time_t published_on; /**< Declared publication date. */ - - char *source_address; /**< Canonical directory server hostname. */ - uint32_t source_addr; /**< Canonical directory server IP. */ - uint16_t source_dirport; /**< Canonical directory server dirport. */ - - unsigned int binds_names:1; /**< True iff this directory server binds - * names. */ - unsigned int recommends_versions:1; /**< True iff this directory server - * recommends client and server software - * versions. */ - unsigned int lists_bad_exits:1; /**< True iff this directory server marks - * malfunctioning exits as bad. */ - /** True iff this directory server marks malfunctioning directories as - * bad. */ - unsigned int lists_bad_directories:1; - - char identity_digest[DIGEST_LEN]; /**< Digest of signing key. */ - char *contact; /**< How to contact directory admin? (may be NULL). */ - crypto_pk_t *signing_key; /**< Key used to sign this directory. */ - char *client_versions; /**< comma-separated list of recommended client - * versions. */ - char *server_versions; /**< comma-separated list of recommended server - * versions. */ - - smartlist_t *entries; /**< List of routerstatus_t*. This list is kept - * sorted by identity_digest. */ -} networkstatus_v2_t; - /** Linked list of microdesc hash lines for a single router in a directory * vote. */ @@ -2344,9 +2456,18 @@ typedef struct vote_routerstatus_t { char *version; /**< The version that the authority says this router is * running. */ unsigned int has_measured_bw:1; /**< The vote had a measured bw */ + /** True iff the vote included an entry for ed25519 ID, or included + * "id ed25519 none" to indicate that there was no ed25519 ID. */ + unsigned int has_ed25519_listing:1; + /** True if the Ed25519 listing here is the consensus-opinion for the + * Ed25519 listing; false if there was no consensus on Ed25519 key status, + * or if this VRS doesn't reflect it. */ + unsigned int ed25519_reflects_consensus:1; uint32_t measured_bw_kb; /**< Measured bandwidth (capacity) of the router */ /** The hash or hashes that the authority claims this microdesc has. */ vote_microdesc_hash_t *microdesc; + /** Ed25519 identity for this router, or zero if it has none. */ + uint8_t ed25519_id[ED25519_PUBKEY_LEN]; } vote_routerstatus_t; /** A signature of some document by an authority. */ @@ -2408,8 +2529,8 @@ typedef enum { /** A common structure to hold a v3 network status vote, or a v3 network * status consensus. */ typedef struct networkstatus_t { - ENUM_BF(networkstatus_type_t) type : 8; /**< Vote, consensus, or opinion? */ - ENUM_BF(consensus_flavor_t) flavor : 8; /**< If a consensus, what kind? */ + networkstatus_type_t type; /**< Vote, consensus, or opinion? */ + consensus_flavor_t flavor; /**< If a consensus, what kind? */ unsigned int has_measured_bws : 1;/**< True iff this networkstatus contains * measured= bandwidth values. */ @@ -2425,6 +2546,9 @@ typedef struct networkstatus_t { /** Vote only: what methods is this voter willing to use? */ smartlist_t *supported_methods; + /** List of 'package' lines describing hashes of downloadable packages */ + smartlist_t *package_lines; + /** How long does this vote/consensus claim that authorities take to * distribute their votes to one another? */ int vote_seconds; @@ -2456,7 +2580,7 @@ typedef struct networkstatus_t { struct authority_cert_t *cert; /**< Vote only: the voter's certificate. */ /** Digests of this document, as signed. */ - digests_t digests; + common_digests_t digests; /** List of router statuses, sorted by identity digest. For a vote, * the elements are vote_routerstatus_t; for a consensus, the elements @@ -2492,10 +2616,6 @@ typedef struct desc_store_t { * filename for a temporary file when rebuilding the store, and .new to this * filename for the journal. */ const char *fname_base; - /** Alternative (obsolete) value for fname_base: if the file named by - * fname_base isn't present, we read from here instead, but we never write - * here. */ - const char *fname_alt_base; /** Human-readable description of what this store contains. */ const char *description; @@ -2550,9 +2670,7 @@ typedef struct extend_info_t { uint16_t port; /**< OR port. */ tor_addr_t addr; /**< IP address. */ crypto_pk_t *onion_key; /**< Current onionskin key. */ -#ifdef CURVE25519_ENABLED curve25519_public_key_t curve25519_onion_key; -#endif } extend_info_t; /** Certificate for v3 directory protocol: binds long-term authority identity @@ -2572,9 +2690,6 @@ typedef struct authority_cert_t { uint32_t addr; /** This authority's directory port. */ uint16_t dir_port; - /** True iff this certificate was cross-certified by signing the identity - * key with the signing key. */ - uint8_t is_cross_certified; } authority_cert_t; /** Bitfield enum type listing types of information that directory authorities @@ -2588,15 +2703,8 @@ typedef struct authority_cert_t { */ typedef enum { NO_DIRINFO = 0, - /** Serves/signs v1 directory information: Big lists of routers, and short - * routerstatus documents. */ - V1_DIRINFO = 1 << 0, - /** Serves/signs v2 directory information: i.e. v2 networkstatus documents */ - V2_DIRINFO = 1 << 1, /** Serves/signs v3 directory information: votes, consensuses, certs */ V3_DIRINFO = 1 << 2, - /** Serves hidden service descriptors. */ - HIDSERV_DIRINFO = 1 << 3, /** Serves bridge descriptors. */ BRIDGE_DIRINFO = 1 << 4, /** Serves extrainfo documents. */ @@ -2719,11 +2827,30 @@ typedef struct { time_t expiry_time; } cpath_build_state_t; +/** "magic" value for an origin_circuit_t */ #define ORIGIN_CIRCUIT_MAGIC 0x35315243u +/** "magic" value for an or_circuit_t */ #define OR_CIRCUIT_MAGIC 0x98ABC04Fu +/** "magic" value for a circuit that would have been freed by circuit_free, + * but which we're keeping around until a cpuworker reply arrives. See + * circuit_free() for more documentation. */ +#define DEAD_CIRCUIT_MAGIC 0xdeadc14c struct create_cell_t; +/** Entry in the cell stats list of a circuit; used only if CELL_STATS + * events are enabled. */ +typedef struct testing_cell_stats_entry_t { + uint8_t command; /**< cell command number. */ + /** Waiting time in centiseconds if this event is for a removed cell, + * or 0 if this event is for adding a cell to the queue. 22 bits can + * store more than 11 hours, enough to assume that a circuit with this + * delay would long have been closed. */ + unsigned int waiting_time:22; + unsigned int removed:1; /**< 0 for added to, 1 for removed from queue. */ + unsigned int exitward:1; /**< 0 for app-ward, 1 for exit-ward. */ +} testing_cell_stats_entry_t; + /** * A circuit is a path over the onion routing * network. Applications can connect to one end of the circuit, and can @@ -2785,6 +2912,13 @@ typedef struct circuit_t { * allowing n_streams to add any more cells. (OR circuit only.) */ unsigned int streams_blocked_on_p_chan : 1; + /** True iff we have queued a delete backwards on this circuit, but not put + * it on the output buffer. */ + unsigned int p_delete_pending : 1; + /** True iff we have queued a delete forwards on this circuit, but not put + * it on the output buffer. */ + unsigned int n_delete_pending : 1; + /** True iff this circuit has received a DESTROY cell in either direction */ unsigned int received_destroy : 1; @@ -2801,6 +2935,9 @@ typedef struct circuit_t { * more. */ int deliver_window; + /** Temporary field used during circuits_handle_oom. */ + uint32_t age_tmp; + /** For storage while n_chan is pending (state CIRCUIT_STATE_CHAN_WAIT). */ struct create_cell_t *n_chan_create_cell; @@ -2830,6 +2967,11 @@ typedef struct circuit_t { * circuits entered certain states. This usage probably won't * interfere with this field's primary purpose, but we should * document it more thoroughly to make sure of that. + * + * XXX027 The SocksPort option KeepaliveIsolateSOCKSAuth will artificially + * adjust this value forward each time a suitable stream is attached to an + * already constructed circuit, potentially keeping the circuit alive + * indefinitely. */ time_t timestamp_dirty; @@ -2838,11 +2980,20 @@ typedef struct circuit_t { * where this circuit was marked.) */ const char *marked_for_close_file; /**< For debugging: in which file was this * circuit marked for close? */ + /** For what reason (See END_CIRC_REASON...) is this circuit being closed? + * This field is set in circuit_mark_for_close and used later in + * circuit_about_to_free. */ + uint16_t marked_for_close_reason; + /** As marked_for_close_reason, but reflects the underlying reason for + * closing this circuit. + */ + uint16_t marked_for_close_orig_reason; /** Unique ID for measuring tunneled network status requests. */ uint64_t dirreq_id; - struct circuit_t *next; /**< Next circuit in linked list of all circuits. */ + /** Index in smartlist of all circuits (global_circuitlist). */ + int global_circuitlist_idx; /** Next circuit in the doubly-linked ring of circuits waiting to add * cells to n_conn. NULL if we have no cells pending, or if we're not @@ -2852,6 +3003,11 @@ typedef struct circuit_t { * cells to n_conn. NULL if we have no cells pending, or if we're not * linked to an OR connection. */ struct circuit_t *prev_active_on_n_chan; + + /** Various statistics about cells being added to or removed from this + * circuit's queues; used only if CELL_STATS events are enabled and + * cleared after being sent to control port. */ + smartlist_t *testing_cell_stats; } circuit_t; /** Largest number of relay_early cells that we can send on a given @@ -2913,6 +3069,7 @@ typedef enum { */ PATH_STATE_ALREADY_COUNTED = 6, } path_state_t; +#define path_state_bitfield_t ENUM_BF(path_state_t) /** An origin_circuit_t holds data necessary to build and use a circuit. */ @@ -2922,6 +3079,17 @@ typedef struct origin_circuit_t { /** Linked list of AP streams (or EXIT streams if hidden service) * associated with this circuit. */ edge_connection_t *p_streams; + + /** Bytes read from any attached stream since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_read_circ_bw; + + /** Bytes written to any attached stream since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_written_circ_bw; + /** Build state for this circuit. It includes the intended path * length, the chosen exit router, rendezvous information, etc. */ @@ -2952,7 +3120,7 @@ typedef struct origin_circuit_t { * circuit building and usage accounting. See path_state_t * for more details. */ - ENUM_BF(path_state_t) path_state : 3; + path_state_bitfield_t path_state : 3; /* If this flag is set, we should not consider attaching any more * connections to this circuit. */ @@ -3099,6 +3267,10 @@ typedef struct or_circuit_t { /** Pointer to an entry on the onion queue, if this circuit is waiting for a * chance to give an onionskin to a cpuworker. Used only in onion.c */ struct onion_queue_t *onionqueue_entry; + /** Pointer to a workqueue entry, if this circuit has given an onionskin to + * a cpuworker and is waiting for a response. Used to decide whether it is + * safe to free a circuit or if it is still in use by a cpuworker. */ + struct workqueue_entry_s *workqueue_entry; /** The circuit_id used in the previous (backward) hop of this circuit. */ circid_t p_circ_id; @@ -3136,20 +3308,8 @@ typedef struct or_circuit_t { * is not marked for close. */ struct or_circuit_t *rend_splice; -#if REND_COOKIE_LEN >= DIGEST_LEN -#define REND_TOKEN_LEN REND_COOKIE_LEN -#else -#define REND_TOKEN_LEN DIGEST_LEN -#endif + struct or_circuit_rendinfo_s *rendinfo; - /** A hash of location-hidden service's PK if purpose is INTRO_POINT, or a - * rendezvous cookie if purpose is REND_POINT_WAITING. Filled with zeroes - * otherwise. - * ???? move to a subtype or adjunct structure? Wastes 20 bytes. -NM - */ - char rend_token[REND_TOKEN_LEN]; - - /* ???? move to a subtype or adjunct structure? Wastes 20 bytes -NM */ /** Stores KH for the handshake. */ char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */ @@ -3163,6 +3323,10 @@ typedef struct or_circuit_t { /** True iff this circuit was made with a CREATE_FAST cell. */ unsigned int is_first_hop : 1; + /** If set, this circuit carries HS traffic. Consider it in any HS + * statistics. */ + unsigned int circuit_carries_hs_traffic_stats : 1; + /** Number of cells that were removed from circuit queue; reset every * time when writing buffer stats to disk. */ uint32_t processed_cells; @@ -3171,24 +3335,70 @@ typedef struct or_circuit_t { * exit-ward queues of this circuit; reset every time when writing * buffer stats to disk. */ uint64_t total_cell_waiting_time; + + /** Maximum cell queue size for a middle relay; this is stored per circuit + * so append_cell_to_circuit_queue() can adjust it if it changes. If set + * to zero, it is initialized to the default value. + */ + uint32_t max_middle_cells; } or_circuit_t; +typedef struct or_circuit_rendinfo_s { + +#if REND_COOKIE_LEN != DIGEST_LEN +#error "The REND_TOKEN_LEN macro assumes REND_COOKIE_LEN == DIGEST_LEN" +#endif +#define REND_TOKEN_LEN DIGEST_LEN + + /** A hash of location-hidden service's PK if purpose is INTRO_POINT, or a + * rendezvous cookie if purpose is REND_POINT_WAITING. Filled with zeroes + * otherwise. + */ + char rend_token[REND_TOKEN_LEN]; + + /** True if this is a rendezvous point circuit; false if this is an + * introduction point. */ + unsigned is_rend_circ; + +} or_circuit_rendinfo_t; + /** Convert a circuit subtype to a circuit_t. */ #define TO_CIRCUIT(x) (&((x)->base_)) /** Convert a circuit_t* to a pointer to the enclosing or_circuit_t. Assert * if the cast is impossible. */ static or_circuit_t *TO_OR_CIRCUIT(circuit_t *); +static const or_circuit_t *CONST_TO_OR_CIRCUIT(const circuit_t *); /** Convert a circuit_t* to a pointer to the enclosing origin_circuit_t. * Assert if the cast is impossible. */ static origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *); +static const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT(const circuit_t *); -static INLINE or_circuit_t *TO_OR_CIRCUIT(circuit_t *x) +/** Return 1 iff <b>node</b> has Exit flag and no BadExit flag. + * Otherwise, return 0. + */ +static inline int node_is_good_exit(const node_t *node) +{ + return node->is_exit && ! node->is_bad_exit; +} + +static inline or_circuit_t *TO_OR_CIRCUIT(circuit_t *x) { tor_assert(x->magic == OR_CIRCUIT_MAGIC); return DOWNCAST(or_circuit_t, x); } -static INLINE origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *x) +static inline const or_circuit_t *CONST_TO_OR_CIRCUIT(const circuit_t *x) +{ + tor_assert(x->magic == OR_CIRCUIT_MAGIC); + return DOWNCAST(or_circuit_t, x); +} +static inline origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *x) +{ + tor_assert(x->magic == ORIGIN_CIRCUIT_MAGIC); + return DOWNCAST(origin_circuit_t, x); +} +static inline const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT( + const circuit_t *x) { tor_assert(x->magic == ORIGIN_CIRCUIT_MAGIC); return DOWNCAST(origin_circuit_t, x); @@ -3251,44 +3461,13 @@ typedef struct port_cfg_t { uint8_t type; /**< One of CONN_TYPE_*_LISTENER */ unsigned is_unix_addr : 1; /**< True iff this is an AF_UNIX address. */ - /* Client port types (socks, dns, trans, natd) only: */ - uint8_t isolation_flags; /**< Zero or more isolation flags */ - int session_group; /**< A session group, or -1 if this port is not in a - * session group. */ - /* Socks only: */ - /** When both no-auth and user/pass are advertised by a SOCKS client, select - * no-auth. */ - unsigned int socks_prefer_no_auth : 1; + unsigned is_group_writable : 1; + unsigned is_world_writable : 1; + unsigned relax_dirmode_check : 1; - /* Server port types (or, dir) only: */ - unsigned int no_advertise : 1; - unsigned int no_listen : 1; - unsigned int all_addrs : 1; - unsigned int bind_ipv4_only : 1; - unsigned int bind_ipv6_only : 1; + entry_port_cfg_t entry_cfg; - /* Client port types only: */ - unsigned int ipv4_traffic : 1; - unsigned int ipv6_traffic : 1; - unsigned int prefer_ipv6 : 1; - - /** For a socks listener: should we cache IPv4/IPv6 DNS information that - * exit nodes tell us? - * - * @{ */ - unsigned int cache_ipv4_answers : 1; - unsigned int cache_ipv6_answers : 1; - /** @} */ - /** For a socks listeners: if we find an answer in our client-side DNS cache, - * should we use it? - * - * @{ */ - unsigned int use_cached_ipv4_answers : 1; - unsigned int use_cached_ipv6_answers : 1; - /** @} */ - /** For socks listeners: When we can automap an address to IPv4 or IPv6, - * do we prefer IPv6? */ - unsigned int prefer_ipv6_virtaddr : 1; + server_port_cfg_t server_cfg; /* Unix sockets only: */ /** Path for an AF_UNIX address */ @@ -3329,9 +3508,10 @@ typedef struct { /** What should the tor process actually do? */ enum { CMD_RUN_TOR=0, CMD_LIST_FINGERPRINT, CMD_HASH_PASSWORD, - CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS + CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS, CMD_DUMP_CONFIG, + CMD_KEYGEN } command; - const char *command_arg; /**< Argument for command-line option. */ + char *command_arg; /**< Argument for command-line option. */ config_line_t *Logs; /**< New-style list of configuration lines * for logs */ @@ -3339,15 +3519,17 @@ typedef struct { int LogMessageDomains; /**< Boolean: Should we log the domain(s) in which * each log message occurs? */ + int TruncateLogFile; /**< Boolean: Should we truncate the log file + before we start writing? */ + char *SyslogIdentityTag; /**< Identity tag to add for syslog logging. */ char *DebugLogFile; /**< Where to send verbose log messages. */ char *DataDirectory; /**< OR only: where to store long-term data. */ + int DataDirectoryGroupReadable; /**< Boolean: Is the DataDirectory g+r? */ char *Nickname; /**< OR only: nickname of this onion router. */ char *Address; /**< OR only: configured address for this onion router. */ char *PidFile; /**< Where to store PID of Tor process. */ - int DynamicDHGroups; /**< Dynamic generation of prime moduli for use in DH.*/ - routerset_t *ExitNodes; /**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as exits. */ @@ -3405,6 +3587,7 @@ typedef struct { config_line_t *RecommendedVersions; config_line_t *RecommendedClientVersions; config_line_t *RecommendedServerVersions; + config_line_t *RecommendedPackages; /** Whether dirservers allow router descriptors with private IPs. */ int DirAllowPrivateAddresses; /** Whether routers accept EXTEND cells to routers with private IPs. */ @@ -3412,10 +3595,21 @@ typedef struct { char *User; /**< Name of user to run Tor as. */ char *Group; /**< Name of group to run Tor as. */ config_line_t *ORPort_lines; /**< Ports to listen on for OR connections. */ + /** Ports to listen on for extended OR connections. */ + config_line_t *ExtORPort_lines; /** Ports to listen on for SOCKS connections. */ config_line_t *SocksPort_lines; /** Ports to listen on for transparent pf/netfilter connections. */ config_line_t *TransPort_lines; + const char *TransProxyType; /**< What kind of transparent proxy + * implementation are we using? */ + /** Parsed value of TransProxyType. */ + enum { + TPT_DEFAULT, + TPT_PF_DIVERT, + TPT_IPFW, + TPT_TPROXY, + } TransProxyType_parsed; config_line_t *NATDPort_lines; /**< Ports to listen on for transparent natd * connections. */ config_line_t *ControlPort_lines; /**< Ports to listen on for control @@ -3424,13 +3618,18 @@ typedef struct { * for control connections. */ int ControlSocketsGroupWritable; /**< Boolean: Are control sockets g+rw? */ + int SocksSocketsGroupWritable; /**< Boolean: Are SOCKS sockets g+rw? */ /** Ports to listen on for directory connections. */ config_line_t *DirPort_lines; config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */ - uint64_t MaxMemInCellQueues; /**< If we have more memory than this allocated - * for circuit cell queues, run the OOM handler - */ + /* MaxMemInQueues value as input by the user. We clean this up to be + * MaxMemInQueues. */ + uint64_t MaxMemInQueues_raw; + uint64_t MaxMemInQueues;/**< If we have more memory than this allocated + * for queues and buffers, run the OOM handler */ + /** Above this value, consider ourselves low on RAM. */ + uint64_t MaxMemInQueues_low_threshold; /** @name port booleans * @@ -3447,20 +3646,13 @@ typedef struct { unsigned int ControlPort_set : 1; unsigned int DirPort_set : 1; unsigned int DNSPort_set : 1; + unsigned int ExtORPort_set : 1; /**@}*/ int AssumeReachable; /**< Whether to publish our descriptor regardless. */ int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */ - int V1AuthoritativeDir; /**< Boolean: is this an authoritative directory - * for version 1 directories? */ - int V2AuthoritativeDir; /**< Boolean: is this an authoritative directory - * for version 2 directories? */ int V3AuthoritativeDir; /**< Boolean: is this an authoritative directory * for version 3 directories? */ - int HSAuthoritativeDir; /**< Boolean: does this an authoritative directory - * handle hidden service requests? */ - int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory - * that's willing to bind names? */ int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative * directory that's willing to recommend * versions? */ @@ -3486,6 +3678,9 @@ typedef struct { /** List of TCP/IP addresses that transports should listen at. */ config_line_t *ServerTransportListenAddr; + /** List of options that must be passed to pluggable transports. */ + config_line_t *ServerTransportOptions; + int BridgeRelay; /**< Boolean: are we acting as a bridge relay? We make * this explicit so we can change how we behave in the * future. */ @@ -3506,12 +3701,7 @@ typedef struct { int PublishHidServDescriptors; int FetchServerDescriptors; /**< Do we fetch server descriptors as normal? */ int FetchHidServDescriptors; /**< and hidden service descriptors? */ - int FetchV2Networkstatus; /**< Do we fetch v2 networkstatus documents when - * we don't need to? */ - int HidServDirectoryV2; /**< Do we participate in the HS DHT? */ - int VoteOnHidServDirectoriesV2; /**< As a directory authority, vote on - * assignment of the HSDir flag? */ int MinUptimeHidServDirectoryV2; /**< As directory authority, accept hidden * service directories after what time? */ @@ -3524,6 +3714,9 @@ typedef struct { * circuits.) */ int Tor2webMode; + /** A routerset that should be used when picking RPs for HS circuits. */ + routerset_t *Tor2webRendezvousPoints; + /** Close hidden service client circuits immediately when they reach * the normal circuit-build timeout, even if they have already sent * an INTRODUCE1 cell on its way to the service. */ @@ -3573,8 +3766,9 @@ typedef struct { * hostname ending with one of the suffixes in * <b>AutomapHostsSuffixes</b>, map it to a * virtual address. */ - smartlist_t *AutomapHostsSuffixes; /**< List of suffixes for - * <b>AutomapHostsOnResolve</b>. */ + /** List of suffixes for <b>AutomapHostsOnResolve</b>. The special value + * "." means "match everything." */ + smartlist_t *AutomapHostsSuffixes; int RendPostPeriod; /**< How often do we post each rendezvous service * descriptor? Remember to publish them independently. */ int KeepalivePeriod; /**< How often do we send padding cells to keep @@ -3593,11 +3787,15 @@ typedef struct { * and try a new circuit if the stream has been * waiting for this many seconds. If zero, use * our default internal timeout schedule. */ - int MaxOnionQueueDelay; /**<DOCDOC*/ + int MaxOnionQueueDelay; /*< DOCDOC */ int NewCircuitPeriod; /**< How long do we use a circuit before building * a new one? */ int MaxCircuitDirtiness; /**< Never use circs that were first used more than this interval ago. */ + int PredictedPortsRelevanceTime; /** How long after we've requested a + * connection for a given port, do we want + * to continue to pick exits that support + * that port? */ uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing * to use in a second? */ uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing @@ -3649,6 +3847,8 @@ typedef struct { /** List of fallback directory servers */ config_line_t *FallbackDir; + /** Whether to use the default hard-coded FallbackDirs */ + int UseDefaultFallbackDirs; /** Weight to apply to all directory authority rates if considering them * along with fallbackdirs */ @@ -3661,15 +3861,10 @@ typedef struct { /** If set, use these bridge authorities and not the default one. */ config_line_t *AlternateBridgeAuthority; - /** If set, use these HS authorities and not the default ones. */ - config_line_t *AlternateHSAuthority; - char *MyFamily; /**< Declared family for this OR. */ config_line_t *NodeFamilies; /**< List of config lines for * node families */ smartlist_t *NodeFamilySets; /**< List of parsed NodeFamilies values. */ - config_line_t *AuthDirBadDir; /**< Address policy for descriptors to - * mark as bad dir mirrors. */ config_line_t *AuthDirBadExit; /**< Address policy for descriptors to * mark as bad exits. */ config_line_t *AuthDirReject; /**< Address policy for descriptors to @@ -3678,29 +3873,25 @@ typedef struct { * never mark as valid. */ /** @name AuthDir...CC * - * Lists of country codes to mark as BadDir, BadExit, or Invalid, or to + * Lists of country codes to mark as BadExit, or Invalid, or to * reject entirely. * * @{ */ - smartlist_t *AuthDirBadDirCCs; smartlist_t *AuthDirBadExitCCs; smartlist_t *AuthDirInvalidCCs; smartlist_t *AuthDirRejectCCs; /**@}*/ - int AuthDirListBadDirs; /**< True iff we should list bad dirs, - * and vote for all other dir mirrors as good. */ int AuthDirListBadExits; /**< True iff we should list bad exits, * and vote for all other exits as good. */ - int AuthDirRejectUnlisted; /**< Boolean: do we reject all routers that - * aren't named in our fingerprint file? */ int AuthDirMaxServersPerAddr; /**< Do not permit more than this * number of servers per IP address. */ int AuthDirMaxServersPerAuthAddr; /**< Do not permit more than this * number of servers per IP address shared * with an authority. */ int AuthDirHasIPv6Connectivity; /**< Boolean: are we on IPv6? */ + int AuthDirPinKeys; /**< Boolean: Do we enforce key-pinning? */ /** If non-zero, always vote the Fast flag for any relay advertising * this amount of capacity or more. */ @@ -3715,6 +3906,13 @@ typedef struct { uint64_t AccountingMax; /**< How many bytes do we allow per accounting * interval before hibernation? 0 for "never * hibernate." */ + /** How do we determine when our AccountingMax has been reached? + * "max" for when in or out reaches AccountingMax + * "sum" for when in plus out reaches AccountingMax + * "in" for when in reaches AccountingMax + * "out" for when out reaches AccountingMax */ + char *AccountingRule_option; + enum { ACCT_MAX, ACCT_SUM, ACCT_IN, ACCT_OUT } AccountingRule; /** Base64-encoded hash of accepted passwords for the control system. */ config_line_t *HashedControlPassword; @@ -3723,8 +3921,13 @@ typedef struct { int CookieAuthentication; /**< Boolean: do we enable cookie-based auth for * the control system? */ - char *CookieAuthFile; /**< Location of a cookie authentication file. */ + char *CookieAuthFile; /**< Filesystem location of a ControlPort + * authentication cookie. */ + char *ExtORPortCookieAuthFile; /**< Filesystem location of Extended + * ORPort authentication cookie. */ int CookieAuthFileGroupReadable; /**< Boolean: Is the CookieAuthFile g+r? */ + int ExtORPortCookieAuthFileGroupReadable; /**< Boolean: Is the + * ExtORPortCookieAuthFile g+r? */ int LeaveStreamsUnattached; /**< Boolean: Does Tor attach new streams to * circuits itself (0), or does it expect a controller * to cope? (1) */ @@ -3745,6 +3948,7 @@ typedef struct { SAFELOG_SCRUB_ALL, SAFELOG_SCRUB_RELAY, SAFELOG_SCRUB_NONE } SafeLogging_; + int Sandbox; /**< Boolean: should sandboxing be enabled? */ int SafeSocks; /**< Boolean: should we outright refuse application * connections that use socks4 or socks5-with-local-dns? */ #define LOG_PROTOCOL_WARN (get_options()->ProtocolWarnings ? \ @@ -3764,6 +3968,12 @@ typedef struct { int NumEntryGuards; /**< How many entry guards do we try to establish? */ int UseEntryGuardsAsDirGuards; /** Boolean: Do we try to get directory info * from a smallish number of fixed nodes? */ + + /** If 1, we use any guardfraction information we see in the + * consensus. If 0, we don't. If -1, let the consensus parameter + * decide. */ + int UseGuardFraction; + int NumDirectoryGuards; /**< How many dir guards do we try to establish? * If 0, use value from NumEntryGuards. */ int RephistTrackTime; /**< How many seconds do we keep rephist info? */ @@ -3776,6 +3986,10 @@ typedef struct { /** Should we fetch our dir info at the start of the consensus period? */ int FetchDirInfoExtraEarly; + int DirCache; /**< Cache all directory documents and accept requests via + * tunnelled dir conns from clients. If 1, enabled (default); + * If 0, disabled. */ + char *VirtualAddrNetworkIPv4; /**< Address and mask to hand out for virtual * MAPADDRESS requests for IPv4 addresses */ char *VirtualAddrNetworkIPv6; /**< Address and mask to hand out for virtual @@ -3807,10 +4021,6 @@ typedef struct { * testing our DNS server. */ int EnforceDistinctSubnets; /**< If true, don't allow multiple routers in the * same network zone in the same circuit. */ - int TunnelDirConns; /**< If true, use BEGIN_DIR rather than BEGIN when - * possible. */ - int PreferTunneledDirConns; /**< If true, avoid dirservers that don't - * support BEGIN_DIR, when possible. */ int PortForwarding; /**< If true, use NAT-PMP or UPnP to automatically * forward the DirPort and ORPort on the NAT device */ char *PortForwardingHelper; /** < Filename or full path of the port @@ -3845,8 +4055,11 @@ typedef struct { * instead of a hostname. */ int WarnUnsafeSocks; - /** If true, the user wants us to collect statistics on clients + /** If true, we're configured to collect statistics on clients * requesting network statuses from us as directory. */ + int DirReqStatistics_option; + /** Internal variable to remember whether we're actually acting on + * DirReqStatistics_option -- yes if it's set and we're a server, else no. */ int DirReqStatistics; /** If true, the user wants us to collect statistics on port usage. */ @@ -3861,6 +4074,10 @@ typedef struct { /** If true, the user wants us to collect statistics as entry node. */ int EntryStatistics; + /** If true, the user wants us to collect statistics as hidden service + * directory, introduction point, or rendezvous point. */ + int HiddenServiceStatistics; + /** If true, include statistics file contents in extra-info documents. */ int ExtraInfoStatistics; @@ -3873,12 +4090,24 @@ typedef struct { * over randomly chosen exits. */ int ClientRejectInternalAddresses; - /** If true, clients may connect over IPv6. XXX we don't really - enforce this -- clients _may_ set up outgoing IPv6 connections - even when this option is not set. */ + /** If true, clients may connect over IPv4. If false, they will avoid + * connecting over IPv4. We enforce this for OR and Dir connections. */ + int ClientUseIPv4; + /** If true, clients may connect over IPv6. If false, they will avoid + * connecting over IPv4. We enforce this for OR and Dir connections. + * Use fascist_firewall_use_ipv6() instead of accessing this value + * directly. */ int ClientUseIPv6; - /** If true, prefer an IPv6 OR port over an IPv4 one. */ + /** If true, prefer an IPv6 OR port over an IPv4 one for entry node + * connections. If auto, bridge clients prefer IPv6, and other clients + * prefer IPv4. Use node_ipv6_or_preferred() instead of accessing this value + * directly. */ int ClientPreferIPv6ORPort; + /** If true, prefer an IPv6 directory port over an IPv4 one for direct + * directory connections. If auto, bridge clients prefer IPv6, and other + * clients prefer IPv4. Use fascist_firewall_prefer_ipv6_dirport() instead of + * accessing this value directly. */ + int ClientPreferIPv6DirPort; /** The length of time that we think a consensus should be fresh. */ int V3AuthVotingInterval; @@ -3896,12 +4125,15 @@ typedef struct { /** Location of bandwidth measurement file */ char *V3BandwidthsFile; + /** Location of guardfraction file */ + char *GuardfractionFile; + /** Authority only: key=value pairs that we add to our networkstatus * consensus vote on the 'params' line. */ char *ConsensusParams; /** Authority only: minimum number of measured bandwidths we must see - * before we only beliee measured bandwidths to assign flags. */ + * before we only believe measured bandwidths to assign flags. */ int MinMeasuredBWsForAuthToIgnoreAdvertised; /** The length of time that we think an initial consensus should be fresh. @@ -3916,6 +4148,10 @@ typedef struct { * signatures. Only altered on testing networks.*/ int TestingV3AuthInitialDistDelay; + /** Offset in seconds added to the starting time for consensus + voting. Only altered on testing networks. */ + int TestingV3AuthVotingStartOffset; + /** If an authority has been around for less than this amount of time, it * does not believe its reachability information is accurate. Only * altered on testing networks. */ @@ -3926,6 +4162,96 @@ typedef struct { * networks. */ int TestingEstimatedDescriptorPropagationTime; + /** Schedule for when servers should download things in general. Only + * altered on testing networks. */ + smartlist_t *TestingServerDownloadSchedule; + + /** Schedule for when clients should download things in general. Only + * altered on testing networks. */ + smartlist_t *TestingClientDownloadSchedule; + + /** Schedule for when servers should download consensuses. Only altered + * on testing networks. */ + smartlist_t *TestingServerConsensusDownloadSchedule; + + /** Schedule for when clients should download consensuses. Only altered + * on testing networks. */ + smartlist_t *TestingClientConsensusDownloadSchedule; + + /** Schedule for when clients should download consensuses from authorities + * if they are bootstrapping (that is, they don't have a usable, reasonably + * live consensus). Only used by clients fetching from a list of fallback + * directory mirrors. + * + * This schedule is incremented by (potentially concurrent) connection + * attempts, unlike other schedules, which are incremented by connection + * failures. Only altered on testing networks. */ + smartlist_t *ClientBootstrapConsensusAuthorityDownloadSchedule; + + /** Schedule for when clients should download consensuses from fallback + * directory mirrors if they are bootstrapping (that is, they don't have a + * usable, reasonably live consensus). Only used by clients fetching from a + * list of fallback directory mirrors. + * + * This schedule is incremented by (potentially concurrent) connection + * attempts, unlike other schedules, which are incremented by connection + * failures. Only altered on testing networks. */ + smartlist_t *ClientBootstrapConsensusFallbackDownloadSchedule; + + /** Schedule for when clients should download consensuses from authorities + * if they are bootstrapping (that is, they don't have a usable, reasonably + * live consensus). Only used by clients which don't have or won't fetch + * from a list of fallback directory mirrors. + * + * This schedule is incremented by (potentially concurrent) connection + * attempts, unlike other schedules, which are incremented by connection + * failures. Only altered on testing networks. */ + smartlist_t *ClientBootstrapConsensusAuthorityOnlyDownloadSchedule; + + /** Schedule for when clients should download bridge descriptors. Only + * altered on testing networks. */ + smartlist_t *TestingBridgeDownloadSchedule; + + /** When directory clients have only a few descriptors to request, they + * batch them until they have more, or until this amount of time has + * passed. Only altered on testing networks. */ + int TestingClientMaxIntervalWithoutRequest; + + /** How long do we let a directory connection stall before expiring + * it? Only altered on testing networks. */ + int TestingDirConnectionMaxStall; + + /** How many times will we try to fetch a consensus before we give + * up? Only altered on testing networks. */ + int TestingConsensusMaxDownloadTries; + + /** How many times will a client try to fetch a consensus while + * bootstrapping using a list of fallback directories, before it gives up? + * Only altered on testing networks. */ + int ClientBootstrapConsensusMaxDownloadTries; + + /** How many times will a client try to fetch a consensus while + * bootstrapping using only a list of authorities, before it gives up? + * Only altered on testing networks. */ + int ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries; + + /** How many simultaneous in-progress connections will we make when trying + * to fetch a consensus before we wait for one to complete, timeout, or + * error out? Only altered on testing networks. */ + int ClientBootstrapConsensusMaxInProgressTries; + + /** How many times will we try to download a router's descriptor before + * giving up? Only altered on testing networks. */ + int TestingDescriptorMaxDownloadTries; + + /** How many times will we try to download a microdescriptor before + * giving up? Only altered on testing networks. */ + int TestingMicrodescMaxDownloadTries; + + /** How many times will we try to fetch a certificate before giving + * up? Only altered on testing networks. */ + int TestingCertMaxDownloadTries; + /** If true, we take part in a testing network. Change the defaults of a * couple of other configuration options and allow to change the values * of certain configuration options. */ @@ -3937,6 +4263,31 @@ typedef struct { /** Minimum value for the Fast flag threshold on testing networks. */ uint64_t TestingMinFastFlagThreshold; + /** Relays in a testing network which should be voted Exit + * regardless of exit policy. */ + routerset_t *TestingDirAuthVoteExit; + int TestingDirAuthVoteExitIsStrict; + + /** Relays in a testing network which should be voted Guard + * regardless of uptime and bandwidth. */ + routerset_t *TestingDirAuthVoteGuard; + int TestingDirAuthVoteGuardIsStrict; + + /** Relays in a testing network which should be voted HSDir + * regardless of uptime and DirPort. + * Respects VoteOnHidServDirectoriesV2. */ + routerset_t *TestingDirAuthVoteHSDir; + int TestingDirAuthVoteHSDirIsStrict; + + /** Enable CONN_BW events. Only altered on testing networks. */ + int TestingEnableConnBwEvent; + + /** Enable CELL_STATS events. Only altered on testing networks. */ + int TestingEnableCellStatsEvent; + + /** Enable TB_EMPTY events. Only altered on testing networks. */ + int TestingEnableTbEmptyEvent; + /** If true, and we have GeoIP data, and we're a bridge, keep a per-country * count of how many client addresses have contacted us so that we can help * the bridge authority guess which countries have blocked access to us. */ @@ -4075,16 +4426,6 @@ typedef struct { /** Fraction: */ double PathsNeededToBuildCircuits; - /** Do we serve v2 directory info at all? This is a temporary option, since - * we'd like to disable v2 directory serving entirely, but we need a way to - * make it temporarily disableable, in order to do fast testing and be - * able to turn it back on if it turns out to be non-workable. - * - * XXXX025 Make this always-on, or always-off. Right now, it's only - * enableable for authorities. - */ - int DisableV2DirectoryInfo_; - /** What expiry time shall we place on our SSL certs? "0" means we * should guess a suitable value. */ int SSLKeyLifetime; @@ -4092,8 +4433,56 @@ typedef struct { /** How long (seconds) do we keep a guard before picking a new one? */ int GuardLifetime; - /** Should we send the timestamps that pre-023 hidden services want? */ - int Support022HiddenServices; + /** Low-water mark for global scheduler - start sending when estimated + * queued size falls below this threshold. + */ + uint64_t SchedulerLowWaterMark__; + /** High-water mark for global scheduler - stop sending when estimated + * queued size exceeds this threshold. + */ + uint64_t SchedulerHighWaterMark__; + /** Flush size for global scheduler - flush this many cells at a time + * when sending. + */ + int SchedulerMaxFlushCells__; + + /** Is this an exit node? This is a tristate, where "1" means "yes, and use + * the default exit policy if none is given" and "0" means "no; exit policy + * is 'reject *'" and "auto" (-1) means "same as 1, but warn the user." + * + * XXXX Eventually, the default will be 0. */ + int ExitRelay; + + /** For how long (seconds) do we declare our singning keys to be valid? */ + int SigningKeyLifetime; + /** For how long (seconds) do we declare our link keys to be valid? */ + int TestingLinkCertLifetime; + /** For how long (seconds) do we declare our auth keys to be valid? */ + int TestingAuthKeyLifetime; + + /** How long before signing keys expire will we try to make a new one? */ + int TestingSigningKeySlop; + /** How long before link keys expire will we try to make a new one? */ + int TestingLinkKeySlop; + /** How long before auth keys expire will we try to make a new one? */ + int TestingAuthKeySlop; + + /** Force use of offline master key features: never generate a master + * ed25519 identity key except from tor --keygen */ + int OfflineMasterKey; + + enum { + FORCE_PASSPHRASE_AUTO=0, + FORCE_PASSPHRASE_ON, + FORCE_PASSPHRASE_OFF + } keygen_force_passphrase; + int use_keygen_passphrase_fd; + int keygen_passphrase_fd; + int change_key_passphrase; + char *master_key_fname; + + /** Autobool: Do we try to retain capabilities if we can? */ + int KeepBindCapabilities; } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -4166,7 +4555,7 @@ typedef struct { /** Change the next_write time of <b>state</b> to <b>when</b>, unless the * state is already scheduled to be written to disk earlier than <b>when</b>. */ -static INLINE void or_state_mark_dirty(or_state_t *state, time_t when) +static inline void or_state_mark_dirty(or_state_t *state, time_t when) { if (state->next_write > when) state->next_write = when; @@ -4184,7 +4573,8 @@ static INLINE void or_state_mark_dirty(or_state_t *state, time_t when) /** Please turn this IP address into an FQDN, privately. */ #define SOCKS_COMMAND_RESOLVE_PTR 0xF1 -#define SOCKS_COMMAND_IS_CONNECT(c) ((c)==SOCKS_COMMAND_CONNECT) +/* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */ +#define SOCKS_COMMAND_IS_CONNECT(c) (((c)==SOCKS_COMMAND_CONNECT) || 0) #define SOCKS_COMMAND_IS_RESOLVE(c) ((c)==SOCKS_COMMAND_RESOLVE || \ (c)==SOCKS_COMMAND_RESOLVE_PTR) @@ -4349,30 +4739,7 @@ typedef struct { int after_firsthop_idx; } network_liveness_t; -/** Structure for circuit build times history */ -typedef struct { - /** The circular array of recorded build times in milliseconds */ - build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE]; - /** Current index in the circuit_build_times circular array */ - int build_times_idx; - /** Total number of build times accumulated. Max CBT_NCIRCUITS_TO_OBSERVE */ - int total_build_times; - /** Information about the state of our local network connection */ - network_liveness_t liveness; - /** Last time we built a circuit. Used to decide to build new test circs */ - time_t last_circ_at; - /** "Minimum" value of our pareto distribution (actually mode) */ - build_time_t Xm; - /** alpha exponent for pareto dist. */ - double alpha; - /** Have we computed a timeout? */ - int have_computed_timeout; - /** The exact value for that timeout in milliseconds. Stored as a double - * to maintain precision from calculations to and from quantile value. */ - double timeout_ms; - /** How long we wait before actually closing the circuit. */ - double close_ms; -} circuit_build_times_t; +typedef struct circuit_build_times_s circuit_build_times_t; /********************************* config.c ***************************/ @@ -4409,6 +4776,7 @@ typedef enum { * did this remapping happen." */ ADDRMAPSRC_NONE } addressmap_entry_source_t; +#define addressmap_entry_source_bitfield_t ENUM_BF(addressmap_entry_source_t) /********************************* control.c ***************************/ @@ -4560,8 +4928,6 @@ typedef enum { GEOIP_CLIENT_CONNECT = 0, /** We've served a networkstatus consensus as a directory server. */ GEOIP_CLIENT_NETWORKSTATUS = 1, - /** We've served a v2 networkstatus consensus as a directory server. */ - GEOIP_CLIENT_NETWORKSTATUS_V2 = 2, } geoip_client_action_t; /** Indicates either a positive reply or a reason for rejectng a network * status request that will be included in geoip statistics. */ @@ -4619,11 +4985,6 @@ typedef struct microdesc_cache_t microdesc_cache_t; /********************************* networkstatus.c *********************/ -/** Location where we found a v2 networkstatus. */ -typedef enum { - NS_FROM_CACHE, NS_FROM_DIR_BY_FP, NS_FROM_DIR_ALL, NS_GENERATED -} v2_networkstatus_source_t; - /** Possible statuses of a version of Tor, given opinions from the directory * servers. */ typedef enum version_status_t { @@ -4686,12 +5047,13 @@ typedef struct rend_encoded_v2_service_descriptor_t { * introduction point. See also rend_intro_point_t.unreachable_count. */ #define MAX_INTRO_POINT_REACHABILITY_FAILURES 5 -/** The maximum number of distinct INTRODUCE2 cells which a hidden - * service's introduction point will receive before it begins to - * expire. - * - * XXX023 Is this number at all sane? */ -#define INTRO_POINT_LIFETIME_INTRODUCTIONS 16384 +/** The minimum and maximum number of distinct INTRODUCE2 cells which a + * hidden service's introduction point will receive before it begins to + * expire. */ +#define INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS 16384 +/* Double the minimum value so the interval is [min, min * 2]. */ +#define INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS \ + (INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS * 2) /** The minimum number of seconds that an introduction point will last * before expiring due to old age. (If it receives @@ -4706,6 +5068,11 @@ typedef struct rend_encoded_v2_service_descriptor_t { * XXX023 Should this be configurable? */ #define INTRO_POINT_LIFETIME_MAX_SECONDS (24*60*60) +/** The maximum number of circuit creation retry we do to an intro point + * before giving up. We try to reuse intro point that fails during their + * lifetime so this is a hard limit on the amount of time we do that. */ +#define MAX_INTRO_POINT_CIRCUIT_RETRIES 3 + /** Introduction point information. Used both in rend_service_t (on * the service side) and in rend_service_descriptor_t (on both the * client and service side). */ @@ -4730,11 +5097,6 @@ typedef struct rend_intro_point_t { * included in the last HS descriptor we generated. */ unsigned int listed_in_last_desc : 1; - /** (Service side only) Flag indicating that - * rend_service_note_removing_intro_point has been called for this - * intro point. */ - unsigned int rend_service_note_removing_intro_point_called : 1; - /** (Service side only) A replay cache recording the RSA-encrypted parts * of INTRODUCE2 cells this intro point's circuit has received. This is * used to prevent replay attacks. */ @@ -4745,6 +5107,12 @@ typedef struct rend_intro_point_t { */ int accepted_introduce2_count; + /** (Service side only) Number of maximum INTRODUCE2 cells that this IP + * will accept. This is a random value between + * INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS and + * INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS. */ + int max_introductions; + /** (Service side only) The time at which this intro point was first * published, or -1 if this intro point has not yet been * published. */ @@ -4755,15 +5123,16 @@ typedef struct rend_intro_point_t { * point should expire. */ time_t time_to_expire; - /** (Service side only) The time at which we decided that this intro - * point should start expiring, or -1 if this intro point is not yet - * expiring. - * - * This field also serves as a flag to indicate that we have decided - * to expire this intro point, in case intro_point_should_expire_now - * flaps (perhaps due to a clock jump; perhaps due to other - * weirdness, or even a (present or future) bug). */ - time_t time_expiring; + /** (Service side only) The amount of circuit creation we've made to this + * intro point. This is incremented every time we do a circuit relaunch on + * this object which is triggered when the circuit dies but the node is + * still in the consensus. After MAX_INTRO_POINT_CIRCUIT_RETRIES, we give + * up on it. */ + unsigned int circuit_retries; + + /** (Service side only) Set if this intro point has an established circuit + * and unset if it doesn't. */ + unsigned int circuit_established:1; } rend_intro_point_t; #define REND_PROTOCOL_VERSION_BITMASK_WIDTH 16 @@ -4774,9 +5143,9 @@ typedef struct rend_service_descriptor_t { crypto_pk_t *pk; /**< This service's public key. */ int version; /**< Version of the descriptor format: 0 or 2. */ time_t timestamp; /**< Time when the descriptor was generated. */ - /** Bitmask: which rendezvous protocols are supported? - * (We allow bits '0', '1', and '2' to be set.) */ - int protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; + /** Bitmask: which introduce/rendezvous protocols are supported? + * (We allow bits '0', '1', '2' and '3' to be set.) */ + unsigned protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; /** List of the service's introduction points. Elements are removed if * introduction attempts fail. */ smartlist_t *intro_nodes; @@ -4789,14 +5158,6 @@ typedef struct rend_service_descriptor_t { smartlist_t *successful_uploads; } rend_service_descriptor_t; -/** A cached rendezvous descriptor. */ -typedef struct rend_cache_entry_t { - size_t len; /**< Length of <b>desc</b> */ - time_t received; /**< When was the descriptor received? */ - char *desc; /**< Service descriptor */ - rend_service_descriptor_t *parsed; /**< Parsed value of 'desc' */ -} rend_cache_entry_t; - /********************************* routerlist.c ***************************/ /** Represents information about a single trusted or fallback directory @@ -4805,9 +5166,13 @@ typedef struct dir_server_t { char *description; char *nickname; char *address; /**< Hostname. */ + /* XX/teor - why do we duplicate the address and port fields here and in + * fake_status? Surely we could just use fake_status (#17867). */ + tor_addr_t ipv6_addr; /**< IPv6 address if present; AF_UNSPEC if not */ uint32_t addr; /**< IPv4 address. */ uint16_t dir_port; /**< Directory port. */ uint16_t or_port; /**< OR port: Used for tunneling connections. */ + uint16_t ipv6_orport; /**< OR port corresponding to ipv6_addr. */ double weight; /** Weight used when selecting this node at random */ char digest[DIGEST_LEN]; /**< Digest of identity key. */ char v3_identity_digest[DIGEST_LEN]; /**< Digest of v3 (authority only, @@ -4824,8 +5189,6 @@ typedef struct dir_server_t { /** What kind of authority is this? (Bitfield.) */ dirinfo_type_t type; - download_status_t v2_ns_dl_status; /**< Status of downloading this server's - * v2 network status. */ time_t addr_current_at; /**< When was the document that we derived the * address information from published? */ @@ -4836,7 +5199,8 @@ typedef struct dir_server_t { **/ } dir_server_t; -#define ROUTER_REQUIRED_MIN_BANDWIDTH (20*1024) +#define RELAY_REQUIRED_MIN_BANDWIDTH (75*1024) +#define BRIDGE_REQUIRED_MIN_BANDWIDTH (50*1024) #define ROUTER_MAX_DECLARED_BANDWIDTH INT32_MAX @@ -4860,22 +5224,19 @@ typedef struct dir_server_t { * or extrainfo documents. * * Passed to router_pick_directory_server (et al) - * - * [XXXX NOTE: This option is only implemented for pick_trusteddirserver, - * not pick_directory_server. If we make it work on pick_directory_server - * too, we could conservatively make it only prevent multiple fetches to - * the same authority, or we could aggressively make it prevent multiple - * fetches to _any_ single directory server.] */ #define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3) +/** Flag to indicate that we should not use any directory authority to which + * we have an existing directory connection for downloading microdescs. + * + * Passed to router_pick_directory_server (et al) + */ #define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4) /** This node is to be chosen as a directory guard, so don't choose any * node that's currently a guard. */ #define PDS_FOR_GUARD (1<<5) -#define PDS_PREFER_TUNNELED_DIR_CONNS_ (1<<16) - /** Possible ways to weight routers when choosing one randomly. See * routerlist_sl_choose_by_bandwidth() for more information.*/ typedef enum bandwidth_weight_rule_t { @@ -4892,19 +5253,43 @@ typedef enum { CRN_ALLOW_INVALID = 1<<3, /* XXXX not used, apparently. */ CRN_WEIGHT_AS_EXIT = 1<<5, - CRN_NEED_DESC = 1<<6 + CRN_NEED_DESC = 1<<6, + /* On clients, only provide nodes that satisfy ClientPreferIPv6OR */ + CRN_PREF_ADDR = 1<<7, + /* On clients, only provide nodes that we can connect to directly, based on + * our firewall rules */ + CRN_DIRECT_CONN = 1<<8 } router_crn_flags_t; /** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */ typedef enum was_router_added_t { + /* Router was added successfully. */ ROUTER_ADDED_SUCCESSFULLY = 1, + /* Router descriptor was added with warnings to submitter. */ ROUTER_ADDED_NOTIFY_GENERATOR = 0, + /* Extrainfo document was rejected because no corresponding router + * descriptor was found OR router descriptor was rejected because + * it was incompatible with its extrainfo document. */ ROUTER_BAD_EI = -1, - ROUTER_WAS_NOT_NEW = -2, + /* Router descriptor was rejected because it is already known. */ + ROUTER_IS_ALREADY_KNOWN = -2, + /* General purpose router was rejected, because it was not listed + * in consensus. */ ROUTER_NOT_IN_CONSENSUS = -3, + /* Router was neither in directory consensus nor in any of + * networkstatus documents. Caching it to access later. + * (Applies to fetched descriptors only.) */ ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS = -4, + /* Router was rejected by directory authority. */ ROUTER_AUTHDIR_REJECTS = -5, - ROUTER_WAS_NOT_WANTED = -6 + /* Bridge descriptor was rejected because such bridge was not one + * of the bridges we have listed in our configuration. */ + ROUTER_WAS_NOT_WANTED = -6, + /* Router descriptor was rejected because it was older than + * OLD_ROUTER_DESC_MAX_AGE. */ + ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */ + /* DOCDOC */ + ROUTER_CERTS_EXPIRED = -8 } was_router_added_t; /********************************* routerparse.c ************************/ diff --git a/src/or/periodic.c b/src/or/periodic.c new file mode 100644 index 0000000000..057fcf672e --- /dev/null +++ b/src/or/periodic.c @@ -0,0 +1,126 @@ +/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file periodic.c + * + * \brief Generic backend for handling periodic events. + */ + +#include "or.h" +#include "compat_libevent.h" +#include "config.h" +#include "periodic.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +/** We disable any interval greater than this number of seconds, on the + * grounds that it is probably an absolute time mistakenly passed in as a + * relative time. + */ +static const int MAX_INTERVAL = 10 * 365 * 86400; + +/** Set the event <b>event</b> to run in <b>next_interval</b> seconds from + * now. */ +static void +periodic_event_set_interval(periodic_event_item_t *event, + time_t next_interval) +{ + tor_assert(next_interval < MAX_INTERVAL); + struct timeval tv; + tv.tv_sec = next_interval; + tv.tv_usec = 0; + event_add(event->ev, &tv); +} + +/** Wraps dispatches for periodic events, <b>data</b> will be a pointer to the + * event that needs to be called */ +static void +periodic_event_dispatch(evutil_socket_t fd, short what, void *data) +{ + (void)fd; + (void)what; + periodic_event_item_t *event = data; + + time_t now = time(NULL); + const or_options_t *options = get_options(); +// log_debug(LD_GENERAL, "Dispatching %s", event->name); + int r = event->fn(now, options); + int next_interval = 0; + + /* update the last run time if action was taken */ + if (r==0) { + log_err(LD_BUG, "Invalid return value for periodic event from %s.", + event->name); + tor_assert(r != 0); + } else if (r > 0) { + event->last_action_time = now; + /* If the event is meant to happen after ten years, that's likely + * a bug, and somebody gave an absolute time rather than an interval. + */ + tor_assert(r < MAX_INTERVAL); + next_interval = r; + } else { + /* no action was taken, it is likely a precondition failed, + * we should reschedule for next second incase the precondition + * passes then */ + next_interval = 1; + } + +// log_debug(LD_GENERAL, "Scheduling %s for %d seconds", event->name, +// next_interval); + struct timeval tv = { next_interval , 0 }; + event_add(event->ev, &tv); +} + +/** Schedules <b>event</b> to run as soon as possible from now. */ +void +periodic_event_reschedule(periodic_event_item_t *event) +{ + periodic_event_set_interval(event, 1); +} + +/** Initializes the libevent backend for a periodic event. */ +void +periodic_event_setup(periodic_event_item_t *event) +{ + if (event->ev) { /* Already setup? This is a bug */ + log_err(LD_BUG, "Initial dispatch should only be done once."); + tor_assert(0); + } + + event->ev = tor_event_new(tor_libevent_get_base(), + -1, 0, + periodic_event_dispatch, + event); + tor_assert(event->ev); +} + +/** Handles initial dispatch for periodic events. It should happen 1 second + * after the events are created to mimic behaviour before #3199's refactor */ +void +periodic_event_launch(periodic_event_item_t *event) +{ + if (! event->ev) { /* Not setup? This is a bug */ + log_err(LD_BUG, "periodic_event_launch without periodic_event_setup"); + tor_assert(0); + } + + // Initial dispatch + periodic_event_dispatch(-1, EV_TIMEOUT, event); +} + +/** Release all storage associated with <b>event</b> */ +void +periodic_event_destroy(periodic_event_item_t *event) +{ + if (!event) + return; + tor_event_free(event->ev); + event->last_action_time = 0; +} + diff --git a/src/or/periodic.h b/src/or/periodic.h new file mode 100644 index 0000000000..021bb4ef5c --- /dev/null +++ b/src/or/periodic.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_PERIODIC_H +#define TOR_PERIODIC_H + +#define PERIODIC_EVENT_NO_UPDATE (-1) + +/** Callback function for a periodic event to take action. The return value +* influences the next time the function will get called. Return +* PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled +* again in the next second. If a positive value is returned it will update the +* interval time. */ +typedef int (*periodic_event_helper_t)(time_t now, + const or_options_t *options); + +struct event; + +/** A single item for the periodic-events-function table. */ +typedef struct periodic_event_item_t { + periodic_event_helper_t fn; /**< The function to run the event */ + time_t last_action_time; /**< The last time the function did something */ + struct event *ev; /**< Libevent callback we're using to implement this */ + const char *name; /**< Name of the function -- for debug */ +} periodic_event_item_t; + +/** events will get their interval from first execution */ +#define PERIODIC_EVENT(fn) { fn##_callback, 0, NULL, #fn } +#define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL } + +void periodic_event_launch(periodic_event_item_t *event); +void periodic_event_setup(periodic_event_item_t *event); +void periodic_event_destroy(periodic_event_item_t *event); +void periodic_event_reschedule(periodic_event_item_t *event); + +#endif + diff --git a/src/or/policies.c b/src/or/policies.c index be4da55061..50fec3a773 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -8,11 +8,15 @@ * \brief Code to parse and use address policies and exit policies. **/ +#define POLICIES_PRIVATE + #include "or.h" #include "config.h" #include "dirserv.h" +#include "networkstatus.h" #include "nodelist.h" #include "policies.h" +#include "router.h" #include "routerparse.h" #include "geoip.h" #include "ht.h" @@ -28,9 +32,6 @@ static smartlist_t *authdir_reject_policy = NULL; * to be marked as valid in our networkstatus. */ static smartlist_t *authdir_invalid_policy = NULL; /** Policy that addresses for incoming router descriptors must <b>not</b> - * match in order to not be marked as BadDirectory. */ -static smartlist_t *authdir_baddir_policy = NULL; -/** Policy that addresses for incoming router descriptors must <b>not</b> * match in order to not be marked as BadExit. */ static smartlist_t *authdir_badexit_policy = NULL; @@ -64,6 +65,16 @@ static const char *private_nets[] = { NULL }; +static int policies_parse_exit_policy_internal( + config_line_t *cfg, + smartlist_t **dest, + int ipv6_exit, + int rejectprivate, + const smartlist_t *configured_addresses, + int reject_interface_addresses, + int reject_configured_port_addresses, + int add_default_policy); + /** Replace all "private" entries in *<b>policy</b> with their expanded * equivalents. */ void @@ -147,7 +158,7 @@ policy_expand_unspec(smartlist_t **policy) } /** - * Given a linked list of config lines containing "allow" and "deny" + * Given a linked list of config lines containing "accept[6]" and "reject[6]" * tokens, parse them and append the result to <b>dest</b>. Return -1 * if any tokens are malformed (and don't append any), else return 0. * @@ -162,6 +173,7 @@ parse_addr_policy(config_line_t *cfg, smartlist_t **dest, smartlist_t *result; smartlist_t *entries; addr_policy_t *item; + int malformed_list; int r = 0; if (!cfg) @@ -174,12 +186,22 @@ parse_addr_policy(config_line_t *cfg, smartlist_t **dest, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH_BEGIN(entries, const char *, ent) { log_debug(LD_CONFIG,"Adding new entry '%s'",ent); - item = router_parse_addr_policy_item_from_string(ent, assume_action); + malformed_list = 0; + item = router_parse_addr_policy_item_from_string(ent, assume_action, + &malformed_list); if (item) { smartlist_add(result, item); - } else { - log_warn(LD_CONFIG,"Malformed policy '%s'.", ent); + } else if (malformed_list) { + /* the error is so severe the entire list should be discarded */ + log_warn(LD_CONFIG, "Malformed policy '%s'. Discarding entire policy " + "list.", ent); r = -1; + } else { + /* the error is minor: don't add the item, but keep processing the + * rest of the policies in the list */ + log_debug(LD_CONFIG, "Ignored policy '%s' due to non-fatal error. " + "The remainder of the policy list will be used.", + ent); } } SMARTLIST_FOREACH_END(ent); SMARTLIST_FOREACH(entries, char *, ent, tor_free(ent)); @@ -249,16 +271,76 @@ parse_reachable_addresses(void) "Error parsing ReachableDirAddresses entry; ignoring."); ret = -1; } + + /* We ignore ReachableAddresses for relays */ + if (!server_mode(options)) { + if ((reachable_or_addr_policy + && policy_is_reject_star(reachable_or_addr_policy, AF_UNSPEC)) + || (reachable_dir_addr_policy + && policy_is_reject_star(reachable_dir_addr_policy, AF_UNSPEC))) { + log_warn(LD_CONFIG, "Tor cannot connect to the Internet if " + "ReachableAddresses, ReachableORAddresses, or " + "ReachableDirAddresses reject all addresses. Please accept " + "some addresses in these options."); + } else if (options->ClientUseIPv4 == 1 + && ((reachable_or_addr_policy + && policy_is_reject_star(reachable_or_addr_policy, AF_INET)) + || (reachable_dir_addr_policy + && policy_is_reject_star(reachable_dir_addr_policy, AF_INET)))) { + log_warn(LD_CONFIG, "You have set ClientUseIPv4 1, but " + "ReachableAddresses, ReachableORAddresses, or " + "ReachableDirAddresses reject all IPv4 addresses. " + "Tor will not connect using IPv4."); + } else if (fascist_firewall_use_ipv6(options) + && ((reachable_or_addr_policy + && policy_is_reject_star(reachable_or_addr_policy, AF_INET6)) + || (reachable_dir_addr_policy + && policy_is_reject_star(reachable_dir_addr_policy, AF_INET6)))) { + log_warn(LD_CONFIG, "You have configured tor to use IPv6 " + "(ClientUseIPv6 1 or UseBridges 1), but " + "ReachableAddresses, ReachableORAddresses, or " + "ReachableDirAddresses reject all IPv6 addresses. " + "Tor will not connect using IPv6."); + } + } + return ret; } -/** Return true iff the firewall options might block any address:port - * combination. +/* Return true iff ClientUseIPv4 0 or ClientUseIPv6 0 might block any OR or Dir + * address:port combination. */ +static int +firewall_is_fascist_impl(void) +{ + const or_options_t *options = get_options(); + /* Assume every non-bridge relay has an IPv4 address. + * Clients which use bridges may only know the IPv6 address of their + * bridge. */ + return (options->ClientUseIPv4 == 0 + || (!fascist_firewall_use_ipv6(options) + && options->UseBridges == 1)); +} + +/** Return true iff the firewall options, including ClientUseIPv4 0 and + * ClientUseIPv6 0, might block any OR address:port combination. + * Address preferences may still change which address is selected even if + * this function returns false. */ int firewall_is_fascist_or(void) { - return reachable_or_addr_policy != NULL; + return (reachable_or_addr_policy != NULL || firewall_is_fascist_impl()); +} + +/** Return true iff the firewall options, including ClientUseIPv4 0 and + * ClientUseIPv6 0, might block any Dir address:port combination. + * Address preferences may still change which address is selected even if + * this function returns false. + */ +int +firewall_is_fascist_dir(void) +{ + return (reachable_dir_addr_policy != NULL || firewall_is_fascist_impl()); } /** Return true iff <b>policy</b> (possibly NULL) will allow a @@ -296,49 +378,618 @@ addr_policy_permits_address(uint32_t addr, uint16_t port, return addr_policy_permits_tor_addr(&a, port, policy); } -/** Return true iff we think our firewall will let us make an OR connection to - * addr:port. */ -int -fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port) +/** Return true iff we think our firewall will let us make a connection to + * addr:port. + * + * If we are configured as a server, ignore any address family preference and + * just use IPv4. + * Otherwise: + * - return false for all IPv4 addresses: + * - if ClientUseIPv4 is 0, or + * if pref_only and pref_ipv6 are both true; + * - return false for all IPv6 addresses: + * - if fascist_firewall_use_ipv6() is 0, or + * - if pref_only is true and pref_ipv6 is false. + * + * Return false if addr is NULL or tor_addr_is_null(), or if port is 0. */ +STATIC int +fascist_firewall_allows_address(const tor_addr_t *addr, + uint16_t port, + smartlist_t *firewall_policy, + int pref_only, int pref_ipv6) { + const or_options_t *options = get_options(); + const int client_mode = !server_mode(options); + + if (!addr || tor_addr_is_null(addr) || !port) { + return 0; + } + + /* Clients stop using IPv4 if it's disabled. In most cases, clients also + * stop using IPv4 if it's not preferred. + * Servers must have IPv4 enabled and preferred. */ + if (tor_addr_family(addr) == AF_INET && client_mode && + (!options->ClientUseIPv4 || (pref_only && pref_ipv6))) { + return 0; + } + + /* Clients and Servers won't use IPv6 unless it's enabled (and in most + * cases, IPv6 must also be preferred before it will be used). */ + if (tor_addr_family(addr) == AF_INET6 && + (!fascist_firewall_use_ipv6(options) || (pref_only && !pref_ipv6))) { + return 0; + } + return addr_policy_permits_tor_addr(addr, port, - reachable_or_addr_policy); + firewall_policy); } -/** Return true iff we think our firewall will let us make an OR connection to - * <b>ri</b>. */ +/** Is this client configured to use IPv6? + * Use node_ipv6_or/dir_preferred() when checking a specific node and OR/Dir + * port: it supports bridge client per-node IPv6 preferences. + */ int -fascist_firewall_allows_or(const routerinfo_t *ri) +fascist_firewall_use_ipv6(const or_options_t *options) +{ + /* Clients use IPv6 if it's set, or they use bridges, or they don't use + * IPv4 */ + return (options->ClientUseIPv6 == 1 || options->UseBridges == 1 + || options->ClientUseIPv4 == 0); +} + +/** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and + * ClientPreferIPv6DirPort? + * If we're unsure, return -1, otherwise, return 1 for IPv6 and 0 for IPv4. + */ +static int +fascist_firewall_prefer_ipv6_impl(const or_options_t *options) { - /* XXXX proposal 118 */ - tor_addr_t addr; - tor_addr_from_ipv4h(&addr, ri->addr); - return fascist_firewall_allows_address_or(&addr, ri->or_port); + /* + Cheap implementation of config options ClientUseIPv4 & ClientUseIPv6 -- + If we're a server or IPv6 is disabled, use IPv4. + If IPv4 is disabled, use IPv6. + */ + + if (server_mode(options) || !fascist_firewall_use_ipv6(options)) { + return 0; + } + + if (!options->ClientUseIPv4) { + return 1; + } + + return -1; } -/** Return true iff we think our firewall will let us make an OR connection to - * <b>node</b>. */ +/** Do we prefer to connect to IPv6 ORPorts? + * Use node_ipv6_or_preferred() whenever possible: it supports bridge client + * per-node IPv6 preferences. + */ int -fascist_firewall_allows_node(const node_t *node) +fascist_firewall_prefer_ipv6_orport(const or_options_t *options) { - if (node->ri) { - return fascist_firewall_allows_or(node->ri); - } else if (node->rs) { - tor_addr_t addr; - tor_addr_from_ipv4h(&addr, node->rs->addr); - return fascist_firewall_allows_address_or(&addr, node->rs->or_port); + int pref_ipv6 = fascist_firewall_prefer_ipv6_impl(options); + + if (pref_ipv6 >= 0) { + return pref_ipv6; + } + + /* We can use both IPv4 and IPv6 - which do we prefer? */ + if (options->ClientPreferIPv6ORPort == 1) { + return 1; + } + + return 0; +} + +/** Do we prefer to connect to IPv6 DirPorts? + * + * (node_ipv6_dir_preferred() doesn't support bridge client per-node IPv6 + * preferences. There's no reason to use it instead of this function.) + */ +int +fascist_firewall_prefer_ipv6_dirport(const or_options_t *options) +{ + int pref_ipv6 = fascist_firewall_prefer_ipv6_impl(options); + + if (pref_ipv6 >= 0) { + return pref_ipv6; + } + + /* We can use both IPv4 and IPv6 - which do we prefer? */ + if (options->ClientPreferIPv6DirPort == 1) { + return 1; + } + + return 0; +} + +/** Return true iff we think our firewall will let us make a connection to + * addr:port. Uses ReachableORAddresses or ReachableDirAddresses based on + * fw_connection. + * If pref_only is true, return true if addr is in the client's preferred + * address family, which is IPv6 if pref_ipv6 is true, and IPv4 otherwise. + * If pref_only is false, ignore pref_ipv6, and return true if addr is allowed. + */ +int +fascist_firewall_allows_address_addr(const tor_addr_t *addr, uint16_t port, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6) +{ + if (fw_connection == FIREWALL_OR_CONNECTION) { + return fascist_firewall_allows_address(addr, port, + reachable_or_addr_policy, + pref_only, pref_ipv6); + } else if (fw_connection == FIREWALL_DIR_CONNECTION) { + return fascist_firewall_allows_address(addr, port, + reachable_dir_addr_policy, + pref_only, pref_ipv6); + } else { + log_warn(LD_BUG, "Bad firewall_connection_t value %d.", + fw_connection); + return 0; + } +} + +/** Return true iff we think our firewall will let us make a connection to + * addr:port (ap). Uses ReachableORAddresses or ReachableDirAddresses based on + * fw_connection. + * pref_only and pref_ipv6 work as in fascist_firewall_allows_address_addr(). + */ +static int +fascist_firewall_allows_address_ap(const tor_addr_port_t *ap, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6) +{ + tor_assert(ap); + return fascist_firewall_allows_address_addr(&ap->addr, ap->port, + fw_connection, pref_only, + pref_ipv6); +} + +/* Return true iff we think our firewall will let us make a connection to + * ipv4h_or_addr:ipv4_or_port. ipv4h_or_addr is interpreted in host order. + * Uses ReachableORAddresses or ReachableDirAddresses based on + * fw_connection. + * pref_only and pref_ipv6 work as in fascist_firewall_allows_address_addr(). + */ +static int +fascist_firewall_allows_address_ipv4h(uint32_t ipv4h_or_addr, + uint16_t ipv4_or_port, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6) +{ + tor_addr_t ipv4_or_addr; + tor_addr_from_ipv4h(&ipv4_or_addr, ipv4h_or_addr); + return fascist_firewall_allows_address_addr(&ipv4_or_addr, ipv4_or_port, + fw_connection, pref_only, + pref_ipv6); +} + +/** Return true iff we think our firewall will let us make a connection to + * ipv4h_addr/ipv6_addr. Uses ipv4_orport/ipv6_orport/ReachableORAddresses or + * ipv4_dirport/ipv6_dirport/ReachableDirAddresses based on IPv4/IPv6 and + * <b>fw_connection</b>. + * pref_only and pref_ipv6 work as in fascist_firewall_allows_address_addr(). + */ +static int +fascist_firewall_allows_base(uint32_t ipv4h_addr, uint16_t ipv4_orport, + uint16_t ipv4_dirport, + const tor_addr_t *ipv6_addr, uint16_t ipv6_orport, + uint16_t ipv6_dirport, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6) +{ + if (fascist_firewall_allows_address_ipv4h(ipv4h_addr, + (fw_connection == FIREWALL_OR_CONNECTION + ? ipv4_orport + : ipv4_dirport), + fw_connection, + pref_only, pref_ipv6)) { + return 1; + } + + if (fascist_firewall_allows_address_addr(ipv6_addr, + (fw_connection == FIREWALL_OR_CONNECTION + ? ipv6_orport + : ipv6_dirport), + fw_connection, + pref_only, pref_ipv6)) { + return 1; + } + + return 0; +} + +/** Like fascist_firewall_allows_base(), but takes ri. */ +static int +fascist_firewall_allows_ri_impl(const routerinfo_t *ri, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6) +{ + if (!ri) { + return 0; + } + + /* Assume IPv4 and IPv6 DirPorts are the same */ + return fascist_firewall_allows_base(ri->addr, ri->or_port, ri->dir_port, + &ri->ipv6_addr, ri->ipv6_orport, + ri->dir_port, fw_connection, pref_only, + pref_ipv6); +} + +/** Like fascist_firewall_allows_rs, but takes pref_ipv6. */ +static int +fascist_firewall_allows_rs_impl(const routerstatus_t *rs, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6) +{ + if (!rs) { + return 0; + } + + /* Assume IPv4 and IPv6 DirPorts are the same */ + return fascist_firewall_allows_base(rs->addr, rs->or_port, rs->dir_port, + &rs->ipv6_addr, rs->ipv6_orport, + rs->dir_port, fw_connection, pref_only, + pref_ipv6); +} + +/** Like fascist_firewall_allows_base(), but takes rs. + * When rs is a fake_status from a dir_server_t, it can have a reachable + * address, even when the corresponding node does not. + * nodes can be missing addresses when there's no consensus (IPv4 and IPv6), + * or when there is a microdescriptor consensus, but no microdescriptors + * (microdescriptors have IPv6, the microdesc consensus does not). */ +int +fascist_firewall_allows_rs(const routerstatus_t *rs, + firewall_connection_t fw_connection, int pref_only) +{ + if (!rs) { + return 0; + } + + /* We don't have access to the node-specific IPv6 preference, so use the + * generic IPv6 preference instead. */ + const or_options_t *options = get_options(); + int pref_ipv6 = (fw_connection == FIREWALL_OR_CONNECTION + ? fascist_firewall_prefer_ipv6_orport(options) + : fascist_firewall_prefer_ipv6_dirport(options)); + + return fascist_firewall_allows_rs_impl(rs, fw_connection, pref_only, + pref_ipv6); +} + +/** Return true iff we think our firewall will let us make a connection to + * ipv6_addr:ipv6_orport based on ReachableORAddresses. + * If <b>fw_connection</b> is FIREWALL_DIR_CONNECTION, returns 0. + * pref_only and pref_ipv6 work as in fascist_firewall_allows_address_addr(). + */ +static int +fascist_firewall_allows_md_impl(const microdesc_t *md, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6) +{ + if (!md) { + return 0; + } + + /* Can't check dirport, it doesn't have one */ + if (fw_connection == FIREWALL_DIR_CONNECTION) { + return 0; + } + + /* Also can't check IPv4, doesn't have that either */ + return fascist_firewall_allows_address_addr(&md->ipv6_addr, md->ipv6_orport, + fw_connection, pref_only, + pref_ipv6); +} + +/** Like fascist_firewall_allows_base(), but takes node, and looks up pref_ipv6 + * from node_ipv6_or/dir_preferred(). */ +int +fascist_firewall_allows_node(const node_t *node, + firewall_connection_t fw_connection, + int pref_only) +{ + if (!node) { + return 0; + } + + node_assert_ok(node); + + const int pref_ipv6 = (fw_connection == FIREWALL_OR_CONNECTION + ? node_ipv6_or_preferred(node) + : node_ipv6_dir_preferred(node)); + + /* Sometimes, the rs is missing the IPv6 address info, and we need to go + * all the way to the md */ + if (node->ri && fascist_firewall_allows_ri_impl(node->ri, fw_connection, + pref_only, pref_ipv6)) { + return 1; + } else if (node->rs && fascist_firewall_allows_rs_impl(node->rs, + fw_connection, + pref_only, + pref_ipv6)) { + return 1; + } else if (node->md && fascist_firewall_allows_md_impl(node->md, + fw_connection, + pref_only, + pref_ipv6)) { + return 1; } else { + /* If we know nothing, assume it's unreachable, we'll never get an address + * to connect to. */ + return 0; + } +} + +/** Like fascist_firewall_allows_rs(), but takes ds. */ +int +fascist_firewall_allows_dir_server(const dir_server_t *ds, + firewall_connection_t fw_connection, + int pref_only) +{ + if (!ds) { + return 0; + } + + /* A dir_server_t always has a fake_status. As long as it has the same + * addresses/ports in both fake_status and dir_server_t, this works fine. + * (See #17867.) + * fascist_firewall_allows_rs only checks the addresses in fake_status. */ + return fascist_firewall_allows_rs(&ds->fake_status, fw_connection, + pref_only); +} + +/** If a and b are both valid and allowed by fw_connection, + * choose one based on want_a and return it. + * Otherwise, return whichever is allowed. + * Otherwise, return NULL. + * pref_only and pref_ipv6 work as in fascist_firewall_allows_address_addr(). + */ +static const tor_addr_port_t * +fascist_firewall_choose_address_impl(const tor_addr_port_t *a, + const tor_addr_port_t *b, + int want_a, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6) +{ + const tor_addr_port_t *use_a = NULL; + const tor_addr_port_t *use_b = NULL; + + if (fascist_firewall_allows_address_ap(a, fw_connection, pref_only, + pref_ipv6)) { + use_a = a; + } + + if (fascist_firewall_allows_address_ap(b, fw_connection, pref_only, + pref_ipv6)) { + use_b = b; + } + + /* If both are allowed */ + if (use_a && use_b) { + /* Choose a if we want it */ + return (want_a ? use_a : use_b); + } else { + /* Choose a if we have it */ + return (use_a ? use_a : use_b); + } +} + +/** If a and b are both valid and preferred by fw_connection, + * choose one based on want_a and return it. + * Otherwise, return whichever is preferred. + * If neither are preferred, and pref_only is false: + * - If a and b are both allowed by fw_connection, + * choose one based on want_a and return it. + * - Otherwise, return whichever is preferred. + * Otherwise, return NULL. */ +STATIC const tor_addr_port_t * +fascist_firewall_choose_address(const tor_addr_port_t *a, + const tor_addr_port_t *b, + int want_a, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6) +{ + const tor_addr_port_t *pref = fascist_firewall_choose_address_impl( + a, b, want_a, + fw_connection, + 1, pref_ipv6); + if (pref_only || pref) { + /* If there is a preferred address, use it. If we can only use preferred + * addresses, and neither address is preferred, pref will be NULL, and we + * want to return NULL, so return it. */ + return pref; + } else { + /* If there's no preferred address, and we can return addresses that are + * not preferred, use an address that's allowed */ + return fascist_firewall_choose_address_impl(a, b, want_a, fw_connection, + 0, pref_ipv6); + } +} + +/** Copy an address and port into <b>ap</b> that we think our firewall will + * let us connect to. Uses ipv4_addr/ipv6_addr and + * ipv4_orport/ipv6_orport/ReachableORAddresses or + * ipv4_dirport/ipv6_dirport/ReachableDirAddresses based on IPv4/IPv6 and + * <b>fw_connection</b>. + * If pref_only, only choose preferred addresses. In either case, choose + * a preferred address before an address that's not preferred. + * If both addresses could be chosen (they are both preferred or both allowed) + * choose IPv6 if pref_ipv6 is true, otherwise choose IPv4. + * If neither address is chosen, return 0, else return 1. */ +static int +fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr, + uint16_t ipv4_orport, + uint16_t ipv4_dirport, + const tor_addr_t *ipv6_addr, + uint16_t ipv6_orport, + uint16_t ipv6_dirport, + firewall_connection_t fw_connection, + int pref_only, + int pref_ipv6, + tor_addr_port_t* ap) +{ + const tor_addr_port_t *result = NULL; + const int want_ipv4 = !pref_ipv6; + + tor_assert(ipv6_addr); + tor_assert(ap); + + tor_addr_port_t ipv4_ap; + tor_addr_copy(&ipv4_ap.addr, ipv4_addr); + ipv4_ap.port = (fw_connection == FIREWALL_OR_CONNECTION + ? ipv4_orport + : ipv4_dirport); + + tor_addr_port_t ipv6_ap; + tor_addr_copy(&ipv6_ap.addr, ipv6_addr); + ipv6_ap.port = (fw_connection == FIREWALL_OR_CONNECTION + ? ipv6_orport + : ipv6_dirport); + + result = fascist_firewall_choose_address(&ipv4_ap, &ipv6_ap, + want_ipv4, + fw_connection, pref_only, + pref_ipv6); + + if (result) { + tor_addr_copy(&ap->addr, &result->addr); + ap->port = result->port; return 1; + } else { + return 0; } } -/** Return true iff we think our firewall will let us make a directory - * connection to addr:port. */ +/** Like fascist_firewall_choose_address_base(), but takes a host-order IPv4 + * address as the first parameter. */ +static int +fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr, + uint16_t ipv4_orport, + uint16_t ipv4_dirport, + const tor_addr_t *ipv6_addr, + uint16_t ipv6_orport, + uint16_t ipv6_dirport, + firewall_connection_t fw_connection, + int pref_only, + int pref_ipv6, + tor_addr_port_t* ap) +{ + tor_addr_t ipv4_addr; + tor_addr_from_ipv4h(&ipv4_addr, ipv4h_addr); + return fascist_firewall_choose_address_base(&ipv4_addr, ipv4_orport, + ipv4_dirport, ipv6_addr, + ipv6_orport, ipv6_dirport, + fw_connection, pref_only, + pref_ipv6, ap); +} + +/** Like fascist_firewall_choose_address_base(), but takes <b>rs</b>. + * Consults the corresponding node, then falls back to rs if node is NULL. + * This should only happen when there's no valid consensus, and rs doesn't + * correspond to a bridge client's bridge. + */ int -fascist_firewall_allows_address_dir(const tor_addr_t *addr, uint16_t port) +fascist_firewall_choose_address_rs(const routerstatus_t *rs, + firewall_connection_t fw_connection, + int pref_only, tor_addr_port_t* ap) { - return addr_policy_permits_tor_addr(addr, port, - reachable_dir_addr_policy); + if (!rs) { + return 0; + } + + tor_assert(ap); + + const node_t *node = node_get_by_id(rs->identity_digest); + + if (node) { + return fascist_firewall_choose_address_node(node, fw_connection, pref_only, + ap); + } else { + /* There's no node-specific IPv6 preference, so use the generic IPv6 + * preference instead. */ + const or_options_t *options = get_options(); + int pref_ipv6 = (fw_connection == FIREWALL_OR_CONNECTION + ? fascist_firewall_prefer_ipv6_orport(options) + : fascist_firewall_prefer_ipv6_dirport(options)); + + /* Assume IPv4 and IPv6 DirPorts are the same. + * Assume the IPv6 OR and Dir addresses are the same. */ + return fascist_firewall_choose_address_ipv4h(rs->addr, + rs->or_port, + rs->dir_port, + &rs->ipv6_addr, + rs->ipv6_orport, + rs->dir_port, + fw_connection, + pref_only, + pref_ipv6, + ap); + } +} + +/** Like fascist_firewall_choose_address_base(), but takes <b>node</b>, and + * looks up the node's IPv6 preference rather than taking an argument + * for pref_ipv6. */ +int +fascist_firewall_choose_address_node(const node_t *node, + firewall_connection_t fw_connection, + int pref_only, tor_addr_port_t *ap) +{ + if (!node) { + return 0; + } + + node_assert_ok(node); + + const int pref_ipv6_node = (fw_connection == FIREWALL_OR_CONNECTION + ? node_ipv6_or_preferred(node) + : node_ipv6_dir_preferred(node)); + + tor_addr_port_t ipv4_or_ap; + node_get_prim_orport(node, &ipv4_or_ap); + tor_addr_port_t ipv4_dir_ap; + node_get_prim_dirport(node, &ipv4_dir_ap); + + tor_addr_port_t ipv6_or_ap; + node_get_pref_ipv6_orport(node, &ipv6_or_ap); + tor_addr_port_t ipv6_dir_ap; + node_get_pref_ipv6_dirport(node, &ipv6_dir_ap); + + /* Assume the IPv6 OR and Dir addresses are the same. */ + return fascist_firewall_choose_address_base(&ipv4_or_ap.addr, + ipv4_or_ap.port, + ipv4_dir_ap.port, + &ipv6_or_ap.addr, + ipv6_or_ap.port, + ipv6_dir_ap.port, + fw_connection, + pref_only, + pref_ipv6_node, + ap); +} + +/** Like fascist_firewall_choose_address_rs(), but takes <b>ds</b>. */ +int +fascist_firewall_choose_address_dir_server(const dir_server_t *ds, + firewall_connection_t fw_connection, + int pref_only, + tor_addr_port_t *ap) +{ + if (!ds) { + return 0; + } + + /* A dir_server_t always has a fake_status. As long as it has the same + * addresses/ports in both fake_status and dir_server_t, this works fine. + * (See #17867.) + * This function relies on fascist_firewall_choose_address_rs looking up the + * node if it can, because that will get the latest info for the relay. */ + return fascist_firewall_choose_address_rs(&ds->fake_status, fw_connection, + pref_only, ap); } /** Return 1 if <b>addr</b> is permitted to connect to our dir port, @@ -399,17 +1050,6 @@ authdir_policy_valid_address(uint32_t addr, uint16_t port) return !addr_is_in_cc_list(addr, get_options()->AuthDirInvalidCCs); } -/** Return 1 if <b>addr</b>:<b>port</b> should be marked as a bad dir, - * based on <b>authdir_baddir_policy</b>. Else return 0. - */ -int -authdir_policy_baddir_address(uint32_t addr, uint16_t port) -{ - if (! addr_policy_permits_address(addr, port, authdir_baddir_policy)) - return 1; - return addr_is_in_cc_list(addr, get_options()->AuthDirBadDirCCs); -} - /** Return 1 if <b>addr</b>:<b>port</b> should be marked as a bad exit, * based on <b>authdir_badexit_policy</b>. Else return 0. */ @@ -436,11 +1076,36 @@ validate_addr_policies(const or_options_t *options, char **msg) smartlist_t *addr_policy=NULL; *msg = NULL; - if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, - options->IPv6Exit, - options->ExitPolicyRejectPrivate, NULL, - !options->BridgeRelay)) + if (policies_parse_exit_policy_from_options(options,0,NULL,&addr_policy)) { REJECT("Error in ExitPolicy entry."); + } + + static int warned_about_exitrelay = 0; + + const int exitrelay_setting_is_auto = options->ExitRelay == -1; + const int policy_accepts_something = + ! (policy_is_reject_star(addr_policy, AF_INET) && + policy_is_reject_star(addr_policy, AF_INET6)); + + if (server_mode(options) && + ! warned_about_exitrelay && + exitrelay_setting_is_auto && + policy_accepts_something) { + /* Policy accepts something */ + warned_about_exitrelay = 1; + log_warn(LD_CONFIG, + "Tor is running as an exit relay%s. If you did not want this " + "behavior, please set the ExitRelay option to 0. If you do " + "want to run an exit Relay, please set the ExitRelay option " + "to 1 to disable this warning, and for forward compatibility.", + options->ExitPolicy == NULL ? + " with the default exit policy" : ""); + if (options->ExitPolicy == NULL) { + log_warn(LD_CONFIG, + "In a future version of Tor, ExitRelay 0 may become the " + "default when no ExitPolicy is given."); + } + } /* The rest of these calls *append* to addr_policy. So don't actually * use the results for anything other than checking if they parse! */ @@ -454,9 +1119,6 @@ validate_addr_policies(const or_options_t *options, char **msg) if (parse_addr_policy(options->AuthDirInvalid, &addr_policy, ADDR_POLICY_REJECT)) REJECT("Error in AuthDirInvalid entry."); - if (parse_addr_policy(options->AuthDirBadDir, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirBadDir entry."); if (parse_addr_policy(options->AuthDirBadExit, &addr_policy, ADDR_POLICY_REJECT)) REJECT("Error in AuthDirBadExit entry."); @@ -482,10 +1144,12 @@ validate_addr_policies(const or_options_t *options, char **msg) * Ignore port specifiers. */ static int -load_policy_from_option(config_line_t *config, smartlist_t **policy, +load_policy_from_option(config_line_t *config, const char *option_name, + smartlist_t **policy, int assume_action) { int r; + int killed_any_ports = 0; addr_policy_list_free(*policy); *policy = NULL; r = parse_addr_policy(config, policy, assume_action); @@ -504,9 +1168,13 @@ load_policy_from_option(config_line_t *config, smartlist_t **policy, c = addr_policy_get_canonical_entry(&newp); SMARTLIST_REPLACE_CURRENT(*policy, n, c); addr_policy_free(n); + killed_any_ports = 1; } } SMARTLIST_FOREACH_END(n); } + if (killed_any_ports) { + log_warn(LD_CONFIG, "Ignoring ports in %s option.", option_name); + } return 0; } @@ -516,20 +1184,19 @@ int policies_parse_from_options(const or_options_t *options) { int ret = 0; - if (load_policy_from_option(options->SocksPolicy, &socks_policy, -1) < 0) + if (load_policy_from_option(options->SocksPolicy, "SocksPolicy", + &socks_policy, -1) < 0) ret = -1; - if (load_policy_from_option(options->DirPolicy, &dir_policy, -1) < 0) + if (load_policy_from_option(options->DirPolicy, "DirPolicy", + &dir_policy, -1) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirReject, + if (load_policy_from_option(options->AuthDirReject, "AuthDirReject", &authdir_reject_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirInvalid, + if (load_policy_from_option(options->AuthDirInvalid, "AuthDirInvalid", &authdir_invalid_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirBadDir, - &authdir_baddir_policy, ADDR_POLICY_REJECT) < 0) - ret = -1; - if (load_policy_from_option(options->AuthDirBadExit, + if (load_policy_from_option(options->AuthDirBadExit, "AuthDirBadExit", &authdir_badexit_policy, ADDR_POLICY_REJECT) < 0) ret = -1; if (parse_reachable_addresses() < 0) @@ -547,6 +1214,8 @@ cmp_single_addr_policy(addr_policy_t *a, addr_policy_t *b) return r; if ((r=((int)a->is_private - (int)b->is_private))) return r; + /* refcnt and is_canonical are irrelevant to equality, + * they are hash table implementation details */ if ((r=tor_addr_compare(&a->addr, &b->addr, CMP_EXACT))) return r; if ((r=((int)a->maskbits - (int)b->maskbits))) @@ -589,7 +1258,7 @@ typedef struct policy_map_ent_t { static HT_HEAD(policy_map, policy_map_ent_t) policy_root = HT_INITIALIZER(); /** Return true iff a and b are equal. */ -static INLINE int +static inline int policy_eq(policy_map_ent_t *a, policy_map_ent_t *b) { return cmp_single_addr_policy(a->policy, b->policy) == 0; @@ -597,27 +1266,31 @@ policy_eq(policy_map_ent_t *a, policy_map_ent_t *b) /** Return a hashcode for <b>ent</b> */ static unsigned int -policy_hash(policy_map_ent_t *ent) +policy_hash(const policy_map_ent_t *ent) { - addr_policy_t *a = ent->policy; - unsigned int r; - if (a->is_private) - r = 0x1234abcd; - else - r = tor_addr_hash(&a->addr); - r += a->prt_min << 8; - r += a->prt_max << 16; - r += a->maskbits; - if (a->policy_type == ADDR_POLICY_REJECT) - r ^= 0xffffffff; + const addr_policy_t *a = ent->policy; + addr_policy_t aa; + memset(&aa, 0, sizeof(aa)); + + aa.prt_min = a->prt_min; + aa.prt_max = a->prt_max; + aa.maskbits = a->maskbits; + aa.policy_type = a->policy_type; + aa.is_private = a->is_private; + + if (a->is_private) { + aa.is_private = 1; + } else { + tor_addr_copy_tight(&aa.addr, &a->addr); + } - return r; + return (unsigned) siphash24g(&aa, sizeof(aa)); } HT_PROTOTYPE(policy_map, policy_map_ent_t, node, policy_hash, policy_eq) -HT_GENERATE(policy_map, policy_map_ent_t, node, policy_hash, - policy_eq, 0.6, malloc, realloc, free) +HT_GENERATE2(policy_map, policy_map_ent_t, node, policy_hash, + policy_eq, 0.6, tor_reallocarray_, tor_free_) /** Given a pointer to an addr_policy_t, return a copy of the pointer to the * "canonical" copy of that addr_policy_t; the canonical copy is a single @@ -653,6 +1326,10 @@ compare_known_tor_addr_to_addr_policy(const tor_addr_t *addr, uint16_t port, /* We know the address and port, and we know the policy, so we can just * compute an exact match. */ SMARTLIST_FOREACH_BEGIN(policy, addr_policy_t *, tmpe) { + if (tmpe->addr.family == AF_UNSPEC) { + log_warn(LD_BUG, "Policy contains an AF_UNSPEC address, which only " + "matches other AF_UNSPEC addresses."); + } /* Address is known */ if (!tor_addr_compare_masked(addr, &tmpe->addr, tmpe->maskbits, CMP_EXACT)) { @@ -680,6 +1357,10 @@ compare_known_tor_addr_to_addr_policy_noport(const tor_addr_t *addr, int maybe_accept = 0, maybe_reject = 0; SMARTLIST_FOREACH_BEGIN(policy, addr_policy_t *, tmpe) { + if (tmpe->addr.family == AF_UNSPEC) { + log_warn(LD_BUG, "Policy contains an AF_UNSPEC address, which only " + "matches other AF_UNSPEC addresses."); + } if (!tor_addr_compare_masked(addr, &tmpe->addr, tmpe->maskbits, CMP_EXACT)) { if (tmpe->prt_min <= 1 && tmpe->prt_max >= 65535) { @@ -719,6 +1400,10 @@ compare_unknown_tor_addr_to_addr_policy(uint16_t port, int maybe_accept = 0, maybe_reject = 0; SMARTLIST_FOREACH_BEGIN(policy, addr_policy_t *, tmpe) { + if (tmpe->addr.family == AF_UNSPEC) { + log_warn(LD_BUG, "Policy contains an AF_UNSPEC address, which only " + "matches other AF_UNSPEC addresses."); + } if (tmpe->prt_min <= port && port <= tmpe->prt_max) { if (tmpe->maskbits == 0) { /* Definitely matches, since it covers all addresses. */ @@ -756,9 +1441,9 @@ compare_unknown_tor_addr_to_addr_policy(uint16_t port, * We could do better by assuming that some ranges never match typical * addresses (127.0.0.1, and so on). But we'll try this for now. */ -addr_policy_result_t -compare_tor_addr_to_addr_policy(const tor_addr_t *addr, uint16_t port, - const smartlist_t *policy) +MOCK_IMPL(addr_policy_result_t, +compare_tor_addr_to_addr_policy,(const tor_addr_t *addr, uint16_t port, + const smartlist_t *policy)) { if (!policy) { /* no policy? accept all. */ @@ -824,7 +1509,7 @@ addr_policy_intersects(addr_policy_t *a, addr_policy_t *b) /** Add the exit policy described by <b>more</b> to <b>policy</b>. */ -static void +STATIC void append_exit_policy_string(smartlist_t **policy, const char *more) { config_line_t tmp; @@ -841,6 +1526,9 @@ append_exit_policy_string(smartlist_t **policy, const char *more) void addr_policy_append_reject_addr(smartlist_t **dest, const tor_addr_t *addr) { + tor_assert(dest); + tor_assert(addr); + addr_policy_t p, *add; memset(&p, 0, sizeof(p)); p.policy_type = ADDR_POLICY_REJECT; @@ -853,6 +1541,71 @@ addr_policy_append_reject_addr(smartlist_t **dest, const tor_addr_t *addr) if (!*dest) *dest = smartlist_new(); smartlist_add(*dest, add); + log_debug(LD_CONFIG, "Adding a reject ExitPolicy 'reject %s:*'", + fmt_addr(addr)); +} + +/* Is addr public for the purposes of rejection? */ +static int +tor_addr_is_public_for_reject(const tor_addr_t *addr) +{ + return (!tor_addr_is_null(addr) && !tor_addr_is_internal(addr, 0) + && !tor_addr_is_multicast(addr)); +} + +/* Add "reject <b>addr</b>:*" to <b>dest</b>, creating the list as needed. + * Filter the address, only adding an IPv4 reject rule if ipv4_rules + * is true, and similarly for ipv6_rules. Check each address returns true for + * tor_addr_is_public_for_reject before adding it. + */ +static void +addr_policy_append_reject_addr_filter(smartlist_t **dest, + const tor_addr_t *addr, + int ipv4_rules, + int ipv6_rules) +{ + tor_assert(dest); + tor_assert(addr); + + /* Only reject IP addresses which are public */ + if (tor_addr_is_public_for_reject(addr)) { + + /* Reject IPv4 addresses and IPv6 addresses based on the filters */ + int is_ipv4 = tor_addr_is_v4(addr); + if ((is_ipv4 && ipv4_rules) || (!is_ipv4 && ipv6_rules)) { + addr_policy_append_reject_addr(dest, addr); + } + } +} + +/** Add "reject addr:*" to <b>dest</b>, for each addr in addrs, creating the + * list as needed. */ +void +addr_policy_append_reject_addr_list(smartlist_t **dest, + const smartlist_t *addrs) +{ + tor_assert(dest); + tor_assert(addrs); + + SMARTLIST_FOREACH_BEGIN(addrs, tor_addr_t *, addr) { + addr_policy_append_reject_addr(dest, addr); + } SMARTLIST_FOREACH_END(addr); +} + +/** Add "reject addr:*" to <b>dest</b>, for each addr in addrs, creating the + * list as needed. Filter using */ +static void +addr_policy_append_reject_addr_list_filter(smartlist_t **dest, + const smartlist_t *addrs, + int ipv4_rules, + int ipv6_rules) +{ + tor_assert(dest); + tor_assert(addrs); + + SMARTLIST_FOREACH_BEGIN(addrs, tor_addr_t *, addr) { + addr_policy_append_reject_addr_filter(dest, addr, ipv4_rules, ipv6_rules); + } SMARTLIST_FOREACH_END(addr); } /** Detect and excise "dead code" from the policy *<b>dest</b>. */ @@ -939,41 +1692,197 @@ exit_policy_remove_redundancies(smartlist_t *dest) } } +/** Reject private helper for policies_parse_exit_policy_internal: rejects + * publicly routable addresses on this exit relay. + * + * Add reject entries to the linked list *<b>dest</b>: + * <ul> + * <li>if configured_addresses is non-NULL, add entries that reject each + * tor_addr_t in the list as a destination. + * <li>if reject_interface_addresses is true, add entries that reject each + * public IPv4 and IPv6 address of each interface on this machine. + * <li>if reject_configured_port_addresses is true, add entries that reject + * each IPv4 and IPv6 address configured for a port. + * </ul> + * + * IPv6 entries are only added if ipv6_exit is true. (All IPv6 addresses are + * already blocked by policies_parse_exit_policy_internal if ipv6_exit is + * false.) + * + * The list in <b>dest</b> is created as needed. + */ +void +policies_parse_exit_policy_reject_private( + smartlist_t **dest, + int ipv6_exit, + const smartlist_t *configured_addresses, + int reject_interface_addresses, + int reject_configured_port_addresses) +{ + tor_assert(dest); + + /* Reject configured addresses, if they are from public netblocks. */ + if (configured_addresses) { + addr_policy_append_reject_addr_list_filter(dest, configured_addresses, + 1, ipv6_exit); + } + + /* Reject configured port addresses, if they are from public netblocks. */ + if (reject_configured_port_addresses) { + const smartlist_t *port_addrs = get_configured_ports(); + + SMARTLIST_FOREACH_BEGIN(port_addrs, port_cfg_t *, port) { + + /* Only reject port IP addresses, not port unix sockets */ + if (!port->is_unix_addr) { + addr_policy_append_reject_addr_filter(dest, &port->addr, 1, ipv6_exit); + } + } SMARTLIST_FOREACH_END(port); + } + + /* Reject local addresses from public netblocks on any interface. */ + if (reject_interface_addresses) { + smartlist_t *public_addresses = NULL; + + /* Reject public IPv4 addresses on any interface */ + public_addresses = get_interface_address6_list(LOG_INFO, AF_INET, 0); + addr_policy_append_reject_addr_list_filter(dest, public_addresses, 1, 0); + free_interface_address6_list(public_addresses); + + /* Don't look for IPv6 addresses if we're configured as IPv4-only */ + if (ipv6_exit) { + /* Reject public IPv6 addresses on any interface */ + public_addresses = get_interface_address6_list(LOG_INFO, AF_INET6, 0); + addr_policy_append_reject_addr_list_filter(dest, public_addresses, 0, 1); + free_interface_address6_list(public_addresses); + } + } + + /* If addresses were added multiple times, remove all but one of them. */ + if (*dest) { + exit_policy_remove_redundancies(*dest); + } +} + +/** + * Iterate through <b>policy</b> looking for redundant entries. Log a + * warning message with the first redundant entry, if any is found. + */ +static void +policies_log_first_redundant_entry(const smartlist_t *policy) +{ + int found_final_effective_entry = 0; + int first_redundant_entry = 0; + tor_assert(policy); + SMARTLIST_FOREACH_BEGIN(policy, const addr_policy_t *, p) { + sa_family_t family; + int found_ipv4_wildcard = 0, found_ipv6_wildcard = 0; + const int i = p_sl_idx; + + /* Look for accept/reject *[4|6|]:* entires */ + if (p->prt_min <= 1 && p->prt_max == 65535 && p->maskbits == 0) { + family = tor_addr_family(&p->addr); + /* accept/reject *:* may have already been expanded into + * accept/reject *4:*,accept/reject *6:* + * But handle both forms. + */ + if (family == AF_INET || family == AF_UNSPEC) { + found_ipv4_wildcard = 1; + } + if (family == AF_INET6 || family == AF_UNSPEC) { + found_ipv6_wildcard = 1; + } + } + + /* We also find accept *4:*,reject *6:* ; and + * accept *4:*,<other policies>,accept *6:* ; and similar. + * That's ok, because they make any subsequent entries redundant. */ + if (found_ipv4_wildcard && found_ipv6_wildcard) { + found_final_effective_entry = 1; + /* if we're not on the final entry in the list */ + if (i < smartlist_len(policy) - 1) { + first_redundant_entry = i + 1; + } + break; + } + } SMARTLIST_FOREACH_END(p); + + /* Work out if there are redundant trailing entries in the policy list */ + if (found_final_effective_entry && first_redundant_entry > 0) { + const addr_policy_t *p; + /* Longest possible policy is + * "accept6 ffff:ffff:..255/128:10000-65535", + * which contains a max-length IPv6 address, plus 24 characters. */ + char line[TOR_ADDR_BUF_LEN + 32]; + + tor_assert(first_redundant_entry < smartlist_len(policy)); + p = smartlist_get(policy, first_redundant_entry); + /* since we've already parsed the policy into an addr_policy_t struct, + * we might not log exactly what the user typed in */ + policy_write_item(line, TOR_ADDR_BUF_LEN + 32, p, 0); + log_warn(LD_DIR, "Exit policy '%s' and all following policies are " + "redundant, as it follows accept/reject *:* rules for both " + "IPv4 and IPv6. They will be removed from the exit policy. (Use " + "accept/reject *:* as the last entry in any exit policy.)", + line); + } +} + #define DEFAULT_EXIT_POLICY \ "reject *:25,reject *:119,reject *:135-139,reject *:445," \ "reject *:563,reject *:1214,reject *:4661-4666," \ "reject *:6346-6429,reject *:6699,reject *:6881-6999,accept *:*" -/** Parse the exit policy <b>cfg</b> into the linked list *<b>dest</b>. If - * cfg doesn't end in an absolute accept or reject and if +/** Parse the exit policy <b>cfg</b> into the linked list *<b>dest</b>. + * + * If <b>ipv6_exit</b> is false, prepend "reject *6:*" to the policy. + * + * If <b>rejectprivate</b> is true: + * - prepend "reject private:*" to the policy. + * - prepend entries that reject publicly routable addresses on this exit + * relay by calling policies_parse_exit_policy_reject_private + * + * If cfg doesn't end in an absolute accept or reject and if * <b>add_default_policy</b> is true, add the default exit - * policy afterwards. If <b>rejectprivate</b> is true, prepend - * "reject private:*" to the policy. Return -1 if we can't parse cfg, - * else return 0. + * policy afterwards. + * + * Return -1 if we can't parse cfg, else return 0. * * This function is used to parse the exit policy from our torrc. For * the functions used to parse the exit policy from a router descriptor, * see router_add_exit_policy. */ -int -policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int ipv6_exit, - int rejectprivate, const char *local_address, - int add_default_policy) +static int +policies_parse_exit_policy_internal(config_line_t *cfg, + smartlist_t **dest, + int ipv6_exit, + int rejectprivate, + const smartlist_t *configured_addresses, + int reject_interface_addresses, + int reject_configured_port_addresses, + int add_default_policy) { if (!ipv6_exit) { append_exit_policy_string(dest, "reject *6:*"); } if (rejectprivate) { + /* Reject IPv4 and IPv6 reserved private netblocks */ append_exit_policy_string(dest, "reject private:*"); - if (local_address) { - char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject %s:*", local_address); - append_exit_policy_string(dest, buf); - } + /* Reject IPv4 and IPv6 publicly routable addresses on this exit relay */ + policies_parse_exit_policy_reject_private( + dest, ipv6_exit, + configured_addresses, + reject_interface_addresses, + reject_configured_port_addresses); } if (parse_addr_policy(cfg, dest, -1)) return -1; + + /* Before we add the default policy and final rejects, check to see if + * there are any lines after accept *:* or reject *:*. These lines have no + * effect, and are most likely an error. */ + policies_log_first_redundant_entry(*dest); + if (add_default_policy) { append_exit_policy_string(dest, DEFAULT_EXIT_POLICY); } else { @@ -985,6 +1894,157 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, return 0; } +/** Parse exit policy in <b>cfg</b> into <b>dest</b> smartlist. + * + * Prepend an entry that rejects all IPv6 destinations unless + * <b>EXIT_POLICY_IPV6_ENABLED</b> bit is set in <b>options</b> bitmask. + * + * If <b>EXIT_POLICY_REJECT_PRIVATE</b> bit is set in <b>options</b>: + * - prepend an entry that rejects all destinations in all netblocks + * reserved for private use. + * - prepend entries that reject publicly routable addresses on this exit + * relay by calling policies_parse_exit_policy_internal + * + * If <b>EXIT_POLICY_ADD_DEFAULT</b> bit is set in <b>options</b>, append + * default exit policy entries to <b>result</b> smartlist. + */ +int +policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, + exit_policy_parser_cfg_t options, + const smartlist_t *configured_addresses) +{ + int ipv6_enabled = (options & EXIT_POLICY_IPV6_ENABLED) ? 1 : 0; + int reject_private = (options & EXIT_POLICY_REJECT_PRIVATE) ? 1 : 0; + int add_default = (options & EXIT_POLICY_ADD_DEFAULT) ? 1 : 0; + + return policies_parse_exit_policy_internal(cfg,dest,ipv6_enabled, + reject_private, + configured_addresses, + reject_private, + reject_private, + add_default); +} + +/** Helper function that adds a copy of addr to a smartlist as long as it is + * non-NULL and not tor_addr_is_null(). + * + * The caller is responsible for freeing all the tor_addr_t* in the smartlist. + */ +static void +policies_copy_addr_to_smartlist(smartlist_t *addr_list, const tor_addr_t *addr) +{ + if (addr && !tor_addr_is_null(addr)) { + tor_addr_t *addr_copy = tor_malloc(sizeof(tor_addr_t)); + tor_addr_copy(addr_copy, addr); + smartlist_add(addr_list, addr_copy); + } +} + +/** Helper function that adds ipv4h_addr to a smartlist as a tor_addr_t *, + * as long as it is not tor_addr_is_null(), by converting it to a tor_addr_t + * and passing it to policies_add_addr_to_smartlist. + * + * The caller is responsible for freeing all the tor_addr_t* in the smartlist. + */ +static void +policies_copy_ipv4h_to_smartlist(smartlist_t *addr_list, uint32_t ipv4h_addr) +{ + if (ipv4h_addr) { + tor_addr_t ipv4_tor_addr; + tor_addr_from_ipv4h(&ipv4_tor_addr, ipv4h_addr); + policies_copy_addr_to_smartlist(addr_list, &ipv4_tor_addr); + } +} + +/** Helper function that adds copies of + * or_options->OutboundBindAddressIPv[4|6]_ to a smartlist as tor_addr_t *, as + * long as or_options is non-NULL, and the addresses are not + * tor_addr_is_null(), by passing them to policies_add_addr_to_smartlist. + * + * The caller is responsible for freeing all the tor_addr_t* in the smartlist. + */ +static void +policies_copy_outbound_addresses_to_smartlist(smartlist_t *addr_list, + const or_options_t *or_options) +{ + if (or_options) { + policies_copy_addr_to_smartlist(addr_list, + &or_options->OutboundBindAddressIPv4_); + policies_copy_addr_to_smartlist(addr_list, + &or_options->OutboundBindAddressIPv6_); + } +} + +/** Parse <b>ExitPolicy</b> member of <b>or_options</b> into <b>result</b> + * smartlist. + * If <b>or_options->IPv6Exit</b> is false, prepend an entry that + * rejects all IPv6 destinations. + * + * If <b>or_options->ExitPolicyRejectPrivate</b> is true: + * - prepend an entry that rejects all destinations in all netblocks reserved + * for private use. + * - if local_address is non-zero, treat it as a host-order IPv4 address, and + * add it to the list of configured addresses. + * - if ipv6_local_address is non-NULL, and not the null tor_addr_t, add it + * to the list of configured addresses. + * - if or_options->OutboundBindAddressIPv4_ is not the null tor_addr_t, add + * it to the list of configured addresses. + * - if or_options->OutboundBindAddressIPv6_ is not the null tor_addr_t, add + * it to the list of configured addresses. + * + * If <b>or_options->BridgeRelay</b> is false, append entries of default + * Tor exit policy into <b>result</b> smartlist. + * + * If or_options->ExitRelay is false, then make our exit policy into + * "reject *:*" regardless. + */ +int +policies_parse_exit_policy_from_options(const or_options_t *or_options, + uint32_t local_address, + const tor_addr_t *ipv6_local_address, + smartlist_t **result) +{ + exit_policy_parser_cfg_t parser_cfg = 0; + smartlist_t *configured_addresses = NULL; + int rv = 0; + + /* Short-circuit for non-exit relays */ + if (or_options->ExitRelay == 0) { + append_exit_policy_string(result, "reject *4:*"); + append_exit_policy_string(result, "reject *6:*"); + return 0; + } + + configured_addresses = smartlist_new(); + + /* Configure the parser */ + if (or_options->IPv6Exit) { + parser_cfg |= EXIT_POLICY_IPV6_ENABLED; + } + + if (or_options->ExitPolicyRejectPrivate) { + parser_cfg |= EXIT_POLICY_REJECT_PRIVATE; + } + + if (!or_options->BridgeRelay) { + parser_cfg |= EXIT_POLICY_ADD_DEFAULT; + } + + /* Copy the configured addresses into the tor_addr_t* list */ + policies_copy_ipv4h_to_smartlist(configured_addresses, local_address); + policies_copy_addr_to_smartlist(configured_addresses, ipv6_local_address); + policies_copy_outbound_addresses_to_smartlist(configured_addresses, + or_options); + + rv = policies_parse_exit_policy(or_options->ExitPolicy, result, parser_cfg, + configured_addresses); + + SMARTLIST_FOREACH(configured_addresses, tor_addr_t *, a, tor_free(a)); + smartlist_free(configured_addresses); + + return rv; +} + /** Add "reject *:*" to the end of the policy in *<b>dest</b>, allocating * *<b>dest</b> as needed. */ void @@ -1090,7 +2150,7 @@ policy_is_reject_star(const smartlist_t *policy, sa_family_t family) /** Write a single address policy to the buf_len byte buffer at buf. Return * the number of characters written, or -1 on failure. */ int -policy_write_item(char *buf, size_t buflen, addr_policy_t *policy, +policy_write_item(char *buf, size_t buflen, const addr_policy_t *policy, int format_for_desc) { size_t written = 0; @@ -1126,9 +2186,9 @@ policy_write_item(char *buf, size_t buflen, addr_policy_t *policy, if (result < 0) return -1; written += strlen(buf); - /* If the maskbits is 32 we don't need to give it. If the mask is 0, - * we already wrote "*". */ - if (policy->maskbits < 32 && policy->maskbits > 0) { + /* If the maskbits is 32 (IPv4) or 128 (IPv6) we don't need to give it. If + the mask is 0, we already wrote "*". */ + if (policy->maskbits < (is_ip6?128:32) && policy->maskbits > 0) { if (tor_snprintf(buf+written, buflen-written, "/%d", policy->maskbits)<0) return -1; written += strlen(buf+written); @@ -1321,9 +2381,9 @@ policy_summary_add_item(smartlist_t *summary, addr_policy_t *p) * The summary will either be an "accept" plus a comma-separated list of port * ranges or a "reject" plus port-ranges, depending on which is shorter. * - * If no exits are allowed at all then NULL is returned, if no ports - * are blocked instead of "reject " we return "accept 1-65535" (this - * is an exception to the shorter-representation-wins rule). + * If no exits are allowed at all then "reject 1-65535" is returned. If no + * ports are blocked instead of "reject " we return "accept 1-65535". (These + * are an exception to the shorter-representation-wins rule). */ char * policy_summarize(smartlist_t *policy, sa_family_t family) @@ -1608,7 +2668,7 @@ compare_tor_addr_to_short_policy(const tor_addr_t *addr, uint16_t port, * allows exit enclaving. Trying it anyway would open up a cool attack * where the node refuses due to exitpolicy, the client reacts in * surprise by rewriting the node's exitpolicy to reject *:*, and then - * a bad guy targets users by causing them to attempt such connections + * an adversary targets users by causing them to attempt such connections * to 98% of the exits. * * Once microdescriptors can handle addresses in special cases (e.g. if @@ -1669,6 +2729,53 @@ compare_tor_addr_to_node_policy(const tor_addr_t *addr, uint16_t port, } } +/** + * Given <b>policy_list</b>, a list of addr_policy_t, produce a string + * representation of the list. + * If <b>include_ipv4</b> is true, include IPv4 entries. + * If <b>include_ipv6</b> is true, include IPv6 entries. + */ +char * +policy_dump_to_string(const smartlist_t *policy_list, + int include_ipv4, + int include_ipv6) +{ + smartlist_t *policy_string_list; + char *policy_string = NULL; + + policy_string_list = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(policy_list, addr_policy_t *, tmpe) { + char *pbuf; + int bytes_written_to_pbuf; + if ((tor_addr_family(&tmpe->addr) == AF_INET6) && (!include_ipv6)) { + continue; /* Don't include IPv6 parts of address policy */ + } + if ((tor_addr_family(&tmpe->addr) == AF_INET) && (!include_ipv4)) { + continue; /* Don't include IPv4 parts of address policy */ + } + + pbuf = tor_malloc(POLICY_BUF_LEN); + bytes_written_to_pbuf = policy_write_item(pbuf,POLICY_BUF_LEN, tmpe, 1); + + if (bytes_written_to_pbuf < 0) { + log_warn(LD_BUG, "policy_dump_to_string ran out of room!"); + tor_free(pbuf); + goto done; + } + + smartlist_add(policy_string_list,pbuf); + } SMARTLIST_FOREACH_END(tmpe); + + policy_string = smartlist_join_strings(policy_string_list, "\n", 0, NULL); + + done: + SMARTLIST_FOREACH(policy_string_list, char *, str, tor_free(str)); + smartlist_free(policy_string_list); + + return policy_string; +} + /** Implementation for GETINFO control command: knows the answer for questions * about "exit-policy/..." */ int @@ -1680,6 +2787,79 @@ getinfo_helper_policies(control_connection_t *conn, (void) errmsg; if (!strcmp(question, "exit-policy/default")) { *answer = tor_strdup(DEFAULT_EXIT_POLICY); + } else if (!strcmp(question, "exit-policy/reject-private/default")) { + smartlist_t *private_policy_strings; + const char **priv = private_nets; + + private_policy_strings = smartlist_new(); + + while (*priv != NULL) { + /* IPv6 addresses are in "[]" and contain ":", + * IPv4 addresses are not in "[]" and contain "." */ + smartlist_add_asprintf(private_policy_strings, "reject %s:*", *priv); + priv++; + } + + *answer = smartlist_join_strings(private_policy_strings, + ",", 0, NULL); + + SMARTLIST_FOREACH(private_policy_strings, char *, str, tor_free(str)); + smartlist_free(private_policy_strings); + } else if (!strcmp(question, "exit-policy/reject-private/relay")) { + const or_options_t *options = get_options(); + const routerinfo_t *me = router_get_my_routerinfo(); + + if (!me) { + *errmsg = "router_get_my_routerinfo returned NULL"; + return -1; + } + + if (!options->ExitPolicyRejectPrivate) { + *answer = tor_strdup(""); + return 0; + } + + smartlist_t *private_policy_list = smartlist_new(); + smartlist_t *configured_addresses = smartlist_new(); + + /* Copy the configured addresses into the tor_addr_t* list */ + policies_copy_ipv4h_to_smartlist(configured_addresses, me->addr); + policies_copy_addr_to_smartlist(configured_addresses, &me->ipv6_addr); + policies_copy_outbound_addresses_to_smartlist(configured_addresses, + options); + + policies_parse_exit_policy_reject_private( + &private_policy_list, + options->IPv6Exit, + configured_addresses, + 1, 1); + *answer = policy_dump_to_string(private_policy_list, 1, 1); + + addr_policy_list_free(private_policy_list); + SMARTLIST_FOREACH(configured_addresses, tor_addr_t *, a, tor_free(a)); + smartlist_free(configured_addresses); + } else if (!strcmpstart(question, "exit-policy/")) { + const routerinfo_t *me = router_get_my_routerinfo(); + + int include_ipv4 = 0; + int include_ipv6 = 0; + + if (!strcmp(question, "exit-policy/ipv4")) { + include_ipv4 = 1; + } else if (!strcmp(question, "exit-policy/ipv6")) { + include_ipv6 = 1; + } else if (!strcmp(question, "exit-policy/full")) { + include_ipv4 = include_ipv6 = 1; + } else { + return 0; /* No such key. */ + } + + if (!me) { + *errmsg = "router_get_my_routerinfo returned NULL"; + return -1; + } + + *answer = router_dump_exit_policy_to_string(me,include_ipv4,include_ipv6); } return 0; } @@ -1731,8 +2911,6 @@ policies_free_all(void) authdir_reject_policy = NULL; addr_policy_list_free(authdir_invalid_policy); authdir_invalid_policy = NULL; - addr_policy_list_free(authdir_baddir_policy); - authdir_baddir_policy = NULL; addr_policy_list_free(authdir_badexit_policy); authdir_badexit_policy = NULL; diff --git a/src/or/policies.h b/src/or/policies.h index facbbb6b5a..aaa6fa0a4e 100644 --- a/src/or/policies.h +++ b/src/or/policies.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -18,16 +18,52 @@ */ #define POLICY_BUF_LEN 72 +#define EXIT_POLICY_IPV6_ENABLED (1 << 0) +#define EXIT_POLICY_REJECT_PRIVATE (1 << 1) +#define EXIT_POLICY_ADD_DEFAULT (1 << 2) + +typedef enum firewall_connection_t { + FIREWALL_OR_CONNECTION = 0, + FIREWALL_DIR_CONNECTION = 1 +} firewall_connection_t; + +typedef int exit_policy_parser_cfg_t; + int firewall_is_fascist_or(void); -int fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port); -int fascist_firewall_allows_or(const routerinfo_t *ri); -int fascist_firewall_allows_node(const node_t *node); -int fascist_firewall_allows_address_dir(const tor_addr_t *addr, uint16_t port); +int firewall_is_fascist_dir(void); +int fascist_firewall_use_ipv6(const or_options_t *options); +int fascist_firewall_prefer_ipv6_orport(const or_options_t *options); +int fascist_firewall_prefer_ipv6_dirport(const or_options_t *options); + +int fascist_firewall_allows_address_addr(const tor_addr_t *addr, + uint16_t port, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6); + +int fascist_firewall_allows_rs(const routerstatus_t *rs, + firewall_connection_t fw_connection, + int pref_only); +int fascist_firewall_allows_node(const node_t *node, + firewall_connection_t fw_connection, + int pref_only); +int fascist_firewall_allows_dir_server(const dir_server_t *ds, + firewall_connection_t fw_connection, + int pref_only); + +int fascist_firewall_choose_address_rs(const routerstatus_t *rs, + firewall_connection_t fw_connection, + int pref_only, tor_addr_port_t* ap); +int fascist_firewall_choose_address_node(const node_t *node, + firewall_connection_t fw_connection, + int pref_only, tor_addr_port_t* ap); +int fascist_firewall_choose_address_dir_server(const dir_server_t *ds, + firewall_connection_t fw_connection, + int pref_only, tor_addr_port_t* ap); + int dir_policy_permits_address(const tor_addr_t *addr); int socks_policy_permits_address(const tor_addr_t *addr); int authdir_policy_permits_address(uint32_t addr, uint16_t port); int authdir_policy_valid_address(uint32_t addr, uint16_t port); -int authdir_policy_baddir_address(uint32_t addr, uint16_t port); int authdir_policy_badexit_address(uint32_t addr, uint16_t port); int validate_addr_policies(const or_options_t *options, char **msg); @@ -37,26 +73,40 @@ int policies_parse_from_options(const or_options_t *options); addr_policy_t *addr_policy_get_canonical_entry(addr_policy_t *ent); int cmp_addr_policies(smartlist_t *a, smartlist_t *b); -addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, - uint16_t port, const smartlist_t *policy); - +MOCK_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy, + (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy)); addr_policy_result_t compare_tor_addr_to_node_policy(const tor_addr_t *addr, uint16_t port, const node_t *node); +int policies_parse_exit_policy_from_options( + const or_options_t *or_options, + uint32_t local_address, + const tor_addr_t *ipv6_local_address, + smartlist_t **result); int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int ipv6exit, - int rejectprivate, const char *local_address, - int add_default_policy); + exit_policy_parser_cfg_t options, + const smartlist_t *configured_addresses); +void policies_parse_exit_policy_reject_private( + smartlist_t **dest, + int ipv6_exit, + const smartlist_t *configured_addresses, + int reject_interface_addresses, + int reject_configured_port_addresses); void policies_exit_policy_append_reject_star(smartlist_t **dest); void addr_policy_append_reject_addr(smartlist_t **dest, const tor_addr_t *addr); +void addr_policy_append_reject_addr_list(smartlist_t **dest, + const smartlist_t *addrs); void policies_set_node_exitpolicy_to_reject_all(node_t *exitrouter); int exit_policy_is_general_exit(smartlist_t *policy); int policy_is_reject_star(const smartlist_t *policy, sa_family_t family); +char * policy_dump_to_string(const smartlist_t *policy_list, + int include_ipv4, + int include_ipv6); int getinfo_helper_policies(control_connection_t *conn, const char *question, char **answer, const char **errmsg); -int policy_write_item(char *buf, size_t buflen, addr_policy_t *item, +int policy_write_item(char *buf, size_t buflen, const addr_policy_t *item, int format_for_desc); void addr_policy_list_free(smartlist_t *p); @@ -73,5 +123,20 @@ addr_policy_result_t compare_tor_addr_to_short_policy( const tor_addr_t *addr, uint16_t port, const short_policy_t *policy); +#ifdef POLICIES_PRIVATE +STATIC void append_exit_policy_string(smartlist_t **policy, const char *more); +STATIC int fascist_firewall_allows_address(const tor_addr_t *addr, + uint16_t port, + smartlist_t *firewall_policy, + int pref_only, int pref_ipv6); +STATIC const tor_addr_port_t * fascist_firewall_choose_address( + const tor_addr_port_t *a, + const tor_addr_port_t *b, + int want_a, + firewall_connection_t fw_connection, + int pref_only, int pref_ipv6); + +#endif + #endif diff --git a/src/or/reasons.c b/src/or/reasons.c index 0674474e72..36921cafcd 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -231,6 +231,8 @@ orconn_end_reason_to_control_string(int r) return "RESOURCELIMIT"; case END_OR_CONN_REASON_MISC: return "MISC"; + case END_OR_CONN_REASON_PT_MISSING: + return "PT_MISSING"; case 0: return ""; default: @@ -348,6 +350,8 @@ circuit_end_reason_to_control_string(int reason) return "NOSUCHSERVICE"; case END_CIRC_REASON_MEASUREMENT_EXPIRED: return "MEASUREMENT_EXPIRED"; + case END_CIRC_REASON_IP_NOW_REDUNDANT: + return "IP_NOW_REDUNDANT"; default: if (is_remote) { /* @@ -365,7 +369,7 @@ circuit_end_reason_to_control_string(int reason) } } -/** Return a string corresponding to a SOCKS4 reponse code. */ +/** Return a string corresponding to a SOCKS4 response code. */ const char * socks4_response_code_to_string(uint8_t code) { @@ -383,7 +387,7 @@ socks4_response_code_to_string(uint8_t code) } } -/** Return a string corresponding to a SOCKS5 reponse code. */ +/** Return a string corresponding to a SOCKS5 response code. */ const char * socks5_response_code_to_string(uint8_t code) { diff --git a/src/or/reasons.h b/src/or/reasons.h index fe7e67722a..2e12c93728 100644 --- a/src/or/reasons.h +++ b/src/or/reasons.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/relay.c b/src/or/relay.c index 7f06c6e145..3f7751826c 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,6 +15,7 @@ #include "addressmap.h" #include "buffers.h" #include "channel.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -25,17 +26,18 @@ #include "control.h" #include "geoip.h" #include "main.h" -#include "mempool.h" #include "networkstatus.h" #include "nodelist.h" #include "onion.h" #include "policies.h" #include "reasons.h" #include "relay.h" +#include "rendcache.h" #include "rendcommon.h" #include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "scheduler.h" static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -58,6 +60,9 @@ static void adjust_exit_policy_from_exitpolicy_failure(origin_circuit_t *circ, entry_connection_t *conn, node_t *node, const tor_addr_t *addr); +#if 0 +static int get_max_middle_cells(void); +#endif /** Stop reading on edge connections when we have this many cells * waiting on the appropriate queue. */ @@ -105,14 +110,14 @@ relay_set_digest(crypto_digest_t *digest, cell_t *cell) static int relay_digest_matches(crypto_digest_t *digest, cell_t *cell) { - char received_integrity[4], calculated_integrity[4]; + uint32_t received_integrity, calculated_integrity; relay_header_t rh; crypto_digest_t *backup_digest=NULL; backup_digest = crypto_digest_dup(digest); relay_header_unpack(&rh, cell->payload); - memcpy(received_integrity, rh.integrity, 4); + memcpy(&received_integrity, rh.integrity, 4); memset(rh.integrity, 0, 4); relay_header_pack(cell->payload, &rh); @@ -121,15 +126,15 @@ relay_digest_matches(crypto_digest_t *digest, cell_t *cell) // received_integrity[2], received_integrity[3]); crypto_digest_add_bytes(digest, (char*) cell->payload, CELL_PAYLOAD_SIZE); - crypto_digest_get_digest(digest, calculated_integrity, 4); + crypto_digest_get_digest(digest, (char*) &calculated_integrity, 4); - if (tor_memneq(received_integrity, calculated_integrity, 4)) { + if (calculated_integrity != received_integrity) { // log_fn(LOG_INFO,"Recognized=0 but bad digest. Not recognizing."); // (%d vs %d).", received_integrity, calculated_integrity); /* restore digest to its old form */ crypto_digest_assign(digest, backup_digest); /* restore the relay header */ - memcpy(rh.integrity, received_integrity, 4); + memcpy(rh.integrity, &received_integrity, 4); relay_header_pack(cell->payload, &rh); crypto_digest_free(backup_digest); return 0; @@ -143,20 +148,15 @@ relay_digest_matches(crypto_digest_t *digest, cell_t *cell) * * If <b>encrypt_mode</b> is 1 then encrypt, else decrypt. * - * Return -1 if the crypto fails, else return 0. + * Returns 0. */ static int relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in, int encrypt_mode) { - int r; (void)encrypt_mode; - r = crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE); + crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE); - if (r) { - log_warn(LD_BUG,"Error during relay encryption"); - return -1; - } return 0; } @@ -204,8 +204,7 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, return 0; } - conn = relay_lookup_conn(circ, cell, cell_direction, - layer_hint); + conn = relay_lookup_conn(circ, cell, cell_direction, layer_hint); if (cell_direction == CELL_DIRECTION_OUT) { ++stats_n_relay_cells_delivered; log_debug(LD_OR,"Sending away from origin."); @@ -517,6 +516,7 @@ relay_header_unpack(relay_header_t *dest, const uint8_t *src) static const char * relay_command_to_string(uint8_t command) { + static char buf[64]; switch (command) { case RELAY_COMMAND_BEGIN: return "BEGIN"; case RELAY_COMMAND_DATA: return "DATA"; @@ -541,7 +541,12 @@ relay_command_to_string(uint8_t command) case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED: return "RENDEZVOUS_ESTABLISHED"; case RELAY_COMMAND_INTRODUCE_ACK: return "INTRODUCE_ACK"; - default: return "(unrecognized)"; + case RELAY_COMMAND_EXTEND2: return "EXTEND2"; + case RELAY_COMMAND_EXTENDED2: return "EXTENDED2"; + default: + tor_snprintf(buf, sizeof(buf), "Unrecognized relay command %u", + (unsigned)command); + return buf; } } @@ -791,8 +796,10 @@ connection_ap_process_end_not_open( return 0; } - if ((tor_addr_family(&addr) == AF_INET && !conn->ipv4_traffic_ok) || - (tor_addr_family(&addr) == AF_INET6 && !conn->ipv6_traffic_ok)) { + if ((tor_addr_family(&addr) == AF_INET && + !conn->entry_cfg.ipv4_traffic) || + (tor_addr_family(&addr) == AF_INET6 && + !conn->entry_cfg.ipv6_traffic)) { log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got an EXITPOLICY failure on a connection with a " "mismatched family. Closing."); @@ -821,7 +828,7 @@ connection_ap_process_end_not_open( } } } - /* check if he *ought* to have allowed it */ + /* check if the exit *ought* to have allowed it */ adjust_exit_policy_from_exitpolicy_failure(circ, conn, @@ -968,7 +975,7 @@ remap_event_helper(entry_connection_t *conn, const tor_addr_t *new_addr) * <b>addr_out</b> to the address we're connected to, and <b>ttl_out</b> to * the ttl of that address, in seconds, and return 0. On failure, return * -1. */ -int +STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out) { @@ -1005,6 +1012,254 @@ connected_cell_parse(const relay_header_t *rh, const cell_t *cell, return 0; } +/** Drop all storage held by <b>addr</b>. */ +STATIC void +address_ttl_free(address_ttl_t *addr) +{ + if (!addr) + return; + tor_free(addr->hostname); + tor_free(addr); +} + +/** Parse a resolved cell in <b>cell</b>, with parsed header in <b>rh</b>. + * Return -1 on parse error. On success, add one or more newly allocated + * address_ttl_t to <b>addresses_out</b>; set *<b>errcode_out</b> to + * one of 0, RESOLVED_TYPE_ERROR, or RESOLVED_TYPE_ERROR_TRANSIENT, and + * return 0. */ +STATIC int +resolved_cell_parse(const cell_t *cell, const relay_header_t *rh, + smartlist_t *addresses_out, int *errcode_out) +{ + const uint8_t *cp; + uint8_t answer_type; + size_t answer_len; + address_ttl_t *addr; + size_t remaining; + int errcode = 0; + smartlist_t *addrs; + + tor_assert(cell); + tor_assert(rh); + tor_assert(addresses_out); + tor_assert(errcode_out); + + *errcode_out = 0; + + if (rh->length > RELAY_PAYLOAD_SIZE) + return -1; + + addrs = smartlist_new(); + + cp = cell->payload + RELAY_HEADER_SIZE; + + remaining = rh->length; + while (remaining) { + const uint8_t *cp_orig = cp; + if (remaining < 2) + goto err; + answer_type = *cp++; + answer_len = *cp++; + if (remaining < 2 + answer_len + 4) { + goto err; + } + if (answer_type == RESOLVED_TYPE_IPV4) { + if (answer_len != 4) { + goto err; + } + addr = tor_malloc_zero(sizeof(*addr)); + tor_addr_from_ipv4n(&addr->addr, get_uint32(cp)); + cp += 4; + addr->ttl = ntohl(get_uint32(cp)); + cp += 4; + smartlist_add(addrs, addr); + } else if (answer_type == RESOLVED_TYPE_IPV6) { + if (answer_len != 16) + goto err; + addr = tor_malloc_zero(sizeof(*addr)); + tor_addr_from_ipv6_bytes(&addr->addr, (const char*) cp); + cp += 16; + addr->ttl = ntohl(get_uint32(cp)); + cp += 4; + smartlist_add(addrs, addr); + } else if (answer_type == RESOLVED_TYPE_HOSTNAME) { + if (answer_len == 0) { + goto err; + } + addr = tor_malloc_zero(sizeof(*addr)); + addr->hostname = tor_memdup_nulterm(cp, answer_len); + cp += answer_len; + addr->ttl = ntohl(get_uint32(cp)); + cp += 4; + smartlist_add(addrs, addr); + } else if (answer_type == RESOLVED_TYPE_ERROR_TRANSIENT || + answer_type == RESOLVED_TYPE_ERROR) { + errcode = answer_type; + /* Ignore the error contents */ + cp += answer_len + 4; + } else { + cp += answer_len + 4; + } + tor_assert(((ssize_t)remaining) >= (cp - cp_orig)); + remaining -= (cp - cp_orig); + } + + if (errcode && smartlist_len(addrs) == 0) { + /* Report an error only if there were no results. */ + *errcode_out = errcode; + } + + smartlist_add_all(addresses_out, addrs); + smartlist_free(addrs); + + return 0; + + err: + /* On parse error, don't report any results */ + SMARTLIST_FOREACH(addrs, address_ttl_t *, a, address_ttl_free(a)); + smartlist_free(addrs); + return -1; +} + +/** Helper for connection_edge_process_resolved_cell: given an error code, + * an entry_connection, and a list of address_ttl_t *, report the best answer + * to the entry_connection. */ +static void +connection_ap_handshake_socks_got_resolved_cell(entry_connection_t *conn, + int error_code, + smartlist_t *results) +{ + address_ttl_t *addr_ipv4 = NULL; + address_ttl_t *addr_ipv6 = NULL; + address_ttl_t *addr_hostname = NULL; + address_ttl_t *addr_best = NULL; + + /* If it's an error code, that's easy. */ + if (error_code) { + tor_assert(error_code == RESOLVED_TYPE_ERROR || + error_code == RESOLVED_TYPE_ERROR_TRANSIENT); + connection_ap_handshake_socks_resolved(conn, + error_code,0,NULL,-1,-1); + return; + } + + /* Get the first answer of each type. */ + SMARTLIST_FOREACH_BEGIN(results, address_ttl_t *, addr) { + if (addr->hostname) { + if (!addr_hostname) { + addr_hostname = addr; + } + } else if (tor_addr_family(&addr->addr) == AF_INET) { + if (!addr_ipv4 && conn->entry_cfg.ipv4_traffic) { + addr_ipv4 = addr; + } + } else if (tor_addr_family(&addr->addr) == AF_INET6) { + if (!addr_ipv6 && conn->entry_cfg.ipv6_traffic) { + addr_ipv6 = addr; + } + } + } SMARTLIST_FOREACH_END(addr); + + /* Now figure out which type we wanted to deliver. */ + if (conn->socks_request->command == SOCKS_COMMAND_RESOLVE_PTR) { + if (addr_hostname) { + connection_ap_handshake_socks_resolved(conn, + RESOLVED_TYPE_HOSTNAME, + strlen(addr_hostname->hostname), + (uint8_t*)addr_hostname->hostname, + addr_hostname->ttl,-1); + } else { + connection_ap_handshake_socks_resolved(conn, + RESOLVED_TYPE_ERROR,0,NULL,-1,-1); + } + return; + } + + if (conn->entry_cfg.prefer_ipv6) { + addr_best = addr_ipv6 ? addr_ipv6 : addr_ipv4; + } else { + addr_best = addr_ipv4 ? addr_ipv4 : addr_ipv6; + } + + /* Now convert it to the ugly old interface */ + if (! addr_best) { + connection_ap_handshake_socks_resolved(conn, + RESOLVED_TYPE_ERROR,0,NULL,-1,-1); + return; + } + + connection_ap_handshake_socks_resolved_addr(conn, + &addr_best->addr, + addr_best->ttl, + -1); + + remap_event_helper(conn, &addr_best->addr); +} + +/** Handle a RELAY_COMMAND_RESOLVED cell that we received on a non-open AP + * stream. */ +STATIC int +connection_edge_process_resolved_cell(edge_connection_t *conn, + const cell_t *cell, + const relay_header_t *rh) +{ + entry_connection_t *entry_conn = EDGE_TO_ENTRY_CONN(conn); + smartlist_t *resolved_addresses = NULL; + int errcode = 0; + + if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { + log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got a 'resolved' cell while " + "not in state resolve_wait. Dropping."); + return 0; + } + tor_assert(SOCKS_COMMAND_IS_RESOLVE(entry_conn->socks_request->command)); + + resolved_addresses = smartlist_new(); + if (resolved_cell_parse(cell, rh, resolved_addresses, &errcode)) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Dropping malformed 'resolved' cell"); + connection_mark_unattached_ap(entry_conn, END_STREAM_REASON_TORPROTOCOL); + goto done; + } + + if (get_options()->ClientDNSRejectInternalAddresses) { + int orig_len = smartlist_len(resolved_addresses); + SMARTLIST_FOREACH_BEGIN(resolved_addresses, address_ttl_t *, addr) { + if (addr->hostname == NULL && tor_addr_is_internal(&addr->addr, 0)) { + log_info(LD_APP, "Got a resolved cell with answer %s; dropping that " + "answer.", + safe_str_client(fmt_addr(&addr->addr))); + address_ttl_free(addr); + SMARTLIST_DEL_CURRENT(resolved_addresses, addr); + } + } SMARTLIST_FOREACH_END(addr); + if (orig_len && smartlist_len(resolved_addresses) == 0) { + log_info(LD_APP, "Got a resolved cell with only private addresses; " + "dropping it."); + connection_ap_handshake_socks_resolved(entry_conn, + RESOLVED_TYPE_ERROR_TRANSIENT, + 0, NULL, 0, TIME_MAX); + connection_mark_unattached_ap(entry_conn, + END_STREAM_REASON_TORPROTOCOL); + goto done; + } + } + + connection_ap_handshake_socks_got_resolved_cell(entry_conn, + errcode, + resolved_addresses); + + connection_mark_unattached_ap(entry_conn, + END_STREAM_REASON_DONE | + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); + + done: + SMARTLIST_FOREACH(resolved_addresses, address_ttl_t *, addr, + address_ttl_free(addr)); + smartlist_free(resolved_addresses); + return 0; +} + /** An incoming relay cell has arrived from circuit <b>circ</b> to * stream <b>conn</b>. * @@ -1044,8 +1299,12 @@ connection_edge_process_relay_cell_not_open( "Got 'connected' while not in state connect_wait. Dropping."); return 0; } + CONNECTION_AP_EXPECT_NONPENDING(entry_conn); conn->base_.state = AP_CONN_STATE_OPEN; - log_info(LD_APP,"'connected' received after %d seconds.", + log_info(LD_APP,"'connected' received for circid %u streamid %d " + "after %d seconds.", + (unsigned)circ->n_circ_id, + rh->stream_id, (int)(time(NULL) - conn->base_.timestamp_lastread)); if (connected_cell_parse(rh, cell, &addr, &ttl) < 0) { log_fn(LOG_PROTOCOL_WARN, LD_APP, @@ -1066,8 +1325,8 @@ connection_edge_process_relay_cell_not_open( return 0; } - if ((family == AF_INET && ! entry_conn->ipv4_traffic_ok) || - (family == AF_INET6 && ! entry_conn->ipv6_traffic_ok)) { + if ((family == AF_INET && ! entry_conn->entry_cfg.ipv4_traffic) || + (family == AF_INET6 && ! entry_conn->entry_cfg.ipv6_traffic)) { log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got a connected cell to %s with unsupported address family." " Closing.", fmt_addr(&addr)); @@ -1106,8 +1365,9 @@ connection_edge_process_relay_cell_not_open( break; case DIR_PURPOSE_FETCH_SERVERDESC: case DIR_PURPOSE_FETCH_MICRODESC: - control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, - count_loading_descriptors_progress()); + if (TO_DIR_CONN(dirconn)->router_purpose == ROUTER_PURPOSE_GENERAL) + control_event_bootstrap(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, + count_loading_descriptors_progress()); break; } } @@ -1128,67 +1388,7 @@ connection_edge_process_relay_cell_not_open( } if (conn->base_.type == CONN_TYPE_AP && rh->command == RELAY_COMMAND_RESOLVED) { - int ttl; - int answer_len; - uint8_t answer_type; - entry_connection_t *entry_conn = EDGE_TO_ENTRY_CONN(conn); - if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { - log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got a 'resolved' cell while " - "not in state resolve_wait. Dropping."); - return 0; - } - tor_assert(SOCKS_COMMAND_IS_RESOLVE(entry_conn->socks_request->command)); - answer_len = cell->payload[RELAY_HEADER_SIZE+1]; - if (rh->length < 2 || answer_len+2>rh->length) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Dropping malformed 'resolved' cell"); - connection_mark_unattached_ap(entry_conn, END_STREAM_REASON_TORPROTOCOL); - return 0; - } - answer_type = cell->payload[RELAY_HEADER_SIZE]; - if (rh->length >= answer_len+6) - ttl = (int)ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+ - 2+answer_len)); - else - ttl = -1; - if (answer_type == RESOLVED_TYPE_IPV4 || - answer_type == RESOLVED_TYPE_IPV6) { - tor_addr_t addr; - if (decode_address_from_payload(&addr, cell->payload+RELAY_HEADER_SIZE, - rh->length) && - tor_addr_is_internal(&addr, 0) && - get_options()->ClientDNSRejectInternalAddresses) { - log_info(LD_APP,"Got a resolve with answer %s. Rejecting.", - fmt_addr(&addr)); - connection_ap_handshake_socks_resolved(entry_conn, - RESOLVED_TYPE_ERROR_TRANSIENT, - 0, NULL, 0, TIME_MAX); - connection_mark_unattached_ap(entry_conn, - END_STREAM_REASON_TORPROTOCOL); - return 0; - } - } - connection_ap_handshake_socks_resolved(entry_conn, - answer_type, - cell->payload[RELAY_HEADER_SIZE+1], /*answer_len*/ - cell->payload+RELAY_HEADER_SIZE+2, /*answer*/ - ttl, - -1); - if (answer_type == RESOLVED_TYPE_IPV4 && answer_len == 4) { - tor_addr_t addr; - tor_addr_from_ipv4n(&addr, - get_uint32(cell->payload+RELAY_HEADER_SIZE+2)); - remap_event_helper(entry_conn, &addr); - } else if (answer_type == RESOLVED_TYPE_IPV6 && answer_len == 16) { - tor_addr_t addr; - tor_addr_from_ipv6_bytes(&addr, - (char*)(cell->payload+RELAY_HEADER_SIZE+2)); - remap_event_helper(entry_conn, &addr); - } - connection_mark_unattached_ap(entry_conn, - END_STREAM_REASON_DONE | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return 0; + return connection_edge_process_resolved_cell(conn, cell, rh); } log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, @@ -1242,7 +1442,6 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, switch (rh.command) { case RELAY_COMMAND_BEGIN: case RELAY_COMMAND_CONNECTED: - case RELAY_COMMAND_DATA: case RELAY_COMMAND_END: case RELAY_COMMAND_RESOLVE: case RELAY_COMMAND_RESOLVED: @@ -1267,6 +1466,9 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING. * This speeds up HTTP, for example. */ optimistic_data = 1; + } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) { + log_warn(LD_BUG, "Somehow I had a connection that matched a " + "data cell with stream ID 0."); } else { return connection_edge_process_relay_cell_not_open( &rh, cell, circ, conn, layer_hint); @@ -1327,7 +1529,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, circuit_consider_sending_sendme(circ, layer_hint); - if (!conn) { + if (rh.stream_id == 0) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero " + "stream_id. Dropping."); + return 0; + } else if (!conn) { log_info(domain,"data cell dropped, unknown stream (streamid %d).", rh.stream_id); return 0; @@ -1436,8 +1642,9 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } if ((reason = circuit_finish_handshake(TO_ORIGIN_CIRCUIT(circ), &extended_cell.created_cell)) < 0) { - log_warn(domain,"circuit_finish_handshake failed."); - return reason; + circuit_mark_for_close(circ, -reason); + return 0; /* We don't want to cause a warning, so we mark the circuit + * here. */ } } if ((reason=circuit_send_next_onion_skin(TO_ORIGIN_CIRCUIT(circ)))<0) { @@ -1490,14 +1697,17 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, return -END_CIRC_REASON_TORPROTOCOL; } log_info(domain, - "'connected' received, no conn attached anymore. Ignoring."); + "'connected' received on circid %u for streamid %d, " + "no conn attached anymore. Ignoring.", + (unsigned)circ->n_circ_id, rh.stream_id); return 0; case RELAY_COMMAND_SENDME: if (!rh.stream_id) { if (layer_hint) { if (layer_hint->package_window + CIRCWINDOW_INCREMENT > CIRCWINDOW_START_MAX) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL, "Unexpected sendme cell from exit relay. " "Closing circ."); return -END_CIRC_REASON_TORPROTOCOL; @@ -1509,7 +1719,8 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } else { if (circ->package_window + CIRCWINDOW_INCREMENT > CIRCWINDOW_START_MAX) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&client_warn_ratelim,LOG_PROTOCOL_WARN, LD_PROTOCOL, "Unexpected sendme cell from client. " "Closing circ (window %d).", circ->package_window); @@ -2036,60 +2247,31 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) #define assert_cmux_ok_paranoid(chan) #endif -/** The total number of cells we have allocated from the memory pool. */ +/** The total number of cells we have allocated. */ static size_t total_cells_allocated = 0; -/** A memory pool to allocate packed_cell_t objects. */ -static mp_pool_t *cell_pool = NULL; - -/** Allocate structures to hold cells. */ -void -init_cell_pool(void) -{ - tor_assert(!cell_pool); - cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); -} - -/** Free all storage used to hold cells (and insertion times if we measure - * cell statistics). */ -void -free_cell_pool(void) -{ - /* Maybe we haven't called init_cell_pool yet; need to check for it. */ - if (cell_pool) { - mp_pool_destroy(cell_pool); - cell_pool = NULL; - } -} - -/** Free excess storage in cell pool. */ -void -clean_cell_pool(void) -{ - tor_assert(cell_pool); - mp_pool_clean(cell_pool, 0, 1); -} - /** Release storage held by <b>cell</b>. */ -static INLINE void +static inline void packed_cell_free_unchecked(packed_cell_t *cell) { --total_cells_allocated; - mp_pool_release(cell); + tor_free(cell); } /** Allocate and return a new packed_cell_t. */ -static INLINE packed_cell_t * +STATIC packed_cell_t * packed_cell_new(void) { ++total_cells_allocated; - return mp_pool_get(cell_pool); + return tor_malloc_zero(sizeof(packed_cell_t)); } /** Return a packed cell used outside by channel_t lower layer */ void packed_cell_free(packed_cell_t *cell) { + if (!cell) + return; packed_cell_free_unchecked(cell); } @@ -2098,28 +2280,26 @@ packed_cell_free(packed_cell_t *cell) void dump_cell_pool_usage(int severity) { - circuit_t *c; int n_circs = 0; int n_cells = 0; - for (c = circuit_get_global_list_(); c; c = c->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, c) { n_cells += c->n_chan_cells.n; if (!CIRCUIT_IS_ORIGIN(c)) n_cells += TO_OR_CIRCUIT(c)->p_chan_cells.n; ++n_circs; } + SMARTLIST_FOREACH_END(c); tor_log(severity, LD_MM, "%d cells allocated on %d circuits. %d cells leaked.", n_cells, n_circs, (int)total_cells_allocated - n_cells); - mp_pool_log_status(cell_pool, severity); } /** Allocate a new copy of packed <b>cell</b>. */ -static INLINE packed_cell_t * +static inline packed_cell_t * packed_cell_copy(const cell_t *cell, int wide_circ_ids) { packed_cell_t *c = packed_cell_new(); cell_pack(c, cell, wide_circ_ids); - c->next = NULL; return c; } @@ -2127,58 +2307,61 @@ packed_cell_copy(const cell_t *cell, int wide_circ_ids) void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell) { - if (queue->tail) { - tor_assert(!queue->tail->next); - queue->tail->next = cell; - } else { - queue->head = cell; - } - queue->tail = cell; - cell->next = NULL; + TOR_SIMPLEQ_INSERT_TAIL(&queue->head, cell, next); ++queue->n; } -/** Append a newly allocated copy of <b>cell</b> to the end of <b>queue</b> */ +/** Append a newly allocated copy of <b>cell</b> to the end of the + * <b>exitward</b> (or app-ward) <b>queue</b> of <b>circ</b>. If + * <b>use_stats</b> is true, record statistics about the cell. + */ void -cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell, - int wide_circ_ids) +cell_queue_append_packed_copy(circuit_t *circ, cell_queue_t *queue, + int exitward, const cell_t *cell, + int wide_circ_ids, int use_stats) { struct timeval now; packed_cell_t *copy = packed_cell_copy(cell, wide_circ_ids); - tor_gettimeofday_cached(&now); + (void)circ; + (void)exitward; + (void)use_stats; + tor_gettimeofday_cached_monotonic(&now); + copy->inserted_time = (uint32_t)tv_to_msec(&now); cell_queue_append(queue, copy); } +/** Initialize <b>queue</b> as an empty cell queue. */ +void +cell_queue_init(cell_queue_t *queue) +{ + memset(queue, 0, sizeof(cell_queue_t)); + TOR_SIMPLEQ_INIT(&queue->head); +} + /** Remove and free every cell in <b>queue</b>. */ void cell_queue_clear(cell_queue_t *queue) { - packed_cell_t *cell, *next; - cell = queue->head; - while (cell) { - next = cell->next; + packed_cell_t *cell; + while ((cell = TOR_SIMPLEQ_FIRST(&queue->head))) { + TOR_SIMPLEQ_REMOVE_HEAD(&queue->head, next); packed_cell_free_unchecked(cell); - cell = next; } - queue->head = queue->tail = NULL; + TOR_SIMPLEQ_INIT(&queue->head); queue->n = 0; } /** Extract and return the cell at the head of <b>queue</b>; return NULL if * <b>queue</b> is empty. */ -static INLINE packed_cell_t * +STATIC packed_cell_t * cell_queue_pop(cell_queue_t *queue) { - packed_cell_t *cell = queue->head; + packed_cell_t *cell = TOR_SIMPLEQ_FIRST(&queue->head); if (!cell) return NULL; - queue->head = cell->next; - if (cell == queue->tail) { - tor_assert(!queue->head); - queue->tail = NULL; - } + TOR_SIMPLEQ_REMOVE_HEAD(&queue->head, next); --queue->n; return cell; } @@ -2188,22 +2371,61 @@ cell_queue_pop(cell_queue_t *queue) size_t packed_cell_mem_cost(void) { - return sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD; + return sizeof(packed_cell_t); +} + +/* DOCDOC */ +STATIC size_t +cell_queues_get_total_allocation(void) +{ + return total_cells_allocated * packed_cell_mem_cost(); } +/** How long after we've been low on memory should we try to conserve it? */ +#define MEMORY_PRESSURE_INTERVAL (30*60) + +/** The time at which we were last low on memory. */ +static time_t last_time_under_memory_pressure = 0; + /** Check whether we've got too much space used for cells. If so, * call the OOM handler and return 1. Otherwise, return 0. */ -static int +STATIC int cell_queues_check_size(void) { - size_t alloc = total_cells_allocated * packed_cell_mem_cost(); - if (alloc >= get_options()->MaxMemInCellQueues) { - circuits_handle_oom(alloc); - return 1; + size_t alloc = cell_queues_get_total_allocation(); + alloc += buf_get_total_allocation(); + alloc += tor_zlib_get_total_allocation(); + const size_t rend_cache_total = rend_cache_get_total_allocation(); + alloc += rend_cache_total; + if (alloc >= get_options()->MaxMemInQueues_low_threshold) { + last_time_under_memory_pressure = approx_time(); + if (alloc >= get_options()->MaxMemInQueues) { + /* If we're spending over 20% of the memory limit on hidden service + * descriptors, free them until we're down to 10%. + */ + if (rend_cache_total > get_options()->MaxMemInQueues / 5) { + const size_t bytes_to_remove = + rend_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); + rend_cache_clean_v2_descs_as_dir(time(NULL), bytes_to_remove); + alloc -= rend_cache_total; + alloc += rend_cache_get_total_allocation(); + } + circuits_handle_oom(alloc); + return 1; + } } return 0; } +/** Return true if we've been under memory pressure in the last + * MEMORY_PRESSURE_INTERVAL seconds. */ +int +have_been_under_memory_pressure(void) +{ + return last_time_under_memory_pressure + MEMORY_PRESSURE_INTERVAL + < approx_time(); +} + /** * Update the number of cells available on the circuit's n_chan or p_chan's * circuit mux. @@ -2252,14 +2474,18 @@ update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, assert_cmux_ok_paranoid(chan); } -/** Remove all circuits from the cmux on <b>chan</b>. */ +/** Remove all circuits from the cmux on <b>chan</b>. + * + * If <b>circuits_out</b> is non-NULL, add all detached circuits to + * <b>circuits_out</b>. + **/ void -channel_unlink_all_circuits(channel_t *chan) +channel_unlink_all_circuits(channel_t *chan, smartlist_t *circuits_out) { tor_assert(chan); tor_assert(chan->cmux); - circuitmux_detach_all_circuits(chan->cmux); + circuitmux_detach_all_circuits(chan->cmux, circuits_out); chan->num_n_circuits = 0; chan->num_p_circuits = 0; } @@ -2318,16 +2544,38 @@ set_streams_blocked_on_circ(circuit_t *circ, channel_t *chan, return n; } +/** Extract the command from a packed cell. */ +static uint8_t +packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids) +{ + if (wide_circ_ids) { + return get_uint8(cell->body+4); + } else { + return get_uint8(cell->body+2); + } +} + +/** Extract the circuit ID from a packed cell. */ +circid_t +packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids) +{ + if (wide_circ_ids) { + return ntohl(get_uint32(cell->body)); + } else { + return ntohs(get_uint16(cell->body)); + } +} + /** Pull as many cells as possible (but no more than <b>max</b>) from the * queue of the first active circuit on <b>chan</b>, and write them to * <b>chan</b>->outbuf. Return the number of cells written. Advance * the active circuit pointer to the next active circuit in the ring. */ -int -channel_flush_from_first_active_circuit(channel_t *chan, int max) +MOCK_IMPL(int, +channel_flush_from_first_active_circuit, (channel_t *chan, int max)) { circuitmux_t *cmux = NULL; int n_flushed = 0; - cell_queue_t *queue; + cell_queue_t *queue, *destroy_queue=NULL; circuit_t *circ; or_circuit_t *or_circ; int streams_blocked; @@ -2340,7 +2588,18 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) /* Main loop: pick a circuit, send a cell, update the cmux */ while (n_flushed < max) { - circ = circuitmux_get_first_active_circuit(cmux); + circ = circuitmux_get_first_active_circuit(cmux, &destroy_queue); + if (destroy_queue) { + /* this code is duplicated from some of the logic below. Ugly! XXXX */ + tor_assert(destroy_queue->n > 0); + cell = cell_queue_pop(destroy_queue); + channel_write_packed_cell(chan, cell); + /* Update the cmux destroy counter */ + circuitmux_notify_xmit_destroy(cmux); + cell = NULL; + ++n_flushed; + continue; + } /* If it returns NULL, no cells left to send */ if (!circ) break; assert_cmux_ok_paranoid(chan); @@ -2356,6 +2615,15 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) } /* Circuitmux told us this was active, so it should have cells */ + if (/*BUG(*/ queue->n == 0 /*)*/) { + log_warn(LD_BUG, "Found a supposedly active circuit with no cells " + "to send. Trying to recover."); + circuitmux_set_num_cells(cmux, circ, 0); + if (! circ->marked_for_close) + circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL); + continue; + } + tor_assert(queue->n > 0); /* @@ -2366,15 +2634,33 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) cell = cell_queue_pop(queue); /* Calculate the exact time that this cell has spent in the queue. */ - if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + if (get_options()->CellStatistics || + get_options()->TestingEnableCellStatsEvent) { uint32_t msec_waiting; struct timeval tvnow; - or_circ = TO_OR_CIRCUIT(circ); tor_gettimeofday_cached(&tvnow); msec_waiting = ((uint32_t)tv_to_msec(&tvnow)) - cell->inserted_time; - or_circ->total_cell_waiting_time += msec_waiting; - or_circ->processed_cells++; + if (get_options()->CellStatistics && !CIRCUIT_IS_ORIGIN(circ)) { + or_circ = TO_OR_CIRCUIT(circ); + or_circ->total_cell_waiting_time += msec_waiting; + or_circ->processed_cells++; + } + + if (get_options()->TestingEnableCellStatsEvent) { + uint8_t command = packed_cell_get_command(cell, chan->wide_circ_ids); + + testing_cell_stats_entry_t *ent = + tor_malloc_zero(sizeof(testing_cell_stats_entry_t)); + ent->command = command; + ent->waiting_time = msec_waiting / 10; + ent->removed = 1; + if (circ->n_chan == chan) + ent->exitward = 1; + if (!circ->testing_cell_stats) + circ->testing_cell_stats = smartlist_new(); + smartlist_add(circ->testing_cell_stats, ent); + } } /* If we just flushed our queue and this circuit is used for a @@ -2420,6 +2706,20 @@ channel_flush_from_first_active_circuit(channel_t *chan, int max) return n_flushed; } +#if 0 +/** Indicate the current preferred cap for middle circuits; zero disables + * the cap. Right now it's just a constant, ORCIRC_MAX_MIDDLE_CELLS, but + * the logic in append_cell_to_circuit_queue() is written to be correct + * if we want to base it on a consensus param or something that might change + * in the future. + */ +static int +get_max_middle_cells(void) +{ + return ORCIRC_MAX_MIDDLE_CELLS; +} +#endif + /** Add <b>cell</b> to the queue of <b>circ</b> writing to <b>chan</b> * transmitting in <b>direction</b>. */ void @@ -2430,11 +2730,16 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, or_circuit_t *orcirc = NULL; cell_queue_t *queue; int streams_blocked; +#if 0 + uint32_t tgt_max_middle_cells, p_len, n_len, tmp, hard_max_middle_cells; +#endif + int exitward; if (circ->marked_for_close) return; - if (direction == CELL_DIRECTION_OUT) { + exitward = (direction == CELL_DIRECTION_OUT); + if (exitward) { queue = &circ->n_chan_cells; streams_blocked = circ->streams_blocked_on_n_chan; } else { @@ -2451,28 +2756,82 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, if ((circ->n_chan != NULL) && CIRCUIT_IS_ORCIRC(circ)) { orcirc = TO_OR_CIRCUIT(circ); if (orcirc->p_chan) { - if (queue->n + 1 >= ORCIRC_MAX_MIDDLE_CELLS) { - /* Queueing this cell would put queue over the cap */ - log_warn(LD_CIRC, - "Got a cell exceeding the cap of %u in the %s direction " - "on middle circ ID %u on chan ID " U64_FORMAT - "; killing the circuit.", - ORCIRC_MAX_MIDDLE_CELLS, - (direction == CELL_DIRECTION_OUT) ? "n" : "p", - (direction == CELL_DIRECTION_OUT) ? - circ->n_circ_id : orcirc->p_circ_id, - U64_PRINTF_ARG( + /* We are a middle circuit if we have both n_chan and p_chan */ + /* We'll need to know the current preferred maximum */ + tgt_max_middle_cells = get_max_middle_cells(); + if (tgt_max_middle_cells > 0) { + /* Do we need to initialize middle_max_cells? */ + if (orcirc->max_middle_cells == 0) { + orcirc->max_middle_cells = tgt_max_middle_cells; + } else { + if (tgt_max_middle_cells > orcirc->max_middle_cells) { + /* If we want to increase the cap, we can do so right away */ + orcirc->max_middle_cells = tgt_max_middle_cells; + } else if (tgt_max_middle_cells < orcirc->max_middle_cells) { + /* + * If we're shrinking the cap, we can't shrink past either queue; + * compare tgt_max_middle_cells rather than tgt_max_middle_cells * + * ORCIRC_MAX_MIDDLE_KILL_THRESH so the queues don't shrink enough + * to generate spurious warnings, either. + */ + n_len = circ->n_chan_cells.n; + p_len = orcirc->p_chan_cells.n; + tmp = tgt_max_middle_cells; + if (tmp < n_len) tmp = n_len; + if (tmp < p_len) tmp = p_len; + orcirc->max_middle_cells = tmp; + } + /* else no change */ + } + } else { + /* tgt_max_middle_cells == 0 indicates we should disable the cap */ + orcirc->max_middle_cells = 0; + } + + /* Now we know orcirc->max_middle_cells is set correctly */ + if (orcirc->max_middle_cells > 0) { + hard_max_middle_cells = + (uint32_t)(((double)orcirc->max_middle_cells) * + ORCIRC_MAX_MIDDLE_KILL_THRESH); + + if ((unsigned)queue->n + 1 >= hard_max_middle_cells) { + /* Queueing this cell would put queue over the kill theshold */ + log_warn(LD_CIRC, + "Got a cell exceeding the hard cap of %u in the " + "%s direction on middle circ ID %u on chan ID " + U64_FORMAT "; killing the circuit.", + hard_max_middle_cells, + (direction == CELL_DIRECTION_OUT) ? "n" : "p", + (direction == CELL_DIRECTION_OUT) ? + circ->n_circ_id : orcirc->p_circ_id, + U64_PRINTF_ARG( + (direction == CELL_DIRECTION_OUT) ? + circ->n_chan->global_identifier : + orcirc->p_chan->global_identifier)); + circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); + return; + } else if ((unsigned)queue->n + 1 == orcirc->max_middle_cells) { + /* Only use ==, not >= for this test so we don't spam the log */ + log_warn(LD_CIRC, + "While trying to queue a cell, reached the soft cap of %u " + "in the %s direction on middle circ ID %u " + "on chan ID " U64_FORMAT ".", + orcirc->max_middle_cells, + (direction == CELL_DIRECTION_OUT) ? "n" : "p", (direction == CELL_DIRECTION_OUT) ? - circ->n_chan->global_identifier : - orcirc->p_chan->global_identifier)); - circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); - return; + circ->n_circ_id : orcirc->p_circ_id, + U64_PRINTF_ARG( + (direction == CELL_DIRECTION_OUT) ? + circ->n_chan->global_identifier : + orcirc->p_chan->global_identifier)); + } } } } #endif - cell_queue_append_packed_copy(queue, cell, chan->wide_circ_ids); + cell_queue_append_packed_copy(circ, queue, exitward, cell, + chan->wide_circ_ids, 1); if (PREDICT_UNLIKELY(cell_queues_check_size())) { /* We ran the OOM handler */ @@ -2497,14 +2856,8 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, log_debug(LD_GENERAL, "Made a circuit active."); } - if (!channel_has_queued_writes(chan)) { - /* There is no data at all waiting to be sent on the outbuf. Add a - * cell, so that we can notice when it gets flushed, flushed_some can - * get called, and we can start putting more data onto the buffer then. - */ - log_debug(LD_GENERAL, "Primed a buffer."); - channel_flush_from_first_active_circuit(chan, 1); - } + /* New way: mark this as having waiting cells for the scheduler */ + scheduler_channel_has_waiting_cells(chan); } /** Append an encoded value of <b>addr</b> to <b>payload_out</b>, which must diff --git a/src/or/relay.h b/src/or/relay.h index 1fef10a7da..e15551ca51 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -42,25 +42,27 @@ extern uint64_t stats_n_data_bytes_packaged; extern uint64_t stats_n_data_cells_received; extern uint64_t stats_n_data_bytes_received; -void init_cell_pool(void); -void free_cell_pool(void); -void clean_cell_pool(void); void dump_cell_pool_usage(int severity); size_t packed_cell_mem_cost(void); +int have_been_under_memory_pressure(void); + /* For channeltls.c */ void packed_cell_free(packed_cell_t *cell); +void cell_queue_init(cell_queue_t *queue); void cell_queue_clear(cell_queue_t *queue); void cell_queue_append(cell_queue_t *queue, packed_cell_t *cell); -void cell_queue_append_packed_copy(cell_queue_t *queue, const cell_t *cell, - int wide_circ_ids); +void cell_queue_append_packed_copy(circuit_t *circ, cell_queue_t *queue, + int exitward, const cell_t *cell, + int wide_circ_ids, int use_stats); void append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, cell_t *cell, cell_direction_t direction, streamid_t fromstream); -void channel_unlink_all_circuits(channel_t *chan); -int channel_flush_from_first_active_circuit(channel_t *chan, int max); +void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); +MOCK_DECL(int, channel_flush_from_first_active_circuit, + (channel_t *chan, int max)); void assert_circuit_mux_okay(channel_t *chan); void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, const char *file, int lineno); @@ -75,11 +77,30 @@ void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan); void stream_choice_seed_weak_rng(void); -#ifdef RELAY_PRIVATE int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t **layer_hint, char *recognized); -int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, + +circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids); + +#ifdef RELAY_PRIVATE +STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out); +/** An address-and-ttl tuple as yielded by resolved_cell_parse */ +typedef struct address_ttl_s { + tor_addr_t addr; + char *hostname; + int ttl; +} address_ttl_t; +STATIC void address_ttl_free(address_ttl_t *addr); +STATIC int resolved_cell_parse(const cell_t *cell, const relay_header_t *rh, + smartlist_t *addresses_out, int *errcode_out); +STATIC int connection_edge_process_resolved_cell(edge_connection_t *conn, + const cell_t *cell, + const relay_header_t *rh); +STATIC packed_cell_t *packed_cell_new(void); +STATIC packed_cell_t *cell_queue_pop(cell_queue_t *queue); +STATIC size_t cell_queues_get_total_allocation(void); +STATIC int cell_queues_check_size(void); #endif #endif diff --git a/src/or/rendcache.c b/src/or/rendcache.c new file mode 100644 index 0000000000..f9ae6d1173 --- /dev/null +++ b/src/or/rendcache.c @@ -0,0 +1,1013 @@ +/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendcache.c + * \brief Hidden service descriptor cache. + **/ + +#define RENDCACHE_PRIVATE +#include "rendcache.h" + +#include "config.h" +#include "rephist.h" +#include "routerlist.h" +#include "routerparse.h" +#include "rendcommon.h" + +/** Map from service id (as generated by rend_get_service_id) to + * rend_cache_entry_t. */ +STATIC strmap_t *rend_cache = NULL; + +/** Map from service id to rend_cache_entry_t; only for hidden services. */ +static strmap_t *rend_cache_local_service = NULL; + +/** Map from descriptor id to rend_cache_entry_t; only for hidden service + * directories. */ +STATIC digestmap_t *rend_cache_v2_dir = NULL; + +/** (Client side only) Map from service id to rend_cache_failure_t. This + * cache is used to track intro point(IP) failures so we know when to keep + * or discard a new descriptor we just fetched. Here is a description of the + * cache behavior. + * + * Everytime tor discards an IP (ex: receives a NACK), we add an entry to + * this cache noting the identity digest of the IP and it's failure type for + * the service ID. The reason we indexed this cache by service ID is to + * differentiate errors that can occur only for a specific service like a + * NACK for instance. It applies for one but maybe not for the others. + * + * Once a service descriptor is fetched and considered valid, each IP is + * looked up in this cache and if present, it is discarded from the fetched + * descriptor. At the end, all IP(s) in the cache, for a specific service + * ID, that were NOT present in the descriptor are removed from this cache. + * Which means that if at least one IP was not in this cache, thus usuable, + * it's considered a new descriptor so we keep it. Else, if all IPs were in + * this cache, we discard the descriptor as it's considered unsuable. + * + * Once a descriptor is removed from the rend cache or expires, the entry + * in this cache is also removed for the service ID. + * + * This scheme allows us to not realy on the descriptor's timestamp (which + * is rounded down to the hour) to know if we have a newer descriptor. We + * only rely on the usability of intro points from an internal state. */ +STATIC strmap_t *rend_cache_failure = NULL; + +/* DOCDOC */ +STATIC size_t rend_cache_total_allocation = 0; + +/** Initializes the service descriptor cache. +*/ +void +rend_cache_init(void) +{ + rend_cache = strmap_new(); + rend_cache_v2_dir = digestmap_new(); + rend_cache_local_service = strmap_new(); + rend_cache_failure = strmap_new(); +} + +/** Return the approximate number of bytes needed to hold <b>e</b>. */ +STATIC size_t +rend_cache_entry_allocation(const rend_cache_entry_t *e) +{ + if (!e) + return 0; + + /* This doesn't count intro_nodes or key size */ + return sizeof(*e) + e->len + sizeof(*e->parsed); +} + +/* DOCDOC */ +size_t +rend_cache_get_total_allocation(void) +{ + return rend_cache_total_allocation; +} + +/** Decrement the total bytes attributed to the rendezvous cache by n. */ +STATIC void +rend_cache_decrement_allocation(size_t n) +{ + static int have_underflowed = 0; + + if (rend_cache_total_allocation >= n) { + rend_cache_total_allocation -= n; + } else { + rend_cache_total_allocation = 0; + if (! have_underflowed) { + have_underflowed = 1; + log_warn(LD_BUG, "Underflow in rend_cache_decrement_allocation"); + } + } +} + +/** Increase the total bytes attributed to the rendezvous cache by n. */ +STATIC void +rend_cache_increment_allocation(size_t n) +{ + static int have_overflowed = 0; + if (rend_cache_total_allocation <= SIZE_MAX - n) { + rend_cache_total_allocation += n; + } else { + rend_cache_total_allocation = SIZE_MAX; + if (! have_overflowed) { + have_overflowed = 1; + log_warn(LD_BUG, "Overflow in rend_cache_increment_allocation"); + } + } +} + +/** Helper: free a rend cache failure intro object. */ +STATIC void +rend_cache_failure_intro_entry_free(rend_cache_failure_intro_t *entry) +{ + if (entry == NULL) { + return; + } + tor_free(entry); +} + +static void +rend_cache_failure_intro_entry_free_(void *entry) +{ + rend_cache_failure_intro_entry_free(entry); +} + +/** Allocate a rend cache failure intro object and return it. <b>failure</b> + * is set into the object. This function can not fail. */ +STATIC rend_cache_failure_intro_t * +rend_cache_failure_intro_entry_new(rend_intro_point_failure_t failure) +{ + rend_cache_failure_intro_t *entry = tor_malloc(sizeof(*entry)); + entry->failure_type = failure; + entry->created_ts = time(NULL); + return entry; +} + +/** Helper: free a rend cache failure object. */ +STATIC void +rend_cache_failure_entry_free(rend_cache_failure_t *entry) +{ + if (entry == NULL) { + return; + } + + /* Free and remove every intro failure object. */ + digestmap_free(entry->intro_failures, + rend_cache_failure_intro_entry_free_); + + tor_free(entry); +} + +/** Helper: deallocate a rend_cache_failure_t. (Used with strmap_free(), + * which requires a function pointer whose argument is void*). */ +STATIC void +rend_cache_failure_entry_free_(void *entry) +{ + rend_cache_failure_entry_free(entry); +} + +/** Allocate a rend cache failure object and return it. This function can + * not fail. */ +STATIC rend_cache_failure_t * +rend_cache_failure_entry_new(void) +{ + rend_cache_failure_t *entry = tor_malloc(sizeof(*entry)); + entry->intro_failures = digestmap_new(); + return entry; +} + +/** Remove failure cache entry for the service ID in the given descriptor + * <b>desc</b>. */ +STATIC void +rend_cache_failure_remove(rend_service_descriptor_t *desc) +{ + char service_id[REND_SERVICE_ID_LEN_BASE32 + 1]; + rend_cache_failure_t *entry; + + if (desc == NULL) { + return; + } + if (rend_get_service_id(desc->pk, service_id) < 0) { + return; + } + entry = strmap_get_lc(rend_cache_failure, service_id); + if (entry != NULL) { + strmap_remove_lc(rend_cache_failure, service_id); + rend_cache_failure_entry_free(entry); + } +} + +/** Helper: free storage held by a single service descriptor cache entry. */ +STATIC void +rend_cache_entry_free(rend_cache_entry_t *e) +{ + if (!e) + return; + rend_cache_decrement_allocation(rend_cache_entry_allocation(e)); + /* We are about to remove a descriptor from the cache so remove the entry + * in the failure cache. */ + rend_cache_failure_remove(e->parsed); + rend_service_descriptor_free(e->parsed); + tor_free(e->desc); + tor_free(e); +} + +/** Helper: deallocate a rend_cache_entry_t. (Used with strmap_free(), which + * requires a function pointer whose argument is void*). */ +static void +rend_cache_entry_free_(void *p) +{ + rend_cache_entry_free(p); +} + +/** Free all storage held by the service descriptor cache. */ +void +rend_cache_free_all(void) +{ + strmap_free(rend_cache, rend_cache_entry_free_); + digestmap_free(rend_cache_v2_dir, rend_cache_entry_free_); + strmap_free(rend_cache_local_service, rend_cache_entry_free_); + strmap_free(rend_cache_failure, rend_cache_failure_entry_free_); + rend_cache = NULL; + rend_cache_v2_dir = NULL; + rend_cache_local_service = NULL; + rend_cache_failure = NULL; + rend_cache_total_allocation = 0; +} + +/** Remove all entries that re REND_CACHE_FAILURE_MAX_AGE old. This is + * called every second. + * + * We have to clean these regurlarly else if for whatever reasons an hidden + * service goes offline and a client tries to connect to it during that + * time, a failure entry is created and the client will be unable to connect + * for a while even though the service has return online. */ +void +rend_cache_failure_clean(time_t now) +{ + time_t cutoff = now - REND_CACHE_FAILURE_MAX_AGE; + STRMAP_FOREACH_MODIFY(rend_cache_failure, key, + rend_cache_failure_t *, ent) { + /* Free and remove every intro failure object that match the cutoff. */ + DIGESTMAP_FOREACH_MODIFY(ent->intro_failures, ip_key, + rend_cache_failure_intro_t *, ip_ent) { + if (ip_ent->created_ts < cutoff) { + rend_cache_failure_intro_entry_free(ip_ent); + MAP_DEL_CURRENT(ip_key); + } + } DIGESTMAP_FOREACH_END; + /* If the entry is now empty of intro point failures, remove it. */ + if (digestmap_isempty(ent->intro_failures)) { + rend_cache_failure_entry_free(ent); + MAP_DEL_CURRENT(key); + } + } STRMAP_FOREACH_END; +} + +/** Removes all old entries from the client or service descriptor cache. +*/ +void +rend_cache_clean(time_t now, rend_cache_type_t cache_type) +{ + strmap_iter_t *iter; + const char *key; + void *val; + rend_cache_entry_t *ent; + time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW; + strmap_t *cache = NULL; + + if (cache_type == REND_CACHE_TYPE_CLIENT) { + cache = rend_cache; + } else if (cache_type == REND_CACHE_TYPE_SERVICE) { + cache = rend_cache_local_service; + } + tor_assert(cache); + + for (iter = strmap_iter_init(cache); !strmap_iter_done(iter); ) { + strmap_iter_get(iter, &key, &val); + ent = (rend_cache_entry_t*)val; + if (ent->parsed->timestamp < cutoff) { + iter = strmap_iter_next_rmv(cache, iter); + rend_cache_entry_free(ent); + } else { + iter = strmap_iter_next(cache, iter); + } + } +} + +/** Remove ALL entries from the rendezvous service descriptor cache. +*/ +void +rend_cache_purge(void) +{ + if (rend_cache) { + log_info(LD_REND, "Purging HS descriptor cache"); + strmap_free(rend_cache, rend_cache_entry_free_); + } + rend_cache = strmap_new(); +} + +/** Remove ALL entries from the failure cache. This is also called when a + * NEWNYM signal is received. */ +void +rend_cache_failure_purge(void) +{ + if (rend_cache_failure) { + log_info(LD_REND, "Purging HS failure cache"); + strmap_free(rend_cache_failure, rend_cache_failure_entry_free_); + } + rend_cache_failure = strmap_new(); +} + +/** Lookup the rend failure cache using a relay identity digest in + * <b>identity</b> which has DIGEST_LEN bytes and service ID <b>service_id</b> + * which is a null-terminated string. If found, the intro failure is set in + * <b>intro_entry</b> else it stays untouched. Return 1 iff found else 0. */ +STATIC int +cache_failure_intro_lookup(const uint8_t *identity, const char *service_id, + rend_cache_failure_intro_t **intro_entry) +{ + rend_cache_failure_t *elem; + rend_cache_failure_intro_t *intro_elem; + + tor_assert(rend_cache_failure); + + if (intro_entry) { + *intro_entry = NULL; + } + + /* Lookup descriptor and return it. */ + elem = strmap_get_lc(rend_cache_failure, service_id); + if (elem == NULL) { + goto not_found; + } + intro_elem = digestmap_get(elem->intro_failures, (char *) identity); + if (intro_elem == NULL) { + goto not_found; + } + if (intro_entry) { + *intro_entry = intro_elem; + } + return 1; + not_found: + return 0; +} + +/** Allocate a new cache failure intro object and copy the content from + * <b>entry</b> to this newly allocated object. Return it. */ +static rend_cache_failure_intro_t * +cache_failure_intro_dup(const rend_cache_failure_intro_t *entry) +{ + rend_cache_failure_intro_t *ent_dup = + rend_cache_failure_intro_entry_new(entry->failure_type); + ent_dup->created_ts = entry->created_ts; + return ent_dup; +} + +/** Add an intro point failure to the failure cache using the relay + * <b>identity</b> and service ID <b>service_id</b>. Record the + * <b>failure</b> in that object. */ +STATIC void +cache_failure_intro_add(const uint8_t *identity, const char *service_id, + rend_intro_point_failure_t failure) +{ + rend_cache_failure_t *fail_entry; + rend_cache_failure_intro_t *entry, *old_entry; + + /* Make sure we have a failure object for this service ID and if not, + * create it with this new intro failure entry. */ + fail_entry = strmap_get_lc(rend_cache_failure, service_id); + if (fail_entry == NULL) { + fail_entry = rend_cache_failure_entry_new(); + /* Add failure entry to global rend failure cache. */ + strmap_set_lc(rend_cache_failure, service_id, fail_entry); + } + entry = rend_cache_failure_intro_entry_new(failure); + old_entry = digestmap_set(fail_entry->intro_failures, + (char *) identity, entry); + /* This _should_ be NULL, but in case it isn't, free it. */ + rend_cache_failure_intro_entry_free(old_entry); +} + +/** Using a parsed descriptor <b>desc</b>, check if the introduction points + * are present in the failure cache and if so they are removed from the + * descriptor and kept into the failure cache. Then, each intro points that + * are NOT in the descriptor but in the failure cache for the given + * <b>service_id</b> are removed from the failure cache. */ +STATIC void +validate_intro_point_failure(const rend_service_descriptor_t *desc, + const char *service_id) +{ + rend_cache_failure_t *new_entry, *cur_entry; + /* New entry for the service ID that will be replacing the one in the + * failure cache since we have a new descriptor. In the case where all + * intro points are removed, we are assured that the new entry is the same + * as the current one. */ + new_entry = tor_malloc(sizeof(*new_entry)); + new_entry->intro_failures = digestmap_new(); + + tor_assert(desc); + + SMARTLIST_FOREACH_BEGIN(desc->intro_nodes, rend_intro_point_t *, intro) { + int found; + rend_cache_failure_intro_t *entry; + const uint8_t *identity = + (uint8_t *) intro->extend_info->identity_digest; + + found = cache_failure_intro_lookup(identity, service_id, &entry); + if (found) { + /* Dup here since it will be freed at the end when removing the + * original entry in the cache. */ + rend_cache_failure_intro_t *ent_dup = cache_failure_intro_dup(entry); + /* This intro point is in our cache, discard it from the descriptor + * because chances are that it's unusable. */ + SMARTLIST_DEL_CURRENT(desc->intro_nodes, intro); + /* Keep it for our new entry. */ + digestmap_set(new_entry->intro_failures, (char *) identity, ent_dup); + /* Only free it when we're done looking at it. */ + rend_intro_point_free(intro); + continue; + } + } SMARTLIST_FOREACH_END(intro); + + /* Swap the failure entry in the cache and free the current one. */ + cur_entry = strmap_get_lc(rend_cache_failure, service_id); + if (cur_entry != NULL) { + rend_cache_failure_entry_free(cur_entry); + } + strmap_set_lc(rend_cache_failure, service_id, new_entry); +} + +/** Note down an intro failure in the rend failure cache using the type of + * failure in <b>failure</b> for the relay identity digest in + * <b>identity</b> and service ID <b>service_id</b>. If an entry already + * exists in the cache, the failure type is changed with <b>failure</b>. */ +void +rend_cache_intro_failure_note(rend_intro_point_failure_t failure, + const uint8_t *identity, + const char *service_id) +{ + int found; + rend_cache_failure_intro_t *entry; + + found = cache_failure_intro_lookup(identity, service_id, &entry); + if (!found) { + cache_failure_intro_add(identity, service_id, failure); + } else { + /* Replace introduction point failure with this one. */ + entry->failure_type = failure; + } +} + +/** Remove all old v2 descriptors and those for which this hidden service + * directory is not responsible for any more. + * + * If at all possible, remove at least <b>force_remove</b> bytes of data. + */ +void +rend_cache_clean_v2_descs_as_dir(time_t now, size_t force_remove) +{ + digestmap_iter_t *iter; + time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW; + const int LAST_SERVED_CUTOFF_STEP = 1800; + time_t last_served_cutoff = cutoff; + size_t bytes_removed = 0; + do { + for (iter = digestmap_iter_init(rend_cache_v2_dir); + !digestmap_iter_done(iter); ) { + const char *key; + void *val; + rend_cache_entry_t *ent; + digestmap_iter_get(iter, &key, &val); + ent = val; + if (ent->parsed->timestamp < cutoff || + ent->last_served < last_served_cutoff) { + char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); + log_info(LD_REND, "Removing descriptor with ID '%s' from cache", + safe_str_client(key_base32)); + bytes_removed += rend_cache_entry_allocation(ent); + iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); + rend_cache_entry_free(ent); + } else { + iter = digestmap_iter_next(rend_cache_v2_dir, iter); + } + } + + /* In case we didn't remove enough bytes, advance the cutoff a little. */ + last_served_cutoff += LAST_SERVED_CUTOFF_STEP; + if (last_served_cutoff > now) + break; + } while (bytes_removed < force_remove); +} + +/** Lookup in the client cache the given service ID <b>query</b> for + * <b>version</b>. + * + * Return 0 if found and if <b>e</b> is non NULL, set it with the entry + * found. Else, a negative value is returned and <b>e</b> is untouched. + * -EINVAL means that <b>query</b> is not a valid service id. + * -ENOENT means that no entry in the cache was found. */ +int +rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) +{ + int ret = 0; + char key[REND_SERVICE_ID_LEN_BASE32 + 2]; /* <version><query>\0 */ + rend_cache_entry_t *entry = NULL; + static const int default_version = 2; + + tor_assert(rend_cache); + tor_assert(query); + + if (!rend_valid_service_id(query)) { + ret = -EINVAL; + goto end; + } + + switch (version) { + case 0: + log_warn(LD_REND, "Cache lookup of a v0 renddesc is deprecated."); + break; + case 2: + /* Default is version 2. */ + default: + tor_snprintf(key, sizeof(key), "%d%s", default_version, query); + entry = strmap_get_lc(rend_cache, key); + break; + } + if (!entry) { + ret = -ENOENT; + goto end; + } + tor_assert(entry->parsed && entry->parsed->intro_nodes); + + if (e) { + *e = entry; + } + + end: + return ret; +} + +/* + * Lookup the v2 service descriptor with the service ID <b>query</b> in the + * local service descriptor cache. Return 0 if found and if <b>e</b> is + * non NULL, set it with the entry found. Else, a negative value is returned + * and <b>e</b> is untouched. + * -EINVAL means that <b>query</b> is not a valid service id. + * -ENOENT means that no entry in the cache was found. */ +int +rend_cache_lookup_v2_desc_as_service(const char *query, rend_cache_entry_t **e) +{ + int ret = 0; + rend_cache_entry_t *entry = NULL; + + tor_assert(rend_cache_local_service); + tor_assert(query); + + if (!rend_valid_service_id(query)) { + ret = -EINVAL; + goto end; + } + + /* Lookup descriptor and return. */ + entry = strmap_get_lc(rend_cache_local_service, query); + if (!entry) { + ret = -ENOENT; + goto end; + } + + if (e) { + *e = entry; + } + + end: + return ret; +} + +/** Lookup the v2 service descriptor with base32-encoded <b>desc_id</b> and + * copy the pointer to it to *<b>desc</b>. Return 1 on success, 0 on + * well-formed-but-not-found, and -1 on failure. + */ +int +rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc) +{ + rend_cache_entry_t *e; + char desc_id_digest[DIGEST_LEN]; + tor_assert(rend_cache_v2_dir); + if (base32_decode(desc_id_digest, DIGEST_LEN, + desc_id, REND_DESC_ID_V2_LEN_BASE32) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_REND, + "Rejecting v2 rendezvous descriptor request -- descriptor ID " + "contains illegal characters: %s", + safe_str(desc_id)); + return -1; + } + /* Lookup descriptor and return. */ + e = digestmap_get(rend_cache_v2_dir, desc_id_digest); + if (e) { + *desc = e->desc; + e->last_served = approx_time(); + return 1; + } + return 0; +} + +/** Parse the v2 service descriptor(s) in <b>desc</b> and store it/them to the + * local rend cache. Don't attempt to decrypt the included list of introduction + * points (as we don't have a descriptor cookie for it). + * + * If we have a newer descriptor with the same ID, ignore this one. + * If we have an older descriptor with the same ID, replace it. + * + * Return 0 on success, or -1 if we couldn't parse any of them. + * + * We should only call this function for public (e.g. non bridge) relays. + */ +int +rend_cache_store_v2_desc_as_dir(const char *desc) +{ + const or_options_t *options = get_options(); + rend_service_descriptor_t *parsed; + char desc_id[DIGEST_LEN]; + char *intro_content; + size_t intro_size; + size_t encoded_size; + char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + int number_parsed = 0, number_stored = 0; + const char *current_desc = desc; + const char *next_desc; + rend_cache_entry_t *e; + time_t now = time(NULL); + tor_assert(rend_cache_v2_dir); + tor_assert(desc); + while (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content, + &intro_size, &encoded_size, + &next_desc, current_desc, 1) >= 0) { + number_parsed++; + /* We don't care about the introduction points. */ + tor_free(intro_content); + /* For pretty log statements. */ + base32_encode(desc_id_base32, sizeof(desc_id_base32), + desc_id, DIGEST_LEN); + /* Is descriptor too old? */ + if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { + log_info(LD_REND, "Service descriptor with desc ID %s is too old.", + safe_str(desc_id_base32)); + goto skip; + } + /* Is descriptor too far in the future? */ + if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { + log_info(LD_REND, "Service descriptor with desc ID %s is too far in the " + "future.", + safe_str(desc_id_base32)); + goto skip; + } + /* Do we already have a newer descriptor? */ + e = digestmap_get(rend_cache_v2_dir, desc_id); + if (e && e->parsed->timestamp > parsed->timestamp) { + log_info(LD_REND, "We already have a newer service descriptor with the " + "same desc ID %s and version.", + safe_str(desc_id_base32)); + goto skip; + } + /* Do we already have this descriptor? */ + if (e && !strcmp(desc, e->desc)) { + log_info(LD_REND, "We already have this service descriptor with desc " + "ID %s.", safe_str(desc_id_base32)); + goto skip; + } + /* Store received descriptor. */ + if (!e) { + e = tor_malloc_zero(sizeof(rend_cache_entry_t)); + digestmap_set(rend_cache_v2_dir, desc_id, e); + /* Treat something just uploaded as having been served a little + * while ago, so that flooding with new descriptors doesn't help + * too much. + */ + e->last_served = approx_time() - 3600; + } else { + rend_cache_decrement_allocation(rend_cache_entry_allocation(e)); + rend_service_descriptor_free(e->parsed); + tor_free(e->desc); + } + e->parsed = parsed; + e->desc = tor_strndup(current_desc, encoded_size); + e->len = encoded_size; + rend_cache_increment_allocation(rend_cache_entry_allocation(e)); + log_info(LD_REND, "Successfully stored service descriptor with desc ID " + "'%s' and len %d.", + safe_str(desc_id_base32), (int)encoded_size); + /* Statistics: Note down this potentially new HS. */ + if (options->HiddenServiceStatistics) { + rep_hist_stored_maybe_new_hs(e->parsed->pk); + } + + number_stored++; + goto advance; + skip: + rend_service_descriptor_free(parsed); + advance: + /* advance to next descriptor, if available. */ + current_desc = next_desc; + /* check if there is a next descriptor. */ + if (!current_desc || + strcmpstart(current_desc, "rendezvous-service-descriptor ")) + break; + } + if (!number_parsed) { + log_info(LD_REND, "Could not parse any descriptor."); + return -1; + } + log_info(LD_REND, "Parsed %d and added %d descriptor%s.", + number_parsed, number_stored, number_stored != 1 ? "s" : ""); + return 0; +} + +/** Parse the v2 service descriptor in <b>desc</b> and store it to the +* local service rend cache. Don't attempt to decrypt the included list of +* introduction points. +* +* If we have a newer descriptor with the same ID, ignore this one. +* If we have an older descriptor with the same ID, replace it. +* +* Return 0 on success, or -1 if we couldn't understand the descriptor. +*/ +int +rend_cache_store_v2_desc_as_service(const char *desc) +{ + rend_service_descriptor_t *parsed = NULL; + char desc_id[DIGEST_LEN]; + char *intro_content = NULL; + size_t intro_size; + size_t encoded_size; + const char *next_desc; + char service_id[REND_SERVICE_ID_LEN_BASE32+1]; + rend_cache_entry_t *e; + int retval = -1; + tor_assert(rend_cache_local_service); + tor_assert(desc); + + /* Parse the descriptor. */ + if (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content, + &intro_size, &encoded_size, + &next_desc, desc, 0) < 0) { + log_warn(LD_REND, "Could not parse descriptor."); + goto err; + } + /* Compute service ID from public key. */ + if (rend_get_service_id(parsed->pk, service_id)<0) { + log_warn(LD_REND, "Couldn't compute service ID."); + goto err; + } + + /* Do we already have a newer descriptor? Allow new descriptors with a + rounded timestamp equal to or newer than the current descriptor */ + e = (rend_cache_entry_t*) strmap_get_lc(rend_cache_local_service, + service_id); + if (e && e->parsed->timestamp > parsed->timestamp) { + log_info(LD_REND, "We already have a newer service descriptor for " + "service ID %s.", safe_str_client(service_id)); + goto okay; + } + /* We don't care about the introduction points. */ + tor_free(intro_content); + if (!e) { + e = tor_malloc_zero(sizeof(rend_cache_entry_t)); + strmap_set_lc(rend_cache_local_service, service_id, e); + } else { + rend_cache_decrement_allocation(rend_cache_entry_allocation(e)); + rend_service_descriptor_free(e->parsed); + tor_free(e->desc); + } + e->parsed = parsed; + e->desc = tor_malloc_zero(encoded_size + 1); + strlcpy(e->desc, desc, encoded_size + 1); + e->len = encoded_size; + rend_cache_increment_allocation(rend_cache_entry_allocation(e)); + log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", + safe_str_client(service_id), (int)encoded_size); + return 0; + + okay: + retval = 0; + + err: + rend_service_descriptor_free(parsed); + tor_free(intro_content); + return retval; +} + +/** Parse the v2 service descriptor in <b>desc</b>, decrypt the included list + * of introduction points with <b>descriptor_cookie</b> (which may also be + * <b>NULL</b> if decryption is not necessary), and store the descriptor to + * the local cache under its version and service id. + * + * If we have a newer v2 descriptor with the same ID, ignore this one. + * If we have an older descriptor with the same ID, replace it. + * If the descriptor's service ID does not match + * <b>rend_query</b>-\>onion_address, reject it. + * + * If the descriptor's descriptor ID doesn't match <b>desc_id_base32</b>, + * reject it. + * + * Return 0 on success, or -1 if we rejected the descriptor. + * If entry is not NULL, set it with the cache entry pointer of the descriptor. + */ +int +rend_cache_store_v2_desc_as_client(const char *desc, + const char *desc_id_base32, + const rend_data_t *rend_query, + rend_cache_entry_t **entry) +{ + /*XXXX this seems to have a bit of duplicate code with + * rend_cache_store_v2_desc_as_dir(). Fix that. */ + /* Though having similar elements, both functions were separated on + * purpose: + * - dirs don't care about encoded/encrypted introduction points, clients + * do. + * - dirs store descriptors in a separate cache by descriptor ID, whereas + * clients store them by service ID; both caches are different data + * structures and have different access methods. + * - dirs store a descriptor only if they are responsible for its ID, + * clients do so in every way (because they have requested it before). + * - dirs can process multiple concatenated descriptors which is required + * for replication, whereas clients only accept a single descriptor. + * Thus, combining both methods would result in a lot of if statements + * which probably would not improve, but worsen code readability. -KL */ + rend_service_descriptor_t *parsed = NULL; + char desc_id[DIGEST_LEN]; + char *intro_content = NULL; + size_t intro_size; + size_t encoded_size; + const char *next_desc; + time_t now = time(NULL); + char key[REND_SERVICE_ID_LEN_BASE32+2]; + char service_id[REND_SERVICE_ID_LEN_BASE32+1]; + char want_desc_id[DIGEST_LEN]; + rend_cache_entry_t *e; + int retval = -1; + tor_assert(rend_cache); + tor_assert(desc); + tor_assert(desc_id_base32); + memset(want_desc_id, 0, sizeof(want_desc_id)); + if (entry) { + *entry = NULL; + } + if (base32_decode(want_desc_id, sizeof(want_desc_id), + desc_id_base32, strlen(desc_id_base32)) != 0) { + log_warn(LD_BUG, "Couldn't decode base32 %s for descriptor id.", + escaped_safe_str_client(desc_id_base32)); + goto err; + } + /* Parse the descriptor. */ + if (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content, + &intro_size, &encoded_size, + &next_desc, desc, 0) < 0) { + log_warn(LD_REND, "Could not parse descriptor."); + goto err; + } + /* Compute service ID from public key. */ + if (rend_get_service_id(parsed->pk, service_id)<0) { + log_warn(LD_REND, "Couldn't compute service ID."); + goto err; + } + if (rend_query->onion_address[0] != '\0' && + strcmp(rend_query->onion_address, service_id)) { + log_warn(LD_REND, "Received service descriptor for service ID %s; " + "expected descriptor for service ID %s.", + service_id, safe_str(rend_query->onion_address)); + goto err; + } + if (tor_memneq(desc_id, want_desc_id, DIGEST_LEN)) { + log_warn(LD_REND, "Received service descriptor for %s with incorrect " + "descriptor ID.", service_id); + goto err; + } + + /* Decode/decrypt introduction points. */ + if (intro_content && intro_size > 0) { + int n_intro_points; + if (rend_query->auth_type != REND_NO_AUTH && + !tor_mem_is_zero(rend_query->descriptor_cookie, + sizeof(rend_query->descriptor_cookie))) { + char *ipos_decrypted = NULL; + size_t ipos_decrypted_size; + if (rend_decrypt_introduction_points(&ipos_decrypted, + &ipos_decrypted_size, + rend_query->descriptor_cookie, + intro_content, + intro_size) < 0) { + log_warn(LD_REND, "Failed to decrypt introduction points. We are " + "probably unable to parse the encoded introduction points."); + } else { + /* Replace encrypted with decrypted introduction points. */ + log_info(LD_REND, "Successfully decrypted introduction points."); + tor_free(intro_content); + intro_content = ipos_decrypted; + intro_size = ipos_decrypted_size; + } + } + n_intro_points = rend_parse_introduction_points(parsed, intro_content, + intro_size); + if (n_intro_points <= 0) { + log_warn(LD_REND, "Failed to parse introduction points. Either the " + "service has published a corrupt descriptor or you have " + "provided invalid authorization data, or (maybe!) the " + "server is deliberately serving broken data in an attempt " + "to crash you with bug 21018."); + goto err; + } else if (n_intro_points > MAX_INTRO_POINTS) { + log_warn(LD_REND, "Found too many introduction points on a hidden " + "service descriptor for %s. This is probably a (misguided) " + "attempt to improve reliability, but it could also be an " + "attempt to do a guard enumeration attack. Rejecting.", + safe_str_client(service_id)); + + goto err; + } + } else { + log_info(LD_REND, "Descriptor does not contain any introduction points."); + parsed->intro_nodes = smartlist_new(); + } + /* We don't need the encoded/encrypted introduction points any longer. */ + tor_free(intro_content); + /* Is descriptor too old? */ + if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { + log_warn(LD_REND, "Service descriptor with service ID %s is too old.", + safe_str_client(service_id)); + goto err; + } + /* Is descriptor too far in the future? */ + if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { + log_warn(LD_REND, "Service descriptor with service ID %s is too far in " + "the future.", safe_str_client(service_id)); + goto err; + } + /* Do we have the same exact copy already in our cache? */ + tor_snprintf(key, sizeof(key), "2%s", service_id); + e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); + if (e && !strcmp(desc, e->desc)) { + log_info(LD_REND,"We already have this service descriptor %s.", + safe_str_client(service_id)); + goto okay; + } + /* Verify that we are not replacing an older descriptor. It's important to + * avoid an evil HSDir serving old descriptor. We validate if the + * timestamp is greater than and not equal because it's a rounded down + * timestamp to the hour so if the descriptor changed in the same hour, + * the rend cache failure will tells us if we have a new descriptor. */ + if (e && e->parsed->timestamp > parsed->timestamp) { + log_info(LD_REND, "We already have a new enough service descriptor for " + "service ID %s with the same desc ID and version.", + safe_str_client(service_id)); + goto okay; + } + /* Lookup our failure cache for intro point that might be unsuable. */ + validate_intro_point_failure(parsed, service_id); + /* It's now possible that our intro point list is empty, this means that + * this descriptor is useless to us because intro points have all failed + * somehow before. Discard the descriptor. */ + if (smartlist_len(parsed->intro_nodes) == 0) { + log_info(LD_REND, "Service descriptor with service ID %s, every " + "intro points are unusable. Discarding it.", + safe_str_client(service_id)); + goto err; + } + /* Now either purge the current one and replace it's content or create a + * new one and add it to the rend cache. */ + if (!e) { + e = tor_malloc_zero(sizeof(rend_cache_entry_t)); + strmap_set_lc(rend_cache, key, e); + } else { + rend_cache_decrement_allocation(rend_cache_entry_allocation(e)); + rend_cache_failure_remove(e->parsed); + rend_service_descriptor_free(e->parsed); + tor_free(e->desc); + } + e->parsed = parsed; + e->desc = tor_malloc_zero(encoded_size + 1); + strlcpy(e->desc, desc, encoded_size + 1); + e->len = encoded_size; + rend_cache_increment_allocation(rend_cache_entry_allocation(e)); + log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", + safe_str_client(service_id), (int)encoded_size); + if (entry) { + *entry = e; + } + return 0; + + okay: + if (entry) { + *entry = e; + } + retval = 0; + + err: + rend_service_descriptor_free(parsed); + tor_free(intro_content); + return retval; +} + diff --git a/src/or/rendcache.h b/src/or/rendcache.h new file mode 100644 index 0000000000..0e8b918753 --- /dev/null +++ b/src/or/rendcache.h @@ -0,0 +1,108 @@ +/* Copyright (c) 2015-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file rendcache.h + * \brief Header file for rendcache.c + **/ + +#ifndef TOR_RENDCACHE_H +#define TOR_RENDCACHE_H + +#include "or.h" +#include "rendcommon.h" + +/** How old do we let hidden service descriptors get before discarding + * them as too old? */ +#define REND_CACHE_MAX_AGE (2*24*60*60) +/** How wrong do we assume our clock may be when checking whether hidden + * services are too old or too new? */ +#define REND_CACHE_MAX_SKEW (24*60*60) +/** How old do we keep an intro point failure entry in the failure cache? */ +#define REND_CACHE_FAILURE_MAX_AGE (5*60) + +/* Do not allow more than this many introduction points in a hidden service + * descriptor */ +#define MAX_INTRO_POINTS 10 + +/** A cached rendezvous descriptor. */ +typedef struct rend_cache_entry_t { + size_t len; /**< Length of <b>desc</b> */ + time_t last_served; /**< When did we last write this one to somebody? + * (HSDir only) */ + char *desc; /**< Service descriptor */ + rend_service_descriptor_t *parsed; /**< Parsed value of 'desc' */ +} rend_cache_entry_t; + +/* Introduction point failure type. */ +typedef struct rend_cache_failure_intro_t { + /* When this intro point failure occured thus we allocated this object and + * cache it. */ + time_t created_ts; + rend_intro_point_failure_t failure_type; +} rend_cache_failure_intro_t; + +/** Cache failure object indexed by service ID. */ +typedef struct rend_cache_failure_t { + /* Contains rend_cache_failure_intro_t indexed by identity digest. */ + digestmap_t *intro_failures; +} rend_cache_failure_t; + +typedef enum { + REND_CACHE_TYPE_CLIENT = 1, + REND_CACHE_TYPE_SERVICE = 2, +} rend_cache_type_t; + +void rend_cache_init(void); +void rend_cache_clean(time_t now, rend_cache_type_t cache_type); +void rend_cache_failure_clean(time_t now); +void rend_cache_clean_v2_descs_as_dir(time_t now, size_t min_to_remove); +void rend_cache_purge(void); +void rend_cache_free_all(void); +int rend_cache_lookup_entry(const char *query, int version, + rend_cache_entry_t **entry_out); +int rend_cache_lookup_v2_desc_as_service(const char *query, + rend_cache_entry_t **entry_out); +int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); + +int rend_cache_store_v2_desc_as_dir(const char *desc); +int rend_cache_store_v2_desc_as_service(const char *desc); +int rend_cache_store_v2_desc_as_client(const char *desc, + const char *desc_id_base32, + const rend_data_t *rend_query, + rend_cache_entry_t **entry); +size_t rend_cache_get_total_allocation(void); + +void rend_cache_intro_failure_note(rend_intro_point_failure_t failure, + const uint8_t *identity, + const char *service_id); +void rend_cache_failure_purge(void); + +#ifdef RENDCACHE_PRIVATE + +STATIC size_t rend_cache_entry_allocation(const rend_cache_entry_t *e); +STATIC void rend_cache_entry_free(rend_cache_entry_t *e); +STATIC void rend_cache_failure_intro_entry_free(rend_cache_failure_intro_t + *entry); +STATIC void rend_cache_failure_entry_free(rend_cache_failure_t *entry); +STATIC int cache_failure_intro_lookup(const uint8_t *identity, + const char *service_id, + rend_cache_failure_intro_t + **intro_entry); +STATIC void rend_cache_decrement_allocation(size_t n); +STATIC void rend_cache_increment_allocation(size_t n); +STATIC rend_cache_failure_intro_t *rend_cache_failure_intro_entry_new( + rend_intro_point_failure_t failure); +STATIC rend_cache_failure_t *rend_cache_failure_entry_new(void); +STATIC void rend_cache_failure_remove(rend_service_descriptor_t *desc); +STATIC void cache_failure_intro_add(const uint8_t *identity, + const char *service_id, + rend_intro_point_failure_t failure); +STATIC void validate_intro_point_failure(const rend_service_descriptor_t *desc, + const char *service_id); + +STATIC void rend_cache_failure_entry_free_(void *entry); +#endif + +#endif /* TOR_RENDCACHE_H */ + diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 7abbfd6fc5..609c45c71d 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -8,6 +8,7 @@ **/ #include "or.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" @@ -25,6 +26,7 @@ #include "router.h" #include "routerlist.h" #include "routerset.h" +#include "control.h" static extend_info_t *rend_client_get_random_intro_impl( const rend_cache_entry_t *rend_query, @@ -36,6 +38,7 @@ void rend_client_purge_state(void) { rend_cache_purge(); + rend_cache_failure_purge(); rend_client_cancel_descriptor_fetches(); rend_client_purge_last_hid_serv_requests(); } @@ -49,7 +52,7 @@ rend_client_introcirc_has_opened(origin_circuit_t *circ) tor_assert(circ->cpath); log_info(LD_REND,"introcirc is open"); - connection_ap_attach_pending(); + connection_ap_attach_pending(1); } /** Send the establish-rendezvous cell along a rendezvous circuit. if @@ -62,11 +65,7 @@ rend_client_send_establish_rendezvous(origin_circuit_t *circ) tor_assert(circ->rend_data); log_info(LD_REND, "Sending an ESTABLISH_RENDEZVOUS cell"); - if (crypto_rand(circ->rend_data->rend_cookie, REND_COOKIE_LEN) < 0) { - log_warn(LD_BUG, "Internal error: Couldn't produce random cookie."); - circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); - return -1; - } + crypto_rand(circ->rend_data->rend_cookie, REND_COOKIE_LEN); /* Set timestamp_dirty, because circuit_expire_building expects it, * and the rend cookie also means we've used the circ. */ @@ -128,16 +127,6 @@ rend_client_reextend_intro_circuit(origin_circuit_t *circ) return result; } -/** Return true iff we should send timestamps in our INTRODUCE1 cells */ -static int -rend_client_should_send_timestamp(void) -{ - if (get_options()->Support022HiddenServices >= 0) - return get_options()->Support022HiddenServices; - - return networkstatus_get_param(NULL, "Support022HiddenServices", 1, 0, 1); -} - /** Called when we're trying to connect an ap conn; sends an INTRODUCE1 cell * down introcirc if possible. */ @@ -149,7 +138,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, int r, v3_shift = 0; char payload[RELAY_PAYLOAD_SIZE]; char tmp[RELAY_PAYLOAD_SIZE]; - rend_cache_entry_t *entry; + rend_cache_entry_t *entry = NULL; crypt_path_t *cpath; off_t dh_offset; crypto_pk_t *intro_key = NULL; @@ -166,8 +155,13 @@ rend_client_send_introduction(origin_circuit_t *introcirc, tor_assert(!(rendcirc->build_state->onehop_tunnel)); #endif - if (rend_cache_lookup_entry(introcirc->rend_data->onion_address, -1, - &entry) < 1) { + r = rend_cache_lookup_entry(introcirc->rend_data->onion_address, -1, + &entry); + /* An invalid onion address is not possible else we have a big issue. */ + tor_assert(r != -EINVAL); + if (r < 0 || !rend_client_any_intro_points_usable(entry)) { + /* If the descriptor is not found or the intro points are not usable + * anymore, trigger a fetch. */ log_info(LD_REND, "query %s didn't have valid rend desc in cache. " "Refetching descriptor.", @@ -179,6 +173,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, introcirc->rend_data->onion_address))) { + connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn)); conn->state = AP_CONN_STATE_RENDDESC_WAIT; } } @@ -187,7 +182,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, goto cleanup; } - /* first 20 bytes of payload are the hash of Bob's pk */ + /* first 20 bytes of payload are the hash of the service's pk */ intro_key = NULL; SMARTLIST_FOREACH(entry->parsed->intro_nodes, rend_intro_point_t *, intro, { @@ -249,14 +244,8 @@ rend_client_send_introduction(origin_circuit_t *introcirc, REND_DESC_COOKIE_LEN); v3_shift += 2+REND_DESC_COOKIE_LEN; } - if (rend_client_should_send_timestamp()) { - uint32_t now = (uint32_t)time(NULL); - now += 300; - now -= now % 600; - set_uint32(tmp+v3_shift+1, htonl(now)); - } else { - set_uint32(tmp+v3_shift+1, 0); - } + /* Once this held a timestamp. */ + set_uint32(tmp+v3_shift+1, 0); v3_shift += 4; } /* if version 2 only write version number */ else if (entry->parsed->protocols & (1<<2)) { @@ -368,15 +357,13 @@ rend_client_rendcirc_has_opened(origin_circuit_t *circ) } /** - * Called to close other intro circuits we launched in parallel - * due to timeout. + * Called to close other intro circuits we launched in parallel. */ static void rend_client_close_other_intros(const char *onion_address) { - circuit_t *c; /* abort parallel intro circs, if any */ - for (c = circuit_get_global_list_(); c; c = c->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, c) { if ((c->purpose == CIRCUIT_PURPOSE_C_INTRODUCING || c->purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) && !c->marked_for_close && CIRCUIT_IS_ORIGIN(c)) { @@ -387,10 +374,11 @@ rend_client_close_other_intros(const char *onion_address) log_info(LD_REND|LD_CIRC, "Closing introduction circuit %d that we " "built in parallel (Purpose %d).", oc->global_identifier, c->purpose); - circuit_mark_for_close(c, END_CIRC_REASON_TIMEOUT); + circuit_mark_for_close(c, END_CIRC_REASON_IP_NOW_REDUNDANT); } } } + SMARTLIST_FOREACH_END(c); } /** Called when get an ACK or a NAK for a REND_INTRODUCE1 cell. @@ -466,6 +454,13 @@ rend_client_introduction_acked(origin_circuit_t *circ, /* XXXX If that call failed, should we close the rend circuit, * too? */ return result; + } else { + /* Close circuit because no more intro points are usable thus not + * useful anymore. Change it's purpose before so we don't report an + * intro point failure again triggering an extra descriptor fetch. */ + circuit_change_purpose(TO_CIRCUIT(circ), + CIRCUIT_PURPOSE_C_INTRODUCE_ACKED); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); } } return 0; @@ -477,9 +472,8 @@ rend_client_introduction_acked(origin_circuit_t *circ, /** Contains the last request times to hidden service directories for * certain queries; each key is a string consisting of the - * concatenation of a base32-encoded HS directory identity digest, a - * base32-encoded HS descriptor ID, and a hidden service address - * (without the ".onion" part); each value is a pointer to a time_t + * concatenation of a base32-encoded HS directory identity digest and + * base32-encoded HS descriptor ID; each value is a pointer to a time_t * holding the time of the last request for that descriptor ID to that * HS directory. */ static strmap_t *last_hid_serv_requests_ = NULL; @@ -495,19 +489,16 @@ get_last_hid_serv_requests(void) } #define LAST_HID_SERV_REQUEST_KEY_LEN (REND_DESC_ID_V2_LEN_BASE32 + \ - REND_DESC_ID_V2_LEN_BASE32 + \ - REND_SERVICE_ID_LEN_BASE32) + REND_DESC_ID_V2_LEN_BASE32) /** Look up the last request time to hidden service directory <b>hs_dir</b> - * for descriptor ID <b>desc_id_base32</b> for the service specified in - * <b>rend_query</b>. If <b>set</b> is non-zero, - * assign the current time <b>now</b> and return that. Otherwise, return - * the most recent request time, or 0 if no such request has been sent - * before. */ + * for descriptor ID <b>desc_id_base32</b>. If <b>set</b> is non-zero, + * assign the current time <b>now</b> and return that. Otherwise, return the + * most recent request time, or 0 if no such request has been sent before. + */ static time_t lookup_last_hid_serv_request(routerstatus_t *hs_dir, const char *desc_id_base32, - const rend_data_t *rend_query, time_t now, int set) { char hsdir_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; @@ -516,10 +507,9 @@ lookup_last_hid_serv_request(routerstatus_t *hs_dir, strmap_t *last_hid_serv_requests = get_last_hid_serv_requests(); base32_encode(hsdir_id_base32, sizeof(hsdir_id_base32), hs_dir->identity_digest, DIGEST_LEN); - tor_snprintf(hsdir_desc_comb_id, sizeof(hsdir_desc_comb_id), "%s%s%s", + tor_snprintf(hsdir_desc_comb_id, sizeof(hsdir_desc_comb_id), "%s%s", hsdir_id_base32, - desc_id_base32, - rend_query->onion_address); + desc_id_base32); /* XXX023 tor_assert(strlen(hsdir_desc_comb_id) == LAST_HID_SERV_REQUEST_KEY_LEN); */ if (set) { @@ -560,15 +550,23 @@ directory_clean_last_hid_serv_requests(time_t now) } } -/** Remove all requests related to the hidden service named - * <b>onion_address</b> from the history of times of requests to - * hidden service directories. */ +/** Remove all requests related to the descriptor ID <b>desc_id</b> from the + * history of times of requests to hidden service directories. + * <b>desc_id</b> is an unencoded descriptor ID of size DIGEST_LEN. + * + * This is called from rend_client_note_connection_attempt_ended(), which + * must be idempotent, so any future changes to this function must leave it + * idempotent too. */ static void -purge_hid_serv_from_last_hid_serv_requests(const char *onion_address) +purge_hid_serv_from_last_hid_serv_requests(const char *desc_id) { strmap_iter_t *iter; strmap_t *last_hid_serv_requests = get_last_hid_serv_requests(); - /* XXX023 tor_assert(strlen(onion_address) == REND_SERVICE_ID_LEN_BASE32); */ + char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + + /* Key is stored with the base32 encoded desc_id. */ + base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id, + DIGEST_LEN); for (iter = strmap_iter_init(last_hid_serv_requests); !strmap_iter_done(iter); ) { const char *key; @@ -576,9 +574,9 @@ purge_hid_serv_from_last_hid_serv_requests(const char *onion_address) strmap_iter_get(iter, &key, &val); /* XXX023 tor_assert(strlen(key) == LAST_HID_SERV_REQUEST_KEY_LEN); */ if (tor_memeq(key + LAST_HID_SERV_REQUEST_KEY_LEN - - REND_SERVICE_ID_LEN_BASE32, - onion_address, - REND_SERVICE_ID_LEN_BASE32)) { + REND_DESC_ID_V2_LEN_BASE32, + desc_id_base32, + REND_DESC_ID_V2_LEN_BASE32)) { iter = strmap_iter_next_rmv(last_hid_serv_requests, iter); tor_free(val); } else { @@ -607,68 +605,128 @@ rend_client_purge_last_hid_serv_requests(void) } } -/** Determine the responsible hidden service directories for <b>desc_id</b> - * and fetch the descriptor with that ID from one of them. Only - * send a request to a hidden service directory that we have not yet tried - * during this attempt to connect to this hidden service; on success, return 1, - * in the case that no hidden service directory is left to ask for the - * descriptor, return 0, and in case of a failure -1. */ -static int -directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) +/** This returns a good valid hs dir that should be used for the given + * descriptor id. + * + * Return NULL on error else the hsdir node pointer. */ +static routerstatus_t * +pick_hsdir(const char *desc_id, const char *desc_id_base32) { smartlist_t *responsible_dirs = smartlist_new(); + smartlist_t *usable_responsible_dirs = smartlist_new(); + const or_options_t *options = get_options(); routerstatus_t *hs_dir; - char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; time_t now = time(NULL); - char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64]; - int tor2web_mode = get_options()->Tor2webMode; - tor_assert(desc_id); - tor_assert(rend_query); - /* Determine responsible dirs. Even if we can't get all we want, - * work with the ones we have. If it's empty, we'll notice below. */ - hid_serv_get_responsible_directories(responsible_dirs, desc_id); + int excluded_some; - base32_encode(desc_id_base32, sizeof(desc_id_base32), - desc_id, DIGEST_LEN); + tor_assert(desc_id); + tor_assert(desc_id_base32); - /* Only select those hidden service directories to which we did not send - * a request recently and for which we have a router descriptor here. */ + /* Determine responsible dirs. Even if we can't get all we want, work with + * the ones we have. If it's empty, we'll notice below. */ + hid_serv_get_responsible_directories(responsible_dirs, desc_id); /* Clean request history first. */ directory_clean_last_hid_serv_requests(now); - SMARTLIST_FOREACH(responsible_dirs, routerstatus_t *, dir, { - time_t last = lookup_last_hid_serv_request( - dir, desc_id_base32, rend_query, 0, 0); - const node_t *node = node_get_by_id(dir->identity_digest); - if (last + REND_HID_SERV_DIR_REQUERY_PERIOD >= now || - !node || !node_has_descriptor(node)) + /* Only select those hidden service directories to which we did not send a + * request recently and for which we have a router descriptor here. */ + SMARTLIST_FOREACH_BEGIN(responsible_dirs, routerstatus_t *, dir) { + time_t last = lookup_last_hid_serv_request(dir, desc_id_base32, + 0, 0); + const node_t *node = node_get_by_id(dir->identity_digest); + if (last + REND_HID_SERV_DIR_REQUERY_PERIOD >= now || + !node || !node_has_descriptor(node)) { SMARTLIST_DEL_CURRENT(responsible_dirs, dir); - }); + continue; + } + if (!routerset_contains_node(options->ExcludeNodes, node)) { + smartlist_add(usable_responsible_dirs, dir); + } + } SMARTLIST_FOREACH_END(dir); + + excluded_some = + smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs); + + hs_dir = smartlist_choose(usable_responsible_dirs); + if (!hs_dir && !options->StrictNodes) { + hs_dir = smartlist_choose(responsible_dirs); + } - hs_dir = smartlist_choose(responsible_dirs); smartlist_free(responsible_dirs); + smartlist_free(usable_responsible_dirs); if (!hs_dir) { log_info(LD_REND, "Could not pick one of the responsible hidden " "service directories, because we requested them all " "recently without success."); - return 0; + if (options->StrictNodes && excluded_some) { + log_warn(LD_REND, "Could not pick a hidden service directory for the " + "requested hidden service: they are all either down or " + "excluded, and StrictNodes is set."); + } + } else { + /* Remember that we are requesting a descriptor from this hidden service + * directory now. */ + lookup_last_hid_serv_request(hs_dir, desc_id_base32, now, 1); } - /* Remember that we are requesting a descriptor from this hidden service - * directory now. */ - lookup_last_hid_serv_request(hs_dir, desc_id_base32, rend_query, now, 1); + return hs_dir; +} + +/** Determine the responsible hidden service directories for <b>desc_id</b> + * and fetch the descriptor with that ID from one of them. Only + * send a request to a hidden service directory that we have not yet tried + * during this attempt to connect to this hidden service; on success, return 1, + * in the case that no hidden service directory is left to ask for the + * descriptor, return 0, and in case of a failure -1. */ +static int +directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query, + routerstatus_t *rs_hsdir) +{ + routerstatus_t *hs_dir = rs_hsdir; + char *hsdir_fp; + char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64]; +#ifdef ENABLE_TOR2WEB_MODE + const int tor2web_mode = get_options()->Tor2webMode; + const int how_to_fetch = tor2web_mode ? DIRIND_ONEHOP : DIRIND_ANONYMOUS; +#else + const int how_to_fetch = DIRIND_ANONYMOUS; +#endif + + tor_assert(desc_id); + + base32_encode(desc_id_base32, sizeof(desc_id_base32), + desc_id, DIGEST_LEN); - /* Encode descriptor cookie for logging purposes. */ + /* Automatically pick an hs dir if none given. */ + if (!rs_hsdir) { + hs_dir = pick_hsdir(desc_id, desc_id_base32); + if (!hs_dir) { + /* No suitable hs dir can be found, stop right now. */ + return 0; + } + } + + /* Add a copy of the HSDir identity digest to the query so we can track it + * on the control port. */ + hsdir_fp = tor_memdup(hs_dir->identity_digest, + sizeof(hs_dir->identity_digest)); + smartlist_add(rend_query->hsdirs_fp, hsdir_fp); + + /* Encode descriptor cookie for logging purposes. Also, if the cookie is + * malformed, no fetch is triggered thus this needs to be done before the + * fetch request. */ if (rend_query->auth_type != REND_NO_AUTH) { if (base64_encode(descriptor_cookie_base64, sizeof(descriptor_cookie_base64), - rend_query->descriptor_cookie, REND_DESC_COOKIE_LEN)<0) { + rend_query->descriptor_cookie, REND_DESC_COOKIE_LEN, + 0)<0) { log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); return 0; } - /* Remove == signs and newline. */ - descriptor_cookie_base64[strlen(descriptor_cookie_base64)-3] = '\0'; + /* Remove == signs. */ + descriptor_cookie_base64[strlen(descriptor_cookie_base64)-2] = '\0'; } else { strlcpy(descriptor_cookie_base64, "(none)", sizeof(descriptor_cookie_base64)); @@ -680,7 +738,7 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) directory_initiate_command_routerstatus_rend(hs_dir, DIR_PURPOSE_FETCH_RENDDESC_V2, ROUTER_PURPOSE_GENERAL, - tor2web_mode?DIRIND_ONEHOP:DIRIND_ANONYMOUS, + how_to_fetch, desc_id_base32, NULL, 0, 0, rend_query); @@ -693,19 +751,149 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) (rend_query->auth_type == REND_NO_AUTH ? "[none]" : escaped_safe_str_client(descriptor_cookie_base64)), routerstatus_describe(hs_dir)); + control_event_hs_descriptor_requested(rend_query, + hs_dir->identity_digest, + desc_id_base32); return 1; } +/** Fetch a v2 descriptor using the given descriptor id. If any hsdir(s) are + * given, they will be used instead. + * + * On success, 1 is returned. If no hidden service is left to ask, return 0. + * On error, -1 is returned. */ +static int +fetch_v2_desc_by_descid(const char *desc_id, const rend_data_t *rend_query, + smartlist_t *hsdirs) +{ + int ret; + + tor_assert(rend_query); + + if (!hsdirs) { + ret = directory_get_from_hs_dir(desc_id, rend_query, NULL); + goto end; /* either success or failure, but we're done */ + } + + /* Using the given hsdir list, trigger a fetch on each of them. */ + SMARTLIST_FOREACH_BEGIN(hsdirs, routerstatus_t *, hs_dir) { + /* This should always be a success. */ + ret = directory_get_from_hs_dir(desc_id, rend_query, hs_dir); + tor_assert(ret); + } SMARTLIST_FOREACH_END(hs_dir); + + /* Everything went well. */ + ret = 0; + + end: + return ret; +} + +/** Fetch a v2 descriptor using the onion address in the given query object. + * This will compute the descriptor id for each replicas and fetch it on the + * given hsdir(s) if any or the responsible ones that are choosen + * automatically. + * + * On success, 1 is returned. If no hidden service is left to ask, return 0. + * On error, -1 is returned. */ +static int +fetch_v2_desc_by_addr(rend_data_t *query, smartlist_t *hsdirs) +{ + char descriptor_id[DIGEST_LEN]; + int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS]; + int i, tries_left, ret; + + tor_assert(query); + + /* Randomly iterate over the replicas until a descriptor can be fetched + * from one of the consecutive nodes, or no options are left. */ + for (i = 0; i < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; i++) { + replicas_left_to_try[i] = i; + } + + tries_left = REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; + while (tries_left > 0) { + int rand = crypto_rand_int(tries_left); + int chosen_replica = replicas_left_to_try[rand]; + replicas_left_to_try[rand] = replicas_left_to_try[--tries_left]; + + ret = rend_compute_v2_desc_id(descriptor_id, query->onion_address, + query->auth_type == REND_STEALTH_AUTH ? + query->descriptor_cookie : NULL, + time(NULL), chosen_replica); + if (ret < 0) { + /* Normally, on failure the descriptor_id is untouched but let's be + * safe in general in case the function changes at some point. */ + goto end; + } + + if (tor_memcmp(descriptor_id, query->descriptor_id[chosen_replica], + sizeof(descriptor_id)) != 0) { + /* Not equal from what we currently have so purge the last hid serv + * request cache and update the descriptor ID with the new value. */ + purge_hid_serv_from_last_hid_serv_requests( + query->descriptor_id[chosen_replica]); + memcpy(query->descriptor_id[chosen_replica], descriptor_id, + sizeof(query->descriptor_id[chosen_replica])); + } + + /* Trigger the fetch with the computed descriptor ID. */ + ret = fetch_v2_desc_by_descid(descriptor_id, query, hsdirs); + if (ret != 0) { + /* Either on success or failure, as long as we tried a fetch we are + * done here. */ + goto end; + } + } + + /* If we come here, there are no hidden service directories left. */ + log_info(LD_REND, "Could not pick one of the responsible hidden " + "service directories to fetch descriptors, because " + "we already tried them all unsuccessfully."); + ret = 0; + + end: + memwipe(descriptor_id, 0, sizeof(descriptor_id)); + return ret; +} + +/** Fetch a v2 descriptor using the given query. If any hsdir are specified, + * use them for the fetch. + * + * On success, 1 is returned. If no hidden service is left to ask, return 0. + * On error, -1 is returned. */ +int +rend_client_fetch_v2_desc(rend_data_t *query, smartlist_t *hsdirs) +{ + int ret; + + tor_assert(query); + + /* Depending on what's available in the rend data query object, we will + * trigger a fetch by HS address or using a descriptor ID. */ + + if (query->onion_address[0] != '\0') { + ret = fetch_v2_desc_by_addr(query, hsdirs); + } else if (!tor_digest_is_zero(query->desc_id_fetch)) { + ret = fetch_v2_desc_by_descid(query->desc_id_fetch, query, hsdirs); + } else { + /* Query data is invalid. */ + ret = -1; + goto error; + } + + error: + return ret; +} + /** Unless we already have a descriptor for <b>rend_query</b> with at least * one (possibly) working introduction point in it, start a connection to a * hidden service directory to fetch a v2 rendezvous service descriptor. */ void -rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) +rend_client_refetch_v2_renddesc(rend_data_t *rend_query) { - char descriptor_id[DIGEST_LEN]; - int replicas_left_to_try[REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS]; - int i, tries_left; rend_cache_entry_t *e = NULL; + tor_assert(rend_query); /* Are we configured to fetch descriptors? */ if (!get_options()->FetchHidServDescriptors) { @@ -714,7 +902,7 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) return; } /* Before fetching, check if we already have a usable descriptor here. */ - if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) > 0 && + if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) == 0 && rend_client_any_intro_points_usable(e)) { log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we " "already have a usable descriptor here. Not fetching."); @@ -722,44 +910,11 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) } log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s", safe_str_client(rend_query->onion_address)); - /* Randomly iterate over the replicas until a descriptor can be fetched - * from one of the consecutive nodes, or no options are left. */ - tries_left = REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; - for (i = 0; i < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS; i++) - replicas_left_to_try[i] = i; - while (tries_left > 0) { - int rand = crypto_rand_int(tries_left); - int chosen_replica = replicas_left_to_try[rand]; - replicas_left_to_try[rand] = replicas_left_to_try[--tries_left]; - - if (rend_compute_v2_desc_id(descriptor_id, rend_query->onion_address, - rend_query->auth_type == REND_STEALTH_AUTH ? - rend_query->descriptor_cookie : NULL, - time(NULL), chosen_replica) < 0) { - log_warn(LD_REND, "Internal error: Computing v2 rendezvous " - "descriptor ID did not succeed."); - /* - * Hmm, can this write anything to descriptor_id and still fail? - * Let's clear it just to be safe. - * - * From here on, any returns should goto done which clears - * descriptor_id so we don't leave key-derived material on the stack. - */ - goto done; - } - if (directory_get_from_hs_dir(descriptor_id, rend_query) != 0) - goto done; /* either success or failure, but we're done */ - } - /* If we come here, there are no hidden service directories left. */ - log_info(LD_REND, "Could not pick one of the responsible hidden " - "service directories to fetch descriptors, because " - "we already tried them all unsuccessfully."); - /* Close pending connections. */ - rend_client_desc_trynow(rend_query->onion_address); - - done: - memwipe(descriptor_id, 0, sizeof(descriptor_id)); + rend_client_fetch_v2_desc(rend_query, NULL); + /* We don't need to look the error code because either on failure or + * success, the necessary steps to continue the HS connection will be + * triggered once the descriptor arrives or if all fetch failed. */ return; } @@ -772,8 +927,7 @@ rend_client_cancel_descriptor_fetches(void) SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) { if (conn->type == CONN_TYPE_DIR && - (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC || - conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2)) { + conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { /* It's a rendezvous descriptor fetch in progress -- cancel it * by marking the connection for close. * @@ -821,7 +975,7 @@ rend_client_cancel_descriptor_fetches(void) */ int rend_client_report_intro_point_failure(extend_info_t *failed_intro, - const rend_data_t *rend_query, + rend_data_t *rend_query, unsigned int failure_type) { int i, r; @@ -829,17 +983,26 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro, connection_t *conn; r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent); - if (r<0) { - log_warn(LD_BUG, "Malformed service ID %s.", - escaped_safe_str_client(rend_query->onion_address)); - return -1; - } - if (r==0) { - log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", - escaped_safe_str_client(rend_query->onion_address)); - rend_client_refetch_v2_renddesc(rend_query); - return 0; + if (r < 0) { + /* Either invalid onion address or cache entry not found. */ + switch (-r) { + case EINVAL: + log_warn(LD_BUG, "Malformed service ID %s.", + escaped_safe_str_client(rend_query->onion_address)); + return -1; + case ENOENT: + log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", + escaped_safe_str_client(rend_query->onion_address)); + rend_client_refetch_v2_renddesc(rend_query); + return 0; + default: + log_warn(LD_BUG, "Unknown cache lookup returned code: %d", r); + return -1; + } } + /* The intro points are not checked here if they are usable or not because + * this is called when an intro point circuit is closed thus there must be + * at least one intro point that is usable and is about to be flagged. */ for (i = 0; i < smartlist_len(ent->parsed->intro_nodes); i++) { rend_intro_point_t *intro = smartlist_get(ent->parsed->intro_nodes, i); @@ -852,6 +1015,9 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro, tor_fragile_assert(); /* fall through */ case INTRO_POINT_FAILURE_GENERIC: + rend_cache_intro_failure_note(failure_type, + (uint8_t *)failed_intro->identity_digest, + rend_query->onion_address); rend_intro_point_free(intro); smartlist_del(ent->parsed->intro_nodes, i); break; @@ -867,6 +1033,10 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro, intro->unreachable_count, zap_intro_point ? " Removing from descriptor.": ""); if (zap_intro_point) { + rend_cache_intro_failure_note( + failure_type, + (uint8_t *) failed_intro->identity_digest, + rend_query->onion_address); rend_intro_point_free(intro); smartlist_del(ent->parsed->intro_nodes, i); } @@ -884,9 +1054,11 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro, rend_client_refetch_v2_renddesc(rend_query); /* move all pending streams back to renddesc_wait */ + /* NOTE: We can now do this faster, if we use pending_entry_connections */ while ((conn = connection_get_by_type_state_rendquery(CONN_TYPE_AP, AP_CONN_STATE_CIRCUIT_WAIT, rend_query->onion_address))) { + connection_ap_mark_as_non_pending_circuit(TO_ENTRY_CONN(conn)); conn->state = AP_CONN_STATE_RENDDESC_WAIT; } @@ -922,9 +1094,9 @@ rend_client_rendezvous_acked(origin_circuit_t *circ, const uint8_t *request, circ->base_.timestamp_dirty = time(NULL); /* From a path bias point of view, this circuit is now successfully used. - * Waiting any longer opens us up to attacks from Bob. He could induce - * Alice to attempt to connect to his hidden service and never reply - * to her rend requests */ + * Waiting any longer opens us up to attacks from malicious hidden services. + * They could induce the client to attempt to connect to their hidden + * service and never reply to the client's rend requests */ pathbias_mark_use_success(circ); /* XXXX This is a pretty brute-force approach. It'd be better to @@ -932,11 +1104,11 @@ rend_client_rendezvous_acked(origin_circuit_t *circ, const uint8_t *request, * than trying to attach them all. See comments bug 743. */ /* If we already have the introduction circuit built, make sure we send * the INTRODUCE cell _now_ */ - connection_ap_attach_pending(); + connection_ap_attach_pending(1); return 0; } -/** Bob sent us a rendezvous cell; join the circuits. */ +/** The service sent us a rendezvous cell; join the circuits. */ int rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, size_t request_len) @@ -961,7 +1133,8 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, log_info(LD_REND,"Got RENDEZVOUS2 cell from hidden service."); - /* first DH_KEY_LEN bytes are g^y from bob. Finish the dh handshake...*/ + /* first DH_KEY_LEN bytes are g^y from the service. Finish the dh + * handshake...*/ tor_assert(circ->build_state); tor_assert(circ->build_state->pending_final_cpath); hop = circ->build_state->pending_final_cpath; @@ -990,7 +1163,7 @@ rend_client_receive_rendezvous(origin_circuit_t *circ, const uint8_t *request, circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_REND_JOINED); hop->state = CPATH_STATE_OPEN; /* set the windows to default. these are the windows - * that alice thinks bob has. + * that the client thinks the service has. */ hop->package_window = circuit_initial_package_window(); hop->deliver_window = CIRCWINDOW_START; @@ -1038,7 +1211,7 @@ rend_client_desc_trynow(const char *query) continue; assert_connection_ok(base_conn, now); if (rend_cache_lookup_entry(rend_data->onion_address, -1, - &entry) == 1 && + &entry) == 0 && rend_client_any_intro_points_usable(entry)) { /* either this fetch worked, or it failed but there was a * valid entry from before which we should reuse */ @@ -1051,35 +1224,34 @@ rend_client_desc_trynow(const char *query) base_conn->timestamp_lastread = now; base_conn->timestamp_lastwritten = now; - if (connection_ap_handshake_attach_circuit(conn) < 0) { - /* it will never work */ - log_warn(LD_REND,"Rendezvous attempt failed. Closing."); - if (!base_conn->marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - } + connection_ap_mark_as_pending_circuit(conn); } else { /* 404, or fetch didn't get that far */ log_notice(LD_REND,"Closing stream for '%s.onion': hidden service is " "unavailable (try again later).", safe_str_client(query)); connection_mark_unattached_ap(conn, END_STREAM_REASON_RESOLVEFAILED); - rend_client_note_connection_attempt_ended(query); + rend_client_note_connection_attempt_ended(rend_data); } } SMARTLIST_FOREACH_END(base_conn); } -/** Clear temporary state used only during an attempt to connect to - * the hidden service named <b>onion_address</b>. Called when a - * connection attempt has ended; may be called occasionally at other - * times, and should be reasonably harmless. */ +/** Clear temporary state used only during an attempt to connect to the + * hidden service with <b>rend_data</b>. Called when a connection attempt + * has ended; it is possible for this to be called multiple times while + * handling an ended connection attempt, and any future changes to this + * function must ensure it remains idempotent. */ void -rend_client_note_connection_attempt_ended(const char *onion_address) +rend_client_note_connection_attempt_ended(const rend_data_t *rend_data) { + unsigned int have_onion = 0; rend_cache_entry_t *cache_entry = NULL; - rend_cache_lookup_entry(onion_address, -1, &cache_entry); - log_info(LD_REND, "Connection attempt for %s has ended; " - "cleaning up temporary state.", - safe_str_client(onion_address)); + if (*rend_data->onion_address != '\0') { + /* Ignore return value; we find an entry, or we don't. */ + (void) rend_cache_lookup_entry(rend_data->onion_address, -1, + &cache_entry); + have_onion = 1; + } /* Clear the timed_out flag on all remaining intro points for this HS. */ if (cache_entry != NULL) { @@ -1089,7 +1261,20 @@ rend_client_note_connection_attempt_ended(const char *onion_address) } /* Remove the HS's entries in last_hid_serv_requests. */ - purge_hid_serv_from_last_hid_serv_requests(onion_address); + if (have_onion) { + unsigned int replica; + for (replica = 0; replica < ARRAY_LENGTH(rend_data->descriptor_id); + replica++) { + const char *desc_id = rend_data->descriptor_id[replica]; + purge_hid_serv_from_last_hid_serv_requests(desc_id); + } + log_info(LD_REND, "Connection attempt for %s has ended; " + "cleaning up temporary state.", + safe_str_client(rend_data->onion_address)); + } else { + /* We only have an ID for a fetch. Probably used by HSFETCH. */ + purge_hid_serv_from_last_hid_serv_requests(rend_data->desc_id_fetch); + } } /** Return a newly allocated extend_info_t* for a randomly chosen introduction @@ -1099,13 +1284,17 @@ rend_client_note_connection_attempt_ended(const char *onion_address) extend_info_t * rend_client_get_random_intro(const rend_data_t *rend_query) { + int ret; extend_info_t *result; rend_cache_entry_t *entry; - if (rend_cache_lookup_entry(rend_query->onion_address, -1, &entry) < 1) { - log_warn(LD_REND, - "Query '%s' didn't have valid rend desc in cache. Failing.", - safe_str_client(rend_query->onion_address)); + ret = rend_cache_lookup_entry(rend_query->onion_address, -1, &entry); + if (ret < 0 || !rend_client_any_intro_points_usable(entry)) { + log_warn(LD_REND, + "Query '%s' didn't have valid rend desc in cache. Failing.", + safe_str_client(rend_query->onion_address)); + /* XXX: Should we refetch the descriptor here if the IPs are not usable + * anymore ?. */ return NULL; } @@ -1176,11 +1365,19 @@ rend_client_get_random_intro_impl(const rend_cache_entry_t *entry, smartlist_del(usable_nodes, i); goto again; } +#ifdef ENABLE_TOR2WEB_MODE + new_extend_info = extend_info_from_node(node, options->Tor2webMode); +#else new_extend_info = extend_info_from_node(node, 0); +#endif if (!new_extend_info) { + const char *alternate_reason = ""; +#ifdef ENABLE_TOR2WEB_MODE + alternate_reason = ", or we cannot connect directly to it"; +#endif log_info(LD_REND, "We don't have a descriptor for the intro-point relay " - "'%s'; trying another.", - extend_info_describe(intro->extend_info)); + "'%s'%s; trying another.", + extend_info_describe(intro->extend_info), alternate_reason); smartlist_del(usable_nodes, i); goto again; } else { diff --git a/src/or/rendclient.h b/src/or/rendclient.h index 1f731d0ae5..e90dac07ab 100644 --- a/src/or/rendclient.h +++ b/src/or/rendclient.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,6 +12,8 @@ #ifndef TOR_RENDCLIENT_H #define TOR_RENDCLIENT_H +#include "rendcache.h" + void rend_client_purge_state(void); void rend_client_introcirc_has_opened(origin_circuit_t *circ); @@ -19,16 +21,13 @@ void rend_client_rendcirc_has_opened(origin_circuit_t *circ); int rend_client_introduction_acked(origin_circuit_t *circ, const uint8_t *request, size_t request_len); -void rend_client_refetch_v2_renddesc(const rend_data_t *rend_query); +void rend_client_refetch_v2_renddesc(rend_data_t *rend_query); +int rend_client_fetch_v2_desc(rend_data_t *query, smartlist_t *hsdirs); void rend_client_cancel_descriptor_fetches(void); void rend_client_purge_last_hid_serv_requests(void); -#define INTRO_POINT_FAILURE_GENERIC 0 -#define INTRO_POINT_FAILURE_TIMEOUT 1 -#define INTRO_POINT_FAILURE_UNREACHABLE 2 - int rend_client_report_intro_point_failure(extend_info_t *failed_intro, - const rend_data_t *rend_query, + rend_data_t *rend_query, unsigned int failure_type); int rend_client_rendezvous_acked(origin_circuit_t *circ, @@ -39,7 +38,7 @@ int rend_client_receive_rendezvous(origin_circuit_t *circ, size_t request_len); void rend_client_desc_trynow(const char *query); -void rend_client_note_connection_attempt_ended(const char *onion_address); +void rend_client_note_connection_attempt_ended(const rend_data_t *rend_data); extend_info_t *rend_client_get_random_intro(const rend_data_t *rend_query); int rend_client_any_intro_points_usable(const rend_cache_entry_t *entry); @@ -51,7 +50,6 @@ int rend_parse_service_authorization(const or_options_t *options, rend_service_authorization_t *rend_client_lookup_service_authorization( const char *onion_address); void rend_service_authorization_free_all(void); -rend_data_t *rend_data_dup(const rend_data_t *request); #endif diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 296df55664..438fbc4d9a 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,13 +11,16 @@ #include "or.h" #include "circuitbuild.h" #include "config.h" +#include "control.h" #include "rendclient.h" #include "rendcommon.h" #include "rendmid.h" #include "rendservice.h" #include "rephist.h" +#include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "networkstatus.h" /** Return 0 if one and two are the same service ids, else -1 or 1 */ int @@ -155,10 +158,10 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, } /* Calculate current time-period. */ time_period = get_time_period(now, 0, service_id_binary); - /* Calculate secret-id-part = h(time-period + replica). */ + /* Calculate secret-id-part = h(time-period | desc-cookie | replica). */ get_secret_id_part_bytes(secret_id_part, time_period, descriptor_cookie, replica); - /* Calculate descriptor ID. */ + /* Calculate descriptor ID: H(permanent-id | secret-id-part) */ rend_get_descriptor_id_bytes(desc_id_out, service_id_binary, secret_id_part); return 0; } @@ -268,11 +271,7 @@ rend_encrypt_v2_intro_points_basic(char **encrypted_out, tor_assert(client_cookies && smartlist_len(client_cookies) > 0); /* Generate session key. */ - if (crypto_rand(session_key, CIPHER_KEY_LEN) < 0) { - log_warn(LD_REND, "Unable to generate random session key to encrypt " - "introduction point string."); - goto done; - } + crypto_rand(session_key, CIPHER_KEY_LEN); /* Determine length of encrypted introduction points including session * keys. */ @@ -334,11 +333,7 @@ rend_encrypt_v2_intro_points_basic(char **encrypted_out, REND_BASIC_AUTH_CLIENT_MULTIPLE; i < REND_BASIC_AUTH_CLIENT_MULTIPLE - 1; i++) { client_part = tor_malloc_zero(REND_BASIC_AUTH_CLIENT_ENTRY_LEN); - if (crypto_rand(client_part, REND_BASIC_AUTH_CLIENT_ENTRY_LEN) < 0) { - log_warn(LD_REND, "Unable to generate fake client entry."); - tor_free(client_part); - goto done; - } + crypto_rand(client_part, REND_BASIC_AUTH_CLIENT_ENTRY_LEN); smartlist_add(encrypted_session_keys, client_part); } /* Sort smartlist and put elements in result in order. */ @@ -411,7 +406,7 @@ rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc) &test_intro_content, &test_intro_size, &test_encoded_size, - &test_next, desc->desc_str); + &test_next, desc->desc_str, 1); rend_service_descriptor_free(test_parsed); tor_free(test_intro_content); return (res >= 0); @@ -461,6 +456,7 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, smartlist_t *client_cookies) { char service_id[DIGEST_LEN]; + char service_id_base32[REND_SERVICE_ID_LEN_BASE32+1]; uint32_t time_period; char *ipos_base64 = NULL, *ipos = NULL, *ipos_encrypted = NULL, *descriptor_cookie = NULL; @@ -528,8 +524,9 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return -1; } /* Base64-encode introduction points. */ - ipos_base64 = tor_malloc_zero(ipos_len * 2); - if (base64_encode(ipos_base64, ipos_len * 2, ipos, ipos_len)<0) { + ipos_base64 = tor_calloc(ipos_len, 2); + if (base64_encode(ipos_base64, ipos_len * 2, ipos, ipos_len, + BASE64_ENCODE_MULTILINE)<0) { log_warn(LD_REND, "Could not encode introduction point string to " "base64. length=%d", (int)ipos_len); tor_free(ipos_base64); @@ -556,7 +553,7 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, char desc_digest[DIGEST_LEN]; rend_encoded_v2_service_descriptor_t *enc = tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t)); - /* Calculate secret-id-part = h(time-period + cookie + replica). */ + /* Calculate secret-id-part = h(time-period | cookie | replica). */ get_secret_id_part_bytes(secret_id_part, time_period, descriptor_cookie, k); base32_encode(secret_id_part_base32, sizeof(secret_id_part_base32), @@ -646,7 +643,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, rend_encoded_v2_service_descriptor_free(enc); goto err; } - desc_str[written++] = '\n'; desc_str[written++] = 0; /* Check if we can parse our own descriptor. */ if (!rend_desc_v2_is_parsable(enc)) { @@ -655,6 +651,11 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, goto err; } smartlist_add(descs_out, enc); + /* Add the uploaded descriptor to the local service's descriptor cache */ + rend_cache_store_v2_desc_as_service(enc->desc_str); + base32_encode(service_id_base32, sizeof(service_id_base32), + service_id, REND_SERVICE_ID_LEN); + control_event_hs_descriptor_created(service_id_base32, desc_id_base32, k); } log_info(LD_REND, "Successfully encoded a v2 descriptor and " @@ -672,79 +673,6 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return seconds_valid; } -/** Parse a service descriptor at <b>str</b> (<b>len</b> bytes). On - * success, return a newly alloced service_descriptor_t. On failure, - * return NULL. - */ -rend_service_descriptor_t * -rend_parse_service_descriptor(const char *str, size_t len) -{ - rend_service_descriptor_t *result = NULL; - int i, n_intro_points; - size_t keylen, asn1len; - const char *end, *cp, *eos; - rend_intro_point_t *intro; - - result = tor_malloc_zero(sizeof(rend_service_descriptor_t)); - cp = str; - end = str+len; - if (end-cp<2) goto truncated; - result->version = 0; - if (end-cp < 2) goto truncated; - asn1len = ntohs(get_uint16(cp)); - cp += 2; - if ((size_t)(end-cp) < asn1len) goto truncated; - result->pk = crypto_pk_asn1_decode(cp, asn1len); - if (!result->pk) goto truncated; - cp += asn1len; - if (end-cp < 4) goto truncated; - result->timestamp = (time_t) ntohl(get_uint32(cp)); - cp += 4; - result->protocols = 1<<2; /* always use intro format 2 */ - if (end-cp < 2) goto truncated; - n_intro_points = ntohs(get_uint16(cp)); - cp += 2; - - result->intro_nodes = smartlist_new(); - for (i=0;i<n_intro_points;++i) { - if (end-cp < 2) goto truncated; - eos = (const char *)memchr(cp,'\0',end-cp); - if (!eos) goto truncated; - /* Write nickname to extend info, but postpone the lookup whether - * we know that router. It's not part of the parsing process. */ - intro = tor_malloc_zero(sizeof(rend_intro_point_t)); - intro->extend_info = tor_malloc_zero(sizeof(extend_info_t)); - strlcpy(intro->extend_info->nickname, cp, - sizeof(intro->extend_info->nickname)); - smartlist_add(result->intro_nodes, intro); - cp = eos+1; - } - keylen = crypto_pk_keysize(result->pk); - tor_assert(end-cp >= 0); - if ((size_t)(end-cp) < keylen) goto truncated; - if ((size_t)(end-cp) > keylen) { - log_warn(LD_PROTOCOL, - "Signature is %d bytes too long on service descriptor.", - (int)((size_t)(end-cp) - keylen)); - goto error; - } - note_crypto_pk_op(REND_CLIENT); - if (crypto_pk_public_checksig_digest(result->pk, - (char*)str,cp-str, /* data */ - (char*)cp,end-cp /* signature*/ - )<0) { - log_warn(LD_PROTOCOL, "Bad signature on service descriptor."); - goto error; - } - - return result; - truncated: - log_warn(LD_PROTOCOL, "Truncated service descriptor."); - error: - rend_service_descriptor_free(result); - return NULL; -} - /** Sets <b>out</b> to the first 10 bytes of the digest of <b>pk</b>, * base32 encoded. NUL-terminates out. (We use this string to * identify services in directory requests and .onion URLs.) @@ -760,154 +688,6 @@ rend_get_service_id(crypto_pk_t *pk, char *out) return 0; } -/* ==== Rendezvous service descriptor cache. */ - -/** How old do we let hidden service descriptors get before discarding - * them as too old? */ -#define REND_CACHE_MAX_AGE (2*24*60*60) -/** How wrong do we assume our clock may be when checking whether hidden - * services are too old or too new? */ -#define REND_CACHE_MAX_SKEW (24*60*60) - -/** Map from service id (as generated by rend_get_service_id) to - * rend_cache_entry_t. */ -static strmap_t *rend_cache = NULL; - -/** Map from descriptor id to rend_cache_entry_t; only for hidden service - * directories. */ -static digestmap_t *rend_cache_v2_dir = NULL; - -/** Initializes the service descriptor cache. - */ -void -rend_cache_init(void) -{ - rend_cache = strmap_new(); - rend_cache_v2_dir = digestmap_new(); -} - -/** Helper: free storage held by a single service descriptor cache entry. */ -static void -rend_cache_entry_free(rend_cache_entry_t *e) -{ - if (!e) - return; - rend_service_descriptor_free(e->parsed); - tor_free(e->desc); - tor_free(e); -} - -/** Helper: deallocate a rend_cache_entry_t. (Used with strmap_free(), which - * requires a function pointer whose argument is void*). */ -static void -rend_cache_entry_free_(void *p) -{ - rend_cache_entry_free(p); -} - -/** Free all storage held by the service descriptor cache. */ -void -rend_cache_free_all(void) -{ - strmap_free(rend_cache, rend_cache_entry_free_); - digestmap_free(rend_cache_v2_dir, rend_cache_entry_free_); - rend_cache = NULL; - rend_cache_v2_dir = NULL; -} - -/** Removes all old entries from the service descriptor cache. - */ -void -rend_cache_clean(time_t now) -{ - strmap_iter_t *iter; - const char *key; - void *val; - rend_cache_entry_t *ent; - time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW; - for (iter = strmap_iter_init(rend_cache); !strmap_iter_done(iter); ) { - strmap_iter_get(iter, &key, &val); - ent = (rend_cache_entry_t*)val; - if (ent->parsed->timestamp < cutoff) { - iter = strmap_iter_next_rmv(rend_cache, iter); - rend_cache_entry_free(ent); - } else { - iter = strmap_iter_next(rend_cache, iter); - } - } -} - -/** Remove ALL entries from the rendezvous service descriptor cache. - */ -void -rend_cache_purge(void) -{ - if (rend_cache) { - log_info(LD_REND, "Purging client/v0-HS-authority HS descriptor cache"); - strmap_free(rend_cache, rend_cache_entry_free_); - } - rend_cache = strmap_new(); -} - -/** Remove all old v2 descriptors and those for which this hidden service - * directory is not responsible for any more. */ -void -rend_cache_clean_v2_descs_as_dir(time_t now) -{ - digestmap_iter_t *iter; - time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW; - for (iter = digestmap_iter_init(rend_cache_v2_dir); - !digestmap_iter_done(iter); ) { - const char *key; - void *val; - rend_cache_entry_t *ent; - digestmap_iter_get(iter, &key, &val); - ent = val; - if (ent->parsed->timestamp < cutoff || - !hid_serv_responsible_for_desc_id(key)) { - char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; - base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); - log_info(LD_REND, "Removing descriptor with ID '%s' from cache", - safe_str_client(key_base32)); - iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); - rend_cache_entry_free(ent); - } else { - iter = digestmap_iter_next(rend_cache_v2_dir, iter); - } - } -} - -/** Determines whether <b>a</b> is in the interval of <b>b</b> (excluded) and - * <b>c</b> (included) in a circular digest ring; returns 1 if this is the - * case, and 0 otherwise. - */ -int -rend_id_is_in_interval(const char *a, const char *b, const char *c) -{ - int a_b, b_c, c_a; - tor_assert(a); - tor_assert(b); - tor_assert(c); - - /* There are five cases in which a is outside the interval ]b,c]: */ - a_b = tor_memcmp(a,b,DIGEST_LEN); - if (a_b == 0) - return 0; /* 1. a == b (b is excluded) */ - b_c = tor_memcmp(b,c,DIGEST_LEN); - if (b_c == 0) - return 0; /* 2. b == c (interval is empty) */ - else if (a_b <= 0 && b_c < 0) - return 0; /* 3. a b c */ - c_a = tor_memcmp(c,a,DIGEST_LEN); - if (c_a < 0 && a_b <= 0) - return 0; /* 4. c a b */ - else if (b_c < 0 && c_a < 0) - return 0; /* 5. b c a */ - - /* In the other cases (a c b; b a c; c b a), a is inside the interval. */ - return 1; -} - /** Return true iff <b>query</b> is a syntactically valid service ID (as * generated by rend_get_service_id). */ int @@ -922,494 +702,24 @@ rend_valid_service_id(const char *query) return 1; } -/** If we have a cached rend_cache_entry_t for the service ID <b>query</b> - * with <b>version</b>, set *<b>e</b> to that entry and return 1. - * Else return 0. If <b>version</b> is nonnegative, only return an entry - * in that descriptor format version. Otherwise (if <b>version</b> is - * negative), return the most recent format we have. - */ +/** Return true iff <b>query</b> is a syntactically valid descriptor ID. + * (as generated by rend_get_descriptor_id_bytes). */ int -rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) +rend_valid_descriptor_id(const char *query) { - char key[REND_SERVICE_ID_LEN_BASE32+2]; /* <version><query>\0 */ - tor_assert(rend_cache); - if (!rend_valid_service_id(query)) - return -1; - *e = NULL; - if (version != 0) { - tor_snprintf(key, sizeof(key), "2%s", query); - *e = strmap_get_lc(rend_cache, key); + if (strlen(query) != REND_DESC_ID_V2_LEN_BASE32) { + goto invalid; } - if (!*e && version != 2) { - tor_snprintf(key, sizeof(key), "0%s", query); - *e = strmap_get_lc(rend_cache, key); + if (strspn(query, BASE32_CHARS) != REND_DESC_ID_V2_LEN_BASE32) { + goto invalid; } - if (!*e) - return 0; - tor_assert((*e)->parsed && (*e)->parsed->intro_nodes); - /* XXX023 hack for now, to return "not found" if there are no intro - * points remaining. See bug 997. */ - if (! rend_client_any_intro_points_usable(*e)) - return 0; - return 1; -} -/** <b>query</b> is a base32'ed service id. If it's malformed, return -1. - * Else look it up. - * - If it is found, point *desc to it, and write its length into - * *desc_len, and return 1. - * - If it is not found, return 0. - * Note: calls to rend_cache_clean or rend_cache_store may invalidate - * *desc. - */ -int -rend_cache_lookup_desc(const char *query, int version, const char **desc, - size_t *desc_len) -{ - rend_cache_entry_t *e; - int r; - r = rend_cache_lookup_entry(query,version,&e); - if (r <= 0) return r; - *desc = e->desc; - *desc_len = e->len; return 1; -} -/** Lookup the v2 service descriptor with base32-encoded <b>desc_id</b> and - * copy the pointer to it to *<b>desc</b>. Return 1 on success, 0 on - * well-formed-but-not-found, and -1 on failure. - */ -int -rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc) -{ - rend_cache_entry_t *e; - char desc_id_digest[DIGEST_LEN]; - tor_assert(rend_cache_v2_dir); - if (base32_decode(desc_id_digest, DIGEST_LEN, - desc_id, REND_DESC_ID_V2_LEN_BASE32) < 0) { - log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Rejecting v2 rendezvous descriptor request -- descriptor ID " - "contains illegal characters: %s", - safe_str(desc_id)); - return -1; - } - /* Lookup descriptor and return. */ - e = digestmap_get(rend_cache_v2_dir, desc_id_digest); - if (e) { - *desc = e->desc; - return 1; - } + invalid: return 0; } -/* Do not allow more than this many introduction points in a hidden service - * descriptor */ -#define MAX_INTRO_POINTS 10 - -/** Parse *desc, calculate its service id, and store it in the cache. - * If we have a newer v0 descriptor with the same ID, ignore this one. - * If we have an older descriptor with the same ID, replace it. - * If we are acting as client due to the published flag and have any v2 - * descriptor with the same ID, reject this one in order to not get - * confused with having both versions for the same service. - * - * Return -2 if it's malformed or otherwise rejected; return -1 if we - * already have a v2 descriptor here; return 0 if it's the same or older - * than one we've already got; return 1 if it's novel. - * - * The published flag tells us if we store the descriptor - * in our role as directory (1) or if we cache it as client (0). - * - * If <b>service_id</b> is non-NULL and the descriptor is not for that - * service ID, reject it. <b>service_id</b> must be specified if and - * only if <b>published</b> is 0 (we fetched this descriptor). - */ -int -rend_cache_store(const char *desc, size_t desc_len, int published, - const char *service_id) -{ - rend_cache_entry_t *e; - rend_service_descriptor_t *parsed; - char query[REND_SERVICE_ID_LEN_BASE32+1]; - char key[REND_SERVICE_ID_LEN_BASE32+2]; /* 0<query>\0 */ - time_t now; - tor_assert(rend_cache); - parsed = rend_parse_service_descriptor(desc,desc_len); - if (!parsed) { - log_warn(LD_PROTOCOL,"Couldn't parse service descriptor."); - return -2; - } - if (rend_get_service_id(parsed->pk, query)<0) { - log_warn(LD_BUG,"Couldn't compute service ID."); - rend_service_descriptor_free(parsed); - return -2; - } - if ((service_id != NULL) && strcmp(query, service_id)) { - log_warn(LD_REND, "Received service descriptor for service ID %s; " - "expected descriptor for service ID %s.", - query, safe_str(service_id)); - rend_service_descriptor_free(parsed); - return -2; - } - now = time(NULL); - if (parsed->timestamp < now-REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { - log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too old.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -2; - } - if (parsed->timestamp > now+REND_CACHE_MAX_SKEW) { - log_fn(LOG_PROTOCOL_WARN, LD_REND, - "Service descriptor %s is too far in the future.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -2; - } - /* Do we have a v2 descriptor and fetched this descriptor as a client? */ - tor_snprintf(key, sizeof(key), "2%s", query); - if (!published && strmap_get_lc(rend_cache, key)) { - log_info(LD_REND, "We already have a v2 descriptor for service %s.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -1; - } - if (parsed->intro_nodes && - smartlist_len(parsed->intro_nodes) > MAX_INTRO_POINTS) { - log_warn(LD_REND, "Found too many introduction points on a hidden " - "service descriptor for %s. This is probably a (misguided) " - "attempt to improve reliability, but it could also be an " - "attempt to do a guard enumeration attack. Rejecting.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return -2; - } - tor_snprintf(key, sizeof(key), "0%s", query); - e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); - if (e && e->parsed->timestamp > parsed->timestamp) { - log_info(LD_REND,"We already have a newer service descriptor %s with the " - "same ID and version.", - safe_str_client(query)); - rend_service_descriptor_free(parsed); - return 0; - } - if (e && e->len == desc_len && tor_memeq(desc,e->desc,desc_len)) { - log_info(LD_REND,"We already have this service descriptor %s.", - safe_str_client(query)); - e->received = time(NULL); - rend_service_descriptor_free(parsed); - return 0; - } - if (!e) { - e = tor_malloc_zero(sizeof(rend_cache_entry_t)); - strmap_set_lc(rend_cache, key, e); - } else { - rend_service_descriptor_free(e->parsed); - tor_free(e->desc); - } - e->received = time(NULL); - e->parsed = parsed; - e->len = desc_len; - e->desc = tor_malloc(desc_len); - memcpy(e->desc, desc, desc_len); - - log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str_client(query), (int)desc_len); - return 1; -} - -/** Parse the v2 service descriptor(s) in <b>desc</b> and store it/them to the - * local rend cache. Don't attempt to decrypt the included list of introduction - * points (as we don't have a descriptor cookie for it). - * - * If we have a newer descriptor with the same ID, ignore this one. - * If we have an older descriptor with the same ID, replace it. - * Return -2 if we are not acting as hidden service directory; - * return -1 if the descriptor(s) were not parsable; return 0 if all - * descriptors are the same or older than those we've already got; - * return a positive number for the number of novel stored descriptors. - */ -int -rend_cache_store_v2_desc_as_dir(const char *desc) -{ - rend_service_descriptor_t *parsed; - char desc_id[DIGEST_LEN]; - char *intro_content; - size_t intro_size; - size_t encoded_size; - char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; - int number_parsed = 0, number_stored = 0; - const char *current_desc = desc; - const char *next_desc; - rend_cache_entry_t *e; - time_t now = time(NULL); - tor_assert(rend_cache_v2_dir); - tor_assert(desc); - if (!hid_serv_acting_as_directory()) { - /* Cannot store descs, because we are (currently) not acting as - * hidden service directory. */ - log_info(LD_REND, "Cannot store descs: Not acting as hs dir"); - return -2; - } - while (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content, - &intro_size, &encoded_size, - &next_desc, current_desc) >= 0) { - number_parsed++; - /* We don't care about the introduction points. */ - tor_free(intro_content); - /* For pretty log statements. */ - base32_encode(desc_id_base32, sizeof(desc_id_base32), - desc_id, DIGEST_LEN); - /* Is desc ID in the range that we are (directly or indirectly) responsible - * for? */ - if (!hid_serv_responsible_for_desc_id(desc_id)) { - log_info(LD_REND, "Service descriptor with desc ID %s is not in " - "interval that we are responsible for.", - safe_str_client(desc_id_base32)); - goto skip; - } - /* Is descriptor too old? */ - if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { - log_info(LD_REND, "Service descriptor with desc ID %s is too old.", - safe_str(desc_id_base32)); - goto skip; - } - /* Is descriptor too far in the future? */ - if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { - log_info(LD_REND, "Service descriptor with desc ID %s is too far in the " - "future.", - safe_str(desc_id_base32)); - goto skip; - } - /* Do we already have a newer descriptor? */ - e = digestmap_get(rend_cache_v2_dir, desc_id); - if (e && e->parsed->timestamp > parsed->timestamp) { - log_info(LD_REND, "We already have a newer service descriptor with the " - "same desc ID %s and version.", - safe_str(desc_id_base32)); - goto skip; - } - /* Do we already have this descriptor? */ - if (e && !strcmp(desc, e->desc)) { - log_info(LD_REND, "We already have this service descriptor with desc " - "ID %s.", safe_str(desc_id_base32)); - e->received = time(NULL); - goto skip; - } - /* Store received descriptor. */ - if (!e) { - e = tor_malloc_zero(sizeof(rend_cache_entry_t)); - digestmap_set(rend_cache_v2_dir, desc_id, e); - } else { - rend_service_descriptor_free(e->parsed); - tor_free(e->desc); - } - e->received = time(NULL); - e->parsed = parsed; - e->desc = tor_strndup(current_desc, encoded_size); - e->len = encoded_size; - log_info(LD_REND, "Successfully stored service descriptor with desc ID " - "'%s' and len %d.", - safe_str(desc_id_base32), (int)encoded_size); - number_stored++; - goto advance; - skip: - rend_service_descriptor_free(parsed); - advance: - /* advance to next descriptor, if available. */ - current_desc = next_desc; - /* check if there is a next descriptor. */ - if (!current_desc || - strcmpstart(current_desc, "rendezvous-service-descriptor ")) - break; - } - if (!number_parsed) { - log_info(LD_REND, "Could not parse any descriptor."); - return -1; - } - log_info(LD_REND, "Parsed %d and added %d descriptor%s.", - number_parsed, number_stored, number_stored != 1 ? "s" : ""); - return number_stored; -} - -/** Parse the v2 service descriptor in <b>desc</b>, decrypt the included list - * of introduction points with <b>descriptor_cookie</b> (which may also be - * <b>NULL</b> if decryption is not necessary), and store the descriptor to - * the local cache under its version and service id. - * - * If we have a newer v2 descriptor with the same ID, ignore this one. - * If we have an older descriptor with the same ID, replace it. - * If we have any v0 descriptor with the same ID, reject this one in order - * to not get confused with having both versions for the same service. - * If the descriptor's service ID does not match - * <b>rend_query</b>-\>onion_address, reject it. - * Return -2 if it's malformed or otherwise rejected; return -1 if we - * already have a v0 descriptor here; return 0 if it's the same or older - * than one we've already got; return 1 if it's novel. - */ -int -rend_cache_store_v2_desc_as_client(const char *desc, - const rend_data_t *rend_query) -{ - /*XXXX this seems to have a bit of duplicate code with - * rend_cache_store_v2_desc_as_dir(). Fix that. */ - /* Though having similar elements, both functions were separated on - * purpose: - * - dirs don't care about encoded/encrypted introduction points, clients - * do. - * - dirs store descriptors in a separate cache by descriptor ID, whereas - * clients store them by service ID; both caches are different data - * structures and have different access methods. - * - dirs store a descriptor only if they are responsible for its ID, - * clients do so in every way (because they have requested it before). - * - dirs can process multiple concatenated descriptors which is required - * for replication, whereas clients only accept a single descriptor. - * Thus, combining both methods would result in a lot of if statements - * which probably would not improve, but worsen code readability. -KL */ - rend_service_descriptor_t *parsed = NULL; - char desc_id[DIGEST_LEN]; - char *intro_content = NULL; - size_t intro_size; - size_t encoded_size; - const char *next_desc; - time_t now = time(NULL); - char key[REND_SERVICE_ID_LEN_BASE32+2]; - char service_id[REND_SERVICE_ID_LEN_BASE32+1]; - rend_cache_entry_t *e; - int retval; - tor_assert(rend_cache); - tor_assert(desc); - /* Parse the descriptor. */ - if (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content, - &intro_size, &encoded_size, - &next_desc, desc) < 0) { - log_warn(LD_REND, "Could not parse descriptor."); - retval = -2; - goto err; - } - /* Compute service ID from public key. */ - if (rend_get_service_id(parsed->pk, service_id)<0) { - log_warn(LD_REND, "Couldn't compute service ID."); - retval = -2; - goto err; - } - if (strcmp(rend_query->onion_address, service_id)) { - log_warn(LD_REND, "Received service descriptor for service ID %s; " - "expected descriptor for service ID %s.", - service_id, safe_str(rend_query->onion_address)); - retval = -2; - goto err; - } - /* Decode/decrypt introduction points. */ - if (intro_content && intro_size > 0) { - int n_intro_points; - if (rend_query->auth_type != REND_NO_AUTH && - !tor_mem_is_zero(rend_query->descriptor_cookie, - sizeof(rend_query->descriptor_cookie))) { - char *ipos_decrypted = NULL; - size_t ipos_decrypted_size; - if (rend_decrypt_introduction_points(&ipos_decrypted, - &ipos_decrypted_size, - rend_query->descriptor_cookie, - intro_content, - intro_size) < 0) { - log_warn(LD_REND, "Failed to decrypt introduction points. We are " - "probably unable to parse the encoded introduction points."); - } else { - /* Replace encrypted with decrypted introduction points. */ - log_info(LD_REND, "Successfully decrypted introduction points."); - tor_free(intro_content); - intro_content = ipos_decrypted; - intro_size = ipos_decrypted_size; - } - } - n_intro_points = rend_parse_introduction_points(parsed, intro_content, - intro_size); - if (n_intro_points <= 0) { - log_warn(LD_REND, "Failed to parse introduction points. Either the " - "service has published a corrupt descriptor, or you have " - "provided invalid authorization data, or (maybe!) the " - "server is deliberately serving broken data in an attempt " - "to crash you with bug 21018."); - retval = -2; - goto err; - } else if (n_intro_points > MAX_INTRO_POINTS) { - log_warn(LD_REND, "Found too many introduction points on a hidden " - "service descriptor for %s. This is probably a (misguided) " - "attempt to improve reliability, but it could also be an " - "attempt to do a guard enumeration attack. Rejecting.", - safe_str_client(rend_query->onion_address)); - retval = -2; - goto err; - } - } else { - log_info(LD_REND, "Descriptor does not contain any introduction points."); - parsed->intro_nodes = smartlist_new(); - } - /* We don't need the encoded/encrypted introduction points any longer. */ - tor_free(intro_content); - /* Is descriptor too old? */ - if (parsed->timestamp < now - REND_CACHE_MAX_AGE-REND_CACHE_MAX_SKEW) { - log_warn(LD_REND, "Service descriptor with service ID %s is too old.", - safe_str_client(service_id)); - retval = -2; - goto err; - } - /* Is descriptor too far in the future? */ - if (parsed->timestamp > now + REND_CACHE_MAX_SKEW) { - log_warn(LD_REND, "Service descriptor with service ID %s is too far in " - "the future.", safe_str_client(service_id)); - retval = -2; - goto err; - } - /* Do we have a v0 descriptor? */ - tor_snprintf(key, sizeof(key), "0%s", service_id); - if (strmap_get_lc(rend_cache, key)) { - log_info(LD_REND, "We already have a v0 descriptor for service ID %s.", - safe_str_client(service_id)); - retval = -1; - goto err; - } - /* Do we already have a newer descriptor? */ - tor_snprintf(key, sizeof(key), "2%s", service_id); - e = (rend_cache_entry_t*) strmap_get_lc(rend_cache, key); - if (e && e->parsed->timestamp > parsed->timestamp) { - log_info(LD_REND, "We already have a newer service descriptor for " - "service ID %s with the same desc ID and version.", - safe_str_client(service_id)); - retval = 0; - goto err; - } - /* Do we already have this descriptor? */ - if (e && !strcmp(desc, e->desc)) { - log_info(LD_REND,"We already have this service descriptor %s.", - safe_str_client(service_id)); - e->received = time(NULL); - retval = 0; - goto err; - } - if (!e) { - e = tor_malloc_zero(sizeof(rend_cache_entry_t)); - strmap_set_lc(rend_cache, key, e); - } else { - rend_service_descriptor_free(e->parsed); - tor_free(e->desc); - } - e->received = time(NULL); - e->parsed = parsed; - e->desc = tor_malloc_zero(encoded_size + 1); - strlcpy(e->desc, desc, encoded_size + 1); - e->len = encoded_size; - log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", - safe_str_client(service_id), (int)encoded_size); - return 1; - - err: - rend_service_descriptor_free(parsed); - tor_free(intro_content); - return retval; -} - /** Called when we get a rendezvous-related relay cell on circuit * <b>circ</b>. Dispatch on rendezvous relay command. */ void @@ -1447,7 +757,7 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, break; case RELAY_COMMAND_INTRODUCE2: if (origin_circ) - r = rend_service_introduce(origin_circ,payload,length); + r = rend_service_receive_introduction(origin_circ,payload,length); break; case RELAY_COMMAND_INTRODUCE_ACK: if (origin_circ) @@ -1483,7 +793,151 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, rend_data_t * rend_data_dup(const rend_data_t *data) { + rend_data_t *data_dup; tor_assert(data); - return tor_memdup(data, sizeof(rend_data_t)); + data_dup = tor_memdup(data, sizeof(rend_data_t)); + data_dup->hsdirs_fp = smartlist_new(); + SMARTLIST_FOREACH(data->hsdirs_fp, char *, fp, + smartlist_add(data_dup->hsdirs_fp, + tor_memdup(fp, DIGEST_LEN))); + return data_dup; +} + +/** Compute descriptor ID for each replicas and save them. A valid onion + * address must be present in the <b>rend_data</b>. + * + * Return 0 on success else -1. */ +static int +compute_desc_id(rend_data_t *rend_data) +{ + int ret = 0; + unsigned replica; + time_t now = time(NULL); + + tor_assert(rend_data); + + /* Compute descriptor ID for each replicas. */ + for (replica = 0; replica < ARRAY_LENGTH(rend_data->descriptor_id); + replica++) { + ret = rend_compute_v2_desc_id(rend_data->descriptor_id[replica], + rend_data->onion_address, + rend_data->descriptor_cookie, + now, replica); + if (ret < 0) { + goto end; + } + } + + end: + return ret; +} + +/** Allocate and initialize a rend_data_t object for a service using the + * given arguments. Only the <b>onion_address</b> is not optional. + * + * Return a valid rend_data_t pointer. */ +rend_data_t * +rend_data_service_create(const char *onion_address, const char *pk_digest, + const uint8_t *cookie, rend_auth_type_t auth_type) +{ + rend_data_t *rend_data = tor_malloc_zero(sizeof(*rend_data)); + + /* We need at least one else the call is wrong. */ + tor_assert(onion_address != NULL); + + if (pk_digest) { + memcpy(rend_data->rend_pk_digest, pk_digest, + sizeof(rend_data->rend_pk_digest)); + } + if (cookie) { + memcpy(rend_data->rend_cookie, cookie, + sizeof(rend_data->rend_cookie)); + } + + strlcpy(rend_data->onion_address, onion_address, + sizeof(rend_data->onion_address)); + rend_data->auth_type = auth_type; + /* Won't be used but still need to initialize it for rend_data dup and + * free. */ + rend_data->hsdirs_fp = smartlist_new(); + + return rend_data; +} + +/** Allocate and initialize a rend_data_t object for a client request using + * the given arguments. Either an onion address or a descriptor ID is + * needed. Both can be given but only the onion address will be used to make + * the descriptor fetch. + * + * Return a valid rend_data_t pointer or NULL on error meaning the + * descriptor IDs couldn't be computed from the given data. */ +rend_data_t * +rend_data_client_create(const char *onion_address, const char *desc_id, + const char *cookie, rend_auth_type_t auth_type) +{ + rend_data_t *rend_data = tor_malloc_zero(sizeof(*rend_data)); + + /* We need at least one else the call is wrong. */ + tor_assert(onion_address != NULL || desc_id != NULL); + + if (cookie) { + memcpy(rend_data->descriptor_cookie, cookie, + sizeof(rend_data->descriptor_cookie)); + } + if (desc_id) { + memcpy(rend_data->desc_id_fetch, desc_id, + sizeof(rend_data->desc_id_fetch)); + } + if (onion_address) { + strlcpy(rend_data->onion_address, onion_address, + sizeof(rend_data->onion_address)); + if (compute_desc_id(rend_data) < 0) { + goto error; + } + } + + rend_data->auth_type = auth_type; + rend_data->hsdirs_fp = smartlist_new(); + + return rend_data; + + error: + rend_data_free(rend_data); + return NULL; +} + +/** Determine the routers that are responsible for <b>id</b> (binary) and + * add pointers to those routers' routerstatus_t to <b>responsible_dirs</b>. + * Return -1 if we're returning an empty smartlist, else return 0. + */ +int +hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, + const char *id) +{ + int start, found, n_added = 0, i; + networkstatus_t *c = networkstatus_get_latest_consensus(); + if (!c || !smartlist_len(c->routerstatus_list)) { + log_warn(LD_REND, "We don't have a consensus, so we can't perform v2 " + "rendezvous operations."); + return -1; + } + tor_assert(id); + start = networkstatus_vote_find_entry_idx(c, id, &found); + if (start == smartlist_len(c->routerstatus_list)) start = 0; + i = start; + do { + routerstatus_t *r = smartlist_get(c->routerstatus_list, i); + if (r->is_hs_dir) { + smartlist_add(responsible_dirs, r); + if (++n_added == REND_NUMBER_OF_CONSECUTIVE_REPLICAS) + return 0; + } + if (++i == smartlist_len(c->routerstatus_list)) + i = 0; + } while (i != start); + + /* Even though we don't have the desired number of hidden service + * directories, be happy if we got any. */ + return smartlist_len(responsible_dirs) ? 0 : -1; } diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h index f476593d2b..d67552e405 100644 --- a/src/or/rendcommon.h +++ b/src/or/rendcommon.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,10 +12,22 @@ #ifndef TOR_RENDCOMMON_H #define TOR_RENDCOMMON_H +typedef enum rend_intro_point_failure_t { + INTRO_POINT_FAILURE_GENERIC = 0, + INTRO_POINT_FAILURE_TIMEOUT = 1, + INTRO_POINT_FAILURE_UNREACHABLE = 2, +} rend_intro_point_failure_t; + /** Free all storage associated with <b>data</b> */ -static INLINE void +static inline void rend_data_free(rend_data_t *data) { + if (!data) { + return; + } + /* Cleanup the HSDir identity digest. */ + SMARTLIST_FOREACH(data->hsdirs_fp, char *, d, tor_free(d)); + smartlist_free(data->hsdirs_fp); tor_free(data); } @@ -26,29 +38,13 @@ void rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, const uint8_t *payload); void rend_service_descriptor_free(rend_service_descriptor_t *desc); -rend_service_descriptor_t *rend_parse_service_descriptor(const char *str, - size_t len); int rend_get_service_id(crypto_pk_t *pk, char *out); void rend_encoded_v2_service_descriptor_free( rend_encoded_v2_service_descriptor_t *desc); void rend_intro_point_free(rend_intro_point_t *intro); -void rend_cache_init(void); -void rend_cache_clean(time_t now); -void rend_cache_clean_v2_descs_as_dir(time_t now); -void rend_cache_purge(void); -void rend_cache_free_all(void); int rend_valid_service_id(const char *query); -int rend_cache_lookup_desc(const char *query, int version, const char **desc, - size_t *desc_len); -int rend_cache_lookup_entry(const char *query, int version, - rend_cache_entry_t **entry_out); -int rend_cache_lookup_v2_desc_as_dir(const char *query, const char **desc); -int rend_cache_store(const char *desc, size_t desc_len, int published, - const char *service_id); -int rend_cache_store_v2_desc_as_client(const char *desc, - const rend_data_t *rend_query); -int rend_cache_store_v2_desc_as_dir(const char *desc); +int rend_valid_descriptor_id(const char *query); int rend_encode_v2_descriptors(smartlist_t *descs_out, rend_service_descriptor_t *desc, time_t now, uint8_t period, rend_auth_type_t auth_type, @@ -57,10 +53,20 @@ int rend_encode_v2_descriptors(smartlist_t *descs_out, int rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, const char *descriptor_cookie, time_t now, uint8_t replica); -int rend_id_is_in_interval(const char *a, const char *b, const char *c); void rend_get_descriptor_id_bytes(char *descriptor_id_out, const char *service_id, const char *secret_id_part); +int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, + const char *id); +rend_data_t *rend_data_dup(const rend_data_t *data); +rend_data_t *rend_data_client_create(const char *onion_address, + const char *desc_id, + const char *cookie, + rend_auth_type_t auth_type); +rend_data_t *rend_data_service_create(const char *onion_address, + const char *pk_digest, + const uint8_t *cookie, + rend_auth_type_t auth_type); #endif diff --git a/src/or/rendmid.c b/src/or/rendmid.c index 0a005a6312..a33ad92966 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -80,7 +80,7 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, goto err; } - /* The request is valid. First, compute the hash of Bob's PK.*/ + /* The request is valid. First, compute the hash of the service's PK.*/ if (crypto_pk_get_digest(pk, pk_digest)<0) { log_warn(LD_BUG, "Internal error: couldn't hash public key."); goto err; @@ -94,7 +94,7 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, /* Close any other intro circuits with the same pk. */ c = NULL; - while ((c = circuit_get_intro_point(pk_digest))) { + while ((c = circuit_get_intro_point((const uint8_t *)pk_digest))) { log_info(LD_REND, "Replacing old circuit for service %s", safe_str(serviceid)); circuit_mark_for_close(TO_CIRCUIT(c), END_CIRC_REASON_FINISHED); @@ -111,7 +111,7 @@ rend_mid_establish_intro(or_circuit_t *circ, const uint8_t *request, /* Now, set up this circuit. */ circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT); - memcpy(circ->rend_token, pk_digest, DIGEST_LEN); + circuit_set_intro_point_digest(circ, (uint8_t *)pk_digest); log_info(LD_REND, "Established introduction point on circuit %u for service %s", @@ -178,8 +178,9 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1, (char*)request, REND_SERVICE_ID_LEN); - /* The first 20 bytes are all we look at: they have a hash of Bob's PK. */ - intro_circ = circuit_get_intro_point((char*)request); + /* The first 20 bytes are all we look at: they have a hash of the service's + * PK. */ + intro_circ = circuit_get_intro_point((const uint8_t*)request); if (!intro_circ) { log_info(LD_REND, "No intro circ found for INTRODUCE1 cell (%s) from circuit %u; " @@ -202,7 +203,7 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, "Unable to send INTRODUCE2 cell to Tor client."); goto err; } - /* And sent an ack down Alice's circuit. Empty body means succeeded. */ + /* And send an ack down the client's circuit. Empty body means succeeded. */ if (relay_send_command_from_edge(0,TO_CIRCUIT(circ), RELAY_COMMAND_INTRODUCE_ACK, NULL,0,NULL)) { @@ -213,7 +214,7 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, return 0; err: - /* Send the client an NACK */ + /* Send the client a NACK */ nak_body[0] = 1; if (relay_send_command_from_edge(0,TO_CIRCUIT(circ), RELAY_COMMAND_INTRODUCE_ACK, @@ -238,18 +239,26 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, log_info(LD_REND, "Received an ESTABLISH_RENDEZVOUS request on circuit %u", (unsigned)circ->p_circ_id); - if (circ->base_.purpose != CIRCUIT_PURPOSE_OR || circ->base_.n_chan) { + if (circ->base_.purpose != CIRCUIT_PURPOSE_OR) { log_warn(LD_PROTOCOL, - "Tried to establish rendezvous on non-OR or non-edge circuit."); + "Tried to establish rendezvous on non-OR circuit with purpose %s", + circuit_purpose_to_string(circ->base_.purpose)); + goto err; + } + + if (circ->base_.n_chan) { + log_warn(LD_PROTOCOL, + "Tried to establish rendezvous on non-edge circuit"); goto err; } if (request_len != REND_COOKIE_LEN) { - log_warn(LD_PROTOCOL, "Invalid length on ESTABLISH_RENDEZVOUS."); + log_fn(LOG_PROTOCOL_WARN, + LD_PROTOCOL, "Invalid length on ESTABLISH_RENDEZVOUS."); goto err; } - if (circuit_get_rendezvous((char*)request)) { + if (circuit_get_rendezvous(request)) { log_warn(LD_PROTOCOL, "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS."); goto err; @@ -265,7 +274,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, } circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_POINT_WAITING); - memcpy(circ->rend_token, request, REND_COOKIE_LEN); + circuit_set_rendezvous_cookie(circ, request); base16_encode(hexid,9,(char*)request,4); @@ -287,6 +296,7 @@ int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, size_t request_len) { + const or_options_t *options = get_options(); or_circuit_t *rend_circ; char hexid[9]; int reason = END_CIRC_REASON_INTERNAL; @@ -313,7 +323,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, "Got request for rendezvous from circuit %u to cookie %s.", (unsigned)circ->p_circ_id, hexid); - rend_circ = circuit_get_rendezvous((char*)request); + rend_circ = circuit_get_rendezvous(request); if (!rend_circ) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Rejecting RENDEZVOUS1 cell with unrecognized rendezvous cookie %s.", @@ -322,7 +332,13 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, goto err; } - /* Send the RENDEZVOUS2 cell to Alice. */ + /* Statistics: Mark this circuit as an RP circuit so that we collect + stats from it. */ + if (options->HiddenServiceStatistics) { + circ->circuit_carries_hs_traffic_stats = 1; + } + + /* Send the RENDEZVOUS2 cell to the client. */ if (relay_send_command_from_edge(0, TO_CIRCUIT(rend_circ), RELAY_COMMAND_RENDEZVOUS2, (char*)(request+REND_COOKIE_LEN), @@ -341,7 +357,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_REND_ESTABLISHED); circuit_change_purpose(TO_CIRCUIT(rend_circ), CIRCUIT_PURPOSE_REND_ESTABLISHED); - memset(circ->rend_token, 0, REND_COOKIE_LEN); + circuit_set_rendezvous_cookie(circ, NULL); rend_circ->rend_splice = circ; circ->rend_splice = rend_circ; diff --git a/src/or/rendmid.h b/src/or/rendmid.h index 310276ac96..10d1287085 100644 --- a/src/or/rendmid.h +++ b/src/or/rendmid.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendservice.c b/src/or/rendservice.c index 436f2f4b69..b81a01c568 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -10,11 +10,14 @@ #define RENDSERVICE_PRIVATE #include "or.h" +#include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" #include "config.h" +#include "control.h" #include "directory.h" +#include "main.h" #include "networkstatus.h" #include "nodelist.h" #include "rendclient.h" @@ -28,21 +31,29 @@ #include "routerparse.h" #include "routerset.h" +struct rend_service_t; static origin_circuit_t *find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest); static rend_intro_point_t *find_intro_point(origin_circuit_t *circ); +static rend_intro_point_t *find_expiring_intro_point( + struct rend_service_t *service, origin_circuit_t *circ); static extend_info_t *find_rp_for_intro( const rend_intro_cell_t *intro, - uint8_t *need_free_out, char **err_msg_out); + char **err_msg_out); static int intro_point_accepted_intro_count(rend_intro_point_t *intro); static int intro_point_should_expire_now(rend_intro_point_t *intro, time_t now); -struct rend_service_t; +static int rend_service_derive_key_digests(struct rend_service_t *s); static int rend_service_load_keys(struct rend_service_t *s); static int rend_service_load_auth_keys(struct rend_service_t *s, const char *hfname); +static struct rend_service_t *rend_service_get_by_pk_digest( + const char* digest); +static struct rend_service_t *rend_service_get_by_service_id(const char *id); +static const char *rend_service_escaped_dir( + const struct rend_service_t *s); static ssize_t rend_service_parse_intro_for_v0_or_v1( rend_intro_cell_t *intro, @@ -63,16 +74,26 @@ static ssize_t rend_service_parse_intro_for_v3( /** Represents the mapping from a virtual port of a rendezvous service to * a real port on some IP. */ -typedef struct rend_service_port_config_t { +struct rend_service_port_config_s { + /* The incoming HS virtual port we're mapping */ uint16_t virtual_port; + /* Is this an AF_UNIX port? */ + unsigned int is_unix_addr:1; + /* The outgoing TCP port to use, if !is_unix_addr */ uint16_t real_port; + /* The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */ tor_addr_t real_addr; -} rend_service_port_config_t; + /* The socket path to connect to, if is_unix_addr */ + char unix_addr[FLEXIBLE_ARRAY_MEMBER]; +}; /** Try to maintain this many intro points per service by default. */ #define NUM_INTRO_POINTS_DEFAULT 3 -/** Maintain no more than this many intro points per hidden service. */ +/** Maximum number of intro points per service. */ #define NUM_INTRO_POINTS_MAX 10 +/** Number of extra intro points we launch if our set of intro nodes is + * empty. See proposal 155, section 4. */ +#define NUM_INTRO_POINTS_EXTRA 2 /** If we can't build our intro circuits, don't retry for this long. */ #define INTRO_CIRC_RETRY_PERIOD (60*5) @@ -81,19 +102,18 @@ typedef struct rend_service_port_config_t { #define MAX_INTRO_CIRCS_PER_PERIOD 10 /** How many times will a hidden service operator attempt to connect to * a requested rendezvous point before giving up? */ -#define MAX_REND_FAILURES 30 +#define MAX_REND_FAILURES 1 /** How many seconds should we spend trying to connect to a requested * rendezvous point before giving up? */ #define MAX_REND_TIMEOUT 30 -/** How many seconds should we wait for new HS descriptors to reach - * our clients before we close an expiring intro point? */ -#define INTRO_POINT_EXPIRATION_GRACE_PERIOD (5*60) - /** Represents a single hidden service running at this OP. */ typedef struct rend_service_t { /* Fields specified in config file */ - char *directory; /**< where in the filesystem it stores it */ + char *directory; /**< where in the filesystem it stores it. Will be NULL if + * this service is ephemeral. */ + int dir_group_readable; /**< if 1, allow group read + permissions on directory */ smartlist_t *ports; /**< List of rend_service_port_config_t */ rend_auth_type_t auth_type; /**< Client authorization type or 0 if no client * authorization is performed. */ @@ -107,6 +127,10 @@ typedef struct rend_service_t { char pk_digest[DIGEST_LEN]; /**< Hash of permanent hidden-service key. */ smartlist_t *intro_nodes; /**< List of rend_intro_point_t's we have, * or are trying to establish. */ + /** List of rend_intro_point_t that are expiring. They are removed once + * the new descriptor is successfully uploaded. A node in this list CAN + * NOT appear in the intro_nodes list. */ + smartlist_t *expiring_nodes; time_t intro_period_started; /**< Start of the current period to build * introduction points. */ int n_intro_circuits_launched; /**< Count of intro circuits we have @@ -125,8 +149,26 @@ typedef struct rend_service_t { * when they do, this keeps us from launching multiple simultaneous attempts * to connect to the same rend point. */ replaycache_t *accepted_intro_dh_parts; + /** If true, we don't close circuits for making requests to unsupported + * ports. */ + int allow_unknown_ports; + /** The maximum number of simultanious streams-per-circuit that are allowed + * to be established, or 0 if no limit is set. + */ + int max_streams_per_circuit; + /** If true, we close circuits that exceed the max_streams_per_circuit + * limit. */ + int max_streams_close_circuit; } rend_service_t; +/** Returns a escaped string representation of the service, <b>s</b>. + */ +static const char * +rend_service_escaped_dir(const struct rend_service_t *s) +{ + return (s->directory) ? escaped(s->directory) : "[EPHEMERAL]"; +} + /** A list of rend_service_t's for services run on this OP. */ static smartlist_t *rend_service_list = NULL; @@ -140,17 +182,6 @@ num_rend_services(void) return smartlist_len(rend_service_list); } -/** Return a string identifying <b>service</b>, suitable for use in a - * log message. The result does not need to be freed, but may be - * overwritten by the next call to this function. */ -static const char * -rend_service_describe_for_log(rend_service_t *service) -{ - /* XXX024 Use this function throughout rendservice.c. */ - /* XXX024 Return a more useful description? */ - return safe_str_client(service->service_id); -} - /** Helper: free storage held by a single service authorized client entry. */ static void rend_authorized_client_free(rend_authorized_client_t *client) @@ -159,7 +190,7 @@ rend_authorized_client_free(rend_authorized_client_t *client) return; if (client->client_key) crypto_pk_free(client->client_key); - tor_strclear(client->client_name); + memwipe(client->client_name, 0, strlen(client->client_name)); tor_free(client->client_name); memwipe(client->descriptor_cookie, 0, sizeof(client->descriptor_cookie)); tor_free(client); @@ -181,7 +212,8 @@ rend_service_free(rend_service_t *service) return; tor_free(service->directory); - SMARTLIST_FOREACH(service->ports, void*, p, tor_free(p)); + SMARTLIST_FOREACH(service->ports, rend_service_port_config_t*, p, + rend_service_port_config_free(p)); smartlist_free(service->ports); if (service->private_key) crypto_pk_free(service->private_key); @@ -190,6 +222,11 @@ rend_service_free(rend_service_t *service) rend_intro_point_free(intro);); smartlist_free(service->intro_nodes); } + if (service->expiring_nodes) { + SMARTLIST_FOREACH(service->expiring_nodes, rend_intro_point_t *, intro, + rend_intro_point_free(intro);); + smartlist_free(service->expiring_nodes); + } rend_service_descriptor_free(service->desc); if (service->clients) { @@ -218,29 +255,49 @@ rend_service_free_all(void) } /** Validate <b>service</b> and add it to rend_service_list if possible. + * Return 0 on success. On failure, free <b>service</b> and return -1. */ -static void +static int rend_add_service(rend_service_t *service) { int i; rend_service_port_config_t *p; service->intro_nodes = smartlist_new(); + service->expiring_nodes = smartlist_new(); + + if (service->max_streams_per_circuit < 0) { + log_warn(LD_CONFIG, "Hidden service (%s) configured with negative max " + "streams per circuit; ignoring.", + rend_service_escaped_dir(service)); + rend_service_free(service); + return -1; + } + + if (service->max_streams_close_circuit < 0 || + service->max_streams_close_circuit > 1) { + log_warn(LD_CONFIG, "Hidden service (%s) configured with invalid " + "max streams handling; ignoring.", + rend_service_escaped_dir(service)); + rend_service_free(service); + return -1; + } if (service->auth_type != REND_NO_AUTH && smartlist_len(service->clients) == 0) { log_warn(LD_CONFIG, "Hidden service (%s) with client authorization but no " "clients; ignoring.", - escaped(service->directory)); + rend_service_escaped_dir(service)); rend_service_free(service); - return; + return -1; } if (!smartlist_len(service->ports)) { log_warn(LD_CONFIG, "Hidden service (%s) with no ports configured; " "ignoring.", - escaped(service->directory)); + rend_service_escaped_dir(service)); rend_service_free(service); + return -1; } else { int dupe = 0; /* XXX This duplicate check has two problems: @@ -258,56 +315,102 @@ rend_add_service(rend_service_t *service) * lock file. But this is enough to detect a simple mistake that * at least one person has actually made. */ - SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr, - dupe = dupe || - !strcmp(ptr->directory, service->directory)); - if (dupe) { - log_warn(LD_REND, "Another hidden service is already configured for " - "directory %s, ignoring.", service->directory); - rend_service_free(service); - return; + if (service->directory != NULL) { /* Skip dupe for ephemeral services. */ + SMARTLIST_FOREACH(rend_service_list, rend_service_t*, ptr, + dupe = dupe || + !strcmp(ptr->directory, service->directory)); + if (dupe) { + log_warn(LD_REND, "Another hidden service is already configured for " + "directory %s, ignoring.", + rend_service_escaped_dir(service)); + rend_service_free(service); + return -1; + } } smartlist_add(rend_service_list, service); log_debug(LD_REND,"Configuring service with directory \"%s\"", service->directory); for (i = 0; i < smartlist_len(service->ports); ++i) { p = smartlist_get(service->ports, i); - log_debug(LD_REND,"Service maps port %d to %s", - p->virtual_port, fmt_addrport(&p->real_addr, p->real_port)); + if (!(p->is_unix_addr)) { + log_debug(LD_REND, + "Service maps port %d to %s", + p->virtual_port, + fmt_addrport(&p->real_addr, p->real_port)); + } else { +#ifdef HAVE_SYS_UN_H + log_debug(LD_REND, + "Service maps port %d to socket at \"%s\"", + p->virtual_port, p->unix_addr); +#else + log_debug(LD_REND, + "Service maps port %d to an AF_UNIX socket, but we " + "have no AF_UNIX support on this platform. This is " + "probably a bug.", + p->virtual_port); +#endif /* defined(HAVE_SYS_UN_H) */ + } } + return 0; } + /* NOTREACHED */ } -/** Parses a real-port to virtual-port mapping and returns a new - * rend_service_port_config_t. +/** Return a new rend_service_port_config_t with its path set to + * <b>socket_path</b> or empty if <b>socket_path</b> is NULL */ +static rend_service_port_config_t * +rend_service_port_config_new(const char *socket_path) +{ + if (!socket_path) + return tor_malloc_zero(sizeof(rend_service_port_config_t) + 1); + + const size_t pathlen = strlen(socket_path) + 1; + rend_service_port_config_t *conf = + tor_malloc_zero(sizeof(rend_service_port_config_t) + pathlen); + memcpy(conf->unix_addr, socket_path, pathlen); + conf->is_unix_addr = 1; + return conf; +} + +/** Parses a real-port to virtual-port mapping separated by the provided + * separator and returns a new rend_service_port_config_t, or NULL and an + * optional error string on failure. * - * The format is: VirtualPort (IP|RealPort|IP:RealPort)? + * The format is: VirtualPort SEP (IP|RealPort|IP:RealPort|'socket':path)? * * IP defaults to 127.0.0.1; RealPort defaults to VirtualPort. */ -static rend_service_port_config_t * -parse_port_config(const char *string) +rend_service_port_config_t * +rend_service_parse_port_config(const char *string, const char *sep, + char **err_msg_out) { smartlist_t *sl; int virtport; - int realport; + int realport = 0; uint16_t p; tor_addr_t addr; const char *addrport; rend_service_port_config_t *result = NULL; + unsigned int is_unix_addr = 0; + char *socket_path = NULL; + char *err_msg = NULL; sl = smartlist_new(); - smartlist_split_string(sl, string, " ", + smartlist_split_string(sl, string, sep, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (smartlist_len(sl) < 1 || smartlist_len(sl) > 2) { - log_warn(LD_CONFIG, "Bad syntax in hidden service port configuration."); + if (err_msg_out) + err_msg = tor_strdup("Bad syntax in hidden service port configuration."); + goto err; } virtport = (int)tor_parse_long(smartlist_get(sl,0), 10, 1, 65535, NULL,NULL); if (!virtport) { - log_warn(LD_CONFIG, "Missing or invalid port %s in hidden service port " - "configuration", escaped(smartlist_get(sl,0))); + if (err_msg_out) + tor_asprintf(&err_msg, "Missing or invalid port %s in hidden service " + "port configuration", escaped(smartlist_get(sl,0))); + goto err; } @@ -316,11 +419,27 @@ parse_port_config(const char *string) realport = virtport; tor_addr_from_ipv4h(&addr, 0x7F000001u); /* 127.0.0.1 */ } else { + int ret; + addrport = smartlist_get(sl,1); - if (strchr(addrport, ':') || strchr(addrport, '.')) { + ret = config_parse_unix_port(addrport, &socket_path); + if (ret < 0 && ret != -ENOENT) { + if (ret == -EINVAL) + if (err_msg_out) + err_msg = tor_strdup("Empty socket path in hidden service port " + "configuration."); + + goto err; + } + if (socket_path) { + is_unix_addr = 1; + } else if (strchr(addrport, ':') || strchr(addrport, '.')) { + /* else try it as an IP:port pair if it has a : or . in it */ if (tor_addr_port_lookup(addrport, &addr, &p)<0) { - log_warn(LD_CONFIG,"Unparseable address in hidden service port " - "configuration."); + if (err_msg_out) + err_msg = tor_strdup("Unparseable address in hidden service port " + "configuration."); + goto err; } realport = p?p:virtport; @@ -328,24 +447,43 @@ parse_port_config(const char *string) /* No addr:port, no addr -- must be port. */ realport = (int)tor_parse_long(addrport, 10, 1, 65535, NULL, NULL); if (!realport) { - log_warn(LD_CONFIG,"Unparseable or out-of-range port %s in hidden " - "service port configuration.", escaped(addrport)); + if (err_msg_out) + tor_asprintf(&err_msg, "Unparseable or out-of-range port %s in " + "hidden service port configuration.", + escaped(addrport)); + goto err; } tor_addr_from_ipv4h(&addr, 0x7F000001u); /* Default to 127.0.0.1 */ } } - result = tor_malloc(sizeof(rend_service_port_config_t)); + /* Allow room for unix_addr */ + result = rend_service_port_config_new(socket_path); result->virtual_port = virtport; - result->real_port = realport; - tor_addr_copy(&result->real_addr, &addr); + result->is_unix_addr = is_unix_addr; + if (!is_unix_addr) { + result->real_port = realport; + tor_addr_copy(&result->real_addr, &addr); + result->unix_addr[0] = '\0'; + } + err: + if (err_msg_out) *err_msg_out = err_msg; SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); + if (socket_path) tor_free(socket_path); + return result; } +/** Release all storage held in a rend_service_port_config_t. */ +void +rend_service_port_config_free(rend_service_port_config_t *p) +{ + tor_free(p); +} + /** Set up rend_service_list, based on the values of HiddenServiceDir and * HiddenServicePort in <b>options</b>. Return 0 on success and -1 on * failure. (If <b>validate_only</b> is set, parse, warn and return as @@ -358,6 +496,7 @@ rend_config_services(const or_options_t *options, int validate_only) rend_service_t *service = NULL; rend_service_port_config_t *portcfg; smartlist_t *old_service_list = NULL; + int ok = 0; if (!validate_only) { old_service_list = rend_service_list; @@ -368,7 +507,7 @@ rend_config_services(const or_options_t *options, int validate_only) if (!strcasecmp(line->key, "HiddenServiceDir")) { if (service) { /* register the one we just finished parsing */ if (validate_only) - rend_service_free(service); + rend_service_free(service); else rend_add_service(service); } @@ -386,12 +525,87 @@ rend_config_services(const or_options_t *options, int validate_only) return -1; } if (!strcasecmp(line->key, "HiddenServicePort")) { - portcfg = parse_port_config(line->value); + char *err_msg = NULL; + portcfg = rend_service_parse_port_config(line->value, " ", &err_msg); if (!portcfg) { + if (err_msg) + log_warn(LD_CONFIG, "%s", err_msg); + tor_free(err_msg); rend_service_free(service); return -1; } + tor_assert(!err_msg); smartlist_add(service->ports, portcfg); + } else if (!strcasecmp(line->key, "HiddenServiceAllowUnknownPorts")) { + service->allow_unknown_ports = (int)tor_parse_long(line->value, + 10, 0, 1, &ok, NULL); + if (!ok) { + log_warn(LD_CONFIG, + "HiddenServiceAllowUnknownPorts should be 0 or 1, not %s", + line->value); + rend_service_free(service); + return -1; + } + log_info(LD_CONFIG, + "HiddenServiceAllowUnknownPorts=%d for %s", + (int)service->allow_unknown_ports, service->directory); + } else if (!strcasecmp(line->key, + "HiddenServiceDirGroupReadable")) { + service->dir_group_readable = (int)tor_parse_long(line->value, + 10, 0, 1, &ok, NULL); + if (!ok) { + log_warn(LD_CONFIG, + "HiddenServiceDirGroupReadable should be 0 or 1, not %s", + line->value); + rend_service_free(service); + return -1; + } + log_info(LD_CONFIG, + "HiddenServiceDirGroupReadable=%d for %s", + service->dir_group_readable, service->directory); + } else if (!strcasecmp(line->key, "HiddenServiceMaxStreams")) { + service->max_streams_per_circuit = (int)tor_parse_long(line->value, + 10, 0, 65535, &ok, NULL); + if (!ok) { + log_warn(LD_CONFIG, + "HiddenServiceMaxStreams should be between 0 and %d, not %s", + 65535, line->value); + rend_service_free(service); + return -1; + } + log_info(LD_CONFIG, + "HiddenServiceMaxStreams=%d for %s", + service->max_streams_per_circuit, service->directory); + } else if (!strcasecmp(line->key, "HiddenServiceMaxStreamsCloseCircuit")) { + service->max_streams_close_circuit = (int)tor_parse_long(line->value, + 10, 0, 1, &ok, NULL); + if (!ok) { + log_warn(LD_CONFIG, + "HiddenServiceMaxStreamsCloseCircuit should be 0 or 1, " + "not %s", + line->value); + rend_service_free(service); + return -1; + } + log_info(LD_CONFIG, + "HiddenServiceMaxStreamsCloseCircuit=%d for %s", + (int)service->max_streams_close_circuit, service->directory); + } else if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) { + service->n_intro_points_wanted = + (unsigned int) tor_parse_long(line->value, 10, + NUM_INTRO_POINTS_DEFAULT, + NUM_INTRO_POINTS_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_CONFIG, + "HiddenServiceNumIntroductionPoints " + "should be between %d and %d, not %s", + NUM_INTRO_POINTS_DEFAULT, NUM_INTRO_POINTS_MAX, + line->value); + rend_service_free(service); + return -1; + } + log_info(LD_CONFIG, "HiddenServiceNumIntroductionPoints=%d for %s", + service->n_intro_points_wanted, service->directory); } else if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) { /* Parse auth type and comma-separated list of client names and add a * rend_authorized_client_t for each client to the service's list @@ -512,10 +726,21 @@ rend_config_services(const or_options_t *options, int validate_only) } } if (service) { - if (validate_only) + cpd_check_t check_opts = CPD_CHECK_MODE_ONLY|CPD_CHECK; + if (service->dir_group_readable) { + check_opts |= CPD_GROUP_READ; + } + + if (check_private_dir(service->directory, check_opts, options->User) < 0) { rend_service_free(service); - else + return -1; + } + + if (validate_only) { + rend_service_free(service); + } else { rend_add_service(service); + } } /* If this is a reload and there were hidden services configured before, @@ -523,16 +748,40 @@ rend_config_services(const or_options_t *options, int validate_only) * other ones. */ if (old_service_list && !validate_only) { smartlist_t *surviving_services = smartlist_new(); - circuit_t *circ; + + /* Preserve the existing ephemeral services. + * + * This is the ephemeral service equivalent of the "Copy introduction + * points to new services" block, except there's no copy required since + * the service structure isn't regenerated. + * + * After this is done, all ephemeral services will be: + * * Removed from old_service_list, so the equivalent non-ephemeral code + * will not attempt to preserve them. + * * Added to the new rend_service_list (that previously only had the + * services listed in the configuration). + * * Added to surviving_services, which is the list of services that + * will NOT have their intro point closed. + */ + SMARTLIST_FOREACH(old_service_list, rend_service_t *, old, { + if (!old->directory) { + SMARTLIST_DEL_CURRENT(old_service_list, old); + smartlist_add(surviving_services, old); + smartlist_add(rend_service_list, old); + } + }); /* Copy introduction points to new services. */ /* XXXX This is O(n^2), but it's only called on reconfigure, so it's * probably ok? */ SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, new) { SMARTLIST_FOREACH_BEGIN(old_service_list, rend_service_t *, old) { - if (!strcmp(old->directory, new->directory)) { + if (new->directory && old->directory && + !strcmp(old->directory, new->directory)) { smartlist_add_all(new->intro_nodes, old->intro_nodes); smartlist_clear(old->intro_nodes); + smartlist_add_all(new->expiring_nodes, old->expiring_nodes); + smartlist_clear(old->expiring_nodes); smartlist_add(surviving_services, old); break; } @@ -543,7 +792,7 @@ rend_config_services(const or_options_t *options, int validate_only) /* XXXX it would be nicer if we had a nicer abstraction to use here, * so we could just iterate over the list of services to close, but * once again, this isn't critical-path code. */ - for (circ = circuit_get_global_list_(); circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN && (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || @@ -568,6 +817,7 @@ rend_config_services(const or_options_t *options, int validate_only) /* XXXX Is there another reason we should use here? */ } } + SMARTLIST_FOREACH_END(circ); smartlist_free(surviving_services); SMARTLIST_FOREACH(old_service_list, rend_service_t *, ptr, rend_service_free(ptr)); @@ -577,6 +827,124 @@ rend_config_services(const or_options_t *options, int validate_only) return 0; } +/** Add the ephemeral service <b>pk</b>/<b>ports</b> if possible, with + * <b>max_streams_per_circuit</b> streams allowed per rendezvous circuit, + * and circuit closure on max streams being exceeded set by + * <b>max_streams_close_circuit</b>. + * + * Regardless of sucess/failure, callers should not touch pk/ports after + * calling this routine, and may assume that correct cleanup has been done + * on failure. + * + * Return an appropriate rend_service_add_ephemeral_status_t. + */ +rend_service_add_ephemeral_status_t +rend_service_add_ephemeral(crypto_pk_t *pk, + smartlist_t *ports, + int max_streams_per_circuit, + int max_streams_close_circuit, + char **service_id_out) +{ + *service_id_out = NULL; + /* Allocate the service structure, and initialize the key, and key derived + * parameters. + */ + rend_service_t *s = tor_malloc_zero(sizeof(rend_service_t)); + s->directory = NULL; /* This indicates the service is ephemeral. */ + s->private_key = pk; + s->auth_type = REND_NO_AUTH; + s->ports = ports; + s->intro_period_started = time(NULL); + s->n_intro_points_wanted = NUM_INTRO_POINTS_DEFAULT; + s->max_streams_per_circuit = max_streams_per_circuit; + s->max_streams_close_circuit = max_streams_close_circuit; + if (rend_service_derive_key_digests(s) < 0) { + rend_service_free(s); + return RSAE_BADPRIVKEY; + } + + if (!s->ports || smartlist_len(s->ports) == 0) { + log_warn(LD_CONFIG, "At least one VIRTPORT/TARGET must be specified."); + rend_service_free(s); + return RSAE_BADVIRTPORT; + } + + /* Enforcing pk/id uniqueness should be done by rend_service_load_keys(), but + * it's not, see #14828. + */ + if (rend_service_get_by_pk_digest(s->pk_digest)) { + log_warn(LD_CONFIG, "Onion Service private key collides with an " + "existing service."); + rend_service_free(s); + return RSAE_ADDREXISTS; + } + if (rend_service_get_by_service_id(s->service_id)) { + log_warn(LD_CONFIG, "Onion Service id collides with an existing service."); + rend_service_free(s); + return RSAE_ADDREXISTS; + } + + /* Initialize the service. */ + if (rend_add_service(s)) { + return RSAE_INTERNAL; + } + *service_id_out = tor_strdup(s->service_id); + + log_debug(LD_CONFIG, "Added ephemeral Onion Service: %s", s->service_id); + return RSAE_OKAY; +} + +/** Remove the ephemeral service <b>service_id</b> if possible. Returns 0 on + * success, and -1 on failure. + */ +int +rend_service_del_ephemeral(const char *service_id) +{ + rend_service_t *s; + if (!rend_valid_service_id(service_id)) { + log_warn(LD_CONFIG, "Requested malformed Onion Service id for removal."); + return -1; + } + if ((s = rend_service_get_by_service_id(service_id)) == NULL) { + log_warn(LD_CONFIG, "Requested non-existent Onion Service id for " + "removal."); + return -1; + } + if (s->directory) { + log_warn(LD_CONFIG, "Requested non-ephemeral Onion Service for removal."); + return -1; + } + + /* Kill the intro point circuit for the Onion Service, and remove it from + * the list. Closing existing connections is the application's problem. + * + * XXX: As with the comment in rend_config_services(), a nice abstraction + * would be ideal here, but for now just duplicate the code. + */ + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { + if (!circ->marked_for_close && + circ->state == CIRCUIT_STATE_OPEN && + (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || + circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { + origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); + tor_assert(oc->rend_data); + if (!tor_memeq(s->pk_digest, oc->rend_data->rend_pk_digest, DIGEST_LEN)) + continue; + log_debug(LD_REND, "Closing intro point %s for service %s.", + safe_str_client(extend_info_describe( + oc->build_state->chosen_exit)), + oc->rend_data->onion_address); + circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); + } + } SMARTLIST_FOREACH_END(circ); + smartlist_remove(rend_service_list, s); + rend_service_free(s); + + log_debug(LD_CONFIG, "Removed ephemeral Onion Service: %s", service_id); + + return 0; +} + /** Replace the old value of <b>service</b>-\>desc with one that reflects * the other fields in service. */ @@ -605,11 +973,6 @@ rend_service_update_descriptor(rend_service_t *service) /* This intro point won't be listed in the descriptor... */ intro_svc->listed_in_last_desc = 0; - if (intro_svc->time_expiring != -1) { - /* This intro point is expiring. Don't list it. */ - continue; - } - circ = find_intro_circuit(intro_svc, service->pk_digest); if (!circ || circ->base_.purpose != CIRCUIT_PURPOSE_S_INTRO) { /* This intro point's circuit isn't finished yet. Don't list it. */ @@ -655,6 +1018,56 @@ rend_service_load_all_keys(void) return 0; } +/** Add to <b>lst</b> every filename used by <b>s</b>. */ +static void +rend_service_add_filenames_to_list(smartlist_t *lst, const rend_service_t *s) +{ + tor_assert(lst); + tor_assert(s); + tor_assert(s->directory); + smartlist_add_asprintf(lst, "%s"PATH_SEPARATOR"private_key", + s->directory); + smartlist_add_asprintf(lst, "%s"PATH_SEPARATOR"hostname", + s->directory); + smartlist_add_asprintf(lst, "%s"PATH_SEPARATOR"client_keys", + s->directory); +} + +/** Add to <b>open_lst</b> every filename used by a configured hidden service, + * and to <b>stat_lst</b> every directory used by a configured hidden + * service */ +void +rend_services_add_filenames_to_lists(smartlist_t *open_lst, + smartlist_t *stat_lst) +{ + if (!rend_service_list) + return; + SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, s) { + if (s->directory) { + rend_service_add_filenames_to_list(open_lst, s); + smartlist_add(stat_lst, tor_strdup(s->directory)); + } + } SMARTLIST_FOREACH_END(s); +} + +/** Derive all rend_service_t internal material based on the service's key. + * Returns 0 on sucess, -1 on failure. + */ +static int +rend_service_derive_key_digests(struct rend_service_t *s) +{ + if (rend_get_service_id(s->private_key, s->service_id)<0) { + log_warn(LD_BUG, "Internal error: couldn't encode service ID."); + return -1; + } + if (crypto_pk_get_digest(s->private_key, s->pk_digest)<0) { + log_warn(LD_BUG, "Couldn't compute hash of public key."); + return -1; + } + + return 0; +} + /** Load and/or generate private keys for the hidden service <b>s</b>, * possibly including keys for client authorization. Return 0 on success, -1 * on failure. */ @@ -663,10 +1076,23 @@ rend_service_load_keys(rend_service_t *s) { char fname[512]; char buf[128]; + cpd_check_t check_opts = CPD_CREATE; + if (s->dir_group_readable) { + check_opts |= CPD_GROUP_READ; + } /* Check/create directory */ - if (check_private_dir(s->directory, CPD_CREATE, get_options()->User) < 0) + if (check_private_dir(s->directory, check_opts, get_options()->User) < 0) { return -1; + } +#ifndef _WIN32 + if (s->dir_group_readable) { + /* Only new dirs created get new opts, also enforce group read. */ + if (chmod(s->directory, 0750)) { + log_warn(LD_FS,"Unable to make %s group-readable.", s->directory); + } + } +#endif /* Load key */ if (strlcpy(fname,s->directory,sizeof(fname)) >= sizeof(fname) || @@ -676,19 +1102,14 @@ rend_service_load_keys(rend_service_t *s) s->directory); return -1; } - s->private_key = init_key_from_file(fname, 1, LOG_ERR); + s->private_key = init_key_from_file(fname, 1, LOG_ERR, 0); if (!s->private_key) return -1; - /* Create service file */ - if (rend_get_service_id(s->private_key, s->service_id)<0) { - log_warn(LD_BUG, "Internal error: couldn't encode service ID."); + if (rend_service_derive_key_digests(s) < 0) return -1; - } - if (crypto_pk_get_digest(s->private_key, s->pk_digest)<0) { - log_warn(LD_BUG, "Couldn't compute hash of public key."); - return -1; - } + + /* Create service file */ if (strlcpy(fname,s->directory,sizeof(fname)) >= sizeof(fname) || strlcat(fname,PATH_SEPARATOR"hostname",sizeof(fname)) >= sizeof(fname)) { @@ -703,6 +1124,15 @@ rend_service_load_keys(rend_service_t *s) memwipe(buf, 0, sizeof(buf)); return -1; } +#ifndef _WIN32 + if (s->dir_group_readable) { + /* Also verify hostname file created with group read. */ + if (chmod(fname, 0640)) + log_warn(LD_FS,"Unable to make hidden hostname file %s group-readable.", + fname); + } +#endif + memwipe(buf, 0, sizeof(buf)); /* If client authorization is configured, load or generate keys. */ @@ -782,7 +1212,7 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) } if (base64_encode(desc_cook_out, 3*REND_DESC_COOKIE_LEN_BASE64+1, client->descriptor_cookie, - REND_DESC_COOKIE_LEN) < 0) { + REND_DESC_COOKIE_LEN, 0) < 0) { log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); goto err; } @@ -809,7 +1239,6 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) client->client_key = prkey; } /* Add entry to client_keys file. */ - desc_cook_out[strlen(desc_cook_out)-1] = '\0'; /* Remove newline. */ written = tor_snprintf(buf, sizeof(buf), "client-name %s\ndescriptor-cookie %s\n", client->client_name, desc_cook_out); @@ -864,12 +1293,11 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) ((int)s->auth_type - 1) << 4; if (base64_encode(desc_cook_out, 3*REND_DESC_COOKIE_LEN_BASE64+1, extended_desc_cookie, - REND_DESC_COOKIE_LEN+1) < 0) { + REND_DESC_COOKIE_LEN+1, 0) < 0) { log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); goto err; } - desc_cook_out[strlen(desc_cook_out)-3] = '\0'; /* Remove A= and - newline. */ + desc_cook_out[strlen(desc_cook_out)-2] = '\0'; /* Remove A=. */ tor_snprintf(buf, sizeof(buf),"%s.onion %s # client: %s\n", service_id, desc_cook_out, client->client_name); } @@ -893,7 +1321,7 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) abort_writing_to_file(open_hfile); done: if (client_keys_str) { - tor_strclear(client_keys_str); + memwipe(client_keys_str, 0, strlen(client_keys_str)); tor_free(client_keys_str); } strmap_free(parsed_clients, rend_authorized_client_strmap_item_free); @@ -921,6 +1349,20 @@ rend_service_get_by_pk_digest(const char* digest) return NULL; } +/** Return the service whose service id is <b>id</b>, or NULL if no such + * service exists. + */ +static struct rend_service_t * +rend_service_get_by_service_id(const char *id) +{ + tor_assert(strlen(id) == REND_SERVICE_ID_LEN_BASE32); + SMARTLIST_FOREACH(rend_service_list, rend_service_t*, s, { + if (tor_memeq(s->service_id, id, REND_SERVICE_ID_LEN_BASE32)) + return s; + }); + return NULL; +} + /** Return 1 if any virtual port in <b>service</b> wants a circuit * to have good uptime. Else return 0. */ @@ -939,11 +1381,13 @@ rend_service_requires_uptime(rend_service_t *service) return 0; } -/** Check client authorization of a given <b>descriptor_cookie</b> for - * <b>service</b>. Return 1 for success and 0 for failure. */ +/** Check client authorization of a given <b>descriptor_cookie</b> of + * length <b>cookie_len</b> for <b>service</b>. Return 1 for success + * and 0 for failure. */ static int rend_check_authorization(rend_service_t *service, - const char *descriptor_cookie) + const char *descriptor_cookie, + size_t cookie_len) { rend_authorized_client_t *auth_client = NULL; tor_assert(service); @@ -954,6 +1398,13 @@ rend_check_authorization(rend_service_t *service, return 0; } + if (cookie_len != REND_DESC_COOKIE_LEN) { + log_info(LD_REND, "Descriptor cookie is %lu bytes, but we expected " + "%lu bytes. Dropping cell.", + (unsigned long)cookie_len, (unsigned long)REND_DESC_COOKIE_LEN); + return 0; + } + /* Look up client authorization by descriptor cookie. */ SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, client, { if (tor_memeq(client->descriptor_cookie, descriptor_cookie, @@ -965,7 +1416,7 @@ rend_check_authorization(rend_service_t *service, if (!auth_client) { char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64]; base64_encode(descriptor_cookie_base64, sizeof(descriptor_cookie_base64), - descriptor_cookie, REND_DESC_COOKIE_LEN); + descriptor_cookie, REND_DESC_COOKIE_LEN, 0); log_info(LD_REND, "No authorization found for descriptor cookie '%s'! " "Dropping cell!", descriptor_cookie_base64); @@ -973,111 +1424,11 @@ rend_check_authorization(rend_service_t *service, } /* Allow the request. */ - log_debug(LD_REND, "Client %s authorized for service %s.", - auth_client->client_name, service->service_id); + log_info(LD_REND, "Client %s authorized for service %s.", + auth_client->client_name, service->service_id); return 1; } -/** Called when <b>intro</b> will soon be removed from - * <b>service</b>'s list of intro points. */ -static void -rend_service_note_removing_intro_point(rend_service_t *service, - rend_intro_point_t *intro) -{ - time_t now = time(NULL); - - /* Don't process an intro point twice here. */ - if (intro->rend_service_note_removing_intro_point_called) { - return; - } else { - intro->rend_service_note_removing_intro_point_called = 1; - } - - /* Update service->n_intro_points_wanted based on how long intro - * lasted and how many introductions it handled. */ - if (intro->time_published == -1) { - /* This intro point was never used. Don't change - * n_intro_points_wanted. */ - } else { - /* We want to increase the number of introduction points service - * operates if intro was heavily used, or decrease the number of - * intro points if intro was lightly used. - * - * We consider an intro point's target 'usage' to be - * INTRO_POINT_LIFETIME_INTRODUCTIONS introductions in - * INTRO_POINT_LIFETIME_MIN_SECONDS seconds. To calculate intro's - * fraction of target usage, we divide the fraction of - * _LIFETIME_INTRODUCTIONS introductions that it has handled by - * the fraction of _LIFETIME_MIN_SECONDS for which it existed. - * - * Then we multiply that fraction of desired usage by a fudge - * factor of 1.5, to decide how many new introduction points - * should ideally replace intro (which is now closed or soon to be - * closed). In theory, assuming that introduction load is - * distributed equally across all intro points and ignoring the - * fact that different intro points are established and closed at - * different times, that number of intro points should bring all - * of our intro points exactly to our target usage. - * - * Then we clamp that number to a number of intro points we might - * be willing to replace this intro point with and turn it into an - * integer. then we clamp it again to the number of new intro - * points we could establish now, then we adjust - * service->n_intro_points_wanted and let rend_services_introduce - * create the new intro points we want (if any). - */ - const double intro_point_usage = - intro_point_accepted_intro_count(intro) / - (double)(now - intro->time_published); - const double intro_point_target_usage = - INTRO_POINT_LIFETIME_INTRODUCTIONS / - (double)INTRO_POINT_LIFETIME_MIN_SECONDS; - const double fractional_n_intro_points_wanted_to_replace_this_one = - (1.5 * (intro_point_usage / intro_point_target_usage)); - unsigned int n_intro_points_wanted_to_replace_this_one; - unsigned int n_intro_points_wanted_now; - unsigned int n_intro_points_really_wanted_now; - int n_intro_points_really_replacing_this_one; - - if (fractional_n_intro_points_wanted_to_replace_this_one > - NUM_INTRO_POINTS_MAX) { - n_intro_points_wanted_to_replace_this_one = NUM_INTRO_POINTS_MAX; - } else if (fractional_n_intro_points_wanted_to_replace_this_one < 0) { - n_intro_points_wanted_to_replace_this_one = 0; - } else { - n_intro_points_wanted_to_replace_this_one = (unsigned) - fractional_n_intro_points_wanted_to_replace_this_one; - } - - n_intro_points_wanted_now = - service->n_intro_points_wanted + - n_intro_points_wanted_to_replace_this_one - 1; - - if (n_intro_points_wanted_now < NUM_INTRO_POINTS_DEFAULT) { - /* XXXX This should be NUM_INTRO_POINTS_MIN instead. Perhaps - * another use of NUM_INTRO_POINTS_DEFAULT should be, too. */ - n_intro_points_really_wanted_now = NUM_INTRO_POINTS_DEFAULT; - } else if (n_intro_points_wanted_now > NUM_INTRO_POINTS_MAX) { - n_intro_points_really_wanted_now = NUM_INTRO_POINTS_MAX; - } else { - n_intro_points_really_wanted_now = n_intro_points_wanted_now; - } - - n_intro_points_really_replacing_this_one = - n_intro_points_really_wanted_now - service->n_intro_points_wanted + 1; - - log_info(LD_REND, "Replacing closing intro point for service %s " - "with %d new intro points (wanted %g replacements); " - "service will now try to have %u intro points", - rend_service_describe_for_log(service), - n_intro_points_really_replacing_this_one, - fractional_n_intro_points_wanted_to_replace_this_one, - n_intro_points_really_wanted_now); - - service->n_intro_points_wanted = n_intro_points_really_wanted_now; - } -} - /****** * Handle cells ******/ @@ -1086,13 +1437,15 @@ rend_service_note_removing_intro_point(rend_service_t *service, * rendezvous point. */ int -rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, - size_t request_len) +rend_service_receive_introduction(origin_circuit_t *circuit, + const uint8_t *request, + size_t request_len) { /* Global status stuff */ int status = 0, result; const or_options_t *options = get_options(); char *err_msg = NULL; + int err_msg_severity = LOG_WARN; const char *stage_descr = NULL; int reason = END_CIRC_REASON_TORPROTOCOL; /* Service/circuit/key stuff we can learn before parsing */ @@ -1104,13 +1457,6 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, rend_intro_cell_t *parsed_req = NULL; /* Rendezvous point */ extend_info_t *rp = NULL; - /* - * We need to look up and construct the extend_info_t for v0 and v1, - * but all the info is in the cell and it's constructed by the parser - * for v2 and v3, so freeing it would be a double-free. Use this to - * keep track of whether we should free it. - */ - uint8_t need_rp_free = 0; /* XXX not handled yet */ char buf[RELAY_PAYLOAD_SIZE]; char keys[DIGEST_LEN+CPATH_KEY_MATERIAL_LEN]; /* Holds KH, Df, Db, Kf, Kb */ @@ -1154,12 +1500,15 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, intro_point = find_intro_point(circuit); if (intro_point == NULL) { - log_warn(LD_BUG, - "Internal error: Got an INTRODUCE2 cell on an " - "intro circ (for service %s) with no corresponding " - "rend_intro_point_t.", - escaped(serviceid)); - goto err; + intro_point = find_expiring_intro_point(service, circuit); + if (intro_point == NULL) { + log_warn(LD_BUG, + "Internal error: Got an INTRODUCE2 cell on an " + "intro circ (for service %s) with no corresponding " + "rend_intro_point_t.", + escaped(serviceid)); + goto err; + } } log_info(LD_REND, "Received INTRODUCE2 cell for service %s on circ %u.", @@ -1183,17 +1532,6 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, tor_free(err_msg); } - stage_descr = "early validation"; - /* Early validation of pk/ciphertext part */ - result = rend_service_validate_intro_early(parsed_req, &err_msg); - if (result < 0) { - goto log_error; - } else if (err_msg) { - log_info(LD_REND, "%s on circ %u.", err_msg, - (unsigned)circuit->base_.n_circ_id); - tor_free(err_msg); - } - /* make sure service replay caches are present */ if (!service->accepted_intro_dh_parts) { service->accepted_intro_dh_parts = @@ -1208,7 +1546,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, /* check for replay of PK-encrypted portion. */ replay = replaycache_add_test_and_elapsed( intro_point->accepted_intro_rsa_parts, - parsed_req->ciphertext, (int)parsed_req->ciphertext_len, + parsed_req->ciphertext, parsed_req->ciphertext_len, &elapsed); if (replay) { @@ -1258,9 +1596,11 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, ++(intro_point->accepted_introduce2_count); /* Find the rendezvous point */ - rp = find_rp_for_intro(parsed_req, &need_rp_free, &err_msg); - if (!rp) + rp = find_rp_for_intro(parsed_req, &err_msg); + if (!rp) { + err_msg_severity = LOG_PROTOCOL_WARN; goto log_error; + } /* Check if we'd refuse to talk to this router */ if (options->StrictNodes && @@ -1300,7 +1640,8 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, if (service->clients) { if (parsed_req->version == 3 && parsed_req->u.v3.auth_len > 0) { if (rend_check_authorization(service, - (const char*)parsed_req->u.v3.auth_data)) { + (const char*)parsed_req->u.v3.auth_data, + parsed_req->u.v3.auth_len)) { log_info(LD_REND, "Authorization data in INTRODUCE2 cell are valid."); } else { log_info(LD_REND, "The authorization data that are contained in " @@ -1338,7 +1679,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, /* help predict this next time */ rep_hist_note_used_internal(now, circ_needs_uptime, 1); - /* Launch a circuit to alice's chosen rendezvous point. + /* Launch a circuit to the client's chosen rendezvous point. */ for (i=0;i<MAX_REND_FAILURES;i++) { int flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL; @@ -1364,13 +1705,11 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, hexcookie, serviceid); tor_assert(launched->build_state); /* Fill in the circuit's state. */ - launched->rend_data = tor_malloc_zero(sizeof(rend_data_t)); - memcpy(launched->rend_data->rend_pk_digest, - circuit->rend_data->rend_pk_digest, - DIGEST_LEN); - memcpy(launched->rend_data->rend_cookie, parsed_req->rc, REND_COOKIE_LEN); - strlcpy(launched->rend_data->onion_address, service->service_id, - sizeof(launched->rend_data->onion_address)); + + launched->rend_data = + rend_data_service_create(service->service_id, + circuit->rend_data->rend_pk_digest, + parsed_req->rc, service->auth_type); launched->build_state->service_pending_final_cpath_ref = tor_malloc_zero(sizeof(crypt_path_reference_t)); @@ -1399,7 +1738,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, } } - log_warn(LD_REND, "%s on circ %u", err_msg, + log_fn(err_msg_severity, LD_REND, "%s on circ %u", err_msg, (unsigned)circuit->base_.n_circ_id); err: status = -1; @@ -1416,32 +1755,27 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, memwipe(hexcookie, 0, sizeof(hexcookie)); /* Free the parsed cell */ - if (parsed_req) { - rend_service_free_intro(parsed_req); - parsed_req = NULL; - } + rend_service_free_intro(parsed_req); - /* Free rp if we must */ - if (need_rp_free) extend_info_free(rp); + /* Free rp */ + extend_info_free(rp); return status; } /** Given a parsed and decrypted INTRODUCE2, find the rendezvous point or - * return NULL and an error string if we can't. - */ - + * return NULL and an error string if we can't. Return a newly allocated + * extend_info_t* for the rendezvous point. */ static extend_info_t * find_rp_for_intro(const rend_intro_cell_t *intro, - uint8_t *need_free_out, char **err_msg_out) + char **err_msg_out) { extend_info_t *rp = NULL; char *err_msg = NULL; const char *rp_nickname = NULL; const node_t *node = NULL; - uint8_t need_free = 0; - if (!intro || !need_free_out) { + if (!intro) { if (err_msg_out) err_msg = tor_strdup("Bad parameters to find_rp_for_intro()"); @@ -1449,8 +1783,7 @@ find_rp_for_intro(const rend_intro_cell_t *intro, } if (intro->version == 0 || intro->version == 1) { - if (intro->version == 1) rp_nickname = (const char *)(intro->u.v1.rp); - else rp_nickname = (const char *)(intro->u.v0.rp); + rp_nickname = (const char *)(intro->u.v0_v1.rp); node = node_get_by_nickname(rp_nickname, 0); if (!node) { @@ -1467,19 +1800,17 @@ find_rp_for_intro(const rend_intro_cell_t *intro, if (!rp) { if (err_msg_out) { tor_asprintf(&err_msg, - "Could build extend_info_t for router %s named " + "Couldn't build extend_info_t for router %s named " "in INTRODUCE2 cell", escaped_safe_str_client(rp_nickname)); } goto err; - } else { - need_free = 1; } } else if (intro->version == 2) { - rp = intro->u.v2.extend_info; + rp = extend_info_dup(intro->u.v2.extend_info); } else if (intro->version == 3) { - rp = intro->u.v3.extend_info; + rp = extend_info_dup(intro->u.v3.extend_info); } else { if (err_msg_out) { tor_asprintf(&err_msg, @@ -1490,39 +1821,30 @@ find_rp_for_intro(const rend_intro_cell_t *intro, goto err; } + /* Make sure the RP we are being asked to connect to is _not_ a private + * address unless it's allowed. Let's avoid to build a circuit to our + * second middle node and fail right after when extending to the RP. */ + if (!extend_info_addr_is_allowed(&rp->addr)) { + if (err_msg_out) { + tor_asprintf(&err_msg, + "Relay IP in INTRODUCE2 cell is private address."); + } + extend_info_free(rp); + rp = NULL; + goto err; + } goto done; err: - if (err_msg_out) *err_msg_out = err_msg; - else tor_free(err_msg); + if (err_msg_out) + *err_msg_out = err_msg; + else + tor_free(err_msg); done: - if (rp && need_free_out) *need_free_out = need_free; - return rp; } -/** Remove unnecessary parts from a rend_intro_cell_t - the ciphertext if - * already decrypted, the plaintext too if already parsed - */ - -void -rend_service_compact_intro(rend_intro_cell_t *request) -{ - if (!request) return; - - if ((request->plaintext && request->plaintext_len > 0) || - request->parsed) { - tor_free(request->ciphertext); - request->ciphertext_len = 0; - } - - if (request->parsed) { - tor_free(request->plaintext); - request->plaintext_len = 0; - } -} - /** Free a parsed INTRODUCE1 or INTRODUCE2 cell that was allocated by * rend_service_parse_intro(). */ @@ -1530,7 +1852,6 @@ void rend_service_free_intro(rend_intro_cell_t *request) { if (!request) { - log_info(LD_BUG, "rend_service_free_intro() called with NULL request!"); return; } @@ -1639,8 +1960,9 @@ rend_service_begin_parse_intro(const uint8_t *request, goto done; err: - if (rv) rend_service_free_intro(rv); + rend_service_free_intro(rv); rv = NULL; + if (err_msg_out && !err_msg) { tor_asprintf(&err_msg, "unknown INTRODUCE%d error", @@ -1720,11 +2042,7 @@ rend_service_parse_intro_for_v0_or_v1( goto err; } - if (intro->version == 1) { - memcpy(intro->u.v1.rp, rp_nickname, endptr - rp_nickname + 1); - } else { - memcpy(intro->u.v0.rp, rp_nickname, endptr - rp_nickname + 1); - } + memcpy(intro->u.v0_v1.rp, rp_nickname, endptr - rp_nickname + 1); return ver_specific_len; @@ -1748,7 +2066,7 @@ rend_service_parse_intro_for_v2( /* * We accept version 3 too so that the v3 parser can call this with - * and adjusted buffer for the latter part of a v3 cell, which is + * an adjusted buffer for the latter part of a v3 cell, which is * identical to a v2 cell. */ if (!(intro->version == 2 || @@ -1986,7 +2304,7 @@ rend_service_decrypt_intro( char service_id[REND_SERVICE_ID_LEN_BASE32+1]; ssize_t key_len; uint8_t buf[RELAY_PAYLOAD_SIZE]; - int result, status = 0; + int result, status = -1; if (!intro || !key) { if (err_msg_out) { @@ -2065,15 +2383,16 @@ rend_service_decrypt_intro( intro->plaintext = tor_malloc(intro->plaintext_len); memcpy(intro->plaintext, buf, intro->plaintext_len); + status = 0; + goto done; err: if (err_msg_out && !err_msg) { tor_asprintf(&err_msg, "unknown INTRODUCE%d error decrypting encrypted part", - (int)(intro->type)); + intro ? (int)(intro->type) : -1); } - if (status >= 0) status = -1; done: if (err_msg_out) *err_msg_out = err_msg; @@ -2100,7 +2419,7 @@ rend_service_parse_intro_plaintext( char *err_msg = NULL; ssize_t ver_specific_len, ver_invariant_len; uint8_t version; - int status = 0; + int status = -1; if (!intro) { if (err_msg_out) { @@ -2159,6 +2478,7 @@ rend_service_parse_intro_plaintext( (int)(intro->type), (long)(intro->plaintext_len)); status = -6; + goto err; } else { memcpy(intro->rc, intro->plaintext + ver_specific_len, @@ -2171,15 +2491,15 @@ rend_service_parse_intro_plaintext( /* Flag it as being fully parsed */ intro->parsed = 1; + status = 0; goto done; err: if (err_msg_out && !err_msg) { tor_asprintf(&err_msg, "unknown INTRODUCE%d error parsing encrypted part", - (int)(intro->type)); + intro ? (int)(intro->type) : -1); } - if (status >= 0) status = -1; done: if (err_msg_out) *err_msg_out = err_msg; @@ -2188,37 +2508,6 @@ rend_service_parse_intro_plaintext( return status; } -/** Do validity checks on a parsed intro cell before decryption; some of - * these are not done in rend_service_begin_parse_intro() itself because - * they depend on a lot of other state and would make it hard to unit test. - * Returns >= 0 if successful or < 0 if the intro cell is invalid, and - * optionally writes out an error message for logging. If an err_msg - * pointer is provided, it is the caller's responsibility to free any - * provided message. - */ - -int -rend_service_validate_intro_early(const rend_intro_cell_t *intro, - char **err_msg_out) -{ - int status = 0; - - if (!intro) { - if (err_msg_out) - *err_msg_out = - tor_strdup("NULL intro cell passed to " - "rend_service_validate_intro_early()"); - - status = -1; - goto err; - } - - /* TODO */ - - err: - return status; -} - /** Do validity checks on a parsed intro cell after decryption; some of * these are not done in rend_service_parse_intro_plaintext() itself because * they depend on a lot of other state and would make it hard to unit test. @@ -2353,50 +2642,54 @@ rend_service_launch_establish_intro(rend_service_t *service, safe_str_client(extend_info_describe(intro->extend_info))); return -1; } + /* We must have the same exit node even if cannibalized. */ + tor_assert(tor_memeq(intro->extend_info->identity_digest, + launched->build_state->chosen_exit->identity_digest, + DIGEST_LEN)); - if (tor_memneq(intro->extend_info->identity_digest, - launched->build_state->chosen_exit->identity_digest, DIGEST_LEN)) { - char cann[HEX_DIGEST_LEN+1], orig[HEX_DIGEST_LEN+1]; - base16_encode(cann, sizeof(cann), - launched->build_state->chosen_exit->identity_digest, - DIGEST_LEN); - base16_encode(orig, sizeof(orig), - intro->extend_info->identity_digest, DIGEST_LEN); - log_info(LD_REND, "The intro circuit we just cannibalized ends at $%s, " - "but we requested an intro circuit to $%s. Updating " - "our service.", cann, orig); - extend_info_free(intro->extend_info); - intro->extend_info = extend_info_dup(launched->build_state->chosen_exit); - } - - launched->rend_data = tor_malloc_zero(sizeof(rend_data_t)); - strlcpy(launched->rend_data->onion_address, service->service_id, - sizeof(launched->rend_data->onion_address)); - memcpy(launched->rend_data->rend_pk_digest, service->pk_digest, DIGEST_LEN); + launched->rend_data = rend_data_service_create(service->service_id, + service->pk_digest, NULL, + service->auth_type); launched->intro_key = crypto_pk_dup_key(intro->intro_key); if (launched->base_.state == CIRCUIT_STATE_OPEN) rend_service_intro_has_opened(launched); return 0; } -/** Return the number of introduction points that are or have been - * established for the given service address in <b>query</b>. */ -static int -count_established_intro_points(const char *query) +/** Return the number of introduction points that are established for the + * given service. */ +static unsigned int +count_established_intro_points(const rend_service_t *service) +{ + unsigned int num = 0; + + SMARTLIST_FOREACH(service->intro_nodes, rend_intro_point_t *, intro, + num += intro->circuit_established + ); + return num; +} + +/** Return the number of introduction points that are or are being + * established for the given service. This function iterates over all + * circuit and count those that are linked to the service and are waiting + * for the intro point to respond. */ +static unsigned int +count_intro_point_circuits(const rend_service_t *service) { - int num_ipos = 0; - circuit_t *circ; - for (circ = circuit_get_global_list_(); circ; circ = circ->next) { + unsigned int num_ipos = 0; + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN && (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); if (oc->rend_data && - !rend_cmp_service_ids(query, oc->rend_data->onion_address)) + !rend_cmp_service_ids(service->service_id, + oc->rend_data->onion_address)) num_ipos++; } } + SMARTLIST_FOREACH_END(circ); return num_ipos; } @@ -2429,16 +2722,27 @@ rend_service_intro_has_opened(origin_circuit_t *circuit) circuit->rend_data->rend_pk_digest); if (!service) { log_warn(LD_REND, "Unrecognized service ID %s on introduction circuit %u.", - serviceid, (unsigned)circuit->base_.n_circ_id); + safe_str_client(serviceid), (unsigned)circuit->base_.n_circ_id); reason = END_CIRC_REASON_NOSUCHSERVICE; goto err; } /* If we already have enough introduction circuits for this service, - * redefine this one as a general circuit or close it, depending. */ - if (count_established_intro_points(serviceid) > - (int)service->n_intro_points_wanted) { /* XXX023 remove cast */ + * redefine this one as a general circuit or close it, depending. + * Substract the amount of expiring nodes here since the circuits are + * still opened. */ + if ((count_intro_point_circuits(service) - + smartlist_len(service->expiring_nodes)) > + service->n_intro_points_wanted) { const or_options_t *options = get_options(); + /* Remove the intro point associated with this circuit, it's being + * repurposed or closed thus cleanup memory. */ + rend_intro_point_t *intro = find_intro_point(circuit); + if (intro != NULL) { + smartlist_remove(service->intro_nodes, intro); + rend_intro_point_free(intro); + } + if (options->ExcludeNodes) { /* XXXX in some future version, we can test whether the transition is allowed or not given the actual nodes in the circuit. But for now, @@ -2537,6 +2841,7 @@ rend_service_intro_established(origin_circuit_t *circuit, size_t request_len) { rend_service_t *service; + rend_intro_point_t *intro; char serviceid[REND_SERVICE_ID_LEN_BASE32+1]; (void) request; (void) request_len; @@ -2554,6 +2859,19 @@ rend_service_intro_established(origin_circuit_t *circuit, (unsigned)circuit->base_.n_circ_id); goto err; } + /* We've just successfully established a intro circuit to one of our + * introduction point, account for it. */ + intro = find_intro_point(circuit); + if (intro == NULL) { + log_warn(LD_REND, + "Introduction circuit established without a rend_intro_point_t " + "object for service %s on circuit %u", + safe_str_client(serviceid), (unsigned)circuit->base_.n_circ_id); + goto err; + } + intro->circuit_established = 1; + /* We might not have every introduction point ready but at this point we + * know that the descriptor needs to be uploaded. */ service->desc_is_dirty = time(NULL); circuit_change_purpose(TO_CIRCUIT(circuit), CIRCUIT_PURPOSE_S_INTRO); @@ -2669,7 +2987,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit) /* Append the cpath entry. */ hop->state = CPATH_STATE_OPEN; /* set the windows to default. these are the windows - * that bob thinks alice has. + * that the service thinks the client has. */ hop->package_window = circuit_initial_package_window(); hop->deliver_window = CIRCWINDOW_START; @@ -2728,6 +3046,24 @@ find_intro_circuit(rend_intro_point_t *intro, const char *pk_digest) return NULL; } +/** Return the corresponding introdution point using the circuit <b>circ</b> + * found in the <b>service</b>. NULL is returned if not found. */ +static rend_intro_point_t * +find_expiring_intro_point(rend_service_t *service, origin_circuit_t *circ) +{ + tor_assert(service); + tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || + TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO); + + SMARTLIST_FOREACH(service->expiring_nodes, rend_intro_point_t *, + intro_point, + if (crypto_pk_eq_keys(intro_point->intro_key, circ->intro_key)) { + return intro_point; + }); + + return NULL; +} + /** Return a pointer to the rend_intro_point_t corresponding to the * service-side introduction circuit <b>circ</b>. */ static rend_intro_point_t * @@ -2757,14 +3093,16 @@ find_intro_point(origin_circuit_t *circ) return NULL; } -/** Determine the responsible hidden service directories for the - * rend_encoded_v2_service_descriptor_t's in <b>descs</b> and upload them; - * <b>service_id</b> and <b>seconds_valid</b> are only passed for logging - * purposes. */ -static void +/** Upload the rend_encoded_v2_service_descriptor_t's in <b>descs</b> + * associated with the rend_service_descriptor_t <b>renddesc</b> to + * the responsible hidden service directories OR the hidden service + * directories specified by <b>hs_dirs</b>; <b>service_id</b> and + * <b>seconds_valid</b> are only passed for logging purposes. + */ +void directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, - smartlist_t *descs, const char *service_id, - int seconds_valid) + smartlist_t *descs, smartlist_t *hs_dirs, + const char *service_id, int seconds_valid) { int i, j, failed_upload = 0; smartlist_t *responsible_dirs = smartlist_new(); @@ -2772,19 +3110,27 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, routerstatus_t *hs_dir; for (i = 0; i < smartlist_len(descs); i++) { rend_encoded_v2_service_descriptor_t *desc = smartlist_get(descs, i); - /* Determine responsible dirs. */ - if (hid_serv_get_responsible_directories(responsible_dirs, - desc->desc_id) < 0) { - log_warn(LD_REND, "Could not determine the responsible hidden service " - "directories to post descriptors to."); - smartlist_free(responsible_dirs); - smartlist_free(successful_uploads); - return; + /** If any HSDirs are specified, they should be used instead of + * the responsible directories */ + if (hs_dirs && smartlist_len(hs_dirs) > 0) { + smartlist_add_all(responsible_dirs, hs_dirs); + } else { + /* Determine responsible dirs. */ + if (hid_serv_get_responsible_directories(responsible_dirs, + desc->desc_id) < 0) { + log_warn(LD_REND, "Could not determine the responsible hidden service " + "directories to post descriptors to."); + control_event_hs_descriptor_upload(service_id, + "UNKNOWN", + "UNKNOWN"); + goto done; + } } for (j = 0; j < smartlist_len(responsible_dirs); j++) { char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; char *hs_dir_ip; const node_t *node; + rend_data_t *rend_data; hs_dir = smartlist_get(responsible_dirs, j); if (smartlist_contains_digest(renddesc->successful_uploads, hs_dir->identity_digest)) @@ -2800,12 +3146,19 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, continue; } /* Send publish request. */ - directory_initiate_command_routerstatus(hs_dir, + + /* We need the service ID to identify which service did the upload + * request. Lookup is made in rend_service_desc_has_uploaded(). */ + rend_data = rend_data_client_create(service_id, desc->desc_id, NULL, + REND_NO_AUTH); + directory_initiate_command_routerstatus_rend(hs_dir, DIR_PURPOSE_UPLOAD_RENDDESC_V2, - ROUTER_PURPOSE_GENERAL, - DIRIND_ANONYMOUS, NULL, - desc->desc_str, - strlen(desc->desc_str), 0); + ROUTER_PURPOSE_GENERAL, + DIRIND_ANONYMOUS, NULL, + desc->desc_str, + strlen(desc->desc_str), + 0, rend_data); + rend_data_free(rend_data); base32_encode(desc_id_base32, sizeof(desc_id_base32), desc->desc_id, DIGEST_LEN); hs_dir_ip = tor_dup_ip(hs_dir->addr); @@ -2819,6 +3172,9 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, hs_dir->nickname, hs_dir_ip, hs_dir->or_port); + control_event_hs_descriptor_upload(service_id, + hs_dir->identity_digest, + desc_id_base32); tor_free(hs_dir_ip); /* Remember successful upload to this router for next time. */ if (!smartlist_contains_digest(successful_uploads, @@ -2846,6 +3202,7 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, } }); } + done: smartlist_free(responsible_dirs); smartlist_free(successful_uploads); } @@ -2863,39 +3220,72 @@ upload_service_descriptor(rend_service_t *service) rendpostperiod = get_options()->RendPostPeriod; - /* Upload descriptor? */ - if (get_options()->PublishHidServDescriptors) { - networkstatus_t *c = networkstatus_get_latest_consensus(); - if (c && smartlist_len(c->routerstatus_list) > 0) { - int seconds_valid, i, j, num_descs; - smartlist_t *descs = smartlist_new(); - smartlist_t *client_cookies = smartlist_new(); - /* Either upload a single descriptor (including replicas) or one - * descriptor for each authorized client in case of authorization - * type 'stealth'. */ - num_descs = service->auth_type == REND_STEALTH_AUTH ? - smartlist_len(service->clients) : 1; - for (j = 0; j < num_descs; j++) { - crypto_pk_t *client_key = NULL; - rend_authorized_client_t *client = NULL; - smartlist_clear(client_cookies); - switch (service->auth_type) { - case REND_NO_AUTH: - /* Do nothing here. */ - break; - case REND_BASIC_AUTH: - SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, - cl, smartlist_add(client_cookies, cl->descriptor_cookie)); - break; - case REND_STEALTH_AUTH: - client = smartlist_get(service->clients, j); - client_key = client->client_key; - smartlist_add(client_cookies, client->descriptor_cookie); - break; - } - /* Encode the current descriptor. */ + networkstatus_t *c = networkstatus_get_latest_consensus(); + if (c && smartlist_len(c->routerstatus_list) > 0) { + int seconds_valid, i, j, num_descs; + smartlist_t *descs = smartlist_new(); + smartlist_t *client_cookies = smartlist_new(); + /* Either upload a single descriptor (including replicas) or one + * descriptor for each authorized client in case of authorization + * type 'stealth'. */ + num_descs = service->auth_type == REND_STEALTH_AUTH ? + smartlist_len(service->clients) : 1; + for (j = 0; j < num_descs; j++) { + crypto_pk_t *client_key = NULL; + rend_authorized_client_t *client = NULL; + smartlist_clear(client_cookies); + switch (service->auth_type) { + case REND_NO_AUTH: + /* Do nothing here. */ + break; + case REND_BASIC_AUTH: + SMARTLIST_FOREACH(service->clients, rend_authorized_client_t *, + cl, smartlist_add(client_cookies, cl->descriptor_cookie)); + break; + case REND_STEALTH_AUTH: + client = smartlist_get(service->clients, j); + client_key = client->client_key; + smartlist_add(client_cookies, client->descriptor_cookie); + break; + } + /* Encode the current descriptor. */ + seconds_valid = rend_encode_v2_descriptors(descs, service->desc, + now, 0, + service->auth_type, + client_key, + client_cookies); + if (seconds_valid < 0) { + log_warn(LD_BUG, "Internal error: couldn't encode service " + "descriptor; not uploading."); + smartlist_free(descs); + smartlist_free(client_cookies); + return; + } + rend_get_service_id(service->desc->pk, serviceid); + if (get_options()->PublishHidServDescriptors) { + /* Post the current descriptors to the hidden service directories. */ + log_info(LD_REND, "Launching upload for hidden service %s", + serviceid); + directory_post_to_hs_dir(service->desc, descs, NULL, serviceid, + seconds_valid); + } + /* Free memory for descriptors. */ + for (i = 0; i < smartlist_len(descs); i++) + rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); + smartlist_clear(descs); + /* Update next upload time. */ + if (seconds_valid - REND_TIME_PERIOD_OVERLAPPING_V2_DESCS + > rendpostperiod) + service->next_upload_time = now + rendpostperiod; + else if (seconds_valid < REND_TIME_PERIOD_OVERLAPPING_V2_DESCS) + service->next_upload_time = now + seconds_valid + 1; + else + service->next_upload_time = now + seconds_valid - + REND_TIME_PERIOD_OVERLAPPING_V2_DESCS + 1; + /* Post also the next descriptors, if necessary. */ + if (seconds_valid < REND_TIME_PERIOD_OVERLAPPING_V2_DESCS) { seconds_valid = rend_encode_v2_descriptors(descs, service->desc, - now, 0, + now, 1, service->auth_type, client_key, client_cookies); @@ -2906,51 +3296,23 @@ upload_service_descriptor(rend_service_t *service) smartlist_free(client_cookies); return; } - /* Post the current descriptors to the hidden service directories. */ - rend_get_service_id(service->desc->pk, serviceid); - log_info(LD_REND, "Launching upload for hidden service %s", - serviceid); - directory_post_to_hs_dir(service->desc, descs, serviceid, - seconds_valid); + if (get_options()->PublishHidServDescriptors) { + directory_post_to_hs_dir(service->desc, descs, NULL, serviceid, + seconds_valid); + } /* Free memory for descriptors. */ for (i = 0; i < smartlist_len(descs); i++) rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); smartlist_clear(descs); - /* Update next upload time. */ - if (seconds_valid - REND_TIME_PERIOD_OVERLAPPING_V2_DESCS - > rendpostperiod) - service->next_upload_time = now + rendpostperiod; - else if (seconds_valid < REND_TIME_PERIOD_OVERLAPPING_V2_DESCS) - service->next_upload_time = now + seconds_valid + 1; - else - service->next_upload_time = now + seconds_valid - - REND_TIME_PERIOD_OVERLAPPING_V2_DESCS + 1; - /* Post also the next descriptors, if necessary. */ - if (seconds_valid < REND_TIME_PERIOD_OVERLAPPING_V2_DESCS) { - seconds_valid = rend_encode_v2_descriptors(descs, service->desc, - now, 1, - service->auth_type, - client_key, - client_cookies); - if (seconds_valid < 0) { - log_warn(LD_BUG, "Internal error: couldn't encode service " - "descriptor; not uploading."); - smartlist_free(descs); - smartlist_free(client_cookies); - return; - } - directory_post_to_hs_dir(service->desc, descs, serviceid, - seconds_valid); - /* Free memory for descriptors. */ - for (i = 0; i < smartlist_len(descs); i++) - rend_encoded_v2_service_descriptor_free(smartlist_get(descs, i)); - smartlist_clear(descs); - } } - smartlist_free(descs); - smartlist_free(client_cookies); - uploaded = 1; + } + smartlist_free(descs); + smartlist_free(client_cookies); + uploaded = 1; + if (get_options()->PublishHidServDescriptors) { log_info(LD_REND, "Successfully uploaded v2 rend descriptors!"); + } else { + log_info(LD_REND, "Successfully stored created v2 rend descriptors!"); } } @@ -2984,14 +3346,8 @@ intro_point_should_expire_now(rend_intro_point_t *intro, return 0; } - if (intro->time_expiring != -1) { - /* We've already started expiring this intro point. *Don't* let - * this function's result 'flap'. */ - return 1; - } - if (intro_point_accepted_intro_count(intro) >= - INTRO_POINT_LIFETIME_INTRODUCTIONS) { + intro->max_introductions) { /* This intro point has been used too many times. Expire it now. */ return 1; } @@ -3000,9 +3356,8 @@ intro_point_should_expire_now(rend_intro_point_t *intro, /* This intro point has been published, but we haven't picked an * expiration time for it. Pick one now. */ int intro_point_lifetime_seconds = - INTRO_POINT_LIFETIME_MIN_SECONDS + - crypto_rand_int(INTRO_POINT_LIFETIME_MAX_SECONDS - - INTRO_POINT_LIFETIME_MIN_SECONDS); + crypto_rand_int_range(INTRO_POINT_LIFETIME_MIN_SECONDS, + INTRO_POINT_LIFETIME_MAX_SECONDS); /* Start the expiration timer now, rather than when the intro * point was first published. There shouldn't be much of a time @@ -3016,43 +3371,160 @@ intro_point_should_expire_now(rend_intro_point_t *intro, return (now >= intro->time_to_expire); } +/** Iterate over intro points in the given service and remove the invalid + * ones. For an intro point object to be considered invalid, the circuit + * _and_ node need to have disappeared. + * + * If the intro point should expire, it's placed into the expiring_nodes + * list of the service and removed from the active intro nodes list. + * + * If <b>exclude_nodes</b> is not NULL, add the valid nodes to it. + * + * If <b>retry_nodes</b> is not NULL, add the valid node to it if the + * circuit disappeared but the node is still in the consensus. */ +static void +remove_invalid_intro_points(rend_service_t *service, + smartlist_t *exclude_nodes, + smartlist_t *retry_nodes, time_t now) +{ + tor_assert(service); + + SMARTLIST_FOREACH_BEGIN(service->intro_nodes, rend_intro_point_t *, + intro) { + /* Find the introduction point node object. */ + const node_t *node = + node_get_by_id(intro->extend_info->identity_digest); + /* Find the intro circuit, this might be NULL. */ + origin_circuit_t *intro_circ = + find_intro_circuit(intro, service->pk_digest); + + /* Add the valid node to the exclusion list so we don't try to establish + * an introduction point to it again. */ + if (node && exclude_nodes) { + smartlist_add(exclude_nodes, (void*) node); + } + + /* First, make sure we still have a valid circuit for this intro point. + * If we dont, we'll give up on it and make a new one. */ + if (intro_circ == NULL) { + log_info(LD_REND, "Attempting to retry on %s as intro point for %s" + " (circuit disappeared).", + safe_str_client(extend_info_describe(intro->extend_info)), + safe_str_client(service->service_id)); + /* We've lost the circuit for this intro point, flag it so it can be + * accounted for when considiring uploading a descriptor. */ + intro->circuit_established = 0; + + /* Node is gone or we've reached our maximum circuit creationg retry + * count, clean up everything, we'll find a new one. */ + if (node == NULL || + intro->circuit_retries >= MAX_INTRO_POINT_CIRCUIT_RETRIES) { + rend_intro_point_free(intro); + SMARTLIST_DEL_CURRENT(service->intro_nodes, intro); + /* We've just killed the intro point, nothing left to do. */ + continue; + } + + /* The intro point is still alive so let's try to use it again because + * we have a published descriptor containing it. Keep the intro point + * in the intro_nodes list because it's still valid, we are rebuilding + * a circuit to it. */ + if (retry_nodes) { + smartlist_add(retry_nodes, intro); + } + } + /* else, the circuit is valid so in both cases, node being alive or not, + * we leave the circuit and intro point object as is. Closing the + * circuit here would leak new consensus timing and freeing the intro + * point object would make the intro circuit unusable. */ + + /* Now, check if intro point should expire. If it does, queue it so + * it can be cleaned up once it has been replaced properly. */ + if (intro_point_should_expire_now(intro, now)) { + log_info(LD_REND, "Expiring %s as intro point for %s.", + safe_str_client(extend_info_describe(intro->extend_info)), + safe_str_client(service->service_id)); + smartlist_add(service->expiring_nodes, intro); + SMARTLIST_DEL_CURRENT(service->intro_nodes, intro); + /* Intro point is expired, we need a new one thus don't consider it + * anymore has a valid established intro point. */ + intro->circuit_established = 0; + } + } SMARTLIST_FOREACH_END(intro); +} + +/** A new descriptor has been successfully uploaded for the given + * <b>rend_data</b>. Remove and free the expiring nodes from the associated + * service. */ +void +rend_service_desc_has_uploaded(const rend_data_t *rend_data) +{ + rend_service_t *service; + + tor_assert(rend_data); + + service = rend_service_get_by_service_id(rend_data->onion_address); + if (service == NULL) { + return; + } + + SMARTLIST_FOREACH_BEGIN(service->expiring_nodes, rend_intro_point_t *, + intro) { + origin_circuit_t *intro_circ = + find_intro_circuit(intro, service->pk_digest); + if (intro_circ != NULL) { + circuit_mark_for_close(TO_CIRCUIT(intro_circ), + END_CIRC_REASON_FINISHED); + } + SMARTLIST_DEL_CURRENT(service->expiring_nodes, intro); + rend_intro_point_free(intro); + } SMARTLIST_FOREACH_END(intro); +} + /** For every service, check how many intro points it currently has, and: + * - Invalidate introdution points based on specific criteria, see + * remove_invalid_intro_points comments. * - Pick new intro points as necessary. * - Launch circuits to any new intro points. + * + * This is called once a second by the main loop. */ void -rend_services_introduce(void) +rend_consider_services_intro_points(void) { - int i,j,r; - const node_t *node; - rend_service_t *service; - rend_intro_point_t *intro; - int intro_point_set_changed, prev_intro_nodes; - unsigned int n_intro_points_unexpired; - unsigned int n_intro_points_to_open; - smartlist_t *intro_nodes; + int i; time_t now; const or_options_t *options = get_options(); + /* List of nodes we need to _exclude_ when choosing a new node to + * establish an intro point to. */ + smartlist_t *exclude_nodes; + /* List of nodes we need to retry to build a circuit on them because the + * node is valid but circuit died. */ + smartlist_t *retry_nodes; + + if (!have_completed_a_circuit()) + return; - intro_nodes = smartlist_new(); + exclude_nodes = smartlist_new(); + retry_nodes = smartlist_new(); now = time(NULL); - for (i=0; i < smartlist_len(rend_service_list); ++i) { - smartlist_clear(intro_nodes); - service = smartlist_get(rend_service_list, i); - - tor_assert(service); - - /* intro_point_set_changed becomes non-zero iff the set of intro - * points to be published in service's descriptor has changed. */ - intro_point_set_changed = 0; - - /* n_intro_points_unexpired collects the number of non-expiring - * intro points we have, so that we know how many new intro - * circuits we need to launch for this service. */ - n_intro_points_unexpired = 0; - - if (now > service->intro_period_started+INTRO_CIRC_RETRY_PERIOD) { + SMARTLIST_FOREACH_BEGIN(rend_service_list, rend_service_t *, service) { + int r; + /* Number of intro points we want to open and add to the intro nodes + * list of the service. */ + unsigned int n_intro_points_to_open; + /* Have an unsigned len so we can use it to compare values else gcc is + * not happy with unmatching signed comparaison. */ + unsigned int intro_nodes_len; + /* Different service are allowed to have the same introduction point as + * long as they are on different circuit thus why we clear this list. */ + smartlist_clear(exclude_nodes); + smartlist_clear(retry_nodes); + + /* This retry period is important here so we don't stress circuit + * creation. */ + if (now > service->intro_period_started + INTRO_CIRC_RETRY_PERIOD) { /* One period has elapsed; we can try building circuits again. */ service->intro_period_started = now; service->n_intro_circuits_launched = 0; @@ -3063,160 +3535,108 @@ rend_services_introduce(void) continue; } - /* Find out which introduction points we have in progress for this - service. */ - SMARTLIST_FOREACH_BEGIN(service->intro_nodes, rend_intro_point_t *, - intro) { - origin_circuit_t *intro_circ = - find_intro_circuit(intro, service->pk_digest); - - if (intro->time_expiring + INTRO_POINT_EXPIRATION_GRACE_PERIOD > now) { - /* This intro point has completely expired. Remove it, and - * mark the circuit for close if it's still alive. */ - if (intro_circ != NULL && - intro_circ->base_.purpose != CIRCUIT_PURPOSE_PATH_BIAS_TESTING) { - circuit_mark_for_close(TO_CIRCUIT(intro_circ), - END_CIRC_REASON_FINISHED); - } - rend_intro_point_free(intro); - intro = NULL; /* SMARTLIST_DEL_CURRENT takes a name, not a value. */ - SMARTLIST_DEL_CURRENT(service->intro_nodes, intro); - /* We don't need to set intro_point_set_changed here, because - * this intro point wouldn't have been published in a current - * descriptor anyway. */ - continue; - } + /* Cleanup the invalid intro points and save the node objects, if apply, + * in the exclude_nodes and retry_nodes list. */ + remove_invalid_intro_points(service, exclude_nodes, retry_nodes, now); - node = node_get_by_id(intro->extend_info->identity_digest); - if (!node || !intro_circ) { - int removing_this_intro_point_changes_the_intro_point_set = 1; - log_info(LD_REND, "Giving up on %s as intro point for %s" - " (circuit disappeared).", + /* Let's try to rebuild circuit on the nodes we want to retry on. */ + SMARTLIST_FOREACH_BEGIN(retry_nodes, rend_intro_point_t *, intro) { + r = rend_service_launch_establish_intro(service, intro); + if (r < 0) { + log_warn(LD_REND, "Error launching circuit to node %s for service %s.", safe_str_client(extend_info_describe(intro->extend_info)), safe_str_client(service->service_id)); - rend_service_note_removing_intro_point(service, intro); - if (intro->time_expiring != -1) { - log_info(LD_REND, "We were already expiring the intro point; " - "no need to mark the HS descriptor as dirty over this."); - removing_this_intro_point_changes_the_intro_point_set = 0; - } else if (intro->listed_in_last_desc) { - log_info(LD_REND, "The intro point we are giving up on was " - "included in the last published descriptor. " - "Marking current descriptor as dirty."); - service->desc_is_dirty = now; - } + /* Unable to launch a circuit to that intro point, remove it from + * the valid list so we can create a new one. */ + smartlist_remove(service->intro_nodes, intro); rend_intro_point_free(intro); - intro = NULL; /* SMARTLIST_DEL_CURRENT takes a name, not a value. */ - SMARTLIST_DEL_CURRENT(service->intro_nodes, intro); - if (removing_this_intro_point_changes_the_intro_point_set) - intro_point_set_changed = 1; - } - - if (intro != NULL && intro_point_should_expire_now(intro, now)) { - log_info(LD_REND, "Expiring %s as intro point for %s.", - safe_str_client(extend_info_describe(intro->extend_info)), - safe_str_client(service->service_id)); - - rend_service_note_removing_intro_point(service, intro); - - /* The polite (and generally Right) way to expire an intro - * point is to establish a new one to replace it, publish a - * new descriptor that doesn't list any expiring intro points, - * and *then*, once our upload attempts for the new descriptor - * have ended (whether in success or failure), close the - * expiring intro points. - * - * Unfortunately, we can't find out when the new descriptor - * has actually been uploaded, so we'll have to settle for a - * five-minute timer. Start it. XXXX024 This sucks. */ - intro->time_expiring = now; - - intro_point_set_changed = 1; + continue; } - - if (intro != NULL && intro->time_expiring == -1) - ++n_intro_points_unexpired; - - if (node) - smartlist_add(intro_nodes, (void*)node); + intro->circuit_retries++; } SMARTLIST_FOREACH_END(intro); - if (!intro_point_set_changed && - (n_intro_points_unexpired >= service->n_intro_points_wanted)) { + /* Avoid mismatched signed comparaison below. */ + intro_nodes_len = (unsigned int) smartlist_len(service->intro_nodes); + + /* Quiescent state, no node expiring and we have more or the amount of + * wanted node for this service. Proceed to the next service. Could be + * more because we launch two preemptive circuits if our intro nodes + * list is empty. */ + if (smartlist_len(service->expiring_nodes) == 0 && + intro_nodes_len >= service->n_intro_points_wanted) { continue; } - /* Remember how many introduction circuits we started with. - * - * prev_intro_nodes serves a different purpose than - * n_intro_points_unexpired -- this variable tells us where our - * previously-created intro points end and our new ones begin in - * the intro-point list, so we don't have to launch the circuits - * at the same time as we create the intro points they correspond - * to. XXXX This is daft. */ - prev_intro_nodes = smartlist_len(service->intro_nodes); - - /* We have enough directory information to start establishing our - * intro points. We want to end up with n_intro_points_wanted - * intro points, but if we're just starting, we launch two extra - * circuits and use the first n_intro_points_wanted that complete. - * - * The ones after the first three will be converted to 'general' - * internal circuits in rend_service_intro_has_opened(), and then - * we'll drop them from the list of intro points next time we - * go through the above "find out which introduction points we have - * in progress" loop. */ - n_intro_points_to_open = (service->n_intro_points_wanted + - (prev_intro_nodes == 0 ? 2 : 0)); - for (j = (int)n_intro_points_unexpired; - j < (int)n_intro_points_to_open; - ++j) { /* XXXX remove casts */ + /* Number of intro points we want to open which is the wanted amount + * minus the current amount of valid nodes. */ + n_intro_points_to_open = service->n_intro_points_wanted - intro_nodes_len; + if (intro_nodes_len == 0) { + /* We want to end up with n_intro_points_wanted intro points, but if + * we have no intro points at all (chances are they all cycled or we + * are starting up), we launch NUM_INTRO_POINTS_EXTRA extra circuits + * and use the first n_intro_points_wanted that complete. See proposal + * #155, section 4 for the rationale of this which is purely for + * performance. + * + * The ones after the first n_intro_points_to_open will be converted + * to 'general' internal circuits in rend_service_intro_has_opened(), + * and then we'll drop them from the list of intro points. */ + n_intro_points_to_open += NUM_INTRO_POINTS_EXTRA; + } + + for (i = 0; i < (int) n_intro_points_to_open; i++) { + const node_t *node; + rend_intro_point_t *intro; router_crn_flags_t flags = CRN_NEED_UPTIME|CRN_NEED_DESC; if (get_options()->AllowInvalid_ & ALLOW_INVALID_INTRODUCTION) flags |= CRN_ALLOW_INVALID; - node = router_choose_random_node(intro_nodes, + node = router_choose_random_node(exclude_nodes, options->ExcludeNodes, flags); if (!node) { log_warn(LD_REND, - "Could only establish %d introduction points for %s; " + "We only have %d introduction points established for %s; " "wanted %u.", - smartlist_len(service->intro_nodes), service->service_id, + smartlist_len(service->intro_nodes), + safe_str_client(service->service_id), n_intro_points_to_open); break; } - intro_point_set_changed = 1; - smartlist_add(intro_nodes, (void*)node); + /* Add the choosen node to the exclusion list in order to avoid to + * pick it again in the next iteration. */ + smartlist_add(exclude_nodes, (void*)node); intro = tor_malloc_zero(sizeof(rend_intro_point_t)); intro->extend_info = extend_info_from_node(node, 0); intro->intro_key = crypto_pk_new(); - tor_assert(!crypto_pk_generate_key(intro->intro_key)); + const int fail = crypto_pk_generate_key(intro->intro_key); + tor_assert(!fail); intro->time_published = -1; intro->time_to_expire = -1; - intro->time_expiring = -1; + intro->max_introductions = + crypto_rand_int_range(INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS, + INTRO_POINT_MAX_LIFETIME_INTRODUCTIONS); smartlist_add(service->intro_nodes, intro); log_info(LD_REND, "Picked router %s as an intro point for %s.", safe_str_client(node_describe(node)), safe_str_client(service->service_id)); - } - - /* If there's no need to launch new circuits, stop here. */ - if (!intro_point_set_changed) - continue; - - /* Establish new introduction points. */ - for (j=prev_intro_nodes; j < smartlist_len(service->intro_nodes); ++j) { - intro = smartlist_get(service->intro_nodes, j); + /* Establish new introduction circuit to our chosen intro point. */ r = rend_service_launch_establish_intro(service, intro); - if (r<0) { + if (r < 0) { log_warn(LD_REND, "Error launching circuit to node %s for service %s.", safe_str_client(extend_info_describe(intro->extend_info)), safe_str_client(service->service_id)); + /* This funcion will be called again by the main loop so this intro + * point without a intro circuit will be retried on or removed after + * a maximum number of attempts. */ } } - } - smartlist_free(intro_nodes); + } SMARTLIST_FOREACH_END(service); + smartlist_free(exclude_nodes); + smartlist_free(retry_nodes); } +#define MIN_REND_INITIAL_POST_DELAY (30) +#define MIN_REND_INITIAL_POST_DELAY_TESTING (5) + /** Regenerate and upload rendezvous service descriptors for all * services, if necessary. If the descriptor has been dirty enough * for long enough, definitely upload; else only upload when the @@ -3231,24 +3651,29 @@ rend_consider_services_upload(time_t now) int i; rend_service_t *service; int rendpostperiod = get_options()->RendPostPeriod; - - if (!get_options()->PublishHidServDescriptors) - return; + int rendinitialpostdelay = (get_options()->TestingTorNetwork ? + MIN_REND_INITIAL_POST_DELAY_TESTING : + MIN_REND_INITIAL_POST_DELAY); for (i=0; i < smartlist_len(rend_service_list); ++i) { service = smartlist_get(rend_service_list, i); if (!service->next_upload_time) { /* never been uploaded yet */ - /* The fixed lower bound of 30 seconds ensures that the descriptor - * is stable before being published. See comment below. */ + /* The fixed lower bound of rendinitialpostdelay seconds ensures that + * the descriptor is stable before being published. See comment below. */ service->next_upload_time = - now + 30 + crypto_rand_int(2*rendpostperiod); + now + rendinitialpostdelay + crypto_rand_int(2*rendpostperiod); } - if (service->next_upload_time < now || + /* Does every introduction points have been established? */ + unsigned int intro_points_ready = + count_established_intro_points(service) >= + service->n_intro_points_wanted; + if (intro_points_ready && + (service->next_upload_time < now || (service->desc_is_dirty && - service->desc_is_dirty < now-30)) { + service->desc_is_dirty < now-rendinitialpostdelay))) { /* if it's time, or if the directory servers have a wrong service - * descriptor and ours has been stable for 30 seconds, upload a - * new one of each format. */ + * descriptor and ours has been stable for rendinitialpostdelay seconds, + * upload a new one of each format. */ rend_service_update_descriptor(service); upload_service_descriptor(service); } @@ -3327,9 +3752,64 @@ rend_service_dump_stats(int severity) } } +#ifdef HAVE_SYS_UN_H + +/** Given <b>ports</b>, a smarlist containing rend_service_port_config_t, + * add the given <b>p</b>, a AF_UNIX port to the list. Return 0 on success + * else return -ENOSYS if AF_UNIX is not supported (see function in the + * #else statement below). */ +static int +add_unix_port(smartlist_t *ports, rend_service_port_config_t *p) +{ + tor_assert(ports); + tor_assert(p); + tor_assert(p->is_unix_addr); + + smartlist_add(ports, p); + return 0; +} + +/** Given <b>conn</b> set it to use the given port <b>p</b> values. Return 0 + * on success else return -ENOSYS if AF_UNIX is not supported (see function + * in the #else statement below). */ +static int +set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p) +{ + tor_assert(conn); + tor_assert(p); + tor_assert(p->is_unix_addr); + + conn->base_.socket_family = AF_UNIX; + tor_addr_make_unspec(&conn->base_.addr); + conn->base_.port = 1; + conn->base_.address = tor_strdup(p->unix_addr); + return 0; +} + +#else /* defined(HAVE_SYS_UN_H) */ + +static int +set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p) +{ + (void) conn; + (void) p; + return -ENOSYS; +} + +static int +add_unix_port(smartlist_t *ports, rend_service_port_config_t *p) +{ + (void) ports; + (void) p; + return -ENOSYS; +} + +#endif /* HAVE_SYS_UN_H */ + /** Given <b>conn</b>, a rendezvous exit stream, look up the hidden service for * 'circ', and look up the port and address based on conn-\>port. - * Assign the actual conn-\>addr and conn-\>port. Return -1 if failure, + * Assign the actual conn-\>addr and conn-\>port. Return -2 on failure + * for which the circuit should be closed, -1 on other failure, * or 0 for success. */ int @@ -3340,6 +3820,7 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, char serviceid[REND_SERVICE_ID_LEN_BASE32+1]; smartlist_t *matching_ports; rend_service_port_config_t *chosen_port; + unsigned int warn_once = 0; tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED); tor_assert(circ->rend_data); @@ -3352,24 +3833,72 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, log_warn(LD_REND, "Couldn't find any service associated with pk %s on " "rendezvous circuit %u; closing.", serviceid, (unsigned)circ->base_.n_circ_id); - return -1; + return -2; + } + if (service->max_streams_per_circuit > 0) { + /* Enforce the streams-per-circuit limit, and refuse to provide a + * mapping if this circuit will exceed the limit. */ +#define MAX_STREAM_WARN_INTERVAL 600 + static struct ratelim_t stream_ratelim = + RATELIM_INIT(MAX_STREAM_WARN_INTERVAL); + if (circ->rend_data->nr_streams >= service->max_streams_per_circuit) { + log_fn_ratelim(&stream_ratelim, LOG_WARN, LD_REND, + "Maximum streams per circuit limit reached on rendezvous " + "circuit %u; %s. Circuit has %d out of %d streams.", + (unsigned)circ->base_.n_circ_id, + service->max_streams_close_circuit ? + "closing circuit" : + "ignoring open stream request", + circ->rend_data->nr_streams, + service->max_streams_per_circuit); + return service->max_streams_close_circuit ? -2 : -1; + } } matching_ports = smartlist_new(); SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p, { - if (conn->base_.port == p->virtual_port) { + if (conn->base_.port != p->virtual_port) { + continue; + } + if (!(p->is_unix_addr)) { smartlist_add(matching_ports, p); + } else { + if (add_unix_port(matching_ports, p)) { + if (!warn_once) { + /* Unix port not supported so warn only once. */ + log_warn(LD_REND, + "Saw AF_UNIX virtual port mapping for port %d on service " + "%s, which is unsupported on this platform. Ignoring it.", + conn->base_.port, serviceid); + } + warn_once++; + } } }); chosen_port = smartlist_choose(matching_ports); smartlist_free(matching_ports); if (chosen_port) { - tor_addr_copy(&conn->base_.addr, &chosen_port->real_addr); - conn->base_.port = chosen_port->real_port; + if (!(chosen_port->is_unix_addr)) { + /* Get a non-AF_UNIX connection ready for connection_exit_connect() */ + tor_addr_copy(&conn->base_.addr, &chosen_port->real_addr); + conn->base_.port = chosen_port->real_port; + } else { + if (set_unix_port(conn, chosen_port)) { + /* Simply impossible to end up here else we were able to add a Unix + * port without AF_UNIX support... ? */ + tor_assert(0); + } + } return 0; } - log_info(LD_REND, "No virtual port mapping exists for port %d on service %s", - conn->base_.port,serviceid); - return -1; + + log_info(LD_REND, + "No virtual port mapping exists for port %d on service %s", + conn->base_.port, serviceid); + + if (service->allow_unknown_ports) + return -1; + else + return -2; } diff --git a/src/or/rendservice.h b/src/or/rendservice.h index caf88a3d64..101b37e18d 100644 --- a/src/or/rendservice.h +++ b/src/or/rendservice.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,6 +15,7 @@ #include "or.h" typedef struct rend_intro_cell_s rend_intro_cell_t; +typedef struct rend_service_port_config_s rend_service_port_config_t; #ifdef RENDSERVICE_PRIVATE @@ -38,13 +39,9 @@ struct rend_intro_cell_s { /* Version-specific parts */ union { struct { - /* Rendezvous point nickname */ - uint8_t rp[20]; - } v0; - struct { /* Rendezvous point nickname or hex-encoded key digest */ uint8_t rp[42]; - } v1; + } v0_v1; struct { /* The extend_info_t struct has everything v2 uses */ extend_info_t *extend_info; @@ -71,7 +68,9 @@ struct rend_intro_cell_s { int num_rend_services(void); int rend_config_services(const or_options_t *options, int validate_only); int rend_service_load_all_keys(void); -void rend_services_introduce(void); +void rend_services_add_filenames_to_lists(smartlist_t *open_lst, + smartlist_t *stat_lst); +void rend_consider_services_intro_points(void); void rend_consider_services_upload(time_t now); void rend_hsdir_routers_changed(void); void rend_consider_descriptor_republication(void); @@ -81,9 +80,9 @@ int rend_service_intro_established(origin_circuit_t *circuit, const uint8_t *request, size_t request_len); void rend_service_rendezvous_has_opened(origin_circuit_t *circuit); -int rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, - size_t request_len); -void rend_service_compact_intro(rend_intro_cell_t *request); +int rend_service_receive_introduction(origin_circuit_t *circuit, + const uint8_t *request, + size_t request_len); int rend_service_decrypt_intro(rend_intro_cell_t *request, crypto_pk_t *key, char **err_msg_out); @@ -94,8 +93,6 @@ rend_intro_cell_t * rend_service_begin_parse_intro(const uint8_t *request, char **err_msg_out); int rend_service_parse_intro_plaintext(rend_intro_cell_t *intro, char **err_msg_out); -int rend_service_validate_intro_early(const rend_intro_cell_t *intro, - char **err_msg_out); int rend_service_validate_intro_late(const rend_intro_cell_t *intro, char **err_msg_out); void rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc); @@ -104,5 +101,30 @@ int rend_service_set_connection_addr_port(edge_connection_t *conn, void rend_service_dump_stats(int severity); void rend_service_free_all(void); +rend_service_port_config_t *rend_service_parse_port_config(const char *string, + const char *sep, + char **err_msg_out); +void rend_service_port_config_free(rend_service_port_config_t *p); + +/** Return value from rend_service_add_ephemeral. */ +typedef enum { + RSAE_BADVIRTPORT = -4, /**< Invalid VIRTPORT/TARGET(s) */ + RSAE_ADDREXISTS = -3, /**< Onion address collision */ + RSAE_BADPRIVKEY = -2, /**< Invalid public key */ + RSAE_INTERNAL = -1, /**< Internal error */ + RSAE_OKAY = 0 /**< Service added as expected */ +} rend_service_add_ephemeral_status_t; +rend_service_add_ephemeral_status_t rend_service_add_ephemeral(crypto_pk_t *pk, + smartlist_t *ports, + int max_streams_per_circuit, + int max_streams_close_circuit, + char **service_id_out); +int rend_service_del_ephemeral(const char *service_id); + +void directory_post_to_hs_dir(rend_service_descriptor_t *renddesc, + smartlist_t *descs, smartlist_t *hs_dirs, + const char *service_id, int seconds_valid); +void rend_service_desc_has_uploaded(const rend_data_t *rend_data); + #endif diff --git a/src/or/rephist.c b/src/or/rephist.c index 2948bf8f00..04ed7aef0f 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -148,7 +148,7 @@ get_link_history(const char *from_id, const char *to_id) return NULL; if (tor_digest_is_zero(to_id)) return NULL; - lhist = (link_history_t*) digestmap_get(orhist->link_history_map, to_id); + lhist = digestmap_get(orhist->link_history_map, to_id); if (!lhist) { lhist = tor_malloc_zero(sizeof(link_history_t)); rephist_total_alloc += sizeof(link_history_t); @@ -879,126 +879,6 @@ rep_hist_record_mtbf_data(time_t now, int missing_means_down) return -1; } -/** Format the current tracked status of the router in <b>hist</b> at time - * <b>now</b> for analysis; return it in a newly allocated string. */ -static char * -rep_hist_format_router_status(or_history_t *hist, time_t now) -{ - char sor_buf[ISO_TIME_LEN+1]; - char sod_buf[ISO_TIME_LEN+1]; - double wfu; - double mtbf; - int up = 0, down = 0; - char *cp = NULL; - - if (hist->start_of_run) { - format_iso_time(sor_buf, hist->start_of_run); - up = 1; - } - if (hist->start_of_downtime) { - format_iso_time(sod_buf, hist->start_of_downtime); - down = 1; - } - - wfu = get_weighted_fractional_uptime(hist, now); - mtbf = get_stability(hist, now); - tor_asprintf(&cp, - "%s%s%s" - "%s%s%s" - "wfu %0.3f\n" - " weighted-time %lu\n" - " weighted-uptime %lu\n" - "mtbf %0.1f\n" - " weighted-run-length %lu\n" - " total-run-weights %f\n", - up?"uptime-started ":"", up?sor_buf:"", up?" UTC\n":"", - down?"downtime-started ":"", down?sod_buf:"", down?" UTC\n":"", - wfu, - hist->total_weighted_time, - hist->weighted_uptime, - mtbf, - hist->weighted_run_length, - hist->total_run_weights - ); - return cp; -} - -/** The last stability analysis document that we created, or NULL if we never - * have created one. */ -static char *last_stability_doc = NULL; -/** The last time we created a stability analysis document, or 0 if we never - * have created one. */ -static time_t built_last_stability_doc_at = 0; -/** Shortest allowable time between building two stability documents. */ -#define MAX_STABILITY_DOC_BUILD_RATE (3*60) - -/** Return a pointer to a NUL-terminated document describing our view of the - * stability of the routers we've been tracking. Return NULL on failure. */ -const char * -rep_hist_get_router_stability_doc(time_t now) -{ - char *result; - smartlist_t *chunks; - if (built_last_stability_doc_at + MAX_STABILITY_DOC_BUILD_RATE > now) - return last_stability_doc; - - if (!history_map) - return NULL; - - tor_free(last_stability_doc); - chunks = smartlist_new(); - - if (rep_hist_have_measured_enough_stability()) { - smartlist_add(chunks, tor_strdup("we-have-enough-measurements\n")); - } else { - smartlist_add(chunks, tor_strdup("we-do-not-have-enough-measurements\n")); - } - - DIGESTMAP_FOREACH(history_map, id, or_history_t *, hist) { - const node_t *node; - char dbuf[BASE64_DIGEST_LEN+1]; - char *info; - digest_to_base64(dbuf, id); - node = node_get_by_id(id); - if (node) { - char ip[INET_NTOA_BUF_LEN+1]; - char tbuf[ISO_TIME_LEN+1]; - time_t published = node_get_published_on(node); - node_get_address_string(node,ip,sizeof(ip)); - if (published > 0) - format_iso_time(tbuf, published); - else - strlcpy(tbuf, "???", sizeof(tbuf)); - smartlist_add_asprintf(chunks, - "router %s %s %s\n" - "published %s\n" - "relevant-flags %s%s%s\n" - "declared-uptime %ld\n", - dbuf, node_get_nickname(node), ip, - tbuf, - node->is_running ? "Running " : "", - node->is_valid ? "Valid " : "", - node->ri && node->ri->is_hibernating ? "Hibernating " : "", - node_get_declared_uptime(node)); - } else { - smartlist_add_asprintf(chunks, - "router %s {no descriptor}\n", dbuf); - } - info = rep_hist_format_router_status(hist, now); - if (info) - smartlist_add(chunks, info); - - } DIGESTMAP_FOREACH_END; - - result = smartlist_join_strings(chunks, "", 0, NULL); - SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); - smartlist_free(chunks); - - last_stability_doc = result; - built_last_stability_doc_at = time(NULL); - return result; -} - /** Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and * such that no line sl[k] with i <= k < j starts with "R ". Return -1 if no * such line exists. */ @@ -1040,7 +920,7 @@ parse_possibly_bad_iso_time(const char *s, time_t *time_out) * that's about as much before <b>now</b> as <b>t</b> was before * <b>stored_at</b>. */ -static INLINE time_t +static inline time_t correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring) { if (t < started_measuring - 24*60*60*365) @@ -1051,7 +931,7 @@ correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring) return 0; else { long run_length = stored_at - t; - t = now - run_length; + t = (time_t)(now - run_length); if (t < started_measuring) t = started_measuring; return t; @@ -1212,7 +1092,7 @@ rep_hist_load_mtbf_data(time_t now) hist->start_of_run = correct_time(start_of_run, now, stored_at, tracked_since); if (hist->start_of_run < latest_possible_start + wrl) - latest_possible_start = hist->start_of_run - wrl; + latest_possible_start = (time_t)(hist->start_of_run - wrl); hist->weighted_run_length = wrl; hist->total_run_weights = trw; @@ -1251,9 +1131,7 @@ rep_hist_load_mtbf_data(time_t now) * totals? */ #define NUM_SECS_ROLLING_MEASURE 10 /** How large are the intervals for which we track and report bandwidth use? */ -/* XXXX Watch out! Before Tor 0.2.2.21-alpha, using any other value here would - * generate an unparseable state file. */ -#define NUM_SECS_BW_SUM_INTERVAL (15*60) +#define NUM_SECS_BW_SUM_INTERVAL (4*60*60) /** How far in the past do we remember and publish bandwidth use? */ #define NUM_SECS_BW_SUM_IS_VALID (24*60*60) /** How many bandwidth usage intervals do we remember? (derived) */ @@ -1312,7 +1190,7 @@ commit_max(bw_array_t *b) } /** Shift the current observation time of <b>b</b> forward by one second. */ -static INLINE void +static inline void advance_obs(bw_array_t *b) { int nextidx; @@ -1338,7 +1216,7 @@ advance_obs(bw_array_t *b) /** Add <b>n</b> bytes to the number of bytes in <b>b</b> for second * <b>when</b>. */ -static INLINE void +static inline void add_obs(bw_array_t *b, time_t when, uint64_t n) { if (when < b->cur_obs_time) @@ -1372,6 +1250,18 @@ bw_array_new(void) return b; } +/** Free storage held by bandwidth array <b>b</b>. */ +static void +bw_array_free(bw_array_t *b) +{ + if (!b) { + return; + } + + rephist_total_alloc -= sizeof(bw_array_t); + tor_free(b); +} + /** Recent history of bandwidth observations for read operations. */ static bw_array_t *read_array = NULL; /** Recent history of bandwidth observations for write operations. */ @@ -1388,10 +1278,11 @@ static bw_array_t *dir_write_array = NULL; static void bw_arrays_init(void) { - tor_free(read_array); - tor_free(write_array); - tor_free(dir_read_array); - tor_free(dir_write_array); + bw_array_free(read_array); + bw_array_free(write_array); + bw_array_free(dir_read_array); + bw_array_free(dir_write_array); + read_array = bw_array_new(); write_array = bw_array_new(); dir_read_array = bw_array_new(); @@ -1690,7 +1581,7 @@ rep_hist_load_bwhist_state_section(bw_array_t *b, time_t start; uint64_t v, mv; - int i,ok,ok_m; + int i,ok,ok_m = 0; int have_maxima = s_maxima && s_values && (smartlist_len(s_values) == smartlist_len(s_maxima)); @@ -1862,22 +1753,20 @@ rep_hist_note_used_port(time_t now, uint16_t port) add_predicted_port(now, port); } -/** For this long after we've seen a request for a given port, assume that - * we'll want to make connections to the same port in the future. */ -#define PREDICTED_CIRCS_RELEVANCE_TIME (60*60) - /** Return a newly allocated pointer to a list of uint16_t * for ports that * are likely to be asked for in the near future. */ smartlist_t * rep_hist_get_predicted_ports(time_t now) { + int predicted_circs_relevance_time; smartlist_t *out = smartlist_new(); tor_assert(predicted_ports_list); + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; /* clean out obsolete entries */ SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { - if (pp->time + PREDICTED_CIRCS_RELEVANCE_TIME < now) { + if (pp->time + predicted_circs_relevance_time < now) { log_debug(LD_CIRC, "Expiring predicted port %d", pp->port); rephist_total_alloc -= sizeof(predicted_port_t); @@ -1904,6 +1793,7 @@ rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports) SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { if (bitarray_is_set(remove_ports, pp->port)) { tor_free(pp); + rephist_total_alloc -= sizeof(*pp); SMARTLIST_DEL_CURRENT(predicted_ports_list, pp); } } SMARTLIST_FOREACH_END(pp); @@ -1944,14 +1834,17 @@ int rep_hist_get_predicted_internal(time_t now, int *need_uptime, int *need_capacity) { + int predicted_circs_relevance_time; + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + if (!predicted_internal_time) { /* initialize it */ predicted_internal_time = now; predicted_internal_uptime_time = now; predicted_internal_capacity_time = now; } - if (predicted_internal_time + PREDICTED_CIRCS_RELEVANCE_TIME < now) + if (predicted_internal_time + predicted_circs_relevance_time < now) return 0; /* too long ago */ - if (predicted_internal_uptime_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) + if (predicted_internal_uptime_time + predicted_circs_relevance_time >= now) *need_uptime = 1; // Always predict that we need capacity. *need_capacity = 1; @@ -1963,22 +1856,28 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, int any_predicted_circuits(time_t now) { + int predicted_circs_relevance_time; + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + return smartlist_len(predicted_ports_list) || - predicted_internal_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now; + predicted_internal_time + predicted_circs_relevance_time >= now; } /** Return 1 if we have no need for circuits currently, else return 0. */ int rep_hist_circbuilding_dormant(time_t now) { + const or_options_t *options = get_options(); + if (any_predicted_circuits(now)) return 0; /* see if we'll still need to build testing circuits */ - if (server_mode(get_options()) && - (!check_whether_orport_reachable() || !circuit_enough_testing_circs())) + if (server_mode(options) && + (!check_whether_orport_reachable(options) || + !circuit_enough_testing_circs())) return 0; - if (!check_whether_dirport_reachable()) + if (!check_whether_dirport_reachable(options)) return 0; return 1; @@ -2114,12 +2013,9 @@ void rep_hist_exit_stats_init(time_t now) { start_of_exit_stats_interval = now; - exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * - sizeof(uint64_t)); - exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * - sizeof(uint64_t)); - exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * - sizeof(uint32_t)); + exit_bytes_read = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t)); + exit_bytes_written = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t)); + exit_streams = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint32_t)); } /** Reset counters for exit port statistics. */ @@ -2313,7 +2209,7 @@ rep_hist_format_exit_stats(time_t now) time_t rep_hist_exit_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_exit_stats_interval) return 0; /* Not initialized. */ @@ -2329,19 +2225,12 @@ rep_hist_exit_stats_write(time_t now) rep_hist_reset_exit_stats(now); /* Try to write to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "exit-stats", str, "exit port statistics"); } - filename = get_datadir_fname2("stats", "exit-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write exit port statistics to disk!"); done: tor_free(str); - tor_free(statsdir); - tor_free(filename); return start_of_exit_stats_interval + WRITE_STATS_INTERVAL; } @@ -2434,7 +2323,7 @@ rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval) return; start_of_interval = (circ->timestamp_created.tv_sec > start_of_buffer_stats_interval) ? - circ->timestamp_created.tv_sec : + (time_t)circ->timestamp_created.tv_sec : start_of_buffer_stats_interval; interval_length = (int) (end_of_interval - start_of_interval); if (interval_length <= 0) @@ -2597,8 +2486,7 @@ rep_hist_format_buffer_stats(time_t now) time_t rep_hist_buffer_stats_write(time_t now) { - circuit_t *circ; - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_buffer_stats_interval) return 0; /* Not initialized. */ @@ -2606,9 +2494,10 @@ rep_hist_buffer_stats_write(time_t now) goto done; /* Not ready to write */ /* Add open circuits to the history. */ - for (circ = circuit_get_global_list_(); circ; circ = circ->next) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { rep_hist_buffer_stats_add_circ(circ, now); } + SMARTLIST_FOREACH_END(circ); /* Generate history string. */ str = rep_hist_format_buffer_stats(now); @@ -2617,19 +2506,12 @@ rep_hist_buffer_stats_write(time_t now) rep_hist_reset_buffer_stats(now); /* Try to write to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "buffer-stats", str, "buffer statistics"); } - filename = get_datadir_fname2("stats", "buffer-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write buffer stats to disk!"); done: tor_free(str); - tor_free(filename); - tor_free(statsdir); return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL; } @@ -2702,7 +2584,7 @@ rep_hist_format_desc_stats(time_t now) size = digestmap_size(served_descs); if (size > 0) { - vals = tor_malloc(size * sizeof(int)); + vals = tor_calloc(size, sizeof(int)); for (iter = digestmap_iter_init(served_descs); !digestmap_iter_done(iter); iter = digestmap_iter_next(served_descs, iter)) { @@ -2741,7 +2623,7 @@ rep_hist_format_desc_stats(time_t now) time_t rep_hist_desc_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *filename = NULL, *str = NULL; if (!start_of_served_descs_stats_interval) return 0; /* We're not collecting stats. */ @@ -2751,10 +2633,8 @@ rep_hist_desc_stats_write(time_t now) str = rep_hist_format_desc_stats(now); tor_assert(str != NULL); - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (check_or_create_data_subdir("stats") < 0) { + goto done; } filename = get_datadir_fname2("stats", "served-desc-stats"); if (append_bytes_to_file(filename, str, strlen(str), 0) < 0) @@ -2763,7 +2643,6 @@ rep_hist_desc_stats_write(time_t now) rep_hist_reset_desc_stats(now); done: - tor_free(statsdir); tor_free(filename); tor_free(str); return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL; @@ -2860,8 +2739,8 @@ bidi_map_ent_hash(const bidi_map_entry_t *entry) HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, bidi_map_ent_eq); -HT_GENERATE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, - bidi_map_ent_eq, 0.6, malloc, realloc, free); +HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, + bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) /* DOCDOC bidi_map_free */ static void @@ -2981,7 +2860,7 @@ rep_hist_format_conn_stats(time_t now) time_t rep_hist_conn_stats_write(time_t now) { - char *statsdir = NULL, *filename = NULL, *str = NULL; + char *str = NULL; if (!start_of_conn_stats_interval) return 0; /* Not initialized. */ @@ -2995,28 +2874,21 @@ rep_hist_conn_stats_write(time_t now) rep_hist_reset_conn_stats(now); /* Try to write to disk. */ - statsdir = get_datadir_fname("stats"); - if (check_private_dir(statsdir, CPD_CREATE, get_options()->User) < 0) { - log_warn(LD_HIST, "Unable to create stats/ directory!"); - goto done; + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "conn-stats", str, "connection statistics"); } - filename = get_datadir_fname2("stats", "conn-stats"); - if (write_str_to_file(filename, str, 0) < 0) - log_warn(LD_HIST, "Unable to write conn stats to disk!"); done: tor_free(str); - tor_free(filename); - tor_free(statsdir); return start_of_conn_stats_interval + WRITE_STATS_INTERVAL; } /** Internal statistics to track how many requests of each type of - * handshake we've received, and how many we've completed. Useful for - * seeing trends in cpu load. + * handshake we've received, and how many we've assigned to cpuworkers. + * Useful for seeing trends in cpu load. * @{ */ -static int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; -static int onion_handshakes_completed[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; /**@}*/ /** A new onionskin (using the <b>type</b> handshake) has arrived. */ @@ -3030,10 +2902,10 @@ rep_hist_note_circuit_handshake_requested(uint16_t type) /** We've sent an onionskin (using the <b>type</b> handshake) to a * cpuworker. */ void -rep_hist_note_circuit_handshake_completed(uint16_t type) +rep_hist_note_circuit_handshake_assigned(uint16_t type) { if (type <= MAX_ONION_HANDSHAKE_TYPE) - onion_handshakes_completed[type]++; + onion_handshakes_assigned[type]++; } /** Log our onionskin statistics since the last time we were called. */ @@ -3041,31 +2913,298 @@ void rep_hist_log_circuit_handshake_stats(time_t now) { (void)now; - log_notice(LD_HIST, "Circuit handshake stats since last time: " + log_notice(LD_HEARTBEAT, "Circuit handshake stats since last time: " "%d/%d TAP, %d/%d NTor.", - onion_handshakes_completed[ONION_HANDSHAKE_TYPE_TAP], + onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP], onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP], - onion_handshakes_completed[ONION_HANDSHAKE_TYPE_NTOR], + onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR], onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR]); - memset(onion_handshakes_completed, 0, sizeof(onion_handshakes_completed)); + memset(onion_handshakes_assigned, 0, sizeof(onion_handshakes_assigned)); memset(onion_handshakes_requested, 0, sizeof(onion_handshakes_requested)); } +/* Hidden service statistics section */ + +/** Start of the current hidden service stats interval or 0 if we're + * not collecting hidden service statistics. */ +static time_t start_of_hs_stats_interval; + +/** Carries the various hidden service statistics, and any other + * information needed. */ +typedef struct hs_stats_t { + /** How many relay cells have we seen as rendezvous points? */ + int64_t rp_relay_cells_seen; + + /** Set of unique public key digests we've seen this stat period + * (could also be implemented as sorted smartlist). */ + digestmap_t *onions_seen_this_period; +} hs_stats_t; + +/** Our statistics structure singleton. */ +static hs_stats_t *hs_stats = NULL; + +/** Allocate, initialize and return an hs_stats_t structure. */ +static hs_stats_t * +hs_stats_new(void) +{ + hs_stats_t * hs_stats = tor_malloc_zero(sizeof(hs_stats_t)); + hs_stats->onions_seen_this_period = digestmap_new(); + + return hs_stats; +} + +/** Free an hs_stats_t structure. */ +static void +hs_stats_free(hs_stats_t *hs_stats) +{ + if (!hs_stats) { + return; + } + + digestmap_free(hs_stats->onions_seen_this_period, NULL); + tor_free(hs_stats); +} + +/** Initialize hidden service statistics. */ +void +rep_hist_hs_stats_init(time_t now) +{ + if (!hs_stats) { + hs_stats = hs_stats_new(); + } + + start_of_hs_stats_interval = now; +} + +/** Clear history of hidden service statistics and set the measurement + * interval start to <b>now</b>. */ +static void +rep_hist_reset_hs_stats(time_t now) +{ + if (!hs_stats) { + hs_stats = hs_stats_new(); + } + + hs_stats->rp_relay_cells_seen = 0; + + digestmap_free(hs_stats->onions_seen_this_period, NULL); + hs_stats->onions_seen_this_period = digestmap_new(); + + start_of_hs_stats_interval = now; +} + +/** Stop collecting hidden service stats in a way that we can re-start + * doing so in rep_hist_buffer_stats_init(). */ +void +rep_hist_hs_stats_term(void) +{ + rep_hist_reset_hs_stats(0); +} + +/** We saw a new HS relay cell, Count it! */ +void +rep_hist_seen_new_rp_cell(void) +{ + if (!hs_stats) { + return; // We're not collecting stats + } + + hs_stats->rp_relay_cells_seen++; +} + +/** As HSDirs, we saw another hidden service with public key + * <b>pubkey</b>. Check whether we have counted it before, if not + * count it now! */ +void +rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey) +{ + char pubkey_hash[DIGEST_LEN]; + + if (!hs_stats) { + return; // We're not collecting stats + } + + /* Get the digest of the pubkey which will be used to detect whether + we've seen this hidden service before or not. */ + if (crypto_pk_get_digest(pubkey, pubkey_hash) < 0) { + /* This fail should not happen; key has been validated by + descriptor parsing code first. */ + return; + } + + /* Check if this is the first time we've seen this hidden + service. If it is, count it as new. */ + if (!digestmap_get(hs_stats->onions_seen_this_period, + pubkey_hash)) { + digestmap_set(hs_stats->onions_seen_this_period, + pubkey_hash, (void*)(uintptr_t)1); + } +} + +/* The number of cells that are supposed to be hidden from the adversary + * by adding noise from the Laplace distribution. This value, divided by + * EPSILON, is Laplace parameter b. It must be greather than 0. */ +#define REND_CELLS_DELTA_F 2048 +/* Security parameter for obfuscating number of cells with a value between + * ]0.0, 1.0]. Smaller values obfuscate observations more, but at the same + * time make statistics less usable. */ +#define REND_CELLS_EPSILON 0.3 +/* The number of cells that are supposed to be hidden from the adversary + * by rounding up to the next multiple of this number. */ +#define REND_CELLS_BIN_SIZE 1024 +/* The number of service identities that are supposed to be hidden from the + * adversary by adding noise from the Laplace distribution. This value, + * divided by EPSILON, is Laplace parameter b. It must be greater than 0. */ +#define ONIONS_SEEN_DELTA_F 8 +/* Security parameter for obfuscating number of service identities with a + * value between ]0.0, 1.0]. Smaller values obfuscate observations more, but + * at the same time make statistics less usable. */ +#define ONIONS_SEEN_EPSILON 0.3 +/* The number of service identities that are supposed to be hidden from + * the adversary by rounding up to the next multiple of this number. */ +#define ONIONS_SEEN_BIN_SIZE 8 + +/** Allocate and return a string containing hidden service stats that + * are meant to be placed in the extra-info descriptor. */ +static char * +rep_hist_format_hs_stats(time_t now) +{ + char t[ISO_TIME_LEN+1]; + char *hs_stats_string; + int64_t obfuscated_cells_seen; + int64_t obfuscated_onions_seen; + + obfuscated_cells_seen = round_int64_to_next_multiple_of( + hs_stats->rp_relay_cells_seen, + REND_CELLS_BIN_SIZE); + obfuscated_cells_seen = add_laplace_noise(obfuscated_cells_seen, + crypto_rand_double(), + REND_CELLS_DELTA_F, REND_CELLS_EPSILON); + obfuscated_onions_seen = round_int64_to_next_multiple_of(digestmap_size( + hs_stats->onions_seen_this_period), + ONIONS_SEEN_BIN_SIZE); + obfuscated_onions_seen = add_laplace_noise(obfuscated_onions_seen, + crypto_rand_double(), ONIONS_SEEN_DELTA_F, + ONIONS_SEEN_EPSILON); + + format_iso_time(t, now); + tor_asprintf(&hs_stats_string, "hidserv-stats-end %s (%d s)\n" + "hidserv-rend-relayed-cells "I64_FORMAT" delta_f=%d " + "epsilon=%.2f bin_size=%d\n" + "hidserv-dir-onions-seen "I64_FORMAT" delta_f=%d " + "epsilon=%.2f bin_size=%d\n", + t, (unsigned) (now - start_of_hs_stats_interval), + I64_PRINTF_ARG(obfuscated_cells_seen), REND_CELLS_DELTA_F, + REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE, + I64_PRINTF_ARG(obfuscated_onions_seen), + ONIONS_SEEN_DELTA_F, + ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE); + + return hs_stats_string; +} + +/** If 24 hours have passed since the beginning of the current HS + * stats period, write buffer stats to $DATADIR/stats/hidserv-stats + * (possibly overwriting an existing file) and reset counters. Return + * when we would next want to write buffer stats or 0 if we never want to + * write. */ +time_t +rep_hist_hs_stats_write(time_t now) +{ + char *str = NULL; + + if (!start_of_hs_stats_interval) { + return 0; /* Not initialized. */ + } + + if (start_of_hs_stats_interval + WRITE_STATS_INTERVAL > now) { + goto done; /* Not ready to write */ + } + + /* Generate history string. */ + str = rep_hist_format_hs_stats(now); + + /* Reset HS history. */ + rep_hist_reset_hs_stats(now); + + /* Try to write to disk. */ + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "hidserv-stats", str, + "hidden service stats"); + } + + done: + tor_free(str); + return start_of_hs_stats_interval + WRITE_STATS_INTERVAL; +} + +#define MAX_LINK_PROTO_TO_LOG 4 +static uint64_t link_proto_count[MAX_LINK_PROTO_TO_LOG+1][2]; + +/** Note that we negotiated link protocol version <b>link_proto</b>, on + * a connection that started here iff <b>started_here</b> is true. + */ +void +rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here) +{ + started_here = !!started_here; /* force to 0 or 1 */ + if (link_proto > MAX_LINK_PROTO_TO_LOG) { + log_warn(LD_BUG, "Can't log link protocol %u", link_proto); + return; + } + + link_proto_count[link_proto][started_here]++; +} + +/** Log a heartbeat message explaining how many connections of each link + * protocol version we have used. + */ +void +rep_hist_log_link_protocol_counts(void) +{ + log_notice(LD_HEARTBEAT, + "Since startup, we have initiated " + U64_FORMAT" v1 connections, " + U64_FORMAT" v2 connections, " + U64_FORMAT" v3 connections, and " + U64_FORMAT" v4 connections; and received " + U64_FORMAT" v1 connections, " + U64_FORMAT" v2 connections, " + U64_FORMAT" v3 connections, and " + U64_FORMAT" v4 connections.", + U64_PRINTF_ARG(link_proto_count[1][1]), + U64_PRINTF_ARG(link_proto_count[2][1]), + U64_PRINTF_ARG(link_proto_count[3][1]), + U64_PRINTF_ARG(link_proto_count[4][1]), + U64_PRINTF_ARG(link_proto_count[1][0]), + U64_PRINTF_ARG(link_proto_count[2][0]), + U64_PRINTF_ARG(link_proto_count[3][0]), + U64_PRINTF_ARG(link_proto_count[4][0])); +} + /** Free all storage held by the OR/link history caches, by the * bandwidth history arrays, by the port history, or by statistics . */ void rep_hist_free_all(void) { + hs_stats_free(hs_stats); digestmap_free(history_map, free_or_history); - tor_free(read_array); - tor_free(write_array); - tor_free(dir_read_array); - tor_free(dir_write_array); - tor_free(last_stability_doc); + + bw_array_free(read_array); + read_array = NULL; + + bw_array_free(write_array); + write_array = NULL; + + bw_array_free(dir_read_array); + dir_read_array = NULL; + + bw_array_free(dir_write_array); + dir_write_array = NULL; + tor_free(exit_bytes_read); tor_free(exit_bytes_written); tor_free(exit_streams); - built_last_stability_doc_at = 0; predicted_ports_free(); bidi_map_free(); @@ -3077,5 +3216,8 @@ rep_hist_free_all(void) } rep_hist_desc_stats_term(); total_descriptor_downloads = 0; + + tor_assert(rephist_total_alloc == 0); + tor_assert(rephist_total_num == 0); } diff --git a/src/or/rephist.h b/src/or/rephist.h index de824749b4..145da97d02 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -47,7 +47,6 @@ double rep_hist_get_stability(const char *id, time_t when); double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when); long rep_hist_get_weighted_time_known(const char *id, time_t when); int rep_hist_have_measured_enough_stability(void); -const char *rep_hist_get_router_stability_doc(time_t now); void rep_hist_note_used_port(time_t now, uint16_t port); smartlist_t *rep_hist_get_predicted_ports(time_t now); @@ -97,10 +96,21 @@ time_t rep_hist_conn_stats_write(time_t now); void rep_hist_conn_stats_term(void); void rep_hist_note_circuit_handshake_requested(uint16_t type); -void rep_hist_note_circuit_handshake_completed(uint16_t type); +void rep_hist_note_circuit_handshake_assigned(uint16_t type); void rep_hist_log_circuit_handshake_stats(time_t now); +void rep_hist_hs_stats_init(time_t now); +void rep_hist_hs_stats_term(void); +time_t rep_hist_hs_stats_write(time_t now); +char *rep_hist_get_hs_stats_string(void); +void rep_hist_seen_new_rp_cell(void); +void rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey); + void rep_hist_free_all(void); +void rep_hist_note_negotiated_link_proto(unsigned link_proto, + int started_here); +void rep_hist_log_link_protocol_counts(void); + #endif diff --git a/src/or/replaycache.c b/src/or/replaycache.c index 59b98489b7..23a1737b18 100644 --- a/src/or/replaycache.c +++ b/src/or/replaycache.c @@ -1,4 +1,4 @@ - /* Copyright (c) 2012-2013, The Tor Project, Inc. */ + /* Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* @@ -23,7 +23,7 @@ replaycache_free(replaycache_t *r) return; } - if (r->digests_seen) digestmap_free(r->digests_seen, tor_free_); + if (r->digests_seen) digest256map_free(r->digests_seen, tor_free_); tor_free(r); } @@ -54,7 +54,7 @@ replaycache_new(time_t horizon, time_t interval) r->scrub_interval = interval; r->scrubbed = 0; r->horizon = horizon; - r->digests_seen = digestmap_new(); + r->digests_seen = digest256map_new(); err: return r; @@ -63,27 +63,27 @@ replaycache_new(time_t horizon, time_t interval) /** See documentation for replaycache_add_and_test() */ -int +STATIC int replaycache_add_and_test_internal( - time_t present, replaycache_t *r, const void *data, int len, + time_t present, replaycache_t *r, const void *data, size_t len, time_t *elapsed) { int rv = 0; - char digest[DIGEST_LEN]; + uint8_t digest[DIGEST256_LEN]; time_t *access_time; /* sanity check */ - if (present <= 0 || !r || !data || len <= 0) { + if (present <= 0 || !r || !data || len == 0) { log_info(LD_BUG, "replaycache_add_and_test_internal() called with stupid" " parameters; please fix this."); goto done; } /* compute digest */ - crypto_digest(digest, (const char *)data, len); + crypto_digest256((char *)digest, (const char *)data, len, DIGEST_SHA256); /* check map */ - access_time = digestmap_get(r->digests_seen, digest); + access_time = digest256map_get(r->digests_seen, digest); /* seen before? */ if (access_time != NULL) { @@ -114,7 +114,7 @@ replaycache_add_and_test_internal( /* No, so no hit and update the digest map with the current time */ access_time = tor_malloc(sizeof(*access_time)); *access_time = present; - digestmap_set(r->digests_seen, digest, access_time); + digest256map_set(r->digests_seen, digest, access_time); } /* now scrub the cache if it's time */ @@ -127,14 +127,13 @@ replaycache_add_and_test_internal( /** See documentation for replaycache_scrub_if_needed() */ -void +STATIC void replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r) { - digestmap_iter_t *itr = NULL; - const char *digest; + digest256map_iter_t *itr = NULL; + const uint8_t *digest; void *valp; time_t *access_time; - char scrub_this; /* sanity check */ if (!r || !(r->digests_seen)) { @@ -150,29 +149,19 @@ replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r) if (r->horizon == 0) return; /* okay, scrub time */ - itr = digestmap_iter_init(r->digests_seen); - while (!digestmap_iter_done(itr)) { - scrub_this = 0; - digestmap_iter_get(itr, &digest, &valp); + itr = digest256map_iter_init(r->digests_seen); + while (!digest256map_iter_done(itr)) { + digest256map_iter_get(itr, &digest, &valp); access_time = (time_t *)valp; - if (access_time) { - /* aged out yet? */ - if (*access_time < present - r->horizon) scrub_this = 1; - } else { - /* Buh? Get rid of it, anyway */ - log_info(LD_BUG, "replaycache_scrub_if_needed_internal() saw a NULL" - " entry in the digestmap."); - scrub_this = 1; - } - - if (scrub_this) { + /* aged out yet? */ + if (*access_time < present - r->horizon) { /* Advance the iterator and remove this one */ - itr = digestmap_iter_next_rmv(r->digests_seen, itr); + itr = digest256map_iter_next_rmv(r->digests_seen, itr); /* Free the value removed */ tor_free(access_time); } else { /* Just advance the iterator */ - itr = digestmap_iter_next(r->digests_seen, itr); + itr = digest256map_iter_next(r->digests_seen, itr); } } @@ -187,7 +176,7 @@ replaycache_scrub_if_needed_internal(time_t present, replaycache_t *r) */ int -replaycache_add_and_test(replaycache_t *r, const void *data, int len) +replaycache_add_and_test(replaycache_t *r, const void *data, size_t len) { return replaycache_add_and_test_internal(time(NULL), r, data, len, NULL); } @@ -198,7 +187,7 @@ replaycache_add_and_test(replaycache_t *r, const void *data, int len) int replaycache_add_test_and_elapsed( - replaycache_t *r, const void *data, int len, time_t *elapsed) + replaycache_t *r, const void *data, size_t len, time_t *elapsed) { return replaycache_add_and_test_internal(time(NULL), r, data, len, elapsed); } diff --git a/src/or/replaycache.h b/src/or/replaycache.h index de20cab627..64a6caf5f5 100644 --- a/src/or/replaycache.h +++ b/src/or/replaycache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* Copyright (c) 2012-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -26,7 +26,7 @@ struct replaycache_s { /* * Digest map: keys are digests, values are times the digest was last seen */ - digestmap_t *digests_seen; + digest256map_t *digests_seen; }; #endif /* REPLAYCACHE_PRIVATE */ @@ -45,10 +45,10 @@ replaycache_t * replaycache_new(time_t horizon, time_t interval); * testing. For everything else, use the wrappers below instead. */ -int replaycache_add_and_test_internal( - time_t present, replaycache_t *r, const void *data, int len, +STATIC int replaycache_add_and_test_internal( + time_t present, replaycache_t *r, const void *data, size_t len, time_t *elapsed); -void replaycache_scrub_if_needed_internal( +STATIC void replaycache_scrub_if_needed_internal( time_t present, replaycache_t *r); #endif /* REPLAYCACHE_PRIVATE */ @@ -57,9 +57,9 @@ void replaycache_scrub_if_needed_internal( * replaycache_t methods */ -int replaycache_add_and_test(replaycache_t *r, const void *data, int len); +int replaycache_add_and_test(replaycache_t *r, const void *data, size_t len); int replaycache_add_test_and_elapsed( - replaycache_t *r, const void *data, int len, time_t *elapsed); + replaycache_t *r, const void *data, size_t len, time_t *elapsed); void replaycache_scrub_if_needed(replaycache_t *r); #endif diff --git a/src/or/router.c b/src/or/router.c index eabd9c3f59..01316c1bc2 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTER_PRIVATE @@ -26,9 +26,11 @@ #include "relay.h" #include "rephist.h" #include "router.h" +#include "routerkeys.h" #include "routerlist.h" #include "routerparse.h" #include "statefile.h" +#include "torcert.h" #include "transports.h" #include "routerset.h" @@ -55,13 +57,11 @@ static crypto_pk_t *onionkey=NULL; /** Previous private onionskin decryption key: used to decode CREATE cells * generated by clients that have an older version of our descriptor. */ static crypto_pk_t *lastonionkey=NULL; -#ifdef CURVE25519_ENABLED /** Current private ntor secret key: used to perform the ntor handshake. */ static curve25519_keypair_t curve25519_onion_key; /** Previous private ntor secret key: used to perform the ntor handshake * with clients that have an older version of our descriptor. */ static curve25519_keypair_t last_curve25519_onion_key; -#endif /** Private server "identity key": used to sign directory info and TLS * certificates. Never changes. */ static crypto_pk_t *server_identitykey=NULL; @@ -134,7 +134,6 @@ dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last) tor_mutex_release(key_lock); } -#ifdef CURVE25519_ENABLED /** Return the current secret onion key for the ntor handshake. Must only * be called from the main thread. */ static const curve25519_keypair_t * @@ -181,7 +180,6 @@ ntor_key_map_free(di_digest256_map_t *map) return; dimap_free(map, ntor_key_map_free_helper); } -#endif /** Return the time when the onion key was last set. This is either the time * when the process launched, or the time of the most recent key rotation since @@ -208,6 +206,8 @@ set_server_identity_key(crypto_pk_t *k) static void assert_identity_keys_ok(void) { + if (1) + return; tor_assert(client_identitykey); if (public_server_mode(get_options())) { /* assert that we have set the client and server keys to be equal */ @@ -232,7 +232,8 @@ get_server_identity_key(void) return server_identitykey; } -/** Return true iff the server identity key has been set. */ +/** Return true iff we are a server and the server identity key + * has been set. */ int server_identity_key_is_set(void) { @@ -268,8 +269,8 @@ client_identity_key_is_set(void) /** Return the key certificate for this v3 (voting) authority, or NULL * if we have no such certificate. */ -authority_cert_t * -get_my_v3_authority_cert(void) +MOCK_IMPL(authority_cert_t *, +get_my_v3_authority_cert, (void)) { return authority_key_certificate; } @@ -312,12 +313,11 @@ rotate_onion_key(void) char *fname, *fname_prev; crypto_pk_t *prkey = NULL; or_state_t *state = get_or_state(); -#ifdef CURVE25519_ENABLED curve25519_keypair_t new_curve25519_keypair; -#endif time_t now; fname = get_datadir_fname2("keys", "secret_onion_key"); fname_prev = get_datadir_fname2("keys", "secret_onion_key.old"); + /* There isn't much point replacing an old key with an empty file */ if (file_status(fname) == FN_FILE) { if (replace_file(fname, fname_prev)) goto error; @@ -334,13 +334,13 @@ rotate_onion_key(void) log_err(LD_FS,"Couldn't write generated onion key to \"%s\".", fname); goto error; } -#ifdef CURVE25519_ENABLED tor_free(fname); tor_free(fname_prev); fname = get_datadir_fname2("keys", "secret_onion_key_ntor"); fname_prev = get_datadir_fname2("keys", "secret_onion_key_ntor.old"); if (curve25519_keypair_generate(&new_curve25519_keypair, 1) < 0) goto error; + /* There isn't much point replacing an old key with an empty file */ if (file_status(fname) == FN_FILE) { if (replace_file(fname, fname_prev)) goto error; @@ -350,18 +350,15 @@ rotate_onion_key(void) log_err(LD_FS,"Couldn't write curve25519 onion key to \"%s\".",fname); goto error; } -#endif log_info(LD_GENERAL, "Rotating onion key"); tor_mutex_acquire(key_lock); crypto_pk_free(lastonionkey); lastonionkey = onionkey; onionkey = prkey; -#ifdef CURVE25519_ENABLED memcpy(&last_curve25519_onion_key, &curve25519_onion_key, sizeof(curve25519_keypair_t)); memcpy(&curve25519_onion_key, &new_curve25519_keypair, sizeof(curve25519_keypair_t)); -#endif now = time(NULL); state->LastRotatedOnionKey = onionkey_set_at = now; tor_mutex_release(key_lock); @@ -373,20 +370,40 @@ rotate_onion_key(void) if (prkey) crypto_pk_free(prkey); done: -#ifdef CURVE25519_ENABLED memwipe(&new_curve25519_keypair, 0, sizeof(new_curve25519_keypair)); -#endif tor_free(fname); tor_free(fname_prev); } +/** Log greeting message that points to new relay lifecycle document the + * first time this function has been called. + */ +static void +log_new_relay_greeting(void) +{ + static int already_logged = 0; + + if (already_logged) + return; + + tor_log(LOG_NOTICE, LD_GENERAL, "You are running a new relay. " + "Thanks for helping the Tor network! If you wish to know " + "what will happen in the upcoming weeks regarding its usage, " + "have a look at https://blog.torproject.org/blog/lifecycle-of" + "-a-new-relay"); + + already_logged = 1; +} + /** Try to read an RSA key from <b>fname</b>. If <b>fname</b> doesn't exist * and <b>generate</b> is true, create a new RSA key and save it in * <b>fname</b>. Return the read/created key, or NULL on error. Log all - * errors at level <b>severity</b>. + * errors at level <b>severity</b>. If <b>log_greeting</b> is non-zero and a + * new key was created, log_new_relay_greeting() is called. */ crypto_pk_t * -init_key_from_file(const char *fname, int generate, int severity) +init_key_from_file(const char *fname, int generate, int severity, + int log_greeting) { crypto_pk_t *prkey = NULL; @@ -400,7 +417,11 @@ init_key_from_file(const char *fname, int generate, int severity) case FN_ERROR: tor_log(severity, LD_FS,"Can't read key from \"%s\"", fname); goto error; + /* treat empty key files as if the file doesn't exist, and, + * if generate is set, replace the empty file in + * crypto_pk_write_private_key_to_filename() */ case FN_NOENT: + case FN_EMPTY: if (generate) { if (!have_lockfile()) { if (try_locking(get_options(), 0)<0) { @@ -424,6 +445,9 @@ init_key_from_file(const char *fname, int generate, int severity) goto error; } log_info(LD_GENERAL, "Generated key seems valid"); + if (log_greeting) { + log_new_relay_greeting(); + } if (crypto_pk_write_private_key_to_filename(prkey, fname)) { tor_log(severity, LD_FS, "Couldn't write generated key to \"%s\".", fname); @@ -449,12 +473,11 @@ init_key_from_file(const char *fname, int generate, int severity) return NULL; } -#ifdef CURVE25519_ENABLED /** Load a curve25519 keypair from the file <b>fname</b>, writing it into - * <b>keys_out</b>. If the file isn't found and <b>generate</b> is true, - * create a new keypair and write it into the file. If there are errors, log - * them at level <b>severity</b>. Generate files using <b>tag</b> in their - * ASCII wrapper. */ + * <b>keys_out</b>. If the file isn't found, or is empty, and <b>generate</b> + * is true, create a new keypair and write it into the file. If there are + * errors, log them at level <b>severity</b>. Generate files using <b>tag</b> + * in their ASCII wrapper. */ static int init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out, const char *fname, @@ -467,7 +490,10 @@ init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out, case FN_ERROR: tor_log(severity, LD_FS,"Can't read key from \"%s\"", fname); goto error; + /* treat empty key files as if the file doesn't exist, and, if generate + * is set, replace the empty file in curve25519_keypair_write_to_file() */ case FN_NOENT: + case FN_EMPTY: if (generate) { if (!have_lockfile()) { if (try_locking(get_options(), 0)<0) { @@ -487,7 +513,7 @@ init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out, if (curve25519_keypair_write_to_file(keys_out, fname, tag)<0) { tor_log(severity, LD_FS, "Couldn't write generated key to \"%s\".", fname); - memset(keys_out, 0, sizeof(*keys_out)); + memwipe(keys_out, 0, sizeof(*keys_out)); goto error; } } else { @@ -518,7 +544,6 @@ init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out, error: return -1; } -#endif /** Try to load the vote-signing private key and certificate for being a v3 * directory authority, and make sure they match. If <b>legacy</b>, load a @@ -537,7 +562,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out, fname = get_datadir_fname2("keys", legacy ? "legacy_signing_key" : "authority_signing_key"); - signing_key = init_key_from_file(fname, 0, LOG_INFO); + signing_key = init_key_from_file(fname, 0, LOG_INFO, 0); if (!signing_key) { log_warn(LD_DIR, "No version 3 directory key found in %s", fname); goto done; @@ -662,7 +687,9 @@ router_initialize_tls_context(void) if (!lifetime) { /* we should guess a good ssl cert lifetime */ /* choose between 5 and 365 days, and round to the day */ - lifetime = 5*24*3600 + crypto_rand_int(361*24*3600); + unsigned int five_days = 5*24*3600; + unsigned int one_year = 365*24*3600; + lifetime = crypto_rand_int_range(five_days, one_year); lifetime -= lifetime % (24*3600); if (crypto_rand_int(2)) { @@ -683,6 +710,103 @@ router_initialize_tls_context(void) (unsigned int)lifetime); } +/** Compute fingerprint (or hashed fingerprint if hashed is 1) and write + * it to 'fingerprint' (or 'hashed-fingerprint'). Return 0 on success, or + * -1 if Tor should die, + */ +STATIC int +router_write_fingerprint(int hashed) +{ + char *keydir = NULL, *cp = NULL; + const char *fname = hashed ? "hashed-fingerprint" : + "fingerprint"; + char fingerprint[FINGERPRINT_LEN+1]; + const or_options_t *options = get_options(); + char *fingerprint_line = NULL; + int result = -1; + + keydir = get_datadir_fname(fname); + log_info(LD_GENERAL,"Dumping %sfingerprint to \"%s\"...", + hashed ? "hashed " : "", keydir); + if (!hashed) { + if (crypto_pk_get_fingerprint(get_server_identity_key(), + fingerprint, 0) < 0) { + log_err(LD_GENERAL,"Error computing fingerprint"); + goto done; + } + } else { + if (crypto_pk_get_hashed_fingerprint(get_server_identity_key(), + fingerprint) < 0) { + log_err(LD_GENERAL,"Error computing hashed fingerprint"); + goto done; + } + } + + tor_asprintf(&fingerprint_line, "%s %s\n", options->Nickname, fingerprint); + + /* Check whether we need to write the (hashed-)fingerprint file. */ + + cp = read_file_to_str(keydir, RFTS_IGNORE_MISSING, NULL); + if (!cp || strcmp(cp, fingerprint_line)) { + if (write_str_to_file(keydir, fingerprint_line, 0)) { + log_err(LD_FS, "Error writing %sfingerprint line to file", + hashed ? "hashed " : ""); + goto done; + } + } + + log_notice(LD_GENERAL, "Your Tor %s identity key fingerprint is '%s %s'", + hashed ? "bridge's hashed" : "server's", options->Nickname, + fingerprint); + + result = 0; + done: + tor_free(cp); + tor_free(keydir); + tor_free(fingerprint_line); + return result; +} + +static int +init_keys_common(void) +{ + if (!key_lock) + key_lock = tor_mutex_new(); + + /* There are a couple of paths that put us here before we've asked + * openssl to initialize itself. */ + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { + log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); + return -1; + } + + return 0; +} + +int +init_keys_client(void) +{ + crypto_pk_t *prkey; + if (init_keys_common() < 0) + return -1; + + if (!(prkey = crypto_pk_new())) + return -1; + if (crypto_pk_generate_key(prkey)) { + crypto_pk_free(prkey); + return -1; + } + set_client_identity_key(prkey); + /* Create a TLS context. */ + if (router_initialize_tls_context() < 0) { + log_err(LD_GENERAL,"Error creating TLS context for Tor client."); + return -1; + } + return 0; +} + /** Initialize all OR private keys, and the TLS context, as necessary. * On OPs, this only initializes the tls context. Return 0 on success, * or -1 if Tor should die. @@ -691,14 +815,10 @@ int init_keys(void) { char *keydir; - char fingerprint[FINGERPRINT_LEN+1]; - /*nickname<space>fp\n\0 */ - char fingerprint_line[MAX_NICKNAME_LEN+FINGERPRINT_LEN+3]; const char *mydesc; crypto_pk_t *prkey; char digest[DIGEST_LEN]; char v3_digest[DIGEST_LEN]; - char *cp; const or_options_t *options = get_options(); dirinfo_type_t type; time_t now = time(NULL); @@ -706,35 +826,13 @@ init_keys(void) int v3_digest_set = 0; authority_cert_t *cert = NULL; - if (!key_lock) - key_lock = tor_mutex_new(); - - /* There are a couple of paths that put us here before we've asked - * openssl to initialize itself. */ - if (crypto_global_init(get_options()->HardwareAccel, - get_options()->AccelName, - get_options()->AccelDir)) { - log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); - return -1; - } - /* OP's don't need persistent keys; just make up an identity and * initialize the TLS context. */ if (!server_mode(options)) { - if (!(prkey = crypto_pk_new())) - return -1; - if (crypto_pk_generate_key(prkey)) { - crypto_pk_free(prkey); - return -1; - } - set_client_identity_key(prkey); - /* Create a TLS context. */ - if (router_initialize_tls_context() < 0) { - log_err(LD_GENERAL,"Error creating TLS context for Tor client."); - return -1; - } - return 0; + return init_keys_client(); } + if (init_keys_common() < 0) + return -1; /* Make sure DataDirectory exists, and is private. */ if (check_private_dir(options->DataDirectory, CPD_CREATE, options->User)) { return -1; @@ -767,7 +865,7 @@ init_keys(void) /* 1b. Read identity key. Make it if none is found. */ keydir = get_datadir_fname2("keys", "secret_id_key"); log_info(LD_GENERAL,"Reading/making identity key \"%s\"...",keydir); - prkey = init_key_from_file(keydir, 1, LOG_ERR); + prkey = init_key_from_file(keydir, 1, LOG_ERR, 1); tor_free(keydir); if (!prkey) return -1; set_server_identity_key(prkey); @@ -787,10 +885,14 @@ init_keys(void) set_client_identity_key(prkey); } + /* 1d. Load all ed25519 keys */ + if (load_ed_keys(options,now) < 0) + return -1; + /* 2. Read onion key. Make it if none is found. */ keydir = get_datadir_fname2("keys", "secret_onion_key"); log_info(LD_GENERAL,"Reading/making onion key \"%s\"...",keydir); - prkey = init_key_from_file(keydir, 1, LOG_ERR); + prkey = init_key_from_file(keydir, 1, LOG_ERR, 1); tor_free(keydir); if (!prkey) return -1; set_onion_key(prkey); @@ -815,13 +917,14 @@ init_keys(void) keydir = get_datadir_fname2("keys", "secret_onion_key.old"); if (!lastonionkey && file_status(keydir) == FN_FILE) { - prkey = init_key_from_file(keydir, 1, LOG_ERR); /* XXXX Why 1? */ + /* Load keys from non-empty files only. + * Missing old keys won't be replaced with freshly generated keys. */ + prkey = init_key_from_file(keydir, 0, LOG_ERR, 0); if (prkey) lastonionkey = prkey; } tor_free(keydir); -#ifdef CURVE25519_ENABLED { /* 2b. Load curve25519 onion keys. */ int r; @@ -837,12 +940,13 @@ init_keys(void) last_curve25519_onion_key.pubkey.public_key, CURVE25519_PUBKEY_LEN) && file_status(keydir) == FN_FILE) { + /* Load keys from non-empty files only. + * Missing old keys won't be replaced with freshly generated keys. */ init_curve25519_keypair_from_file(&last_curve25519_onion_key, keydir, 0, LOG_ERR, "onion"); } tor_free(keydir); } -#endif /* 3. Initialize link key and TLS context. */ if (router_initialize_tls_context() < 0) { @@ -850,6 +954,13 @@ init_keys(void) return -1; } + /* 3b. Get an ed25519 link certificate. Note that we need to do this + * after we set up the TLS context */ + if (generate_ed_link_cert(options, now) < 0) { + log_err(LD_GENERAL,"Couldn't make link cert"); + return -1; + } + /* 4. Build our router descriptor. */ /* Must be called after keys are initialized. */ mydesc = router_get_my_descriptor(); @@ -857,14 +968,13 @@ init_keys(void) const char *m = NULL; routerinfo_t *ri; /* We need to add our own fingerprint so it gets recognized. */ - if (dirserv_add_own_fingerprint(options->Nickname, - get_server_identity_key())) { - log_err(LD_GENERAL,"Error adding own fingerprint to approved set"); + if (dirserv_add_own_fingerprint(get_server_identity_key())) { + log_err(LD_GENERAL,"Error adding own fingerprint to set of relays"); return -1; } if (mydesc) { was_router_added_t added; - ri = router_parse_entry_from_string(mydesc, NULL, 1, 0, NULL); + ri = router_parse_entry_from_string(mydesc, NULL, 1, 0, NULL, NULL); if (!ri) { log_err(LD_GENERAL,"Generated a routerinfo we couldn't parse."); return -1; @@ -888,40 +998,16 @@ init_keys(void) } } - /* 5. Dump fingerprint to 'fingerprint' */ - keydir = get_datadir_fname("fingerprint"); - log_info(LD_GENERAL,"Dumping fingerprint to \"%s\"...",keydir); - if (crypto_pk_get_fingerprint(get_server_identity_key(), - fingerprint, 0) < 0) { - log_err(LD_GENERAL,"Error computing fingerprint"); - tor_free(keydir); + /* 5. Dump fingerprint and possibly hashed fingerprint to files. */ + if (router_write_fingerprint(0)) { + log_err(LD_FS, "Error writing fingerprint to file"); return -1; } - tor_assert(strlen(options->Nickname) <= MAX_NICKNAME_LEN); - if (tor_snprintf(fingerprint_line, sizeof(fingerprint_line), - "%s %s\n",options->Nickname, fingerprint) < 0) { - log_err(LD_GENERAL,"Error writing fingerprint line"); - tor_free(keydir); + if (!public_server_mode(options) && router_write_fingerprint(1)) { + log_err(LD_FS, "Error writing hashed fingerprint to file"); return -1; } - /* Check whether we need to write the fingerprint file. */ - cp = NULL; - if (file_status(keydir) == FN_FILE) - cp = read_file_to_str(keydir, 0, NULL); - if (!cp || strcmp(cp, fingerprint_line)) { - if (write_str_to_file(keydir, fingerprint_line, 0)) { - log_err(LD_FS, "Error writing fingerprint line to file"); - tor_free(keydir); - tor_free(cp); - return -1; - } - } - tor_free(cp); - tor_free(keydir); - log_notice(LD_GENERAL, - "Your Tor server's identity key fingerprint is '%s %s'", - options->Nickname, fingerprint); if (!authdir_mode(options)) return 0; /* 6. [authdirserver only] load approved-routers file */ @@ -931,18 +1017,16 @@ init_keys(void) } /* 6b. [authdirserver only] add own key to approved directories. */ crypto_pk_get_digest(get_server_identity_key(), digest); - type = ((options->V1AuthoritativeDir ? V1_DIRINFO : NO_DIRINFO) | - (options->V2AuthoritativeDir ? V2_DIRINFO : NO_DIRINFO) | - (options->V3AuthoritativeDir ? + type = ((options->V3AuthoritativeDir ? (V3_DIRINFO|MICRODESC_DIRINFO|EXTRAINFO_DIRINFO) : NO_DIRINFO) | - (options->BridgeAuthoritativeDir ? BRIDGE_DIRINFO : NO_DIRINFO) | - (options->HSAuthoritativeDir ? HIDSERV_DIRINFO : NO_DIRINFO)); + (options->BridgeAuthoritativeDir ? BRIDGE_DIRINFO : NO_DIRINFO)); ds = router_get_trusteddirserver_by_digest(digest); if (!ds) { ds = trusted_dir_server_new(options->Nickname, NULL, router_get_advertised_dir_port(options, 0), router_get_advertised_or_port(options), + NULL, digest, v3_digest, type, 0.0); @@ -995,84 +1079,117 @@ router_reset_reachability(void) can_reach_or_port = can_reach_dir_port = 0; } -/** Return 1 if ORPort is known reachable; else return 0. */ -int -check_whether_orport_reachable(void) +/** Return 1 if we won't do reachability checks, because: + * - AssumeReachable is set, or + * - the network is disabled. + * Otherwise, return 0. + */ +static int +router_reachability_checks_disabled(const or_options_t *options) { - const or_options_t *options = get_options(); return options->AssumeReachable || + net_is_disabled(); +} + +/** Return 0 if we need to do an ORPort reachability check, because: + * - no reachability check has been done yet, or + * - we've initiated reachability checks, but none have succeeded. + * Return 1 if we don't need to do an ORPort reachability check, because: + * - we've seen a successful reachability check, or + * - AssumeReachable is set, or + * - the network is disabled. + */ +int +check_whether_orport_reachable(const or_options_t *options) +{ + int reach_checks_disabled = router_reachability_checks_disabled(options); + return reach_checks_disabled || can_reach_or_port; } -/** Return 1 if we don't have a dirport configured, or if it's reachable. */ +/** Return 0 if we need to do a DirPort reachability check, because: + * - no reachability check has been done yet, or + * - we've initiated reachability checks, but none have succeeded. + * Return 1 if we don't need to do a DirPort reachability check, because: + * - we've seen a successful reachability check, or + * - there is no DirPort set, or + * - AssumeReachable is set, or + * - the network is disabled. + */ int -check_whether_dirport_reachable(void) +check_whether_dirport_reachable(const or_options_t *options) { - const or_options_t *options = get_options(); - return !options->DirPort_set || - options->AssumeReachable || - net_is_disabled() || + int reach_checks_disabled = router_reachability_checks_disabled(options) || + !options->DirPort_set; + return reach_checks_disabled || can_reach_dir_port; } -/** Look at a variety of factors, and return 0 if we don't want to - * advertise the fact that we have a DirPort open. Else return the - * DirPort we want to advertise. - * - * Log a helpful message if we change our mind about whether to publish - * a DirPort. +/** The lower threshold of remaining bandwidth required to advertise (or + * automatically provide) directory services */ +/* XXX Should this be increased? */ +#define MIN_BW_TO_ADVERTISE_DIRSERVER 51200 + +/** Return true iff we have enough configured bandwidth to cache directory + * information. */ +static int +router_has_bandwidth_to_be_dirserver(const or_options_t *options) +{ + if (options->BandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) { + return 0; + } + if (options->RelayBandwidthRate > 0 && + options->RelayBandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) { + return 0; + } + return 1; +} + +/** Helper: Return 1 if we have sufficient resources for serving directory + * requests, return 0 otherwise. + * dir_port is either 0 or the configured DirPort number. + * If AccountingMax is set less than our advertised bandwidth, then don't + * serve requests. Likewise, if our advertised bandwidth is less than + * MIN_BW_TO_ADVERTISE_DIRSERVER, don't bother trying to serve requests. */ static int -decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port) +router_should_be_directory_server(const or_options_t *options, int dir_port) { static int advertising=1; /* start out assuming we will advertise */ int new_choice=1; const char *reason = NULL; - /* Section one: reasons to publish or not publish that aren't - * worth mentioning to the user, either because they're obvious - * or because they're normal behavior. */ - - if (!dir_port) /* short circuit the rest of the function */ - return 0; - if (authdir_mode(options)) /* always publish */ - return dir_port; - if (net_is_disabled()) - return 0; - if (!check_whether_dirport_reachable()) - return 0; - if (!router_get_advertised_dir_port(options, dir_port)) - return 0; - - /* Section two: reasons to publish or not publish that the user - * might find surprising. These are generally config options that - * make us choose not to publish. */ - - if (accounting_is_enabled(options)) { + if (accounting_is_enabled(options) && + get_options()->AccountingRule != ACCT_IN) { /* Don't spend bytes for directory traffic if we could end up hibernating, * but allow DirPort otherwise. Some people set AccountingMax because - * they're confused or to get statistics. */ + * they're confused or to get statistics. Directory traffic has a much + * larger effect on output than input so there is no reason to turn it + * off if using AccountingRule in. */ int interval_length = accounting_get_interval_length(); uint32_t effective_bw = get_effective_bwrate(options); + uint64_t acc_bytes; if (!interval_length) { log_warn(LD_BUG, "An accounting interval is not allowed to be zero " "seconds long. Raising to 1."); interval_length = 1; } - log_info(LD_GENERAL, "Calculating whether to disable dirport: effective " + log_info(LD_GENERAL, "Calculating whether to advertise %s: effective " "bwrate: %u, AccountingMax: "U64_FORMAT", " - "accounting interval length %d", effective_bw, - U64_PRINTF_ARG(options->AccountingMax), + "accounting interval length %d", + dir_port ? "dirport" : "begindir", + effective_bw, U64_PRINTF_ARG(options->AccountingMax), interval_length); + + acc_bytes = options->AccountingMax; + if (get_options()->AccountingRule == ACCT_SUM) + acc_bytes /= 2; if (effective_bw >= - options->AccountingMax / interval_length) { + acc_bytes / interval_length) { new_choice = 0; reason = "AccountingMax enabled"; } -#define MIN_BW_TO_ADVERTISE_DIRPORT 51200 - } else if (options->BandwidthRate < MIN_BW_TO_ADVERTISE_DIRPORT || - (options->RelayBandwidthRate > 0 && - options->RelayBandwidthRate < MIN_BW_TO_ADVERTISE_DIRPORT)) { + } else if (! router_has_bandwidth_to_be_dirserver(options)) { /* if we're advertising a small amount */ new_choice = 0; reason = "BandwidthRate under 50KB"; @@ -1080,15 +1197,91 @@ decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port) if (advertising != new_choice) { if (new_choice == 1) { - log_notice(LD_DIR, "Advertising DirPort as %d", dir_port); + if (dir_port > 0) + log_notice(LD_DIR, "Advertising DirPort as %d", dir_port); + else + log_notice(LD_DIR, "Advertising directory service support"); } else { tor_assert(reason); - log_notice(LD_DIR, "Not advertising DirPort (Reason: %s)", reason); + log_notice(LD_DIR, "Not advertising Dir%s (Reason: %s)", + dir_port ? "Port" : "ectory Service support", reason); } advertising = new_choice; } - return advertising ? dir_port : 0; + return advertising; +} + +/** Return 1 if we are configured to accept either relay or directory requests + * from clients and we aren't at risk of exceeding our bandwidth limits, thus + * we should be a directory server. If not, return 0. + */ +int +dir_server_mode(const or_options_t *options) +{ + if (!options->DirCache) + return 0; + return options->DirPort_set || + (server_mode(options) && router_has_bandwidth_to_be_dirserver(options)); +} + +/** Look at a variety of factors, and return 0 if we don't want to + * advertise the fact that we have a DirPort open or begindir support, else + * return 1. + * + * Where dir_port or supports_tunnelled_dir_requests are not relevant, they + * must be 0. + * + * Log a helpful message if we change our mind about whether to publish. + */ +static int +decide_to_advertise_dir_impl(const or_options_t *options, + uint16_t dir_port, + int supports_tunnelled_dir_requests) +{ + /* Part one: reasons to publish or not publish that aren't + * worth mentioning to the user, either because they're obvious + * or because they're normal behavior. */ + + /* short circuit the rest of the function */ + if (!dir_port && !supports_tunnelled_dir_requests) + return 0; + if (authdir_mode(options)) /* always publish */ + return 1; + if (net_is_disabled()) + return 0; + if (dir_port && !router_get_advertised_dir_port(options, dir_port)) + return 0; + if (supports_tunnelled_dir_requests && + !router_get_advertised_or_port(options)) + return 0; + + /* Part two: consider config options that could make us choose to + * publish or not publish that the user might find surprising. */ + return router_should_be_directory_server(options, dir_port); +} + +/** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to + * advertise the fact that we have a DirPort open, else return the + * DirPort we want to advertise. + */ +static int +decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port) +{ + /* supports_tunnelled_dir_requests is not relevant, pass 0 */ + return decide_to_advertise_dir_impl(options, dir_port, 0) ? dir_port : 0; +} + +/** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to + * advertise the fact that we support begindir requests, else return 1. + */ +static int +decide_to_advertise_begindir(const or_options_t *options, + int supports_tunnelled_dir_requests) +{ + /* dir_port is not relevant, pass 0 */ + return decide_to_advertise_dir_impl(options, 0, + supports_tunnelled_dir_requests); } /** Allocate and return a new extend_info_t that can be used to build @@ -1122,9 +1315,9 @@ void consider_testing_reachability(int test_or, int test_dir) { const routerinfo_t *me = router_get_my_routerinfo(); - int orport_reachable = check_whether_orport_reachable(); - tor_addr_t addr; const or_options_t *options = get_options(); + int orport_reachable = check_whether_orport_reachable(options); + tor_addr_t addr; if (!me) return; @@ -1149,20 +1342,21 @@ consider_testing_reachability(int test_or, int test_dir) /* XXX IPv6 self testing */ log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", !orport_reachable ? "reachability" : "bandwidth", - me->address, me->or_port); + fmt_addr32(me->addr), me->or_port); circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei, CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); extend_info_free(ei); } + /* XXX IPv6 self testing */ tor_addr_from_ipv4h(&addr, me->addr); - if (test_dir && !check_whether_dirport_reachable() && + if (test_dir && !check_whether_dirport_reachable(options) && !connection_get_by_type_addr_port_purpose( CONN_TYPE_DIR, &addr, me->dir_port, DIR_PURPOSE_FETCH_SERVERDESC)) { /* ask myself, via tor, for my server descriptor. */ - directory_initiate_command(me->address, &addr, - me->or_port, me->dir_port, + directory_initiate_command(&addr, me->or_port, + &addr, me->dir_port, me->cache_info.identity_digest, DIR_PURPOSE_FETCH_SERVERDESC, ROUTER_PURPOSE_GENERAL, @@ -1175,16 +1369,25 @@ void router_orport_found_reachable(void) { const routerinfo_t *me = router_get_my_routerinfo(); + const or_options_t *options = get_options(); if (!can_reach_or_port && me) { + char *address = tor_dup_ip(me->addr); log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from " "the outside. Excellent.%s", - get_options()->PublishServerDescriptor_ != NO_DIRINFO ? + options->PublishServerDescriptor_ != NO_DIRINFO + && check_whether_dirport_reachable(options) ? " Publishing server descriptor." : ""); can_reach_or_port = 1; mark_my_descriptor_dirty("ORPort found reachable"); + /* This is a significant enough change to upload immediately, + * at least in a test network */ + if (options->TestingTorNetwork == 1) { + reschedule_descriptor_update_check(); + } control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); + tor_free(address); } } @@ -1193,15 +1396,27 @@ void router_dirport_found_reachable(void) { const routerinfo_t *me = router_get_my_routerinfo(); + const or_options_t *options = get_options(); if (!can_reach_dir_port && me) { + char *address = tor_dup_ip(me->addr); log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable " - "from the outside. Excellent."); + "from the outside. Excellent.%s", + options->PublishServerDescriptor_ != NO_DIRINFO + && check_whether_orport_reachable(options) ? + " Publishing server descriptor." : ""); can_reach_dir_port = 1; - if (decide_to_advertise_dirport(get_options(), me->dir_port)) + if (decide_to_advertise_dirport(options, me->dir_port)) { mark_my_descriptor_dirty("DirPort found reachable"); + /* This is a significant enough change to upload immediately, + * at least in a test network */ + if (options->TestingTorNetwork == 1) { + reschedule_descriptor_update_check(); + } + } control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); + tor_free(address); } } @@ -1236,7 +1451,8 @@ router_perform_bandwidth_test(int num_circs, time_t now) } /** Return true iff our network is in some sense disabled: either we're - * hibernating, entering hibernation, or */ + * hibernating, entering hibernation, or the network is turned off with + * DisableNetwork. */ int net_is_disabled(void) { @@ -1251,22 +1467,6 @@ authdir_mode(const or_options_t *options) { return options->AuthoritativeDir != 0; } -/** Return true iff we believe ourselves to be a v1 authoritative - * directory server. - */ -int -authdir_mode_v1(const or_options_t *options) -{ - return authdir_mode(options) && options->V1AuthoritativeDir != 0; -} -/** Return true iff we believe ourselves to be a v2 authoritative - * directory server. - */ -int -authdir_mode_v2(const or_options_t *options) -{ - return authdir_mode(options) && options->V2AuthoritativeDir != 0; -} /** Return true iff we believe ourselves to be a v3 authoritative * directory server. */ @@ -1275,13 +1475,11 @@ authdir_mode_v3(const or_options_t *options) { return authdir_mode(options) && options->V3AuthoritativeDir != 0; } -/** Return true iff we are a v1, v2, or v3 directory authority. */ +/** Return true iff we are a v3 directory authority. */ int authdir_mode_any_main(const or_options_t *options) { - return options->V1AuthoritativeDir || - options->V2AuthoritativeDir || - options->V3AuthoritativeDir; + return options->V3AuthoritativeDir; } /** Return true if we believe ourselves to be any kind of * authoritative directory beyond just a hidserv authority. */ @@ -1335,8 +1533,8 @@ authdir_mode_bridge(const or_options_t *options) /** Return true iff we are trying to be a server. */ -int -server_mode(const or_options_t *options) +MOCK_IMPL(int, +server_mode,(const or_options_t *options)) { if (options->ClientOnly) return 0; /* XXXX024 I believe we can kill off ORListenAddress here.*/ @@ -1345,8 +1543,8 @@ server_mode(const or_options_t *options) /** Return true iff we are trying to be a non-bridge server. */ -int -public_server_mode(const or_options_t *options) +MOCK_IMPL(int, +public_server_mode,(const or_options_t *options)) { if (!server_mode(options)) return 0; return (!options->BridgeRelay); @@ -1370,8 +1568,8 @@ static int server_is_advertised=0; /** Return true iff we have published our descriptor lately. */ -int -advertised_server_mode(void) +MOCK_IMPL(int, +advertised_server_mode,(void)) { return server_is_advertised; } @@ -1408,7 +1606,10 @@ proxy_mode(const or_options_t *options) * and * - We have ORPort set * and - * - We believe we are reachable from the outside; or + * - We believe our ORPort and DirPort (if present) are reachable from + * the outside; or + * - We believe our ORPort is reachable from the outside, and we can't + * check our DirPort because the consensus has no exits; or * - We are an authoritative directory server. */ static int @@ -1426,8 +1627,15 @@ decide_if_publishable_server(void) return 1; if (!router_get_advertised_or_port(options)) return 0; - - return check_whether_orport_reachable(); + if (!check_whether_orport_reachable(options)) + return 0; + if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL) { + /* All set: there are no exits in the consensus (maybe this is a tiny + * test network), so we can't check our DirPort reachability. */ + return 1; + } else { + return check_whether_dirport_reachable(options); + } } /** Initiate server descriptor upload as reasonable (if server is publishable, @@ -1598,7 +1806,8 @@ router_upload_dir_desc_to_dirservers(int force) int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port) { - if (!router_get_my_routerinfo()) /* make sure desc_routerinfo exists */ + const routerinfo_t *me = router_get_my_routerinfo(); + if (!me) /* make sure routerinfo exists */ return -1; /* make sure it's resolved to something. this way we can't get a @@ -1606,20 +1815,21 @@ router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port) if (tor_addr_is_null(addr)) return -1; - /* look at desc_routerinfo->exit_policy for both the v4 and the v6 - * policies. The exit_policy field in desc_routerinfo is a bit unusual, - * in that it contains IPv6 and IPv6 entries. We don't want to look - * at desc_routerinfio->ipv6_exit_policy, since that's a port summary. */ + /* look at router_get_my_routerinfo()->exit_policy for both the v4 and the + * v6 policies. The exit_policy field in router_get_my_routerinfo() is a + * bit unusual, in that it contains IPv6 and IPv6 entries. We don't want to + * look at router_get_my_routerinfo()->ipv6_exit_policy, since that's a port + * summary. */ if ((tor_addr_family(addr) == AF_INET || tor_addr_family(addr) == AF_INET6)) { return compare_tor_addr_to_addr_policy(addr, port, - desc_routerinfo->exit_policy) != ADDR_POLICY_ACCEPTED; + me->exit_policy) != ADDR_POLICY_ACCEPTED; #if 0 } else if (tor_addr_family(addr) == AF_INET6) { return get_options()->IPv6Exit && desc_routerinfo->ipv6_exit_policy && compare_tor_addr_to_short_policy(addr, port, - desc_routerinfo->ipv6_exit_policy) != ADDR_POLICY_ACCEPTED; + me->ipv6_exit_policy) != ADDR_POLICY_ACCEPTED; #endif } else { return -1; @@ -1628,13 +1838,13 @@ router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port) /** Return true iff my exit policy is reject *:*. Return -1 if we don't * have a descriptor */ -int -router_my_exit_policy_is_reject_star(void) +MOCK_IMPL(int, +router_my_exit_policy_is_reject_star,(void)) { - if (!router_get_my_routerinfo()) /* make sure desc_routerinfo exists */ + if (!router_get_my_routerinfo()) /* make sure routerinfo exists */ return -1; - return desc_routerinfo->policy_is_reject_star; + return router_get_my_routerinfo()->policy_is_reject_star; } /** Return true iff I'm a server and <b>digest</b> is equal to @@ -1674,22 +1884,10 @@ router_is_me(const routerinfo_t *router) return router_digest_is_me(router->cache_info.identity_digest); } -/** Return true iff <b>fp</b> is a hex fingerprint of my identity digest. */ -int -router_fingerprint_is_me(const char *fp) -{ - char digest[DIGEST_LEN]; - if (strlen(fp) == HEX_DIGEST_LEN && - base16_decode(digest, sizeof(digest), fp, HEX_DIGEST_LEN) == 0) - return router_digest_is_me(digest); - - return 0; -} - /** Return a routerinfo for this OR, rebuilding a fresh one if * necessary. Return NULL on error, or if called on an OP. */ -const routerinfo_t * -router_get_my_routerinfo(void) +MOCK_IMPL(const routerinfo_t *, +router_get_my_routerinfo,(void)) { if (!server_mode(get_options())) return NULL; @@ -1705,12 +1903,13 @@ const char * router_get_my_descriptor(void) { const char *body; - if (!router_get_my_routerinfo()) + const routerinfo_t *me = router_get_my_routerinfo(); + if (! me) return NULL; + tor_assert(me->cache_info.saved_location == SAVED_NOWHERE); + body = signed_descriptor_get_body(&me->cache_info); /* Make sure this is nul-terminated. */ - tor_assert(desc_routerinfo->cache_info.saved_location == SAVED_NOWHERE); - body = signed_descriptor_get_body(&desc_routerinfo->cache_info); - tor_assert(!body[desc_routerinfo->cache_info.signed_descriptor_len]); + tor_assert(!body[me->cache_info.signed_descriptor_len]); log_debug(LD_GENERAL,"my desc is '%s'", body); return body; } @@ -1745,8 +1944,8 @@ static int router_guess_address_from_dir_headers(uint32_t *guess); * it's configured in torrc, or because we've learned it from * dirserver headers. Place the answer in *<b>addr</b> and return * 0 on success, else return -1 if we have no guess. */ -int -router_pick_published_address(const or_options_t *options, uint32_t *addr) +MOCK_IMPL(int, +router_pick_published_address,(const or_options_t *options, uint32_t *addr)) { *addr = get_last_resolved_addr(); if (!*addr && @@ -1763,12 +1962,15 @@ router_pick_published_address(const or_options_t *options, uint32_t *addr) return 0; } -/** If <b>force</b> is true, or our descriptor is out-of-date, rebuild a fresh - * routerinfo, signed server descriptor, and extra-info document for this OR. - * Return 0 on success, -1 on temporary error. +/** Build a fresh routerinfo, signed server descriptor, and extra-info document + * for this OR. Set r to the generated routerinfo, e to the generated + * extra-info document. Return 0 on success, -1 on temporary error. Failure to + * generate an extra-info document is not an error and is indicated by setting + * e to NULL. Caller is responsible for freeing generated documents if 0 is + * returned. */ int -router_rebuild_descriptor(int force) +router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) { routerinfo_t *ri; extrainfo_t *ei; @@ -1777,45 +1979,39 @@ router_rebuild_descriptor(int force) int hibernating = we_are_hibernating(); const or_options_t *options = get_options(); - if (desc_clean_since && !force) - return 0; - - if (router_pick_published_address(options, &addr) < 0 || - router_get_advertised_or_port(options) == 0) { - /* Stop trying to rebuild our descriptor every second. We'll - * learn that it's time to try again when ip_address_changed() - * marks it dirty. */ - desc_clean_since = time(NULL); + if (router_pick_published_address(options, &addr) < 0) { + log_warn(LD_CONFIG, "Don't know my address while generating descriptor"); return -1; } - log_info(LD_OR, "Rebuilding relay descriptor%s", force ? " (forced)" : ""); - ri = tor_malloc_zero(sizeof(routerinfo_t)); ri->cache_info.routerlist_index = -1; - ri->address = tor_dup_ip(addr); ri->nickname = tor_strdup(options->Nickname); ri->addr = addr; ri->or_port = router_get_advertised_or_port(options); ri->dir_port = router_get_advertised_dir_port(options, 0); + ri->supports_tunnelled_dir_requests = + directory_permits_begindir_requests(options); ri->cache_info.published_on = time(NULL); ri->onion_pkey = crypto_pk_dup_key(get_onion_key()); /* must invoke from * main thread */ -#ifdef CURVE25519_ENABLED ri->onion_curve25519_pkey = tor_memdup(&get_current_curve25519_keypair()->pubkey, sizeof(curve25519_public_key_t)); -#endif /* For now, at most one IPv6 or-address is being advertised. */ { const port_cfg_t *ipv6_orport = NULL; SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) { if (p->type == CONN_TYPE_OR_LISTENER && - ! p->no_advertise && - ! p->bind_ipv4_only && + ! p->server_cfg.no_advertise && + ! p->server_cfg.bind_ipv4_only && tor_addr_family(&p->addr) == AF_INET6) { - if (! tor_addr_is_internal(&p->addr, 0)) { + /* Like IPv4, if the relay is configured using the default + * authorities, disallow internal IPs. Otherwise, allow them. */ + const int default_auth = (!options->DirAuthorities && + !options->AlternateDirAuthority); + if (! tor_addr_is_internal(&p->addr, 0) || ! default_auth) { ipv6_orport = p; break; } else { @@ -1823,7 +2019,7 @@ router_rebuild_descriptor(int force) log_warn(LD_CONFIG, "Unable to use configured IPv6 address \"%s\" in a " "descriptor. Skipping it. " - "Try specifying a globally reachable address explicitly. ", + "Try specifying a globally reachable address explicitly.", tor_addr_to_str(addrbuf, &p->addr, sizeof(addrbuf), 1)); } } @@ -1840,6 +2036,9 @@ router_rebuild_descriptor(int force) routerinfo_free(ri); return -1; } + ri->cache_info.signing_key_cert = + tor_cert_dup(get_master_signing_key_cert()); + get_platform_str(platform, sizeof(platform)); ri->platform = tor_strdup(platform); @@ -1855,10 +2054,8 @@ router_rebuild_descriptor(int force) /* DNS is screwed up; don't claim to be an exit. */ policies_exit_policy_append_reject_star(&ri->exit_policy); } else { - policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, - options->IPv6Exit, - options->ExitPolicyRejectPrivate, - ri->address, !options->BridgeRelay); + policies_parse_exit_policy_from_options(options,ri->addr,&ri->ipv6_addr, + &ri->exit_policy); } ri->policy_is_reject_star = policy_is_reject_star(ri->exit_policy, AF_INET) && @@ -1871,12 +2068,6 @@ router_rebuild_descriptor(int force) tor_free(p_tmp); } -#if 0 - /* XXXX NM NM I belive this is safe to remove */ - if (authdir_mode(options)) - ri->is_valid = ri->is_named = 1; /* believe in yourself */ -#endif - if (options->MyFamily && ! options->BridgeRelay) { smartlist_t *family; if (!warned_nonexistent_family) @@ -1884,7 +2075,7 @@ router_rebuild_descriptor(int force) family = smartlist_new(); ri->declared_family = smartlist_new(); smartlist_split_string(family, options->MyFamily, ",", - SPLIT_SKIP_SPACE|SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0); SMARTLIST_FOREACH_BEGIN(family, char *, name) { const node_t *member; if (!strcasecmp(name, options->Nickname)) @@ -1938,10 +2129,14 @@ router_rebuild_descriptor(int force) ei->cache_info.is_extrainfo = 1; strlcpy(ei->nickname, get_options()->Nickname, sizeof(ei->nickname)); ei->cache_info.published_on = ri->cache_info.published_on; + ei->cache_info.signing_key_cert = + tor_cert_dup(get_master_signing_key_cert()); + memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest, DIGEST_LEN); if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body, - ei, get_server_identity_key()) < 0) { + ei, get_server_identity_key(), + get_master_signing_keypair()) < 0) { log_warn(LD_BUG, "Couldn't generate extra-info descriptor."); extrainfo_free(ei); ei = NULL; @@ -1951,6 +2146,10 @@ router_rebuild_descriptor(int force) router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body, ei->cache_info.signed_descriptor_len, ei->cache_info.signed_descriptor_digest); + crypto_digest256((char*) ei->digest256, + ei->cache_info.signed_descriptor_body, + ei->cache_info.signed_descriptor_len, + DIGEST_SHA256); } /* Now finish the router descriptor. */ @@ -1958,12 +2157,18 @@ router_rebuild_descriptor(int force) memcpy(ri->cache_info.extra_info_digest, ei->cache_info.signed_descriptor_digest, DIGEST_LEN); + memcpy(ri->cache_info.extra_info_digest256, + ei->digest256, + DIGEST256_LEN); } else { /* ri was allocated with tor_malloc_zero, so there is no need to * zero ri->cache_info.extra_info_digest here. */ } - if (! (ri->cache_info.signed_descriptor_body = router_dump_router_to_string( - ri, get_server_identity_key()))) { + if (! (ri->cache_info.signed_descriptor_body = + router_dump_router_to_string(ri, get_server_identity_key(), + get_onion_key(), + get_current_curve25519_keypair(), + get_master_signing_keypair())) ) { log_warn(LD_BUG, "Couldn't generate router descriptor."); routerinfo_free(ri); extrainfo_free(ei); @@ -1993,7 +2198,44 @@ router_rebuild_descriptor(int force) ri->cache_info.signed_descriptor_digest); if (ei) { - tor_assert(! routerinfo_incompatible_with_extrainfo(ri, ei, NULL, NULL)); + tor_assert(! + routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, + &ri->cache_info, NULL)); + } + + *r = ri; + *e = ei; + return 0; +} + +/** If <b>force</b> is true, or our descriptor is out-of-date, rebuild a fresh + * routerinfo, signed server descriptor, and extra-info document for this OR. + * Return 0 on success, -1 on temporary error. + */ +int +router_rebuild_descriptor(int force) +{ + routerinfo_t *ri; + extrainfo_t *ei; + uint32_t addr; + const or_options_t *options = get_options(); + + if (desc_clean_since && !force) + return 0; + + if (router_pick_published_address(options, &addr) < 0 || + router_get_advertised_or_port(options) == 0) { + /* Stop trying to rebuild our descriptor every second. We'll + * learn that it's time to try again when ip_address_changed() + * marks it dirty. */ + desc_clean_since = time(NULL); + return -1; + } + + log_info(LD_OR, "Rebuilding relay descriptor%s", force ? " (forced)" : ""); + + if (router_build_fresh_descriptor(&ri, &ei) < 0) { + return -1; } routerinfo_free(desc_routerinfo); @@ -2068,7 +2310,8 @@ mark_my_descriptor_dirty(const char *reason) } /** How frequently will we republish our descriptor because of large (factor - * of 2) shifts in estimated bandwidth? */ + * of 2) shifts in estimated bandwidth? Note: We don't use this constant + * if our previous bandwidth estimate was exactly 0. */ #define MAX_BANDWIDTH_CHANGE_FREQ (20*60) /** Check whether bandwidth has changed a lot since the last time we announced @@ -2078,15 +2321,15 @@ check_descriptor_bandwidth_changed(time_t now) { static time_t last_changed = 0; uint64_t prev, cur; - if (!desc_routerinfo) + if (!router_get_my_routerinfo()) return; - prev = desc_routerinfo->bandwidthcapacity; + prev = router_get_my_routerinfo()->bandwidthcapacity; cur = we_are_hibernating() ? 0 : rep_hist_bandwidth_assess(); if ((prev != cur && (!prev || !cur)) || cur > prev*2 || cur < prev/2) { - if (last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now) { + if (last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now || !prev) { log_info(LD_GENERAL, "Measured bandwidth has changed; rebuilding descriptor."); mark_my_descriptor_dirty("bandwidth has changed"); @@ -2135,11 +2378,11 @@ check_descriptor_ipaddress_changed(time_t now) (void) now; - if (!desc_routerinfo) + if (router_get_my_routerinfo() == NULL) return; /* XXXX ipv6 */ - prev = desc_routerinfo->addr; + prev = router_get_my_routerinfo()->addr; if (resolve_my_address(LOG_INFO, options, &cur, &method, &hostname) < 0) { log_info(LD_CONFIG,"options->Address didn't resolve into an IP."); return; @@ -2211,7 +2454,7 @@ router_new_address_suggestion(const char *suggestion, if (tor_addr_eq(&d_conn->base_.addr, &addr)) { /* Don't believe anybody who says our IP is their IP. */ log_debug(LD_DIR, "A directory server told us our IP address is %s, " - "but he's just reporting his own IP address. Ignoring.", + "but they are just reporting their own IP address. Ignoring.", suggestion); return; } @@ -2249,7 +2492,7 @@ router_guess_address_from_dir_headers(uint32_t *guess) * string describing the version of Tor and the operating system we're * currently running on. */ -void +STATIC void get_platform_str(char *platform, size_t len) { tor_snprintf(platform, len, "Tor %s on %s", @@ -2268,23 +2511,29 @@ get_platform_str(char *platform, size_t len) */ char * router_dump_router_to_string(routerinfo_t *router, - crypto_pk_t *ident_key) + const crypto_pk_t *ident_key, + const crypto_pk_t *tap_key, + const curve25519_keypair_t *ntor_keypair, + const ed25519_keypair_t *signing_keypair) { - /* XXXX025 Make this look entirely at its arguments, and not at globals. - */ + char *address = NULL; char *onion_pkey = NULL; /* Onion key, PEM-encoded. */ char *identity_pkey = NULL; /* Identity key, PEM-encoded. */ - char digest[DIGEST_LEN]; + char digest[DIGEST256_LEN]; char published[ISO_TIME_LEN+1]; char fingerprint[FINGERPRINT_LEN+1]; - int has_extra_info_digest; - char extra_info_digest[HEX_DIGEST_LEN+1]; + char *extra_info_line = NULL; size_t onion_pkeylen, identity_pkeylen; char *family_line = NULL; char *extra_or_address = NULL; const or_options_t *options = get_options(); smartlist_t *chunks = NULL; char *output = NULL; + const int emit_ed_sigs = signing_keypair && + router->cache_info.signing_key_cert; + char *ed_cert_line = NULL; + char *rsa_tap_cc_line = NULL; + char *ntor_cc_line = NULL; /* Make sure the identity key matches the one in the routerinfo. */ if (!crypto_pk_eq_keys(ident_key, router->identity_pkey)) { @@ -2292,6 +2541,16 @@ router_dump_router_to_string(routerinfo_t *router, "match router's public key!"); goto err; } + if (emit_ed_sigs) { + if (!router->cache_info.signing_key_cert->signing_key_included || + !ed25519_pubkey_eq(&router->cache_info.signing_key_cert->signed_key, + &signing_keypair->pubkey)) { + log_warn(LD_BUG, "Tried to sign a router descriptor with a mismatched " + "ed25519 key chain %d", + router->cache_info.signing_key_cert->signing_key_included); + goto err; + } + } /* record our fingerprint, so we can include it in the descriptor */ if (crypto_pk_get_fingerprint(router->identity_pkey, fingerprint, 1)<0) { @@ -2299,6 +2558,30 @@ router_dump_router_to_string(routerinfo_t *router, goto err; } + if (emit_ed_sigs) { + /* Encode ed25519 signing cert */ + char ed_cert_base64[256]; + char ed_fp_base64[ED25519_BASE64_LEN+1]; + if (base64_encode(ed_cert_base64, sizeof(ed_cert_base64), + (const char*)router->cache_info.signing_key_cert->encoded, + router->cache_info.signing_key_cert->encoded_len, + BASE64_ENCODE_MULTILINE) < 0) { + log_err(LD_BUG,"Couldn't base64-encode signing key certificate!"); + goto err; + } + if (ed25519_public_to_base64(ed_fp_base64, + &router->cache_info.signing_key_cert->signing_key)<0) { + log_err(LD_BUG,"Couldn't base64-encode identity key\n"); + goto err; + } + tor_asprintf(&ed_cert_line, "identity-ed25519\n" + "-----BEGIN ED25519 CERT-----\n" + "%s" + "-----END ED25519 CERT-----\n" + "master-key-ed25519 %s\n", + ed_cert_base64, ed_fp_base64); + } + /* PEM-encode the onion key */ if (crypto_pk_write_public_key_to_string(router->onion_pkey, &onion_pkey,&onion_pkeylen)<0) { @@ -2313,6 +2596,69 @@ router_dump_router_to_string(routerinfo_t *router, goto err; } + /* Cross-certify with RSA key */ + if (tap_key && router->cache_info.signing_key_cert && + router->cache_info.signing_key_cert->signing_key_included) { + char buf[256]; + int tap_cc_len = 0; + uint8_t *tap_cc = + make_tap_onion_key_crosscert(tap_key, + &router->cache_info.signing_key_cert->signing_key, + router->identity_pkey, + &tap_cc_len); + if (!tap_cc) { + log_warn(LD_BUG,"make_tap_onion_key_crosscert failed!"); + goto err; + } + + if (base64_encode(buf, sizeof(buf), (const char*)tap_cc, tap_cc_len, + BASE64_ENCODE_MULTILINE) < 0) { + log_warn(LD_BUG,"base64_encode(rsa_crosscert) failed!"); + tor_free(tap_cc); + goto err; + } + tor_free(tap_cc); + + tor_asprintf(&rsa_tap_cc_line, + "onion-key-crosscert\n" + "-----BEGIN CROSSCERT-----\n" + "%s" + "-----END CROSSCERT-----\n", buf); + } + + /* Cross-certify with onion keys */ + if (ntor_keypair && router->cache_info.signing_key_cert && + router->cache_info.signing_key_cert->signing_key_included) { + int sign = 0; + char buf[256]; + /* XXXX Base the expiration date on the actual onion key expiration time?*/ + tor_cert_t *cert = + make_ntor_onion_key_crosscert(ntor_keypair, + &router->cache_info.signing_key_cert->signing_key, + router->cache_info.published_on, + MIN_ONION_KEY_LIFETIME, &sign); + if (!cert) { + log_warn(LD_BUG,"make_ntor_onion_key_crosscert failed!"); + goto err; + } + tor_assert(sign == 0 || sign == 1); + + if (base64_encode(buf, sizeof(buf), + (const char*)cert->encoded, cert->encoded_len, + BASE64_ENCODE_MULTILINE)<0) { + log_warn(LD_BUG,"base64_encode(ntor_crosscert) failed!"); + tor_cert_free(cert); + goto err; + } + tor_cert_free(cert); + + tor_asprintf(&ntor_cc_line, + "ntor-onion-key-crosscert %d\n" + "-----BEGIN ED25519 CERT-----\n" + "%s" + "-----END ED25519 CERT-----\n", sign, buf); + } + /* Encode the publication time. */ format_iso_time(published, router->cache_info.published_on); @@ -2325,12 +2671,19 @@ router_dump_router_to_string(routerinfo_t *router, family_line = tor_strdup(""); } - has_extra_info_digest = - ! tor_digest_is_zero(router->cache_info.extra_info_digest); - - if (has_extra_info_digest) { + if (!tor_digest_is_zero(router->cache_info.extra_info_digest)) { + char extra_info_digest[HEX_DIGEST_LEN+1]; base16_encode(extra_info_digest, sizeof(extra_info_digest), router->cache_info.extra_info_digest, DIGEST_LEN); + if (!tor_digest256_is_zero(router->cache_info.extra_info_digest256)) { + char d256_64[BASE64_DIGEST256_LEN+1]; + digest256_to_base64(d256_64, router->cache_info.extra_info_digest256); + tor_asprintf(&extra_info_line, "extra-info-digest %s %s\n", + extra_info_digest, d256_64); + } else { + tor_asprintf(&extra_info_line, "extra-info-digest %s\n", + extra_info_digest); + } } if (router->ipv6_orport && @@ -2345,25 +2698,30 @@ router_dump_router_to_string(routerinfo_t *router, } } + address = tor_dup_ip(router->addr); chunks = smartlist_new(); + /* Generate the easy portion of the router descriptor. */ smartlist_add_asprintf(chunks, "router %s %s %d 0 %d\n" "%s" + "%s" "platform %s\n" "protocols Link 1 2 Circuit 1\n" "published %s\n" "fingerprint %s\n" "uptime %ld\n" "bandwidth %d %d %d\n" - "%s%s%s%s" + "%s%s" "onion-key\n%s" "signing-key\n%s" + "%s%s" "%s%s%s%s", router->nickname, - router->address, + address, router->or_port, decide_to_advertise_dirport(options, router->dir_port), + ed_cert_line ? ed_cert_line : "", extra_or_address ? extra_or_address : "", router->platform, published, @@ -2372,14 +2730,15 @@ router_dump_router_to_string(routerinfo_t *router, (int) router->bandwidthrate, (int) router->bandwidthburst, (int) router->bandwidthcapacity, - has_extra_info_digest ? "extra-info-digest " : "", - has_extra_info_digest ? extra_info_digest : "", - has_extra_info_digest ? "\n" : "", - options->DownloadExtraInfo ? "caches-extra-info\n" : "", + extra_info_line ? extra_info_line : "", + (options->DownloadExtraInfo || options->V3AuthoritativeDir) ? + "caches-extra-info\n" : "", onion_pkey, identity_pkey, + rsa_tap_cc_line ? rsa_tap_cc_line : "", + ntor_cc_line ? ntor_cc_line : "", family_line, we_are_hibernating() ? "hibernating 1\n" : "", - options->HidServDirectoryV2 ? "hidden-service-dir\n" : "", + "hidden-service-dir\n", options->AllowSingleHopExits ? "allow-single-hop-exits\n" : ""); if (options->ContactInfo && strlen(options->ContactInfo)) { @@ -2389,34 +2748,25 @@ router_dump_router_to_string(routerinfo_t *router, smartlist_add_asprintf(chunks, "contact %s\n", ci); } -#ifdef CURVE25519_ENABLED if (router->onion_curve25519_pkey) { char kbuf[128]; base64_encode(kbuf, sizeof(kbuf), (const char *)router->onion_curve25519_pkey->public_key, - CURVE25519_PUBKEY_LEN); + CURVE25519_PUBKEY_LEN, BASE64_ENCODE_MULTILINE); smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf); } -#endif /* Write the exit policy to the end of 's'. */ if (!router->exit_policy || !smartlist_len(router->exit_policy)) { smartlist_add(chunks, tor_strdup("reject *:*\n")); } else if (router->exit_policy) { - int i; - for (i = 0; i < smartlist_len(router->exit_policy); ++i) { - char pbuf[POLICY_BUF_LEN]; - addr_policy_t *tmpe = smartlist_get(router->exit_policy, i); - int result; - if (tor_addr_family(&tmpe->addr) == AF_INET6) - continue; /* Don't include IPv6 parts of address policy */ - result = policy_write_item(pbuf, POLICY_BUF_LEN, tmpe, 1); - if (result < 0) { - log_warn(LD_BUG,"descriptor policy_write_item ran out of room!"); - goto err; - } - smartlist_add_asprintf(chunks, "%s\n", pbuf); - } + char *exit_policy = router_dump_exit_policy_to_string(router,1,0); + + if (!exit_policy) + goto err; + + smartlist_add_asprintf(chunks, "%s\n", exit_policy); + tor_free(exit_policy); } if (router->ipv6_exit_policy) { @@ -2428,7 +2778,29 @@ router_dump_router_to_string(routerinfo_t *router, tor_free(p6); } - /* Sign the descriptor */ + if (decide_to_advertise_begindir(options, + router->supports_tunnelled_dir_requests)) { + smartlist_add(chunks, tor_strdup("tunnelled-dir-server\n")); + } + + /* Sign the descriptor with Ed25519 */ + if (emit_ed_sigs) { + smartlist_add(chunks, tor_strdup("router-sig-ed25519 ")); + crypto_digest_smartlist_prefix(digest, DIGEST256_LEN, + ED_DESC_SIGNATURE_PREFIX, + chunks, "", DIGEST_SHA256); + ed25519_signature_t sig; + char buf[ED25519_SIG_BASE64_LEN+1]; + if (ed25519_sign(&sig, (const uint8_t*)digest, DIGEST256_LEN, + signing_keypair) < 0) + goto err; + if (ed25519_signature_to_base64(buf, &sig) < 0) + goto err; + + smartlist_add_asprintf(chunks, "%s\n", buf); + } + + /* Sign the descriptor with RSA */ smartlist_add(chunks, tor_strdup("router-signature\n")); crypto_digest_smartlist(digest, DIGEST_LEN, chunks, "", DIGEST_SHA1); @@ -2454,7 +2826,7 @@ router_dump_router_to_string(routerinfo_t *router, const char *cp; routerinfo_t *ri_tmp; cp = s_dup = tor_strdup(output); - ri_tmp = router_parse_entry_from_string(cp, NULL, 1, 0, NULL); + ri_tmp = router_parse_entry_from_string(cp, NULL, 1, 0, NULL, NULL); if (!ri_tmp) { log_err(LD_BUG, "We just generated a router descriptor we can't parse."); @@ -2475,14 +2847,38 @@ router_dump_router_to_string(routerinfo_t *router, SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); smartlist_free(chunks); } + tor_free(address); tor_free(family_line); tor_free(onion_pkey); tor_free(identity_pkey); tor_free(extra_or_address); + tor_free(ed_cert_line); + tor_free(rsa_tap_cc_line); + tor_free(ntor_cc_line); + tor_free(extra_info_line); return output; } +/** + * OR only: Given <b>router</b>, produce a string with its exit policy. + * If <b>include_ipv4</b> is true, include IPv4 entries. + * If <b>include_ipv6</b> is true, include IPv6 entries. + */ +char * +router_dump_exit_policy_to_string(const routerinfo_t *router, + int include_ipv4, + int include_ipv6) +{ + if ((!router->exit_policy) || (router->policy_is_reject_star)) { + return tor_strdup("reject *:*"); + } + + return policy_dump_to_string(router->exit_policy, + include_ipv4, + include_ipv6); +} + /** Copy the primary (IPv4) OR port (IP address and TCP port) for * <b>router</b> into *<b>ap_out</b>. */ void @@ -2517,8 +2913,9 @@ router_has_orport(const routerinfo_t *router, const tor_addr_port_t *orport) * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in * the past or more than 1 hour in the future with respect to <b>now</b>, * and write the file contents starting with that line to *<b>out</b>. - * Return 1 for success, 0 if the file does not exist, or -1 if the file - * does not contain a line matching these criteria or other failure. */ + * Return 1 for success, 0 if the file does not exist or is empty, or -1 + * if the file does not contain a line matching these criteria or other + * failure. */ static int load_stats_file(const char *filename, const char *end_line, time_t now, char **out) @@ -2552,7 +2949,9 @@ load_stats_file(const char *filename, const char *end_line, time_t now, notfound: tor_free(contents); break; + /* treat empty stats files as if the file doesn't exist */ case FN_NOENT: + case FN_EMPTY: r = 0; break; case FN_ERROR: @@ -2569,7 +2968,8 @@ load_stats_file(const char *filename, const char *end_line, time_t now, * success, negative on failure. */ int extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, - crypto_pk_t *ident_key) + crypto_pk_t *ident_key, + const ed25519_keypair_t *signing_keypair) { const or_options_t *options = get_options(); char identity[HEX_DIGEST_LEN+1]; @@ -2579,20 +2979,47 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, int result; static int write_stats_to_extrainfo = 1; char sig[DIROBJ_MAX_SIG_LEN+1]; - char *s, *pre, *contents, *cp, *s_dup = NULL; + char *s = NULL, *pre, *contents, *cp, *s_dup = NULL; time_t now = time(NULL); smartlist_t *chunks = smartlist_new(); extrainfo_t *ei_tmp = NULL; + const int emit_ed_sigs = signing_keypair && + extrainfo->cache_info.signing_key_cert; + char *ed_cert_line = NULL; base16_encode(identity, sizeof(identity), extrainfo->cache_info.identity_digest, DIGEST_LEN); format_iso_time(published, extrainfo->cache_info.published_on); bandwidth_usage = rep_hist_get_bandwidth_lines(); + if (emit_ed_sigs) { + if (!extrainfo->cache_info.signing_key_cert->signing_key_included || + !ed25519_pubkey_eq(&extrainfo->cache_info.signing_key_cert->signed_key, + &signing_keypair->pubkey)) { + log_warn(LD_BUG, "Tried to sign a extrainfo descriptor with a " + "mismatched ed25519 key chain %d", + extrainfo->cache_info.signing_key_cert->signing_key_included); + goto err; + } + char ed_cert_base64[256]; + if (base64_encode(ed_cert_base64, sizeof(ed_cert_base64), + (const char*)extrainfo->cache_info.signing_key_cert->encoded, + extrainfo->cache_info.signing_key_cert->encoded_len, + BASE64_ENCODE_MULTILINE) < 0) { + log_err(LD_BUG,"Couldn't base64-encode signing key certificate!"); + goto err; + } + tor_asprintf(&ed_cert_line, "identity-ed25519\n" + "-----BEGIN ED25519 CERT-----\n" + "%s" + "-----END ED25519 CERT-----\n", ed_cert_base64); + } else { + ed_cert_line = tor_strdup(""); + } - tor_asprintf(&pre, "extra-info %s %s\npublished %s\n%s", + tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n%s", extrainfo->nickname, identity, + ed_cert_line, published, bandwidth_usage); - tor_free(bandwidth_usage); smartlist_add(chunks, pre); if (geoip_is_loaded(AF_INET)) @@ -2609,6 +3036,11 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, "dirreq-stats-end", now, &contents) > 0) { smartlist_add(chunks, contents); } + if (options->HiddenServiceStatistics && + load_stats_file("stats"PATH_SEPARATOR"hidserv-stats", + "hidserv-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } if (options->EntryStatistics && load_stats_file("stats"PATH_SEPARATOR"entry-stats", "entry-stats-end", now, &contents) > 0) { @@ -2645,6 +3077,23 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, } } + if (emit_ed_sigs) { + char digest[DIGEST256_LEN]; + smartlist_add(chunks, tor_strdup("router-sig-ed25519 ")); + crypto_digest_smartlist_prefix(digest, DIGEST256_LEN, + ED_DESC_SIGNATURE_PREFIX, + chunks, "", DIGEST_SHA256); + ed25519_signature_t sig; + char buf[ED25519_SIG_BASE64_LEN+1]; + if (ed25519_sign(&sig, (const uint8_t*)digest, DIGEST256_LEN, + signing_keypair) < 0) + goto err; + if (ed25519_signature_to_base64(buf, &sig) < 0) + goto err; + + smartlist_add_asprintf(chunks, "%s\n", buf); + } + smartlist_add(chunks, tor_strdup("router-signature\n")); s = smartlist_join_strings(chunks, "", 0, NULL); @@ -2685,7 +3134,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, s = smartlist_join_strings(chunks, "", 0, NULL); cp = s_dup = tor_strdup(s); - ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); + ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL, NULL); if (!ei_tmp) { if (write_stats_to_extrainfo) { log_warn(LD_GENERAL, "We just generated an extra-info descriptor " @@ -2693,7 +3142,8 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, "adding statistics to this or any future " "extra-info descriptors."); write_stats_to_extrainfo = 0; - result = extrainfo_dump_to_string(s_out, extrainfo, ident_key); + result = extrainfo_dump_to_string(s_out, extrainfo, ident_key, + signing_keypair); goto done; } else { log_warn(LD_BUG, "We just generated an extrainfo descriptor we " @@ -2715,7 +3165,9 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); smartlist_free(chunks); tor_free(s_dup); + tor_free(ed_cert_line); extrainfo_free(ei_tmp); + tor_free(bandwidth_usage); return result; } @@ -3029,10 +3481,8 @@ router_free_all(void) crypto_pk_free(legacy_signing_key); authority_cert_free(legacy_key_certificate); -#ifdef CURVE25519_ENABLED memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key)); memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); -#endif if (warned_nonexistent_family) { SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); @@ -3042,28 +3492,16 @@ router_free_all(void) /** Return a smartlist of tor_addr_port_t's with all the OR ports of <b>ri</b>. Note that freeing of the items in the list as well as - the smartlist itself is the callers responsibility. - - XXX duplicating code from node_get_all_orports(). */ + the smartlist itself is the callers responsibility. */ smartlist_t * router_get_all_orports(const routerinfo_t *ri) { - smartlist_t *sl = smartlist_new(); tor_assert(ri); - - if (ri->addr != 0) { - tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); - tor_addr_from_ipv4h(&ap->addr, ri->addr); - ap->port = ri->or_port; - smartlist_add(sl, ap); - } - if (!tor_addr_is_null(&ri->ipv6_addr)) { - tor_addr_port_t *ap = tor_malloc(sizeof(tor_addr_port_t)); - tor_addr_copy(&ap->addr, &ri->ipv6_addr); - ap->port = ri->or_port; - smartlist_add(sl, ap); - } - - return sl; + node_t fake_node; + memset(&fake_node, 0, sizeof(fake_node)); + /* we don't modify ri, fake_node is passed as a const node_t * + */ + fake_node.ri = (routerinfo_t *)ri; + return node_get_all_orports(&fake_node); } diff --git a/src/or/router.h b/src/or/router.h index 60095d087b..73bfea1faa 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,6 +12,8 @@ #ifndef TOR_ROUTER_H #define TOR_ROUTER_H +#include "testsupport.h" + crypto_pk_t *get_onion_key(void); time_t get_onion_key_set_at(void); void set_server_identity_key(crypto_pk_t *k); @@ -20,26 +22,26 @@ int server_identity_key_is_set(void); void set_client_identity_key(crypto_pk_t *k); crypto_pk_t *get_tlsclient_identity_key(void); int client_identity_key_is_set(void); -authority_cert_t *get_my_v3_authority_cert(void); +MOCK_DECL(authority_cert_t *, get_my_v3_authority_cert, (void)); crypto_pk_t *get_my_v3_authority_signing_key(void); authority_cert_t *get_my_v3_legacy_cert(void); crypto_pk_t *get_my_v3_legacy_signing_key(void); void dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last); void rotate_onion_key(void); crypto_pk_t *init_key_from_file(const char *fname, int generate, - int severity); + int severity, int log_greeting); void v3_authority_check_key_expiry(void); -#ifdef CURVE25519_ENABLED di_digest256_map_t *construct_ntor_key_map(void); void ntor_key_map_free(di_digest256_map_t *map); -#endif int router_initialize_tls_context(void); int init_keys(void); +int init_keys_client(void); -int check_whether_orport_reachable(void); -int check_whether_dirport_reachable(void); +int check_whether_orport_reachable(const or_options_t *options); +int check_whether_dirport_reachable(const or_options_t *options); +int dir_server_mode(const or_options_t *options); void consider_testing_reachability(int test_or, int test_dir); void router_orport_found_reachable(void); void router_dirport_found_reachable(void); @@ -48,8 +50,6 @@ void router_perform_bandwidth_test(int num_circs, time_t now); int net_is_disabled(void); int authdir_mode(const or_options_t *options); -int authdir_mode_v1(const or_options_t *options); -int authdir_mode_v2(const or_options_t *options); int authdir_mode_v3(const or_options_t *options); int authdir_mode_any_main(const or_options_t *options); int authdir_mode_any_nonhidserv(const or_options_t *options); @@ -66,9 +66,9 @@ uint16_t router_get_advertised_or_port_by_af(const or_options_t *options, uint16_t router_get_advertised_dir_port(const or_options_t *options, uint16_t dirport); -int server_mode(const or_options_t *options); -int public_server_mode(const or_options_t *options); -int advertised_server_mode(void); +MOCK_DECL(int, server_mode, (const or_options_t *options)); +MOCK_DECL(int, public_server_mode, (const or_options_t *options)); +MOCK_DECL(int, advertised_server_mode, (void)); int proxy_mode(const or_options_t *options); void consider_publishable_server(int force); int should_refuse_unknown_exits(const or_options_t *options); @@ -81,8 +81,8 @@ void check_descriptor_ipaddress_changed(time_t now); void router_new_address_suggestion(const char *suggestion, const dir_connection_t *d_conn); int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port); -int router_my_exit_policy_is_reject_star(void); -const routerinfo_t *router_get_my_routerinfo(void); +MOCK_DECL(int, router_my_exit_policy_is_reject_star,(void)); +MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void)); extrainfo_t *router_get_my_extrainfo(void); const char *router_get_my_descriptor(void); const char *router_get_descriptor_gen_reason(void); @@ -90,11 +90,18 @@ int router_digest_is_me(const char *digest); const uint8_t *router_get_my_id_digest(void); int router_extrainfo_digest_is_me(const char *digest); int router_is_me(const routerinfo_t *router); -int router_fingerprint_is_me(const char *fp); -int router_pick_published_address(const or_options_t *options, uint32_t *addr); +MOCK_DECL(int,router_pick_published_address,(const or_options_t *options, + uint32_t *addr)); +int router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e); int router_rebuild_descriptor(int force); char *router_dump_router_to_string(routerinfo_t *router, - crypto_pk_t *ident_key); + const crypto_pk_t *ident_key, + const crypto_pk_t *tap_key, + const curve25519_keypair_t *ntor_keypair, + const ed25519_keypair_t *signing_keypair); +char *router_dump_exit_policy_to_string(const routerinfo_t *router, + int include_ipv4, + int include_ipv6); void router_get_prim_orport(const routerinfo_t *router, tor_addr_port_t *addr_port_out); void router_get_pref_orport(const routerinfo_t *router, @@ -106,7 +113,8 @@ int router_has_addr(const routerinfo_t *router, const tor_addr_t *addr); int router_has_orport(const routerinfo_t *router, const tor_addr_port_t *orport); int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo, - crypto_pk_t *ident_key); + crypto_pk_t *ident_key, + const ed25519_keypair_t *signing_keypair); int is_legal_nickname(const char *s); int is_legal_nickname_or_hexdigest(const char *s); int is_legal_hexdigest(const char *s); @@ -146,7 +154,8 @@ smartlist_t *router_get_all_orports(const routerinfo_t *ri); #ifdef ROUTER_PRIVATE /* Used only by router.c and test.c */ -void get_platform_str(char *platform, size_t len); +STATIC void get_platform_str(char *platform, size_t len); +STATIC int router_write_fingerprint(int hashed); #endif #endif diff --git a/src/or/routerkeys.c b/src/or/routerkeys.c new file mode 100644 index 0000000000..fba3491f2b --- /dev/null +++ b/src/or/routerkeys.c @@ -0,0 +1,1147 @@ +/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file routerkeys.c + * + * \brief Functions and structures to handle generating and maintaining the + * set of keypairs necessary to be an OR. (Some of the code in router.c + * belongs here.) + */ + +#include "or.h" +#include "config.h" +#include "router.h" +#include "crypto_pwbox.h" +#include "routerkeys.h" +#include "torcert.h" + +#define ENC_KEY_HEADER "Boxed Ed25519 key" +#define ENC_KEY_TAG "master" + +static ssize_t +do_getpass(const char *prompt, char *buf, size_t buflen, + int twice, const or_options_t *options) +{ + if (options->keygen_force_passphrase == FORCE_PASSPHRASE_OFF) { + tor_assert(buflen); + buf[0] = 0; + return 0; + } + + char *prompt2 = NULL; + char *buf2 = NULL; + int fd = -1; + ssize_t length = -1; + + if (options->use_keygen_passphrase_fd) { + twice = 0; + fd = options->keygen_passphrase_fd; + length = read_all(fd, buf, buflen-1, 0); + if (length >= 0) + buf[length] = 0; + goto done_reading; + } + + if (twice) { + const char msg[] = "One more time:"; + size_t p2len = strlen(prompt) + 1; + if (p2len < sizeof(msg)) + p2len = sizeof(msg); + prompt2 = tor_malloc(strlen(prompt)+1); + memset(prompt2, ' ', p2len); + memcpy(prompt2 + p2len - sizeof(msg), msg, sizeof(msg)); + + buf2 = tor_malloc_zero(buflen); + } + + while (1) { + length = tor_getpass(prompt, buf, buflen); + if (length < 0) + goto done_reading; + + if (! twice) + break; + + ssize_t length2 = tor_getpass(prompt2, buf2, buflen); + + if (length != length2 || tor_memneq(buf, buf2, length)) { + fprintf(stderr, "That didn't match.\n"); + } else { + break; + } + } + + done_reading: + if (twice) { + tor_free(prompt2); + memwipe(buf2, 0, buflen); + tor_free(buf2); + } + + if (options->keygen_force_passphrase == FORCE_PASSPHRASE_ON && length == 0) + return -1; + + return length; +} + +int +read_encrypted_secret_key(ed25519_secret_key_t *out, + const char *fname) +{ + int r = -1; + uint8_t *secret = NULL; + size_t secret_len = 0; + char pwbuf[256]; + uint8_t encrypted_key[256]; + char *tag = NULL; + int saved_errno = 0; + + ssize_t encrypted_len = crypto_read_tagged_contents_from_file(fname, + ENC_KEY_HEADER, + &tag, + encrypted_key, + sizeof(encrypted_key)); + if (encrypted_len < 0) { + saved_errno = errno; + log_info(LD_OR, "%s is missing", fname); + r = 0; + goto done; + } + if (strcmp(tag, ENC_KEY_TAG)) { + saved_errno = EINVAL; + goto done; + } + + while (1) { + ssize_t pwlen = + do_getpass("Enter pasphrase for master key:", pwbuf, sizeof(pwbuf), 0, + get_options()); + if (pwlen < 0) { + saved_errno = EINVAL; + goto done; + } + const int r = crypto_unpwbox(&secret, &secret_len, + encrypted_key, encrypted_len, + pwbuf, pwlen); + if (r == UNPWBOX_CORRUPTED) { + log_err(LD_OR, "%s is corrupted.", fname); + saved_errno = EINVAL; + goto done; + } else if (r == UNPWBOX_OKAY) { + break; + } + + /* Otherwise, passphrase is bad, so try again till user does ctrl-c or gets + * it right. */ + } + + if (secret_len != ED25519_SECKEY_LEN) { + log_err(LD_OR, "%s is corrupted.", fname); + saved_errno = EINVAL; + goto done; + } + memcpy(out->seckey, secret, ED25519_SECKEY_LEN); + r = 1; + + done: + memwipe(encrypted_key, 0, sizeof(encrypted_key)); + memwipe(pwbuf, 0, sizeof(pwbuf)); + tor_free(tag); + if (secret) { + memwipe(secret, 0, secret_len); + tor_free(secret); + } + if (saved_errno) + errno = saved_errno; + return r; +} + +int +write_encrypted_secret_key(const ed25519_secret_key_t *key, + const char *fname) +{ + int r = -1; + char pwbuf0[256]; + uint8_t *encrypted_key = NULL; + size_t encrypted_len = 0; + + if (do_getpass("Enter new passphrase:", pwbuf0, sizeof(pwbuf0), 1, + get_options()) < 0) { + log_warn(LD_OR, "NO/failed passphrase"); + return -1; + } + + if (strlen(pwbuf0) == 0) { + if (get_options()->keygen_force_passphrase == FORCE_PASSPHRASE_ON) + return -1; + else + return 0; + } + + if (crypto_pwbox(&encrypted_key, &encrypted_len, + key->seckey, sizeof(key->seckey), + pwbuf0, strlen(pwbuf0), 0) < 0) { + log_warn(LD_OR, "crypto_pwbox failed!?"); + goto done; + } + if (crypto_write_tagged_contents_to_file(fname, + ENC_KEY_HEADER, + ENC_KEY_TAG, + encrypted_key, encrypted_len) < 0) + goto done; + r = 1; + done: + if (encrypted_key) { + memwipe(encrypted_key, 0, encrypted_len); + tor_free(encrypted_key); + } + memwipe(pwbuf0, 0, sizeof(pwbuf0)); + return r; +} + +static int +write_secret_key(const ed25519_secret_key_t *key, int encrypted, + const char *fname, + const char *fname_tag, + const char *encrypted_fname) +{ + if (encrypted) { + int r = write_encrypted_secret_key(key, encrypted_fname); + if (r == 1) { + /* Success! */ + + /* Try to unlink the unencrypted key, if any existed before */ + if (strcmp(fname, encrypted_fname)) + unlink(fname); + return r; + } else if (r != 0) { + /* Unrecoverable failure! */ + return r; + } + + fprintf(stderr, "Not encrypting the secret key.\n"); + } + return ed25519_seckey_write_to_file(key, fname, fname_tag); +} + +/** + * Read an ed25519 key and associated certificates from files beginning with + * <b>fname</b>, with certificate type <b>cert_type</b>. On failure, return + * NULL; on success return the keypair. + * + * If INIT_ED_KEY_CREATE is set in <b>flags</b>, then create the key (and + * certificate if requested) if it doesn't exist, and save it to disk. + * + * If INIT_ED_KEY_NEEDCERT is set in <b>flags</b>, load/create a certificate + * too and store it in *<b>cert_out</b>. Fail if the cert can't be + * found/created. To create a certificate, <b>signing_key</b> must be set to + * the key that should sign it; <b>now</b> to the current time, and + * <b>lifetime</b> to the lifetime of the key. + * + * If INIT_ED_KEY_REPLACE is set in <b>flags</b>, then create and save new key + * whether we can read the old one or not. + * + * If INIT_ED_KEY_EXTRA_STRONG is set in <b>flags</b>, set the extra_strong + * flag when creating the secret key. + * + * If INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT is set in <b>flags</b>, and + * we create a new certificate, create it with the signing key embedded. + * + * If INIT_ED_KEY_SPLIT is set in <b>flags</b>, and we create a new key, + * store the public key in a separate file from the secret key. + * + * If INIT_ED_KEY_MISSING_SECRET_OK is set in <b>flags</b>, and we find a + * public key file but no secret key file, return successfully anyway. + * + * If INIT_ED_KEY_OMIT_SECRET is set in <b>flags</b>, do not try to load a + * secret key unless no public key is found. Do not return a secret key. (but + * create and save one if needed). + * + * If INIT_ED_KEY_NO_LOAD_SECRET is set in <b>flags</b>, don't try to load + * a secret key, no matter what. + * + * If INIT_ED_KEY_TRY_ENCRYPTED is set, we look for an encrypted secret key + * and consider encrypting any new secret key. + * + * If INIT_ED_KEY_NO_REPAIR is set, and there is any issue loading the keys + * from disk _other than their absence_ (full or partial), we do not try to + * replace them. + * + * If INIT_ED_KEY_SUGGEST_KEYGEN is set, have log messages about failures + * refer to the --keygen option. + * + * If INIT_ED_KEY_EXPLICIT_FNAME is set, use the provided file name for the + * secret key file, encrypted or not. + */ +ed25519_keypair_t * +ed_key_init_from_file(const char *fname, uint32_t flags, + int severity, + const ed25519_keypair_t *signing_key, + time_t now, + time_t lifetime, + uint8_t cert_type, + struct tor_cert_st **cert_out) +{ + char *secret_fname = NULL; + char *encrypted_secret_fname = NULL; + char *public_fname = NULL; + char *cert_fname = NULL; + const char *loaded_secret_fname = NULL; + int created_pk = 0, created_sk = 0, created_cert = 0; + const int try_to_load = ! (flags & INIT_ED_KEY_REPLACE); + const int encrypt_key = !! (flags & INIT_ED_KEY_TRY_ENCRYPTED); + const int norepair = !! (flags & INIT_ED_KEY_NO_REPAIR); + const int split = !! (flags & INIT_ED_KEY_SPLIT); + const int omit_secret = !! (flags & INIT_ED_KEY_OMIT_SECRET); + const int offline_secret = !! (flags & INIT_ED_KEY_OFFLINE_SECRET); + const int explicit_fname = !! (flags & INIT_ED_KEY_EXPLICIT_FNAME); + + /* we don't support setting both of these flags at once. */ + tor_assert((flags & (INIT_ED_KEY_NO_REPAIR|INIT_ED_KEY_NEEDCERT)) != + (INIT_ED_KEY_NO_REPAIR|INIT_ED_KEY_NEEDCERT)); + + char tag[8]; + tor_snprintf(tag, sizeof(tag), "type%d", (int)cert_type); + + tor_cert_t *cert = NULL; + char *got_tag = NULL; + ed25519_keypair_t *keypair = tor_malloc_zero(sizeof(ed25519_keypair_t)); + + if (explicit_fname) { + secret_fname = tor_strdup(fname); + encrypted_secret_fname = tor_strdup(fname); + } else { + tor_asprintf(&secret_fname, "%s_secret_key", fname); + tor_asprintf(&encrypted_secret_fname, "%s_secret_key_encrypted", fname); + } + tor_asprintf(&public_fname, "%s_public_key", fname); + tor_asprintf(&cert_fname, "%s_cert", fname); + + /* Try to read the secret key. */ + int have_secret = 0; + int load_secret = try_to_load && + !offline_secret && + (!omit_secret || file_status(public_fname)==FN_NOENT); + if (load_secret) { + int rv = ed25519_seckey_read_from_file(&keypair->seckey, + &got_tag, secret_fname); + if (rv == 0) { + have_secret = 1; + loaded_secret_fname = secret_fname; + tor_assert(got_tag); + } else { + if (errno != ENOENT && norepair) { + tor_log(severity, LD_OR, "Unable to read %s: %s", secret_fname, + strerror(errno)); + goto err; + } + } + } + + /* Should we try for an encrypted key? */ + int have_encrypted_secret_file = 0; + if (!have_secret && try_to_load && encrypt_key) { + int r = read_encrypted_secret_key(&keypair->seckey, + encrypted_secret_fname); + if (r > 0) { + have_secret = 1; + have_encrypted_secret_file = 1; + tor_free(got_tag); /* convince coverity we aren't leaking */ + got_tag = tor_strdup(tag); + loaded_secret_fname = encrypted_secret_fname; + } else if (errno != ENOENT && norepair) { + tor_log(severity, LD_OR, "Unable to read %s: %s", + encrypted_secret_fname, strerror(errno)); + goto err; + } + } else { + if (try_to_load) { + /* Check if it's there anyway, so we don't replace it. */ + if (file_status(encrypted_secret_fname) != FN_NOENT) + have_encrypted_secret_file = 1; + } + } + + if (have_secret) { + if (strcmp(got_tag, tag)) { + tor_log(severity, LD_OR, "%s has wrong tag", loaded_secret_fname); + goto err; + } + /* Derive the public key */ + if (ed25519_public_key_generate(&keypair->pubkey, &keypair->seckey)<0) { + tor_log(severity, LD_OR, "%s can't produce a public key", + loaded_secret_fname); + goto err; + } + } + + /* If we do split keys here, try to read the pubkey. */ + int found_public = 0; + if (try_to_load && (!have_secret || split)) { + ed25519_public_key_t pubkey_tmp; + tor_free(got_tag); + found_public = ed25519_pubkey_read_from_file(&pubkey_tmp, + &got_tag, public_fname) == 0; + if (!found_public && errno != ENOENT && norepair) { + tor_log(severity, LD_OR, "Unable to read %s: %s", public_fname, + strerror(errno)); + goto err; + } + if (found_public && strcmp(got_tag, tag)) { + tor_log(severity, LD_OR, "%s has wrong tag", public_fname); + goto err; + } + if (found_public) { + if (have_secret) { + /* If we have a secret key and we're reloading the public key, + * the key must match! */ + if (! ed25519_pubkey_eq(&keypair->pubkey, &pubkey_tmp)) { + tor_log(severity, LD_OR, "%s does not match %s! If you are trying " + "to restore from backup, make sure you didn't mix up the " + "key files. If you are absolutely sure that %s is the right " + "key for this relay, delete %s or move it out of the way.", + public_fname, loaded_secret_fname, + loaded_secret_fname, public_fname); + goto err; + } + } else { + /* We only have the public key; better use that. */ + tor_assert(split); + memcpy(&keypair->pubkey, &pubkey_tmp, sizeof(pubkey_tmp)); + } + } else { + /* We have no public key file, but we do have a secret key, make the + * public key file! */ + if (have_secret) { + if (ed25519_pubkey_write_to_file(&keypair->pubkey, public_fname, tag) + < 0) { + tor_log(severity, LD_OR, "Couldn't repair %s", public_fname); + goto err; + } else { + tor_log(LOG_NOTICE, LD_OR, + "Found secret key but not %s. Regenerating.", + public_fname); + } + } + } + } + + /* If the secret key is absent and it's not allowed to be, fail. */ + if (!have_secret && found_public && + !(flags & INIT_ED_KEY_MISSING_SECRET_OK)) { + if (have_encrypted_secret_file) { + tor_log(severity, LD_OR, "We needed to load a secret key from %s, " + "but it was encrypted. Try 'tor --keygen' instead, so you " + "can enter the passphrase.", + secret_fname); + } else if (offline_secret) { + tor_log(severity, LD_OR, "We wanted to load a secret key from %s, " + "but you're keeping it offline. (OfflineMasterKey is set.)", + secret_fname); + } else { + tor_log(severity, LD_OR, "We needed to load a secret key from %s, " + "but couldn't find it. %s", secret_fname, + (flags & INIT_ED_KEY_SUGGEST_KEYGEN) ? + "If you're keeping your master secret key offline, you will " + "need to run 'tor --keygen' to generate new signing keys." : + "Did you forget to copy it over when you copied the rest of the " + "signing key material?"); + } + goto err; + } + + /* If it's absent, and we're not supposed to make a new keypair, fail. */ + if (!have_secret && !found_public && !(flags & INIT_ED_KEY_CREATE)) { + if (split) { + tor_log(severity, LD_OR, "No key found in %s or %s.", + secret_fname, public_fname); + } else { + tor_log(severity, LD_OR, "No key found in %s.", secret_fname); + } + goto err; + } + + /* If the secret key is absent, but the encrypted key would be present, + * that's an error */ + if (!have_secret && !found_public && have_encrypted_secret_file) { + tor_assert(!encrypt_key); + tor_log(severity, LD_OR, "Found an encrypted secret key, " + "but not public key file %s!", public_fname); + goto err; + } + + /* if it's absent, make a new keypair... */ + if (!have_secret && !found_public) { + tor_free(keypair); + keypair = ed_key_new(signing_key, flags, now, lifetime, + cert_type, &cert); + if (!keypair) { + tor_log(severity, LD_OR, "Couldn't create keypair"); + goto err; + } + created_pk = created_sk = created_cert = 1; + } + + /* Write it to disk if we're supposed to do with a new passphrase, or if + * we just created it. */ + if (created_sk || (have_secret && get_options()->change_key_passphrase)) { + if (write_secret_key(&keypair->seckey, + encrypt_key, + secret_fname, tag, encrypted_secret_fname) < 0 + || + (split && + ed25519_pubkey_write_to_file(&keypair->pubkey, public_fname, tag) < 0) + || + (cert && + crypto_write_tagged_contents_to_file(cert_fname, "ed25519v1-cert", + tag, cert->encoded, cert->encoded_len) < 0)) { + tor_log(severity, LD_OR, "Couldn't write keys or cert to file."); + goto err; + } + goto done; + } + + /* If we're not supposed to get a cert, we're done. */ + if (! (flags & INIT_ED_KEY_NEEDCERT)) + goto done; + + /* Read a cert. */ + tor_free(got_tag); + uint8_t certbuf[256]; + ssize_t cert_body_len = crypto_read_tagged_contents_from_file( + cert_fname, "ed25519v1-cert", + &got_tag, certbuf, sizeof(certbuf)); + if (cert_body_len >= 0 && !strcmp(got_tag, tag)) + cert = tor_cert_parse(certbuf, cert_body_len); + + /* If we got it, check it to the extent we can. */ + int bad_cert = 0; + + if (! cert) { + tor_log(severity, LD_OR, "Cert was unparseable"); + bad_cert = 1; + } else if (!tor_memeq(cert->signed_key.pubkey, keypair->pubkey.pubkey, + ED25519_PUBKEY_LEN)) { + tor_log(severity, LD_OR, "Cert was for wrong key"); + bad_cert = 1; + } else if (signing_key && + tor_cert_checksig(cert, &signing_key->pubkey, now) < 0) { + tor_log(severity, LD_OR, "Can't check certificate"); + bad_cert = 1; + } else if (cert->cert_expired) { + tor_log(severity, LD_OR, "Certificate is expired"); + bad_cert = 1; + } else if (signing_key && cert->signing_key_included && + ! ed25519_pubkey_eq(&signing_key->pubkey, &cert->signing_key)) { + tor_log(severity, LD_OR, "Certificate signed by unexpectd key!"); + bad_cert = 1; + } + + if (bad_cert) { + tor_cert_free(cert); + cert = NULL; + } + + /* If we got a cert, we're done. */ + if (cert) + goto done; + + /* If we didn't get a cert, and we're not supposed to make one, fail. */ + if (!signing_key || !(flags & INIT_ED_KEY_CREATE)) { + tor_log(severity, LD_OR, "Without signing key, can't create certificate"); + goto err; + } + + /* We have keys but not a certificate, so make one. */ + uint32_t cert_flags = 0; + if (flags & INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT) + cert_flags |= CERT_FLAG_INCLUDE_SIGNING_KEY; + cert = tor_cert_create(signing_key, cert_type, + &keypair->pubkey, + now, lifetime, + cert_flags); + + if (! cert) { + tor_log(severity, LD_OR, "Couldn't create certificate"); + goto err; + } + + /* Write it to disk. */ + created_cert = 1; + if (crypto_write_tagged_contents_to_file(cert_fname, "ed25519v1-cert", + tag, cert->encoded, cert->encoded_len) < 0) { + tor_log(severity, LD_OR, "Couldn't write cert to disk."); + goto err; + } + + done: + if (cert_out) + *cert_out = cert; + else + tor_cert_free(cert); + + goto cleanup; + + err: + if (keypair) + memwipe(keypair, 0, sizeof(*keypair)); + tor_free(keypair); + tor_cert_free(cert); + if (cert_out) + *cert_out = NULL; + if (created_sk) + unlink(secret_fname); + if (created_pk) + unlink(public_fname); + if (created_cert) + unlink(cert_fname); + + cleanup: + tor_free(encrypted_secret_fname); + tor_free(secret_fname); + tor_free(public_fname); + tor_free(cert_fname); + tor_free(got_tag); + + return keypair; +} + +/** + * Create a new signing key and (optionally) certficiate; do not read or write + * from disk. See ed_key_init_from_file() for more information. + */ +ed25519_keypair_t * +ed_key_new(const ed25519_keypair_t *signing_key, + uint32_t flags, + time_t now, + time_t lifetime, + uint8_t cert_type, + struct tor_cert_st **cert_out) +{ + if (cert_out) + *cert_out = NULL; + + const int extra_strong = !! (flags & INIT_ED_KEY_EXTRA_STRONG); + ed25519_keypair_t *keypair = tor_malloc_zero(sizeof(ed25519_keypair_t)); + if (ed25519_keypair_generate(keypair, extra_strong) < 0) + goto err; + + if (! (flags & INIT_ED_KEY_NEEDCERT)) + return keypair; + + tor_assert(signing_key); + tor_assert(cert_out); + uint32_t cert_flags = 0; + if (flags & INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT) + cert_flags |= CERT_FLAG_INCLUDE_SIGNING_KEY; + tor_cert_t *cert = tor_cert_create(signing_key, cert_type, + &keypair->pubkey, + now, lifetime, + cert_flags); + if (! cert) + goto err; + + *cert_out = cert; + return keypair; + + err: + tor_free(keypair); + return NULL; +} + +static ed25519_keypair_t *master_identity_key = NULL; +static ed25519_keypair_t *master_signing_key = NULL; +static ed25519_keypair_t *current_auth_key = NULL; +static tor_cert_t *signing_key_cert = NULL; +static tor_cert_t *link_cert_cert = NULL; +static tor_cert_t *auth_key_cert = NULL; + +static uint8_t *rsa_ed_crosscert = NULL; +static size_t rsa_ed_crosscert_len = 0; + +/** + * Running as a server: load, reload, or refresh our ed25519 keys and + * certificates, creating and saving new ones as needed. + */ +int +load_ed_keys(const or_options_t *options, time_t now) +{ + ed25519_keypair_t *id = NULL; + ed25519_keypair_t *sign = NULL; + ed25519_keypair_t *auth = NULL; + const ed25519_keypair_t *sign_signing_key_with_id = NULL; + const ed25519_keypair_t *use_signing = NULL; + const tor_cert_t *check_signing_cert = NULL; + tor_cert_t *sign_cert = NULL; + tor_cert_t *auth_cert = NULL; + +#define FAIL(msg) do { \ + log_warn(LD_OR, (msg)); \ + goto err; \ + } while (0) +#define SET_KEY(key, newval) do { \ + if ((key) != (newval)) \ + ed25519_keypair_free(key); \ + key = (newval); \ + } while (0) +#define SET_CERT(cert, newval) do { \ + if ((cert) != (newval)) \ + tor_cert_free(cert); \ + cert = (newval); \ + } while (0) +#define EXPIRES_SOON(cert, interval) \ + (!(cert) || (cert)->valid_until < now + (interval)) + + /* XXXX support encrypted identity keys fully */ + + /* First try to get the signing key to see how it is. */ + { + char *fname = + options_get_datadir_fname2(options, "keys", "ed25519_signing"); + sign = ed_key_init_from_file( + fname, + INIT_ED_KEY_NEEDCERT| + INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT, + LOG_INFO, + NULL, 0, 0, CERT_TYPE_ID_SIGNING, &sign_cert); + tor_free(fname); + check_signing_cert = sign_cert; + use_signing = sign; + } + + if (!use_signing && master_signing_key) { + check_signing_cert = signing_key_cert; + use_signing = master_signing_key; + } + + const int offline_master = + options->OfflineMasterKey && options->command != CMD_KEYGEN; + const int need_new_signing_key = + NULL == use_signing || + EXPIRES_SOON(check_signing_cert, 0) || + (options->command == CMD_KEYGEN && ! options->change_key_passphrase); + const int want_new_signing_key = + need_new_signing_key || + EXPIRES_SOON(check_signing_cert, options->TestingSigningKeySlop); + + /* We can only create a master key if we haven't been told that the + * master key will always be offline. Also, if we have a signing key, + * then we shouldn't make a new master ID key. */ + const int can_make_master_id_key = !offline_master && + NULL == use_signing; + + if (need_new_signing_key) { + log_notice(LD_OR, "It looks like I need to generate and sign a new " + "medium-term signing key, because %s. To do that, I need to " + "load%s the permanent master identity key.", + (NULL == use_signing) ? "I don't have one" : + EXPIRES_SOON(check_signing_cert, 0) ? "the one I have is expired" : + "you asked me to make one with --keygen", + can_make_master_id_key ? " (or create)" : ""); + } else if (want_new_signing_key && !offline_master) { + log_notice(LD_OR, "It looks like I should try to generate and sign a " + "new medium-term signing key, because the one I have is " + "going to expire soon. To do that, I'm going to have to try to " + "load the permanent master identity key."); + } else if (want_new_signing_key) { + log_notice(LD_OR, "It looks like I should try to generate and sign a " + "new medium-term signing key, because the one I have is " + "going to expire soon. But OfflineMasterKey is set, so I " + "won't try to load a permanent master identity key is set. " + "You will need to use 'tor --keygen' make a new signing key " + "and certificate."); + } + + { + uint32_t flags = + (INIT_ED_KEY_SPLIT| + INIT_ED_KEY_EXTRA_STRONG|INIT_ED_KEY_NO_REPAIR); + if (can_make_master_id_key) + flags |= INIT_ED_KEY_CREATE; + if (! need_new_signing_key) + flags |= INIT_ED_KEY_MISSING_SECRET_OK; + if (! want_new_signing_key || offline_master) + flags |= INIT_ED_KEY_OMIT_SECRET; + if (offline_master) + flags |= INIT_ED_KEY_OFFLINE_SECRET; + if (options->command == CMD_KEYGEN) + flags |= INIT_ED_KEY_TRY_ENCRYPTED; + + /* Check the key directory */ + if (check_private_dir(options->DataDirectory, CPD_CREATE, options->User)) { + log_err(LD_OR, "Can't create/check datadirectory %s", + options->DataDirectory); + goto err; + } + char *fname = get_datadir_fname("keys"); + if (check_private_dir(fname, CPD_CREATE, options->User) < 0) { + log_err(LD_OR, "Problem creating/checking key directory %s", fname); + tor_free(fname); + goto err; + } + tor_free(fname); + if (options->master_key_fname) { + fname = tor_strdup(options->master_key_fname); + flags |= INIT_ED_KEY_EXPLICIT_FNAME; + } else { + fname = options_get_datadir_fname2(options, "keys", "ed25519_master_id"); + } + id = ed_key_init_from_file( + fname, + flags, + LOG_WARN, NULL, 0, 0, 0, NULL); + tor_free(fname); + if (!id) { + if (need_new_signing_key) { + if (offline_master) + FAIL("Can't load master identity key; OfflineMasterKey is set."); + else + FAIL("Missing identity key"); + } else { + log_warn(LD_OR, "Master public key was absent; inferring from " + "public key in signing certificate and saving to disk."); + tor_assert(check_signing_cert); + id = tor_malloc_zero(sizeof(*id)); + memcpy(&id->pubkey, &check_signing_cert->signing_key, + sizeof(ed25519_public_key_t)); + fname = options_get_datadir_fname2(options, "keys", + "ed25519_master_id_public_key"); + if (ed25519_pubkey_write_to_file(&id->pubkey, fname, "type0") < 0) { + log_warn(LD_OR, "Error while attempting to write master public key " + "to disk"); + tor_free(fname); + goto err; + } + tor_free(fname); + } + } + if (tor_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey))) + sign_signing_key_with_id = NULL; + else + sign_signing_key_with_id = id; + } + + if (master_identity_key && + !ed25519_pubkey_eq(&id->pubkey, &master_identity_key->pubkey)) { + FAIL("Identity key on disk does not match key we loaded earlier!"); + } + + if (need_new_signing_key && NULL == sign_signing_key_with_id) + FAIL("Can't load master key make a new signing key."); + + if (sign_cert) { + if (! sign_cert->signing_key_included) + FAIL("Loaded a signing cert with no key included!"); + if (! ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey)) + FAIL("The signing cert we have was not signed with the master key " + "we loaded!"); + if (tor_cert_checksig(sign_cert, &id->pubkey, 0) < 0) + FAIL("The signing cert we loaded was not signed correctly!"); + } + + if (want_new_signing_key && sign_signing_key_with_id) { + uint32_t flags = (INIT_ED_KEY_CREATE| + INIT_ED_KEY_REPLACE| + INIT_ED_KEY_EXTRA_STRONG| + INIT_ED_KEY_NEEDCERT| + INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT); + char *fname = + options_get_datadir_fname2(options, "keys", "ed25519_signing"); + ed25519_keypair_free(sign); + tor_cert_free(sign_cert); + sign = ed_key_init_from_file(fname, + flags, LOG_WARN, + sign_signing_key_with_id, now, + options->SigningKeyLifetime, + CERT_TYPE_ID_SIGNING, &sign_cert); + tor_free(fname); + if (!sign) + FAIL("Missing signing key"); + use_signing = sign; + + tor_assert(sign_cert->signing_key_included); + tor_assert(ed25519_pubkey_eq(&sign_cert->signing_key, &id->pubkey)); + tor_assert(ed25519_pubkey_eq(&sign_cert->signed_key, &sign->pubkey)); + } else if (want_new_signing_key) { + static ratelim_t missing_master = RATELIM_INIT(3600); + log_fn_ratelim(&missing_master, LOG_WARN, LD_OR, + "Signing key will expire soon, but I can't load the " + "master key to sign a new one!"); + } + + tor_assert(use_signing); + + /* At this point we no longer need our secret identity key. So wipe + * it, if we loaded it in the first place. */ + memwipe(id->seckey.seckey, 0, sizeof(id->seckey)); + + if (options->command == CMD_KEYGEN) + goto end; + + if (!rsa_ed_crosscert && server_mode(options)) { + uint8_t *crosscert; + ssize_t crosscert_len = tor_make_rsa_ed25519_crosscert(&id->pubkey, + get_server_identity_key(), + now+10*365*86400,/*XXXX*/ + &crosscert); + rsa_ed_crosscert_len = crosscert_len; + rsa_ed_crosscert = crosscert; + } + + if (!current_auth_key || + EXPIRES_SOON(auth_key_cert, options->TestingAuthKeySlop)) { + auth = ed_key_new(use_signing, INIT_ED_KEY_NEEDCERT, + now, + options->TestingAuthKeyLifetime, + CERT_TYPE_SIGNING_AUTH, &auth_cert); + + if (!auth) + FAIL("Can't create auth key"); + } + + /* We've generated or loaded everything. Put them in memory. */ + + end: + if (! master_identity_key) { + SET_KEY(master_identity_key, id); + } else { + tor_free(id); + } + if (sign) { + SET_KEY(master_signing_key, sign); + SET_CERT(signing_key_cert, sign_cert); + } + if (auth) { + SET_KEY(current_auth_key, auth); + SET_CERT(auth_key_cert, auth_cert); + } + + return 0; + err: + ed25519_keypair_free(id); + ed25519_keypair_free(sign); + ed25519_keypair_free(auth); + tor_cert_free(sign_cert); + tor_cert_free(auth_cert); + return -1; +} + +/* DOCDOC */ +int +generate_ed_link_cert(const or_options_t *options, time_t now) +{ + const tor_x509_cert_t *link = NULL, *id = NULL; + tor_cert_t *link_cert = NULL; + + if (tor_tls_get_my_certs(1, &link, &id) < 0 || link == NULL) { + log_warn(LD_OR, "Can't get my x509 link cert."); + return -1; + } + + const common_digests_t *digests = tor_x509_cert_get_cert_digests(link); + + if (link_cert_cert && + ! EXPIRES_SOON(link_cert_cert, options->TestingLinkKeySlop) && + fast_memeq(digests->d[DIGEST_SHA256], link_cert_cert->signed_key.pubkey, + DIGEST256_LEN)) { + return 0; + } + + ed25519_public_key_t dummy_key; + memcpy(dummy_key.pubkey, digests->d[DIGEST_SHA256], DIGEST256_LEN); + + link_cert = tor_cert_create(get_master_signing_keypair(), + CERT_TYPE_SIGNING_LINK, + &dummy_key, + now, + options->TestingLinkCertLifetime, 0); + + if (link_cert) { + SET_CERT(link_cert_cert, link_cert); + } + return 0; +} + +#undef FAIL +#undef SET_KEY +#undef SET_CERT + +int +should_make_new_ed_keys(const or_options_t *options, const time_t now) +{ + if (!master_identity_key || + !master_signing_key || + !current_auth_key || + !link_cert_cert || + EXPIRES_SOON(signing_key_cert, options->TestingSigningKeySlop) || + EXPIRES_SOON(auth_key_cert, options->TestingAuthKeySlop) || + EXPIRES_SOON(link_cert_cert, options->TestingLinkKeySlop)) + return 1; + + const tor_x509_cert_t *link = NULL, *id = NULL; + + if (tor_tls_get_my_certs(1, &link, &id) < 0 || link == NULL) + return 1; + + const common_digests_t *digests = tor_x509_cert_get_cert_digests(link); + + if (!fast_memeq(digests->d[DIGEST_SHA256], + link_cert_cert->signed_key.pubkey, + DIGEST256_LEN)) { + return 1; + } + + return 0; +} + +#undef EXPIRES_SOON + +const ed25519_public_key_t * +get_master_identity_key(void) +{ + if (!master_identity_key) + return NULL; + return &master_identity_key->pubkey; +} + +const ed25519_keypair_t * +get_master_signing_keypair(void) +{ + return master_signing_key; +} + +const struct tor_cert_st * +get_master_signing_key_cert(void) +{ + return signing_key_cert; +} + +const ed25519_keypair_t * +get_current_auth_keypair(void) +{ + return current_auth_key; +} + +const tor_cert_t * +get_current_link_cert_cert(void) +{ + return link_cert_cert; +} + +const tor_cert_t * +get_current_auth_key_cert(void) +{ + return auth_key_cert; +} + +void +get_master_rsa_crosscert(const uint8_t **cert_out, + size_t *size_out) +{ + *cert_out = rsa_ed_crosscert; + *size_out = rsa_ed_crosscert_len; +} + +/** Construct cross-certification for the master identity key with + * the ntor onion key. Store the sign of the corresponding ed25519 public key + * in *<b>sign_out</b>. */ +tor_cert_t * +make_ntor_onion_key_crosscert(const curve25519_keypair_t *onion_key, + const ed25519_public_key_t *master_id_key, time_t now, time_t lifetime, + int *sign_out) +{ + tor_cert_t *cert = NULL; + ed25519_keypair_t ed_onion_key; + + if (ed25519_keypair_from_curve25519_keypair(&ed_onion_key, sign_out, + onion_key) < 0) + goto end; + + cert = tor_cert_create(&ed_onion_key, CERT_TYPE_ONION_ID, master_id_key, + now, lifetime, 0); + + end: + memwipe(&ed_onion_key, 0, sizeof(ed_onion_key)); + return cert; +} + +/** Construct and return an RSA signature for the TAP onion key to + * cross-certify the RSA and Ed25519 identity keys. Set <b>len_out</b> to its + * length. */ +uint8_t * +make_tap_onion_key_crosscert(const crypto_pk_t *onion_key, + const ed25519_public_key_t *master_id_key, + const crypto_pk_t *rsa_id_key, + int *len_out) +{ + uint8_t signature[PK_BYTES]; + uint8_t signed_data[DIGEST_LEN + ED25519_PUBKEY_LEN]; + + *len_out = 0; + crypto_pk_get_digest(rsa_id_key, (char*)signed_data); + memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN); + + int r = crypto_pk_private_sign(onion_key, + (char*)signature, sizeof(signature), + (const char*)signed_data, sizeof(signed_data)); + if (r < 0) + return NULL; + + *len_out = r; + + return tor_memdup(signature, r); +} + +/** Check whether an RSA-TAP cross-certification is correct. Return 0 if it + * is, -1 if it isn't. */ +int +check_tap_onion_key_crosscert(const uint8_t *crosscert, + int crosscert_len, + const crypto_pk_t *onion_pkey, + const ed25519_public_key_t *master_id_pkey, + const uint8_t *rsa_id_digest) +{ + uint8_t *cc = tor_malloc(crypto_pk_keysize(onion_pkey)); + int cc_len = + crypto_pk_public_checksig(onion_pkey, + (char*)cc, + crypto_pk_keysize(onion_pkey), + (const char*)crosscert, + crosscert_len); + if (cc_len < 0) { + goto err; + } + if (cc_len < DIGEST_LEN + ED25519_PUBKEY_LEN) { + log_warn(LD_DIR, "Short signature on cross-certification with TAP key"); + goto err; + } + if (tor_memneq(cc, rsa_id_digest, DIGEST_LEN) || + tor_memneq(cc + DIGEST_LEN, master_id_pkey->pubkey, + ED25519_PUBKEY_LEN)) { + log_warn(LD_DIR, "Incorrect cross-certification with TAP key"); + goto err; + } + + tor_free(cc); + return 0; + err: + tor_free(cc); + return -1; +} + +void +routerkeys_free_all(void) +{ + ed25519_keypair_free(master_identity_key); + ed25519_keypair_free(master_signing_key); + ed25519_keypair_free(current_auth_key); + tor_cert_free(signing_key_cert); + tor_cert_free(link_cert_cert); + tor_cert_free(auth_key_cert); + + master_identity_key = master_signing_key = NULL; + current_auth_key = NULL; + signing_key_cert = link_cert_cert = auth_key_cert = NULL; +} + diff --git a/src/or/routerkeys.h b/src/or/routerkeys.h new file mode 100644 index 0000000000..be9b19aea8 --- /dev/null +++ b/src/or/routerkeys.h @@ -0,0 +1,77 @@ +/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_ROUTERKEYS_H +#define TOR_ROUTERKEYS_H + +#include "crypto_ed25519.h" + +#define INIT_ED_KEY_CREATE (1u<<0) +#define INIT_ED_KEY_REPLACE (1u<<1) +#define INIT_ED_KEY_SPLIT (1u<<2) +#define INIT_ED_KEY_MISSING_SECRET_OK (1u<<3) +#define INIT_ED_KEY_NEEDCERT (1u<<4) +#define INIT_ED_KEY_EXTRA_STRONG (1u<<5) +#define INIT_ED_KEY_INCLUDE_SIGNING_KEY_IN_CERT (1u<<6) +#define INIT_ED_KEY_OMIT_SECRET (1u<<7) +#define INIT_ED_KEY_TRY_ENCRYPTED (1u<<8) +#define INIT_ED_KEY_NO_REPAIR (1u<<9) +#define INIT_ED_KEY_SUGGEST_KEYGEN (1u<<10) +#define INIT_ED_KEY_OFFLINE_SECRET (1u<<11) +#define INIT_ED_KEY_EXPLICIT_FNAME (1u<<12) + +struct tor_cert_st; +ed25519_keypair_t *ed_key_init_from_file(const char *fname, uint32_t flags, + int severity, + const ed25519_keypair_t *signing_key, + time_t now, + time_t lifetime, + uint8_t cert_type, + struct tor_cert_st **cert_out); +ed25519_keypair_t *ed_key_new(const ed25519_keypair_t *signing_key, + uint32_t flags, + time_t now, + time_t lifetime, + uint8_t cert_type, + struct tor_cert_st **cert_out); +const ed25519_public_key_t *get_master_identity_key(void); +const ed25519_keypair_t *get_master_signing_keypair(void); +const struct tor_cert_st *get_master_signing_key_cert(void); + +const ed25519_keypair_t *get_current_auth_keypair(void); +const struct tor_cert_st *get_current_link_cert_cert(void); +const struct tor_cert_st *get_current_auth_key_cert(void); + +void get_master_rsa_crosscert(const uint8_t **cert_out, + size_t *size_out); + +struct tor_cert_st *make_ntor_onion_key_crosscert( + const curve25519_keypair_t *onion_key, + const ed25519_public_key_t *master_id_key, + time_t now, time_t lifetime, + int *sign_out); +uint8_t *make_tap_onion_key_crosscert(const crypto_pk_t *onion_key, + const ed25519_public_key_t *master_id_key, + const crypto_pk_t *rsa_id_key, + int *len_out); + +int check_tap_onion_key_crosscert(const uint8_t *crosscert, + int crosscert_len, + const crypto_pk_t *onion_pkey, + const ed25519_public_key_t *master_id_pkey, + const uint8_t *rsa_id_digest); + +int load_ed_keys(const or_options_t *options, time_t now); +int should_make_new_ed_keys(const or_options_t *options, const time_t now); + +int generate_ed_link_cert(const or_options_t *options, time_t now); + +int read_encrypted_secret_key(ed25519_secret_key_t *out, + const char *fname); +int write_encrypted_secret_key(const ed25519_secret_key_t *out, + const char *fname); + +void routerkeys_free_all(void); + +#endif + diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 9ad763c4d1..64baf4d709 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,6 +13,8 @@ #define ROUTERLIST_PRIVATE #include "or.h" +#include "backtrace.h" +#include "crypto_ed25519.h" #include "circuitstats.h" #include "config.h" #include "connection.h" @@ -37,6 +39,8 @@ #include "routerlist.h" #include "routerparse.h" #include "routerset.h" +#include "sandbox.h" +#include "torcert.h" // #define DEBUG_ROUTERLIST @@ -64,8 +68,6 @@ typedef struct cert_list_t cert_list_t; static int compute_weighted_bandwidths(const smartlist_t *sl, bandwidth_weight_rule_t rule, u64_dbl_t **bandwidths_out); -static const routerstatus_t *router_pick_directory_server_impl( - dirinfo_type_t auth, int flags); static const routerstatus_t *router_pick_trusteddirserver_impl( const smartlist_t *sourcelist, dirinfo_type_t auth, int flags, int *n_busy_out); @@ -79,6 +81,7 @@ static const char *signed_descriptor_get_body_impl( const signed_descriptor_t *desc, int with_annotations); static void list_pending_downloads(digestmap_t *result, + digest256map_t *result256, int purpose, const char *prefix); static void list_pending_fpsk_downloads(fp_pair_map_t *result); static void launch_dummy_descriptor_download_as_needed(time_t now, @@ -98,7 +101,8 @@ static smartlist_t *trusted_dir_servers = NULL; * and all fallback directory servers. */ static smartlist_t *fallback_dir_servers = NULL; -/** List of for a given authority, and download status for latest certificate. +/** List of certificates for a single authority, and download status for + * latest certificate. */ struct cert_list_t { /* @@ -130,16 +134,6 @@ static smartlist_t *warned_nicknames = NULL; * download is low. */ static time_t last_descriptor_download_attempted = 0; -/** When we last computed the weights to use for bandwidths on directory - * requests, what were the total weighted bandwidth, and our share of that - * bandwidth? Used to determine what fraction of directory requests we should - * expect to see. - * - * @{ */ -static uint64_t sl_last_total_weighted_bw = 0, - sl_last_weighted_bw_of_me = 0; -/**@}*/ - /** Return the number of directory authorities whose type matches some bit set * in <b>type</b> */ int @@ -154,6 +148,22 @@ get_n_authorities(dirinfo_type_t type) return n; } +/** Initialise schedule, want_authority, and increment on in the download + * status dlstatus, then call download_status_reset() on it. + * It is safe to call this function or download_status_reset() multiple times + * on a new dlstatus. But it should *not* be called after a dlstatus has been + * used to count download attempts or failures. */ +static void +download_status_cert_init(download_status_t *dlstatus) +{ + dlstatus->schedule = DL_SCHED_CONSENSUS; + dlstatus->want_authority = DL_WANT_ANY_DIRSERVER; + dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE; + + /* Use the new schedule to set next_attempt_at */ + download_status_reset(dlstatus); +} + /** Reset the download status of a specified element in a dsmap */ static void download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest) @@ -174,6 +184,7 @@ download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest) /* Insert before we reset */ dlstatus = tor_malloc_zero(sizeof(*dlstatus)); dsmap_set(cl->dl_status_map, digest, dlstatus); + download_status_cert_init(dlstatus); } tor_assert(dlstatus); /* Go ahead and reset it */ @@ -212,7 +223,7 @@ download_status_is_ready_by_sk_in_cl(cert_list_t *cl, * too. */ dlstatus = tor_malloc_zero(sizeof(*dlstatus)); - download_status_reset(dlstatus); + download_status_cert_init(dlstatus); dsmap_set(cl->dl_status_map, digest, dlstatus); rv = 1; } @@ -220,8 +231,6 @@ download_status_is_ready_by_sk_in_cl(cert_list_t *cl, return rv; } -#define get_n_v2_authorities() get_n_authorities(V2_DIRINFO) - /** Helper: Return the cert_list_t for an authority whose authority ID is * <b>id_digest</b>, allocating a new list if necessary. */ static cert_list_t * @@ -233,7 +242,7 @@ get_cert_list(const char *id_digest) cl = digestmap_get(trusted_dir_certs, id_digest); if (!cl) { cl = tor_malloc_zero(sizeof(cert_list_t)); - cl->dl_status_by_id.schedule = DL_SCHED_CONSENSUS; + download_status_cert_init(&cl->dl_status_by_id); cl->certs = smartlist_new(); cl->dl_status_map = dsmap_new(); digestmap_set(trusted_dir_certs, id_digest, cl); @@ -285,7 +294,7 @@ trusted_dirs_reload_certs(void) /** Helper: return true iff we already have loaded the exact cert * <b>cert</b>. */ -static INLINE int +static inline int already_have_cert(authority_cert_t *cert) { cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest); @@ -449,7 +458,7 @@ trusted_dirs_flush_certs_to_disk(void) } DIGESTMAP_FOREACH_END; filename = get_datadir_fname("cached-certs"); - if (write_chunks_to_file(filename, chunks, 0)) { + if (write_chunks_to_file(filename, chunks, 0, 0)) { log_warn(LD_FS, "Error writing certificates to disk."); } tor_free(filename); @@ -459,46 +468,69 @@ trusted_dirs_flush_certs_to_disk(void) trusted_dir_servers_certs_changed = 0; } -/** Remove all v3 authority certificates that have been superseded for more - * than 48 hours. (If the most recent cert was published more than 48 hours - * ago, then we aren't going to get any consensuses signed with older +static int +compare_certs_by_pubdates(const void **_a, const void **_b) +{ + const authority_cert_t *cert1 = *_a, *cert2=*_b; + + if (cert1->cache_info.published_on < cert2->cache_info.published_on) + return -1; + else if (cert1->cache_info.published_on > cert2->cache_info.published_on) + return 1; + else + return 0; +} + +/** Remove all expired v3 authority certificates that have been superseded for + * more than 48 hours or, if not expired, that were published more than 7 days + * before being superseded. (If the most recent cert was published more than 48 + * hours ago, then we aren't going to get any consensuses signed with older * keys.) */ static void trusted_dirs_remove_old_certs(void) { time_t now = time(NULL); #define DEAD_CERT_LIFETIME (2*24*60*60) -#define OLD_CERT_LIFETIME (7*24*60*60) +#define SUPERSEDED_CERT_LIFETIME (2*24*60*60) if (!trusted_dir_certs) return; DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) { - authority_cert_t *newest = NULL; - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - if (!newest || (cert->cache_info.published_on > - newest->cache_info.published_on)) - newest = cert); - if (newest) { - const time_t newest_published = newest->cache_info.published_on; - SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) { - int expired; - time_t cert_published; - if (newest == cert) - continue; - expired = now > cert->expires; - cert_published = cert->cache_info.published_on; - /* Store expired certs for 48 hours after a newer arrives; + /* Sort the list from first-published to last-published */ + smartlist_sort(cl->certs, compare_certs_by_pubdates); + + SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) { + if (cert_sl_idx == smartlist_len(cl->certs) - 1) { + /* This is the most recently published cert. Keep it. */ + continue; + } + authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1); + const time_t next_cert_published = next_cert->cache_info.published_on; + if (next_cert_published > now) { + /* All later certs are published in the future. Keep everything + * we didn't discard. */ + break; + } + int should_remove = 0; + if (cert->expires + DEAD_CERT_LIFETIME < now) { + /* Certificate has been expired for at least DEAD_CERT_LIFETIME. + * Remove it. */ + should_remove = 1; + } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) { + /* Certificate has been superseded for OLD_CERT_LIFETIME. + * Remove it. */ - if (expired ? - (newest_published + DEAD_CERT_LIFETIME < now) : - (cert_published + OLD_CERT_LIFETIME < newest_published)) { - SMARTLIST_DEL_CURRENT(cl->certs, cert); - authority_cert_free(cert); - trusted_dir_servers_certs_changed = 1; - } - } SMARTLIST_FOREACH_END(cert); - } + should_remove = 1; + } + if (should_remove) { + SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert); + authority_cert_free(cert); + trusted_dir_servers_certs_changed = 1; + } + } SMARTLIST_FOREACH_END(cert); + } DIGESTMAP_FOREACH_END; +#undef DEAD_CERT_LIFETIME #undef OLD_CERT_LIFETIME trusted_dirs_flush_certs_to_disk(); @@ -647,7 +679,7 @@ static const char *BAD_SIGNING_KEYS[] = { NULL, }; -/** DOCDOC */ +/* DOCDOC */ int authority_cert_is_blacklisted(const authority_cert_t *cert) { @@ -681,9 +713,6 @@ authority_cert_dl_looks_uncertain(const char *id_digest) return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER; } -/** How many times will we try to fetch a certificate before giving up? */ -#define MAX_CERT_DL_FAILURES 8 - /** Try to download any v3 authority certificates that we may be missing. If * <b>status</b> is provided, try to get all the ones that were used to sign * <b>status</b>. Additionally, try to have a non-expired certificate for @@ -715,7 +744,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) char id_digest_str[2*DIGEST_LEN+1]; char sk_digest_str[2*DIGEST_LEN+1]; - if (should_delay_dir_fetches(get_options())) + if (should_delay_dir_fetches(get_options(), NULL)) return; pending_cert = fp_pair_map_new(); @@ -727,7 +756,8 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) * First, we get the lists of already pending downloads so we don't * duplicate effort. */ - list_pending_downloads(pending_id, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); + list_pending_downloads(pending_id, NULL, + DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); list_pending_fpsk_downloads(pending_cert); /* @@ -755,7 +785,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) } SMARTLIST_FOREACH_END(cert); if (!found && download_status_is_ready(&(cl->dl_status_by_id), now, - MAX_CERT_DL_FAILURES) && + get_options()->TestingCertMaxDownloadTries) && !digestmap_get(pending_id, ds->v3_identity_digest)) { log_info(LD_DIR, "No current certificate known for authority %s " @@ -817,7 +847,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) } if (download_status_is_ready_by_sk_in_cl( cl, sig->signing_key_digest, - now, MAX_CERT_DL_FAILURES) && + now, get_options()->TestingCertMaxDownloadTries) && !fp_pair_map_get_by_digests(pending_cert, voter->identity_digest, sig->signing_key_digest)) { @@ -883,8 +913,11 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) if (smartlist_len(fps) > 1) { resource = smartlist_join_strings(fps, "", 0, NULL); + /* We want certs from mirrors, because they will almost always succeed. + */ directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, - resource, PDS_RETRY_IF_NO_SERVERS); + resource, PDS_RETRY_IF_NO_SERVERS, + DL_WANT_ANY_DIRSERVER); tor_free(resource); } /* else we didn't add any: they were all pending */ @@ -927,8 +960,11 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) if (smartlist_len(fp_pairs) > 1) { resource = smartlist_join_strings(fp_pairs, "", 0, NULL); + /* We want certs from mirrors, because they will almost always succeed. + */ directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, - resource, PDS_RETRY_IF_NO_SERVERS); + resource, PDS_RETRY_IF_NO_SERVERS, + DL_WANT_ANY_DIRSERVER); tor_free(resource); } /* else they were all pending */ @@ -971,7 +1007,7 @@ router_should_rebuild_store(desc_store_t *store) /** Return the desc_store_t in <b>rl</b> that should be used to store * <b>sd</b>. */ -static INLINE desc_store_t * +static inline desc_store_t * desc_get_store(routerlist_t *rl, const signed_descriptor_t *sd) { if (sd->is_extrainfo) @@ -1103,15 +1139,18 @@ router_rebuild_store(int flags, desc_store_t *store) smartlist_add(chunk_list, c); } SMARTLIST_FOREACH_END(sd); - if (write_chunks_to_file(fname_tmp, chunk_list, 1)<0) { + if (write_chunks_to_file(fname_tmp, chunk_list, 1, 1)<0) { log_warn(LD_FS, "Error writing router store to disk."); goto done; } /* Our mmap is now invalid. */ if (store->mmap) { - tor_munmap_file(store->mmap); + int res = tor_munmap_file(store->mmap); store->mmap = NULL; + if (res != 0) { + log_warn(LD_FS, "Unable to munmap route store in %s", fname); + } } if (replace_file(fname_tmp, fname)<0) { @@ -1178,32 +1217,25 @@ router_rebuild_store(int flags, desc_store_t *store) static int router_reload_router_list_impl(desc_store_t *store) { - char *fname = NULL, *altname = NULL, *contents = NULL; + char *fname = NULL, *contents = NULL; struct stat st; - int read_from_old_location = 0; int extrainfo = (store->type == EXTRAINFO_STORE); - time_t now = time(NULL); store->journal_len = store->store_len = 0; fname = get_datadir_fname(store->fname_base); - if (store->fname_alt_base) - altname = get_datadir_fname(store->fname_alt_base); - if (store->mmap) /* get rid of it first */ - tor_munmap_file(store->mmap); - store->mmap = NULL; + if (store->mmap) { + /* get rid of it first */ + int res = tor_munmap_file(store->mmap); + store->mmap = NULL; + if (res != 0) { + log_warn(LD_FS, "Failed to munmap %s", fname); + tor_free(fname); + return -1; + } + } store->mmap = tor_mmap_file(fname); - if (!store->mmap && altname && file_status(altname) == FN_FILE) { - read_from_old_location = 1; - log_notice(LD_DIR, "Couldn't read %s; trying to load routers from old " - "location %s.", fname, altname); - if ((store->mmap = tor_mmap_file(altname))) - read_from_old_location = 1; - } - if (altname && !read_from_old_location) { - remove_file_if_very_old(altname, now); - } if (store->mmap) { store->store_len = store->mmap->size; if (extrainfo) @@ -1218,16 +1250,9 @@ router_reload_router_list_impl(desc_store_t *store) tor_free(fname); fname = get_datadir_fname_suffix(store->fname_base, ".new"); + /* don't load empty files - we wouldn't get any data, even if we tried */ if (file_status(fname) == FN_FILE) contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st); - if (read_from_old_location) { - tor_free(altname); - altname = get_datadir_fname_suffix(store->fname_alt_base, ".new"); - if (!contents) - contents = read_file_to_str(altname, RFTS_BIN|RFTS_IGNORE_MISSING, &st); - else - remove_file_if_very_old(altname, now); - } if (contents) { if (extrainfo) router_load_extrainfo_from_string(contents, NULL,SAVED_IN_JOURNAL, @@ -1240,9 +1265,8 @@ router_reload_router_list_impl(desc_store_t *store) } tor_free(fname); - tor_free(altname); - if (store->journal_len || read_from_old_location) { + if (store->journal_len) { /* Always clear the journal on startup.*/ router_rebuild_store(RRS_FORCE, store); } else if (!extrainfo) { @@ -1293,8 +1317,8 @@ router_get_fallback_dir_servers(void) /** Try to find a running dirserver that supports operations of <b>type</b>. * * If there are no running dirservers in our routerlist and the - * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the authoritative ones - * as running again, and pick one. + * <b>PDS_RETRY_IF_NO_SERVERS</b> flag is set, set all the fallback ones + * (including authorities) as running again, and pick one. * * If the <b>PDS_IGNORE_FASCISTFIREWALL</b> flag is set, then include * dirservers that we can't reach. @@ -1302,74 +1326,42 @@ router_get_fallback_dir_servers(void) * If the <b>PDS_ALLOW_SELF</b> flag is not set, then don't include ourself * (if we're a dirserver). * - * Don't pick an authority if any non-authority is viable; try to avoid using - * servers that have returned 503 recently. + * Don't pick a fallback directory mirror if any non-fallback is viable; + * (the fallback directory mirrors include the authorities) + * try to avoid using servers that have returned 503 recently. */ const routerstatus_t * router_pick_directory_server(dirinfo_type_t type, int flags) { + int busy = 0; const routerstatus_t *choice; - if (get_options()->PreferTunneledDirConns) - flags |= PDS_PREFER_TUNNELED_DIR_CONNS_; if (!routerlist) return NULL; - choice = router_pick_directory_server_impl(type, flags); + choice = router_pick_directory_server_impl(type, flags, &busy); if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS)) return choice; + if (busy) { + /* If the reason that we got no server is that servers are "busy", + * we must be excluding good servers because we already have serverdesc + * fetches with them. Do not mark down servers up because of this. */ + tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| + PDS_NO_EXISTING_MICRODESC_FETCH))); + return NULL; + } + log_info(LD_DIR, "No reachable router entries for dirservers. " "Trying them all again."); - /* mark all authdirservers as up again */ + /* mark all fallback directory mirrors as up again */ mark_all_dirservers_up(fallback_dir_servers); /* try again */ - choice = router_pick_directory_server_impl(type, flags); + choice = router_pick_directory_server_impl(type, flags, NULL); return choice; } -/** Try to determine which fraction of v2 and v3 directory requests aimed at - * caches will be sent to us. Set *<b>v2_share_out</b> and - * *<b>v3_share_out</b> to the fractions of v2 and v3 protocol shares we - * expect to see, respectively. Return 0 on success, negative on failure. */ -int -router_get_my_share_of_directory_requests(double *v2_share_out, - double *v3_share_out) -{ - const routerinfo_t *me = router_get_my_routerinfo(); - const routerstatus_t *rs; - const int pds_flags = PDS_ALLOW_SELF|PDS_IGNORE_FASCISTFIREWALL; - *v2_share_out = *v3_share_out = 0.0; - if (!me) - return -1; - rs = router_get_consensus_status_by_id(me->cache_info.identity_digest); - if (!rs) - return -1; - - /* Calling for side effect */ - /* XXXX This is a bit of a kludge */ - if (rs->is_v2_dir) { - sl_last_total_weighted_bw = 0; - router_pick_directory_server(V2_DIRINFO, pds_flags); - if (sl_last_total_weighted_bw != 0) { - *v2_share_out = U64_TO_DBL(sl_last_weighted_bw_of_me) / - U64_TO_DBL(sl_last_total_weighted_bw); - } - } - - { - sl_last_total_weighted_bw = 0; - router_pick_directory_server(V3_DIRINFO, pds_flags); - if (sl_last_total_weighted_bw != 0) { - *v3_share_out = U64_TO_DBL(sl_last_weighted_bw_of_me) / - U64_TO_DBL(sl_last_total_weighted_bw); - } - } - - return 0; -} - /** Return the dir_server_t for the directory authority whose identity * key hashes to <b>digest</b>, or NULL if no such authority is known. */ @@ -1389,15 +1381,21 @@ router_get_trusteddirserver_by_digest(const char *digest) } /** Return the dir_server_t for the fallback dirserver whose identity - * key hashes to <b>digest</b>, or NULL if no such authority is known. + * key hashes to <b>digest</b>, or NULL if no such fallback is in the list of + * fallback_dir_servers. (fallback_dir_servers is affected by the FallbackDir + * and UseDefaultFallbackDirs torrc options.) + * The list of fallback directories includes the list of authorities. */ dir_server_t * router_get_fallback_dirserver_by_digest(const char *digest) { - if (!trusted_dir_servers) + if (!fallback_dir_servers) return NULL; - SMARTLIST_FOREACH(trusted_dir_servers, dir_server_t *, ds, + if (!digest) + return NULL; + + SMARTLIST_FOREACH(fallback_dir_servers, dir_server_t *, ds, { if (tor_memeq(ds->digest, digest, DIGEST_LEN)) return ds; @@ -1406,6 +1404,18 @@ router_get_fallback_dirserver_by_digest(const char *digest) return NULL; } +/** Return 1 if any fallback dirserver's identity key hashes to <b>digest</b>, + * or 0 if no such fallback is in the list of fallback_dir_servers. + * (fallback_dir_servers is affected by the FallbackDir and + * UseDefaultFallbackDirs torrc options.) + * The list of fallback directories includes the list of authorities. + */ +int +router_digest_is_fallback_dir(const char *digest) +{ + return (router_get_fallback_dirserver_by_digest(digest) != NULL); +} + /** Return the dir_server_t for the directory authority whose * v3 identity key hashes to <b>digest</b>, or NULL if no such authority * is known. @@ -1435,7 +1445,7 @@ router_pick_trusteddirserver(dirinfo_type_t type, int flags) return router_pick_dirserver_generic(trusted_dir_servers, type, flags); } -/** Try to find a running fallback directory Flags are as for +/** Try to find a running fallback directory. Flags are as for * router_pick_directory_server. */ const routerstatus_t * @@ -1444,7 +1454,7 @@ router_pick_fallback_dirserver(dirinfo_type_t type, int flags) return router_pick_dirserver_generic(fallback_dir_servers, type, flags); } -/** Try to find a running fallback directory Flags are as for +/** Try to find a running fallback directory. Flags are as for * router_pick_directory_server. */ static const routerstatus_t * @@ -1453,8 +1463,6 @@ router_pick_dirserver_generic(smartlist_t *sourcelist, { const routerstatus_t *choice; int busy = 0; - if (get_options()->PreferTunneledDirConns) - flags |= PDS_PREFER_TUNNELED_DIR_CONNS_; choice = router_pick_trusteddirserver_impl(sourcelist, type, flags, &busy); if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS)) @@ -1474,18 +1482,200 @@ router_pick_dirserver_generic(smartlist_t *sourcelist, return router_pick_trusteddirserver_impl(sourcelist, type, flags, NULL); } +/* Check if we already have a directory fetch from ap, for serverdesc + * (including extrainfo) or microdesc documents. + * If so, return 1, if not, return 0. + * Also returns 0 if addr is NULL, tor_addr_is_null(addr), or dir_port is 0. + */ +STATIC int +router_is_already_dir_fetching(const tor_addr_port_t *ap, int serverdesc, + int microdesc) +{ + if (!ap || tor_addr_is_null(&ap->addr) || !ap->port) { + return 0; + } + + /* XX/teor - we're not checking tunnel connections here, see #17848 + */ + if (serverdesc && ( + connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_SERVERDESC) + || connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_EXTRAINFO))) { + return 1; + } + + if (microdesc && ( + connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &ap->addr, ap->port, DIR_PURPOSE_FETCH_MICRODESC))) { + return 1; + } + + return 0; +} + +/* Check if we already have a directory fetch from ds, for serverdesc + * (including extrainfo) or microdesc documents. + * If so, return 1, if not, return 0. + */ +static int +router_is_already_dir_fetching_ds(const dir_server_t *ds, + int serverdesc, + int microdesc) +{ + tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap; + + /* Assume IPv6 DirPort is the same as IPv4 DirPort */ + tor_addr_from_ipv4h(&ipv4_dir_ap.addr, ds->addr); + ipv4_dir_ap.port = ds->dir_port; + tor_addr_copy(&ipv6_dir_ap.addr, &ds->ipv6_addr); + ipv6_dir_ap.port = ds->dir_port; + + return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc) + || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc)); +} + +/* Check if we already have a directory fetch from rs, for serverdesc + * (including extrainfo) or microdesc documents. + * If so, return 1, if not, return 0. + */ +static int +router_is_already_dir_fetching_rs(const routerstatus_t *rs, + int serverdesc, + int microdesc) +{ + tor_addr_port_t ipv4_dir_ap, ipv6_dir_ap; + + /* Assume IPv6 DirPort is the same as IPv4 DirPort */ + tor_addr_from_ipv4h(&ipv4_dir_ap.addr, rs->addr); + ipv4_dir_ap.port = rs->dir_port; + tor_addr_copy(&ipv6_dir_ap.addr, &rs->ipv6_addr); + ipv6_dir_ap.port = rs->dir_port; + + return (router_is_already_dir_fetching(&ipv4_dir_ap, serverdesc, microdesc) + || router_is_already_dir_fetching(&ipv6_dir_ap, serverdesc, microdesc)); +} + +#ifndef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP +#define LOG_FALSE_POSITIVES_DURING_BOOTSTRAP 0 +#endif + +/* Log a message if rs is not found or not a preferred address */ +static void +router_picked_poor_directory_log(const routerstatus_t *rs) +{ + const networkstatus_t *usable_consensus; + usable_consensus = networkstatus_get_reasonably_live_consensus(time(NULL), + usable_consensus_flavor()); + +#if !LOG_FALSE_POSITIVES_DURING_BOOTSTRAP + /* Don't log early in the bootstrap process, it's normal to pick from a + * small pool of nodes. Of course, this won't help if we're trying to + * diagnose bootstrap issues. */ + if (!smartlist_len(nodelist_get_list()) || !usable_consensus + || !router_have_minimum_dir_info()) { + return; + } +#endif + + /* We couldn't find a node, or the one we have doesn't fit our preferences. + * Sometimes this is normal, sometimes it can be a reachability issue. */ + if (!rs) { + /* This happens a lot, so it's at debug level */ + log_debug(LD_DIR, "Wanted to make an outgoing directory connection, but " + "we couldn't find a directory that fit our criteria. " + "Perhaps we will succeed next time with less strict criteria."); + } else if (!fascist_firewall_allows_rs(rs, FIREWALL_OR_CONNECTION, 1) + && !fascist_firewall_allows_rs(rs, FIREWALL_DIR_CONNECTION, 1) + ) { + /* This is rare, and might be interesting to users trying to diagnose + * connection issues on dual-stack machines. */ + log_info(LD_DIR, "Selected a directory %s with non-preferred OR and Dir " + "addresses for launching an outgoing connection: " + "IPv4 %s OR %d Dir %d IPv6 %s OR %d Dir %d", + routerstatus_describe(rs), + fmt_addr32(rs->addr), rs->or_port, + rs->dir_port, fmt_addr(&rs->ipv6_addr), + rs->ipv6_orport, rs->dir_port); + } +} + +#undef LOG_FALSE_POSITIVES_DURING_BOOTSTRAP + /** How long do we avoid using a directory server after it's given us a 503? */ #define DIR_503_TIMEOUT (60*60) +/* Common retry code for router_pick_directory_server_impl and + * router_pick_trusteddirserver_impl. Retry with the non-preferred IP version. + * Must be called before RETRY_WITHOUT_EXCLUDE(). + * + * If we got no result, and we are applying IP preferences, and we are a + * client that could use an alternate IP version, try again with the + * opposite preferences. */ +#define RETRY_ALTERNATE_IP_VERSION(retry_label) \ + STMT_BEGIN \ + if (result == NULL && try_ip_pref && options->ClientUseIPv4 \ + && fascist_firewall_use_ipv6(options) && !server_mode(options) \ + && !n_busy) { \ + n_excluded = 0; \ + n_busy = 0; \ + try_ip_pref = 0; \ + goto retry_label; \ + } \ + STMT_END \ + +/* Common retry code for router_pick_directory_server_impl and + * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with + * the preferred IP version. Must be called after RETRY_ALTERNATE_IP_VERSION(). + * + * If we got no result, and we are excluding nodes, and StrictNodes is + * not set, try again without excluding nodes. */ +#define RETRY_WITHOUT_EXCLUDE(retry_label) \ + STMT_BEGIN \ + if (result == NULL && try_excluding && !options->StrictNodes \ + && n_excluded && !n_busy) { \ + try_excluding = 0; \ + n_excluded = 0; \ + n_busy = 0; \ + try_ip_pref = 1; \ + goto retry_label; \ + } \ + STMT_END + +/* When iterating through the routerlist, can OR address/port preference + * and reachability checks be skipped? + */ +static int +router_skip_or_reachability(const or_options_t *options, int try_ip_pref) +{ + /* Servers always have and prefer IPv4. + * And if clients are checking against the firewall for reachability only, + * but there's no firewall, don't bother checking */ + return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_or()); +} + +/* When iterating through the routerlist, can Dir address/port preference + * and reachability checks be skipped? + */ +static int +router_skip_dir_reachability(const or_options_t *options, int try_ip_pref) +{ + /* Servers always have and prefer IPv4. + * And if clients are checking against the firewall for reachability only, + * but there's no firewall, don't bother checking */ + return server_mode(options) || (!try_ip_pref && !firewall_is_fascist_dir()); +} + /** Pick a random running valid directory server/mirror from our - * routerlist. Arguments are as for router_pick_directory_server(), except - * that RETRY_IF_NO_SERVERS is ignored, and: + * routerlist. Arguments are as for router_pick_directory_server(), except: * - * If the PDS_PREFER_TUNNELED_DIR_CONNS_ flag is set, prefer directory servers - * that we can use with BEGINDIR. + * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of + * directories that we excluded for no other reason than + * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH. */ -static const routerstatus_t * -router_pick_directory_server_impl(dirinfo_type_t type, int flags) +STATIC const routerstatus_t * +router_pick_directory_server_impl(dirinfo_type_t type, int flags, + int *n_busy_out) { const or_options_t *options = get_options(); const node_t *result; @@ -1494,16 +1684,18 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) smartlist_t *overloaded_direct, *overloaded_tunnel; time_t now = time(NULL); const networkstatus_t *consensus = networkstatus_get_latest_consensus(); - int requireother = ! (flags & PDS_ALLOW_SELF); - int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - int prefer_tunnel = (flags & PDS_PREFER_TUNNELED_DIR_CONNS_); - int for_guard = (flags & PDS_FOR_GUARD); - int try_excluding = 1, n_excluded = 0; + const int requireother = ! (flags & PDS_ALLOW_SELF); + const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); + const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); + const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH); + const int for_guard = (flags & PDS_FOR_GUARD); + int try_excluding = 1, n_excluded = 0, n_busy = 0; + int try_ip_pref = 1; if (!consensus) return NULL; - retry_without_exclude: + retry_search: direct = smartlist_new(); tunnel = smartlist_new(); @@ -1512,30 +1704,28 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) overloaded_direct = smartlist_new(); overloaded_tunnel = smartlist_new(); + const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref); + const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref); + const int must_have_or = directory_must_use_begindir(options); + /* Find all the running dirservers we know about. */ SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) { - int is_trusted; + int is_trusted, is_trusted_extrainfo; int is_overloaded; - tor_addr_t addr; const routerstatus_t *status = node->rs; const country_t country = node->country; if (!status) continue; - if (!node->is_running || !status->dir_port || !node->is_valid) - continue; - if (node->is_bad_directory) + if (!node->is_running || !node_is_dir(node) || !node->is_valid) continue; if (requireother && router_digest_is_me(node->identity)) continue; is_trusted = router_digest_is_trusted_dir(node->identity); - if ((type & V2_DIRINFO) && !(node->rs->is_v2_dir || is_trusted)) - continue; + is_trusted_extrainfo = router_digest_is_trusted_dir_type( + node->identity, EXTRAINFO_DIRINFO); if ((type & EXTRAINFO_DIRINFO) && - !router_supports_extrainfo(node->identity, 0)) - continue; - if ((type & MICRODESC_DIRINFO) && !is_trusted && - !node->rs->version_supports_microdesc_cache) + !router_supports_extrainfo(node->identity, is_trusted_extrainfo)) continue; /* Don't make the same node a guard twice */ if (for_guard && node->using_as_guard) { @@ -1552,18 +1742,30 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) continue; } - /* XXXX IP6 proposal 118 */ - tor_addr_from_ipv4h(&addr, node->rs->addr); + if (router_is_already_dir_fetching_rs(status, + no_serverdesc_fetching, + no_microdesc_fetching)) { + ++n_busy; + continue; + } is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now; - if (prefer_tunnel && - (!fascistfirewall || - fascist_firewall_allows_address_or(&addr, status->or_port))) + /* Clients use IPv6 addresses if the server has one and the client + * prefers IPv6. + * Add the router if its preferred address and port are reachable. + * If we don't get any routers, we'll try again with the non-preferred + * address for each router (if any). (To ensure correct load-balancing + * we try routers that only have one address both times.) + */ + if (!fascistfirewall || skip_or_fw || + fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, + try_ip_pref)) smartlist_add(is_trusted ? trusted_tunnel : is_overloaded ? overloaded_tunnel : tunnel, (void*)node); - else if (!fascistfirewall || - fascist_firewall_allows_address_dir(&addr, status->dir_port)) + else if (!must_have_or && (skip_dir_fw || + fascist_firewall_allows_node(node, FIREWALL_DIR_CONNECTION, + try_ip_pref))) smartlist_add(is_trusted ? trusted_direct : is_overloaded ? overloaded_direct : direct, (void*)node); } SMARTLIST_FOREACH_END(node); @@ -1594,13 +1796,14 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) smartlist_free(overloaded_direct); smartlist_free(overloaded_tunnel); - if (result == NULL && try_excluding && !options->StrictNodes && n_excluded) { - /* If we got no result, and we are excluding nodes, and StrictNodes is - * not set, try again without excluding nodes. */ - try_excluding = 0; - n_excluded = 0; - goto retry_without_exclude; - } + RETRY_ALTERNATE_IP_VERSION(retry_search); + + RETRY_WITHOUT_EXCLUDE(retry_search); + + if (n_busy_out) + *n_busy_out = n_busy; + + router_picked_poor_directory_log(result ? result->rs : NULL); return result ? result->rs : NULL; } @@ -1615,7 +1818,7 @@ dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight) u64_dbl_t *weights; const dir_server_t *ds; - weights = tor_malloc(sizeof(u64_dbl_t) * n); + weights = tor_calloc(n, sizeof(u64_dbl_t)); for (i = 0; i < n; ++i) { ds = smartlist_get(servers, i); weights[i].dbl = ds->weight; @@ -1645,7 +1848,6 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist, time_t now = time(NULL); const int requireother = ! (flags & PDS_ALLOW_SELF); const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - const int prefer_tunnel = (flags & PDS_PREFER_TUNNELED_DIR_CONNS_); const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); const int no_microdesc_fetching =(flags & PDS_NO_EXISTING_MICRODESC_FETCH); const double auth_weight = (sourcelist == fallback_dir_servers) ? @@ -1653,30 +1855,36 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist, smartlist_t *pick_from; int n_busy = 0; int try_excluding = 1, n_excluded = 0; + int try_ip_pref = 1; if (!sourcelist) return NULL; - retry_without_exclude: + retry_search: direct = smartlist_new(); tunnel = smartlist_new(); overloaded_direct = smartlist_new(); overloaded_tunnel = smartlist_new(); + const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref); + const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref); + const int must_have_or = directory_must_use_begindir(options); + SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d) { int is_overloaded = d->fake_status.last_dir_503_at + DIR_503_TIMEOUT > now; - tor_addr_t addr; if (!d->is_running) continue; if ((type & d->type) == 0) continue; + int is_trusted_extrainfo = router_digest_is_trusted_dir_type( + d->digest, EXTRAINFO_DIRINFO); if ((type & EXTRAINFO_DIRINFO) && - !router_supports_extrainfo(d->digest, 1)) + !router_supports_extrainfo(d->digest, is_trusted_extrainfo)) continue; if (requireother && me && router_digest_is_me(d->digest)) - continue; + continue; if (try_excluding && routerset_contains_routerstatus(options->ExcludeNodes, &d->fake_status, -1)) { @@ -1684,35 +1892,26 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist, continue; } - /* XXXX IP6 proposal 118 */ - tor_addr_from_ipv4h(&addr, d->addr); - - if (no_serverdesc_fetching) { - if (connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &addr, d->dir_port, DIR_PURPOSE_FETCH_SERVERDESC) - || connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &addr, d->dir_port, DIR_PURPOSE_FETCH_EXTRAINFO)) { - //log_debug(LD_DIR, "We have an existing connection to fetch " - // "descriptor from %s; delaying",d->description); - ++n_busy; - continue; - } - } - if (no_microdesc_fetching) { - if (connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &addr, d->dir_port, DIR_PURPOSE_FETCH_MICRODESC)) { - ++n_busy; - continue; - } + if (router_is_already_dir_fetching_ds(d, no_serverdesc_fetching, + no_microdesc_fetching)) { + ++n_busy; + continue; } - if (prefer_tunnel && - d->or_port && - (!fascistfirewall || - fascist_firewall_allows_address_or(&addr, d->or_port))) + /* Clients use IPv6 addresses if the server has one and the client + * prefers IPv6. + * Add the router if its preferred address and port are reachable. + * If we don't get any routers, we'll try again with the non-preferred + * address for each router (if any). (To ensure correct load-balancing + * we try routers that only have one address both times.) + */ + if (!fascistfirewall || skip_or_fw || + fascist_firewall_allows_dir_server(d, FIREWALL_OR_CONNECTION, + try_ip_pref)) smartlist_add(is_overloaded ? overloaded_tunnel : tunnel, (void*)d); - else if (!fascistfirewall || - fascist_firewall_allows_address_dir(&addr, d->dir_port)) + else if (!must_have_or && (skip_dir_fw || + fascist_firewall_allows_dir_server(d, FIREWALL_DIR_CONNECTION, + try_ip_pref))) smartlist_add(is_overloaded ? overloaded_direct : direct, (void*)d); } SMARTLIST_FOREACH_END(d); @@ -1735,22 +1934,19 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist, result = &selection->fake_status; } - if (n_busy_out) - *n_busy_out = n_busy; - smartlist_free(direct); smartlist_free(tunnel); smartlist_free(overloaded_direct); smartlist_free(overloaded_tunnel); - if (result == NULL && try_excluding && !options->StrictNodes && n_excluded) { - /* If we got no result, and we are excluding nodes, and StrictNodes is - * not set, try again without excluding nodes. */ - try_excluding = 0; - n_excluded = 0; - goto retry_without_exclude; - } + RETRY_ALTERNATE_IP_VERSION(retry_search); + + RETRY_WITHOUT_EXCLUDE(retry_search); + + router_picked_poor_directory_log(result); + if (n_busy_out) + *n_busy_out = n_busy; return result; } @@ -1763,7 +1959,6 @@ mark_all_dirservers_up(smartlist_t *server_list) routerstatus_t *rs; node_t *node; dir->is_running = 1; - download_status_reset(&dir->v2_ns_dl_status); node = node_get_mutable_by_id(dir->digest); if (node) node->is_running = 1; @@ -1818,11 +2013,15 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router) /** Add every suitable node from our nodelist to <b>sl</b>, so that * we can pick a node for a circuit. */ -static void +void router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid, int need_uptime, int need_capacity, - int need_guard, int need_desc) -{ /* XXXX MOVE */ + int need_guard, int need_desc, + int pref_addr, int direct_conn) +{ + const int check_reach = !router_skip_or_reachability(get_options(), + pref_addr); + /* XXXX MOVE */ SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) { if (!node->is_running || (!node->is_valid && !allow_invalid)) @@ -1833,6 +2032,11 @@ router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid, continue; if (node_is_unreliable(node, need_uptime, need_capacity, need_guard)) continue; + /* Choose a node with an OR address that matches the firewall rules, + * if we are making a direct connection */ + if (direct_conn && check_reach && + !fascist_firewall_allows_node(node, FIREWALL_OR_CONNECTION, pref_addr)) + continue; smartlist_add(sl, (void *)node); } SMARTLIST_FOREACH_END(node); @@ -1885,20 +2089,21 @@ router_get_advertised_bandwidth_capped(const routerinfo_t *router) * doubles, convert them to uint64_t, and try to scale them linearly so as to * much of the range of uint64_t. If <b>total_out</b> is provided, set it to * the sum of all elements in the array _before_ scaling. */ -/* private */ void +STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, uint64_t *total_out) { double total = 0.0; - double scale_factor; + double scale_factor = 0.0; int i; /* big, but far away from overflowing an int64_t */ -#define SCALE_TO_U64_MAX (INT64_MAX / 4) +#define SCALE_TO_U64_MAX ((int64_t) (INT64_MAX / 4)) for (i = 0; i < n_entries; ++i) total += entries[i].dbl; - scale_factor = SCALE_TO_U64_MAX / total; + if (total > 0.0) + scale_factor = SCALE_TO_U64_MAX / total; for (i = 0; i < n_entries; ++i) entries[i].u64 = tor_llround(entries[i].dbl * scale_factor); @@ -1914,7 +2119,7 @@ scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, #if SIZEOF_VOID_P == 8 #define gt_i64_timei(a,b) ((a) > (b)) #else -static INLINE int +static inline int gt_i64_timei(uint64_t a, uint64_t b) { int64_t diff = (int64_t) (b - a); @@ -1928,7 +2133,7 @@ gt_i64_timei(uint64_t a, uint64_t b) * value, and return the index of that element. If all elements are 0, choose * an index at random. Return -1 on error. */ -/* private */ int +STATIC int choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries) { int i, i_chosen=-1, n_chosen=0; @@ -1992,7 +2197,7 @@ bridge_get_advertised_bandwidth_bounded(routerinfo_t *router) /** Return bw*1000, unless bw*1000 would overflow, in which case return * INT32_MAX. */ -static INLINE int32_t +static inline int32_t kb_to_bytes(uint32_t bw) { return (bw > (INT32_MAX/1000)) ? INT32_MAX : bw*1000; @@ -2021,8 +2226,7 @@ smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl, if (compute_weighted_bandwidths(sl, rule, &bandwidths) < 0) return NULL; - scale_array_elements_to_u64(bandwidths, smartlist_len(sl), - &sl_last_total_weighted_bw); + scale_array_elements_to_u64(bandwidths, smartlist_len(sl), NULL); { int idx = choose_array_element_by_weight(bandwidths, @@ -2046,6 +2250,7 @@ compute_weighted_bandwidths(const smartlist_t *sl, double Wg = -1, Wm = -1, We = -1, Wd = -1; double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; uint64_t weighted_bw = 0; + guardfraction_bandwidth_t guardfraction_bw; u64_dbl_t *bandwidths; /* Can't choose exit and guard at same time */ @@ -2112,9 +2317,10 @@ compute_weighted_bandwidths(const smartlist_t *sl, if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0 || Web < 0) { log_debug(LD_CIRC, - "Got negative bandwidth weights. Defaulting to old selection" + "Got negative bandwidth weights. Defaulting to naive selection" " algorithm."); - return -1; // Use old algorithm. + Wg = Wm = We = Wd = weight_scale; + Wgb = Wmb = Web = Wdb = weight_scale; } Wg /= weight_scale; @@ -2127,26 +2333,32 @@ compute_weighted_bandwidths(const smartlist_t *sl, Web /= weight_scale; Wdb /= weight_scale; - bandwidths = tor_malloc_zero(sizeof(u64_dbl_t)*smartlist_len(sl)); + bandwidths = tor_calloc(smartlist_len(sl), sizeof(u64_dbl_t)); // Cycle through smartlist and total the bandwidth. + static int warned_missing_bw = 0; SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { - int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0, is_me = 0; + int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0; double weight = 1; + double weight_without_guard_flag = 0; /* Used for guardfraction */ + double final_weight = 0; is_exit = node->is_exit && ! node->is_bad_exit; is_guard = node->is_possible_guard; is_dir = node_is_dir(node); if (node->rs) { if (!node->rs->has_bandwidth) { - tor_free(bandwidths); /* This should never happen, unless all the authorites downgrade * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */ - log_warn(LD_BUG, - "Consensus is not listing bandwidths. Defaulting back to " - "old router selection algorithm."); - return -1; + if (! warned_missing_bw) { + log_warn(LD_BUG, + "Consensus is missing some bandwidths. Using a naive " + "router selection algorithm"); + warned_missing_bw = 1; + } + this_bw = 30000; /* Chosen arbitrarily */ + } else { + this_bw = kb_to_bytes(node->rs->bandwidth_kb); } - this_bw = kb_to_bytes(node->rs->bandwidth_kb); } else if (node->ri) { /* bridge or other descriptor not in our consensus */ this_bw = bridge_get_advertised_bandwidth_bounded(node->ri); @@ -2154,12 +2366,13 @@ compute_weighted_bandwidths(const smartlist_t *sl, /* We can't use this one. */ continue; } - is_me = router_digest_is_me(node->identity); if (is_guard && is_exit) { weight = (is_dir ? Wdb*Wd : Wd); + weight_without_guard_flag = (is_dir ? Web*We : We); } else if (is_guard) { weight = (is_dir ? Wgb*Wg : Wg); + weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm); } else if (is_exit) { weight = (is_dir ? Web*We : We); } else { // middle @@ -2171,10 +2384,43 @@ compute_weighted_bandwidths(const smartlist_t *sl, this_bw = 0; if (weight < 0.0) weight = 0.0; + if (weight_without_guard_flag < 0.0) + weight_without_guard_flag = 0.0; - bandwidths[node_sl_idx].dbl = weight*this_bw + 0.5; - if (is_me) - sl_last_weighted_bw_of_me = (uint64_t) bandwidths[node_sl_idx].dbl; + /* If guardfraction information is available in the consensus, we + * want to calculate this router's bandwidth according to its + * guardfraction. Quoting from proposal236: + * + * Let Wpf denote the weight from the 'bandwidth-weights' line a + * client would apply to N for position p if it had the guard + * flag, Wpn the weight if it did not have the guard flag, and B the + * measured bandwidth of N in the consensus. Then instead of choosing + * N for position p proportionally to Wpf*B or Wpn*B, clients should + * choose N proportionally to F*Wpf*B + (1-F)*Wpn*B. + */ + if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) { + /* XXX The assert should actually check for is_guard. However, + * that crashes dirauths because of #13297. This should be + * equivalent: */ + tor_assert(node->rs->is_possible_guard); + + guard_get_guardfraction_bandwidth(&guardfraction_bw, + this_bw, + node->rs->guardfraction_percentage); + + /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */ + final_weight = + guardfraction_bw.guard_bw * weight + + guardfraction_bw.non_guard_bw * weight_without_guard_flag; + + log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)", + node->rs->nickname, final_weight, weight*this_bw, + bandwidth_weight_rule_to_string(rule)); + } else { /* no guardfraction information. calculate the weight normally. */ + final_weight = weight*this_bw; + } + + bandwidths[node_sl_idx].dbl = final_weight + 0.5; } SMARTLIST_FOREACH_END(node); log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based " @@ -2226,235 +2472,13 @@ frac_nodes_with_descriptors(const smartlist_t *sl, return present / total; } -/** Helper function: - * choose a random node_t element of smartlist <b>sl</b>, weighted by - * the advertised bandwidth of each element. - * - * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all - * nodes' bandwidth equally regardless of their Exit status, since there may - * be some in the list because they exit to obscure ports. If - * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight - * exit-node's bandwidth less depending on the smallness of the fraction of - * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a - * guard node: consider all guard's bandwidth equally. Otherwise, weight - * guards proportionally less. - */ -static const node_t * -smartlist_choose_node_by_bandwidth(const smartlist_t *sl, - bandwidth_weight_rule_t rule) -{ - unsigned int i; - u64_dbl_t *bandwidths; - int is_exit; - int is_guard; - int is_fast; - double total_nonexit_bw = 0, total_exit_bw = 0; - double total_nonguard_bw = 0, total_guard_bw = 0; - double exit_weight; - double guard_weight; - int n_unknown = 0; - bitarray_t *fast_bits; - bitarray_t *exit_bits; - bitarray_t *guard_bits; - int me_idx = -1; - - // This function does not support WEIGHT_FOR_DIR - // or WEIGHT_FOR_MID - if (rule == WEIGHT_FOR_DIR || rule == WEIGHT_FOR_MID) { - rule = NO_WEIGHTING; - } - - /* Can't choose exit and guard at same time */ - tor_assert(rule == NO_WEIGHTING || - rule == WEIGHT_FOR_EXIT || - rule == WEIGHT_FOR_GUARD); - - if (smartlist_len(sl) == 0) { - log_info(LD_CIRC, - "Empty routerlist passed in to old node selection for rule %s", - bandwidth_weight_rule_to_string(rule)); - return NULL; - } - - /* First count the total bandwidth weight, and make a list - * of each value. We use UINT64_MAX to indicate "unknown". */ - bandwidths = tor_malloc_zero(sizeof(u64_dbl_t)*smartlist_len(sl)); - fast_bits = bitarray_init_zero(smartlist_len(sl)); - exit_bits = bitarray_init_zero(smartlist_len(sl)); - guard_bits = bitarray_init_zero(smartlist_len(sl)); - - /* Iterate over all the routerinfo_t or routerstatus_t, and */ - SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { - /* first, learn what bandwidth we think i has */ - int is_known = 1; - uint32_t this_bw = 0; - i = node_sl_idx; - - if (router_digest_is_me(node->identity)) - me_idx = node_sl_idx; - - is_exit = node->is_exit; - is_guard = node->is_possible_guard; - if (node->rs) { - if (node->rs->has_bandwidth) { - this_bw = kb_to_bytes(node->rs->bandwidth_kb); - } else { /* guess */ - is_known = 0; - } - } else if (node->ri) { - /* Must be a bridge if we're willing to use it */ - this_bw = bridge_get_advertised_bandwidth_bounded(node->ri); - } - - if (is_exit) - bitarray_set(exit_bits, i); - if (is_guard) - bitarray_set(guard_bits, i); - if (node->is_fast) - bitarray_set(fast_bits, i); - - if (is_known) { - bandwidths[i].dbl = this_bw; - if (is_guard) - total_guard_bw += this_bw; - else - total_nonguard_bw += this_bw; - if (is_exit) - total_exit_bw += this_bw; - else - total_nonexit_bw += this_bw; - } else { - ++n_unknown; - bandwidths[i].dbl = -1.0; - } - } SMARTLIST_FOREACH_END(node); - -#define EPSILON .1 - - /* Now, fill in the unknown values. */ - if (n_unknown) { - int32_t avg_fast, avg_slow; - if (total_exit_bw+total_nonexit_bw < EPSILON) { - /* if there's some bandwidth, there's at least one known router, - * so no worries about div by 0 here */ - int n_known = smartlist_len(sl)-n_unknown; - avg_fast = avg_slow = (int32_t) - ((total_exit_bw+total_nonexit_bw)/((uint64_t) n_known)); - } else { - avg_fast = 40000; - avg_slow = 20000; - } - for (i=0; i<(unsigned)smartlist_len(sl); ++i) { - if (bandwidths[i].dbl >= 0.0) - continue; - is_fast = bitarray_is_set(fast_bits, i); - is_exit = bitarray_is_set(exit_bits, i); - is_guard = bitarray_is_set(guard_bits, i); - bandwidths[i].dbl = is_fast ? avg_fast : avg_slow; - if (is_exit) - total_exit_bw += bandwidths[i].dbl; - else - total_nonexit_bw += bandwidths[i].dbl; - if (is_guard) - total_guard_bw += bandwidths[i].dbl; - else - total_nonguard_bw += bandwidths[i].dbl; - } - } - - /* If there's no bandwidth at all, pick at random. */ - if (total_exit_bw+total_nonexit_bw < EPSILON) { - tor_free(bandwidths); - tor_free(fast_bits); - tor_free(exit_bits); - tor_free(guard_bits); - return smartlist_choose(sl); - } - - /* Figure out how to weight exits and guards */ - { - double all_bw = U64_TO_DBL(total_exit_bw+total_nonexit_bw); - double exit_bw = U64_TO_DBL(total_exit_bw); - double guard_bw = U64_TO_DBL(total_guard_bw); - /* - * For detailed derivation of this formula, see - * http://archives.seul.org/or/dev/Jul-2007/msg00056.html - */ - if (rule == WEIGHT_FOR_EXIT || total_exit_bw<EPSILON) - exit_weight = 1.0; - else - exit_weight = 1.0 - all_bw/(3.0*exit_bw); - - if (rule == WEIGHT_FOR_GUARD || total_guard_bw<EPSILON) - guard_weight = 1.0; - else - guard_weight = 1.0 - all_bw/(3.0*guard_bw); - - if (exit_weight <= 0.0) - exit_weight = 0.0; - - if (guard_weight <= 0.0) - guard_weight = 0.0; - - sl_last_weighted_bw_of_me = 0; - for (i=0; i < (unsigned)smartlist_len(sl); i++) { - tor_assert(bandwidths[i].dbl >= 0.0); - - is_exit = bitarray_is_set(exit_bits, i); - is_guard = bitarray_is_set(guard_bits, i); - if (is_exit && is_guard) - bandwidths[i].dbl *= exit_weight * guard_weight; - else if (is_guard) - bandwidths[i].dbl *= guard_weight; - else if (is_exit) - bandwidths[i].dbl *= exit_weight; - - if (i == (unsigned) me_idx) - sl_last_weighted_bw_of_me = (uint64_t) bandwidths[i].dbl; - } - } - -#if 0 - log_debug(LD_CIRC, "Total weighted bw = "U64_FORMAT - ", exit bw = "U64_FORMAT - ", nonexit bw = "U64_FORMAT", exit weight = %f " - "(for exit == %d)" - ", guard bw = "U64_FORMAT - ", nonguard bw = "U64_FORMAT", guard weight = %f " - "(for guard == %d)", - U64_PRINTF_ARG(total_bw), - U64_PRINTF_ARG(total_exit_bw), U64_PRINTF_ARG(total_nonexit_bw), - exit_weight, (int)(rule == WEIGHT_FOR_EXIT), - U64_PRINTF_ARG(total_guard_bw), U64_PRINTF_ARG(total_nonguard_bw), - guard_weight, (int)(rule == WEIGHT_FOR_GUARD)); -#endif - - scale_array_elements_to_u64(bandwidths, smartlist_len(sl), - &sl_last_total_weighted_bw); - - { - int idx = choose_array_element_by_weight(bandwidths, - smartlist_len(sl)); - tor_free(bandwidths); - tor_free(fast_bits); - tor_free(exit_bits); - tor_free(guard_bits); - return idx < 0 ? NULL : smartlist_get(sl, idx); - } -} - /** Choose a random element of status list <b>sl</b>, weighted by * the advertised bandwidth of each node */ const node_t * node_sl_choose_by_bandwidth(const smartlist_t *sl, bandwidth_weight_rule_t rule) { /*XXXX MOVE */ - const node_t *ret; - if ((ret = smartlist_choose_node_by_bandwidth_weights(sl, rule))) { - return ret; - } else { - return smartlist_choose_node_by_bandwidth(sl, rule); - } + return smartlist_choose_node_by_bandwidth_weights(sl, rule); } /** Return a random running node from the nodelist. Never @@ -2474,6 +2498,10 @@ node_sl_choose_by_bandwidth(const smartlist_t *sl, * If <b>CRN_NEED_DESC</b> is set in flags, we only consider nodes that * have a routerinfo or microdescriptor -- that is, enough info to be * used to build a circuit. + * If <b>CRN_PREF_ADDR</b> is set in flags, we only consider nodes that + * have an address that is preferred by the ClientPreferIPv6ORPort setting + * (regardless of this flag, we exclude nodes that aren't allowed by the + * firewall, including ClientUseIPv4 0 and fascist_firewall_use_ipv6() == 0). */ const node_t * router_choose_random_node(smartlist_t *excludedsmartlist, @@ -2486,6 +2514,8 @@ router_choose_random_node(smartlist_t *excludedsmartlist, const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; const int weight_for_exit = (flags & CRN_WEIGHT_AS_EXIT) != 0; const int need_desc = (flags & CRN_NEED_DESC) != 0; + const int pref_addr = (flags & CRN_PREF_ADDR) != 0; + const int direct_conn = (flags & CRN_DIRECT_CONN) != 0; smartlist_t *sl=smartlist_new(), *excludednodes=smartlist_new(); @@ -2511,18 +2541,37 @@ router_choose_random_node(smartlist_t *excludedsmartlist, router_add_running_nodes_to_smartlist(sl, allow_invalid, need_uptime, need_capacity, - need_guard, need_desc); + need_guard, need_desc, pref_addr, + direct_conn); + log_debug(LD_CIRC, + "We found %d running nodes.", + smartlist_len(sl)); + smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) + log_debug(LD_CIRC, + "We removed %d excludednodes, leaving %d nodes.", + smartlist_len(excludednodes), + smartlist_len(sl)); + + if (excludedsmartlist) { smartlist_subtract(sl,excludedsmartlist); - if (excludedset) + log_debug(LD_CIRC, + "We removed %d excludedsmartlist, leaving %d nodes.", + smartlist_len(excludedsmartlist), + smartlist_len(sl)); + } + if (excludedset) { routerset_subtract_nodes(sl,excludedset); + log_debug(LD_CIRC, + "We removed excludedset, leaving %d nodes.", + smartlist_len(sl)); + } // Always weight by bandwidth choice = node_sl_choose_by_bandwidth(sl, rule); smartlist_free(sl); - if (!choice && (need_uptime || need_capacity || need_guard)) { + if (!choice && (need_uptime || need_capacity || need_guard || pref_addr)) { /* try once more -- recurse but with fewer restrictions. */ log_info(LD_CIRC, "We couldn't find any live%s%s%s routers; falling back " @@ -2530,7 +2579,8 @@ router_choose_random_node(smartlist_t *excludedsmartlist, need_capacity?", fast":"", need_uptime?", stable":"", need_guard?", guard":""); - flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD); + flags &= ~ (CRN_NEED_UPTIME|CRN_NEED_CAPACITY|CRN_NEED_GUARD| + CRN_PREF_ADDR); choice = router_choose_random_node( excludedsmartlist, excludedset, flags); } @@ -2630,7 +2680,7 @@ router_is_named(const routerinfo_t *router) /** Return true iff <b>digest</b> is the digest of the identity key of a * trusted directory matching at least one bit of <b>type</b>. If <b>type</b> - * is zero, any authority is okay. */ + * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */ int router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type) { @@ -2711,8 +2761,8 @@ router_get_by_descriptor_digest(const char *digest) /** Return the signed descriptor for the router in our routerlist whose * 20-byte extra-info digest is <b>digest</b>. Return NULL if no such router * is known. */ -signed_descriptor_t * -router_get_by_extrainfo_digest(const char *digest) +MOCK_IMPL(signed_descriptor_t *, +router_get_by_extrainfo_digest,(const char *digest)) { tor_assert(digest); @@ -2822,7 +2872,6 @@ router_get_routerlist(void) routerlist->extra_info_map = eimap_new(); routerlist->desc_store.fname_base = "cached-descriptors"; - routerlist->desc_store.fname_alt_base = "cached-routers"; routerlist->extrainfo_store.fname_base = "cached-extrainfo"; routerlist->desc_store.type = ROUTER_STORE; @@ -2842,7 +2891,6 @@ routerinfo_free(routerinfo_t *router) return; tor_free(router->cache_info.signed_descriptor_body); - tor_free(router->address); tor_free(router->nickname); tor_free(router->platform); tor_free(router->contact_info); @@ -2851,6 +2899,7 @@ routerinfo_free(routerinfo_t *router) tor_free(router->onion_curve25519_pkey); if (router->identity_pkey) crypto_pk_free(router->identity_pkey); + tor_cert_free(router->cache_info.signing_key_cert); if (router->declared_family) { SMARTLIST_FOREACH(router->declared_family, char *, s, tor_free(s)); smartlist_free(router->declared_family); @@ -2869,6 +2918,7 @@ extrainfo_free(extrainfo_t *extrainfo) { if (!extrainfo) return; + tor_cert_free(extrainfo->cache_info.signing_key_cert); tor_free(extrainfo->cache_info.signed_descriptor_body); tor_free(extrainfo->pending_sig); @@ -2884,11 +2934,25 @@ signed_descriptor_free(signed_descriptor_t *sd) return; tor_free(sd->signed_descriptor_body); + tor_cert_free(sd->signing_key_cert); memset(sd, 99, sizeof(signed_descriptor_t)); /* Debug bad mem usage */ tor_free(sd); } +/** Copy src into dest, and steal all references inside src so that when + * we free src, we don't mess up dest. */ +static void +signed_descriptor_move(signed_descriptor_t *dest, + signed_descriptor_t *src) +{ + tor_assert(dest != src); + memcpy(dest, src, sizeof(signed_descriptor_t)); + src->signed_descriptor_body = NULL; + src->signing_key_cert = NULL; + dest->routerlist_index = -1; +} + /** Extract a signed_descriptor_t from a general routerinfo, and free the * routerinfo. */ @@ -2898,9 +2962,7 @@ signed_descriptor_from_routerinfo(routerinfo_t *ri) signed_descriptor_t *sd; tor_assert(ri->purpose == ROUTER_PURPOSE_GENERAL); sd = tor_malloc_zero(sizeof(signed_descriptor_t)); - memcpy(sd, &(ri->cache_info), sizeof(signed_descriptor_t)); - sd->routerlist_index = -1; - ri->cache_info.signed_descriptor_body = NULL; + signed_descriptor_move(sd, &ri->cache_info); routerinfo_free(ri); return sd; } @@ -2928,10 +2990,18 @@ routerlist_free(routerlist_t *rl) signed_descriptor_free(sd)); smartlist_free(rl->routers); smartlist_free(rl->old_routers); - if (routerlist->desc_store.mmap) - tor_munmap_file(routerlist->desc_store.mmap); - if (routerlist->extrainfo_store.mmap) - tor_munmap_file(routerlist->extrainfo_store.mmap); + if (rl->desc_store.mmap) { + int res = tor_munmap_file(routerlist->desc_store.mmap); + if (res != 0) { + log_warn(LD_FS, "Failed to munmap routerlist->desc_store.mmap"); + } + } + if (rl->extrainfo_store.mmap) { + int res = tor_munmap_file(routerlist->extrainfo_store.mmap); + if (res != 0) { + log_warn(LD_FS, "Failed to munmap routerlist->extrainfo_store.mmap"); + } + } tor_free(rl); router_dir_info_changed(); @@ -2962,7 +3032,7 @@ dump_routerlist_mem_usage(int severity) * in <b>sl</b> at position <b>idx</b>. Otherwise, search <b>sl</b> for * <b>ri</b>. Return the index of <b>ri</b> in <b>sl</b>, or -1 if <b>ri</b> * is not in <b>sl</b>. */ -static INLINE int +static inline int routerlist_find_elt_(smartlist_t *sl, void *ri, int idx) { if (idx < 0) { @@ -3027,17 +3097,19 @@ routerlist_insert(routerlist_t *rl, routerinfo_t *ri) } /** Adds the extrainfo_t <b>ei</b> to the routerlist <b>rl</b>, if there is a - * corresponding router in rl-\>routers or rl-\>old_routers. Return true iff - * we actually inserted <b>ei</b>. Free <b>ei</b> if it isn't inserted. */ -static int -extrainfo_insert(routerlist_t *rl, extrainfo_t *ei) + * corresponding router in rl-\>routers or rl-\>old_routers. Return the status + * of inserting <b>ei</b>. Free <b>ei</b> if it isn't inserted. */ +MOCK_IMPL(STATIC was_router_added_t, +extrainfo_insert,(routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible)) { - int r = 0; + was_router_added_t r; + const char *compatibility_error_msg; routerinfo_t *ri = rimap_get(rl->identity_map, ei->cache_info.identity_digest); signed_descriptor_t *sd = sdmap_get(rl->desc_by_eid_map, ei->cache_info.signed_descriptor_digest); extrainfo_t *ei_tmp; + const int severity = warn_if_incompatible ? LOG_WARN : LOG_INFO; { extrainfo_t *ei_generated = router_get_my_extrainfo(); @@ -3046,9 +3118,41 @@ extrainfo_insert(routerlist_t *rl, extrainfo_t *ei) if (!ri) { /* This router is unknown; we can't even verify the signature. Give up.*/ + r = ROUTER_NOT_IN_CONSENSUS; + goto done; + } + if (! sd) { + /* The extrainfo router doesn't have a known routerdesc to attach it to. + * This just won't work. */; + static ratelim_t no_sd_ratelim = RATELIM_INIT(1800); + r = ROUTER_BAD_EI; + log_fn_ratelim(&no_sd_ratelim, severity, LD_BUG, + "No entry found in extrainfo map."); + goto done; + } + if (tor_memneq(ei->cache_info.signed_descriptor_digest, + sd->extra_info_digest, DIGEST_LEN)) { + static ratelim_t digest_mismatch_ratelim = RATELIM_INIT(1800); + /* The sd we got from the map doesn't match the digest we used to look + * it up. This makes no sense. */ + r = ROUTER_BAD_EI; + log_fn_ratelim(&digest_mismatch_ratelim, severity, LD_BUG, + "Mismatch in digest in extrainfo map."); goto done; } - if (routerinfo_incompatible_with_extrainfo(ri, ei, sd, NULL)) { + if (routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei, sd, + &compatibility_error_msg)) { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + r = (ri->cache_info.extrainfo_is_bogus) ? + ROUTER_BAD_EI : ROUTER_NOT_IN_CONSENSUS; + + base16_encode(d1, sizeof(d1), ri->cache_info.identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), ei->cache_info.identity_digest, DIGEST_LEN); + + log_fn(severity,LD_DIR, + "router info incompatible with extra info (ri id: %s, ei id %s, " + "reason: %s)", d1, d2, compatibility_error_msg); + goto done; } @@ -3058,7 +3162,7 @@ extrainfo_insert(routerlist_t *rl, extrainfo_t *ei) ei_tmp = eimap_set(rl->extra_info_map, ei->cache_info.signed_descriptor_digest, ei); - r = 1; + r = ROUTER_ADDED_SUCCESSFULLY; if (ei_tmp) { rl->extrainfo_store.bytes_dropped += ei_tmp->cache_info.signed_descriptor_len; @@ -3066,7 +3170,7 @@ extrainfo_insert(routerlist_t *rl, extrainfo_t *ei) } done: - if (r == 0) + if (r != ROUTER_ADDED_SUCCESSFULLY) extrainfo_free(ei); #ifdef DEBUG_ROUTERLIST @@ -3341,19 +3445,21 @@ routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd) ri = router_parse_entry_from_string(body, body+sd->signed_descriptor_len+sd->annotations_len, - 0, 1, NULL); + 0, 1, NULL, NULL); if (!ri) return NULL; - memcpy(&ri->cache_info, sd, sizeof(signed_descriptor_t)); - sd->signed_descriptor_body = NULL; /* Steal reference. */ - ri->cache_info.routerlist_index = -1; + signed_descriptor_move(&ri->cache_info, sd); routerlist_remove_old(rl, sd, -1); return ri; } -/** Free all memory held by the routerlist module. */ +/** Free all memory held by the routerlist module. + * Note: Calling routerlist_free_all() should always be paired with + * a call to nodelist_free_all(). These should only be called during + * cleanup. + */ void routerlist_free_all(void) { @@ -3387,6 +3493,14 @@ routerlist_reset_warnings(void) networkstatus_reset_warnings(); } +/** Return 1 if the signed descriptor of this router is older than + * <b>seconds</b> seconds. Otherwise return 0. */ +MOCK_IMPL(int, +router_descriptor_is_older_than,(const routerinfo_t *router, int seconds)) +{ + return router->cache_info.published_on < approx_time() - seconds; +} + /** Add <b>router</b> to the routerlist, if we don't already have it. Replace * older entries (if any) with the same key. Note: Callers should not hold * their pointers to <b>router</b> if this function fails; <b>router</b> @@ -3418,7 +3532,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, routerinfo_t *old_router; networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(FLAV_NS); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); int in_consensus = 0; tor_assert(msg); @@ -3430,6 +3543,13 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, old_router = router_get_mutable_by_digest(id_digest); + /* Make sure that it isn't expired. */ + if (router->cert_expiration_time < approx_time()) { + routerinfo_free(router); + *msg = "Some certs on this router are expired."; + return ROUTER_CERTS_EXPIRED; + } + /* Make sure that we haven't already got this exact descriptor. */ if (sdmap_get(routerlist->desc_digest_map, router->cache_info.signed_descriptor_digest)) { @@ -3454,7 +3574,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, router_describe(router)); *msg = "Router descriptor was not new."; routerinfo_free(router); - return ROUTER_WAS_NOT_NEW; + return ROUTER_IS_ALREADY_KNOWN; } } @@ -3489,15 +3609,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, } /* We no longer need a router with this descriptor digest. */ - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - routerstatus_t *rs = - networkstatus_v2_find_mutable_entry(ns, id_digest); - if (rs && tor_memeq(rs->descriptor_digest, - router->cache_info.signed_descriptor_digest, - DIGEST_LEN)) - rs->need_to_mirror = 0; - }); if (consensus) { routerstatus_t *rs = networkstatus_vote_find_mutable_entry( consensus, id_digest); @@ -3505,7 +3616,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, router->cache_info.signed_descriptor_digest, DIGEST_LEN)) { in_consensus = 1; - rs->need_to_mirror = 0; } } @@ -3549,7 +3659,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, &routerlist->desc_store); routerlist_insert_old(routerlist, router); *msg = "Router descriptor was not new."; - return ROUTER_WAS_NOT_NEW; + return ROUTER_IS_ALREADY_KNOWN; } else { /* Same key, and either new, or listed in the consensus. */ log_debug(LD_DIR, "Replacing entry for router %s", @@ -3559,7 +3669,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); } - directory_set_dirty(); *msg = authdir_believes_valid ? "Valid server updated" : ("Invalid server updated. (This dirserver is marking your " "server as unapproved.)"); @@ -3568,10 +3677,10 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, } if (!in_consensus && from_cache && - router->cache_info.published_on < time(NULL) - OLD_ROUTER_DESC_MAX_AGE) { + router_descriptor_is_older_than(router, OLD_ROUTER_DESC_MAX_AGE)) { *msg = "Router descriptor was really old."; routerinfo_free(router); - return ROUTER_WAS_NOT_NEW; + return ROUTER_WAS_TOO_OLD; } /* We haven't seen a router with this identity before. Add it to the end of @@ -3581,7 +3690,6 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, signed_desc_append_to_journal(&router->cache_info, &routerlist->desc_store); } - directory_set_dirty(); return ROUTER_ADDED_SUCCESSFULLY; } @@ -3593,21 +3701,18 @@ was_router_added_t router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, int from_cache, int from_fetch) { - int inserted; + was_router_added_t inserted; (void)from_fetch; if (msg) *msg = NULL; /*XXXX023 Do something with msg */ - inserted = extrainfo_insert(router_get_routerlist(), ei); + inserted = extrainfo_insert(router_get_routerlist(), ei, !from_cache); - if (inserted && !from_cache) + if (WRA_WAS_ADDED(inserted) && !from_cache) signed_desc_append_to_journal(&ei->cache_info, &routerlist->extrainfo_store); - if (inserted) - return ROUTER_ADDED_SUCCESSFULLY; - else - return ROUTER_BAD_EI; + return inserted; } /** Sorting helper: return <0, 0, or >0 depending on whether the @@ -3677,9 +3782,9 @@ routerlist_remove_old_cached_routers_with_id(time_t now, n_extra = n - mdpr; } - lifespans = tor_malloc_zero(sizeof(struct duration_idx_t)*n); - rmv = tor_malloc_zero(sizeof(uint8_t)*n); - must_keep = tor_malloc_zero(sizeof(uint8_t)*n); + lifespans = tor_calloc(n, sizeof(struct duration_idx_t)); + rmv = tor_calloc(n, sizeof(uint8_t)); + must_keep = tor_calloc(n, sizeof(uint8_t)); /* Set lifespans to contain the lifespan and index of each server. */ /* Set rmv[i-lo]=1 if we're going to remove a server for being too old. */ for (i = lo; i <= hi; ++i) { @@ -3744,11 +3849,7 @@ routerlist_remove_old_routers(void) routerinfo_t *router; signed_descriptor_t *sd; digestset_t *retain; - int caches = directory_caches_dir_info(get_options()); const networkstatus_t *consensus = networkstatus_get_latest_consensus(); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - int have_enough_v2; - const or_options_t *options = get_options(); trusted_dirs_remove_old_certs(); @@ -3764,38 +3865,10 @@ routerlist_remove_old_routers(void) { /* We'll probably retain everything in the consensus. */ int n_max_retain = smartlist_len(consensus->routerstatus_list); - if (caches && networkstatus_v2_list) { - /* If we care about v2 statuses, we'll retain at most as many as are - listed any of the v2 statues. This will be at least the length of - the largest v2 networkstatus, and in the worst case, this set will be - equal to the sum of the lengths of all v2 consensuses. Take the - worst case. - */ - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - n_max_retain += smartlist_len(ns->entries)); - } retain = digestset_new(n_max_retain); } cutoff = now - OLD_ROUTER_DESC_MAX_AGE; - /* Build a list of all the descriptors that _anybody_ lists. */ - if (caches && networkstatus_v2_list) { - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - /* XXXX The inner loop here gets pretty expensive, and actually shows up - * on some profiles. It may be the reason digestmap_set shows up in - * profiles too. If instead we kept a per-descriptor digest count of - * how many networkstatuses recommended each descriptor, and changed - * that only when the networkstatuses changed, that would be a speed - * improvement, possibly 1-4% if it also removes digestmap_set from the - * profile. Not worth it for 0.1.2.x, though. The new directory - * system will obsolete this whole thing in 0.2.0.x. */ - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t *, rs) { - if (rs->published_on >= cutoff) - digestset_add(retain, rs->descriptor_digest); - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); - } - /* Retain anything listed in the consensus. */ if (consensus) { SMARTLIST_FOREACH(consensus->routerstatus_list, routerstatus_t *, rs, @@ -3803,18 +3876,11 @@ routerlist_remove_old_routers(void) digestset_add(retain, rs->descriptor_digest)); } - /* If we have a consensus, and nearly as many v2 networkstatuses as we want, - * we should consider pruning current routers that are too old and that - * nobody recommends. (If we don't have a consensus or enough v2 - * networkstatuses, then we should get more before we decide to kill - * routers.) */ - /* we set this to true iff we don't care about v2 info, or we have enough. */ - have_enough_v2 = !caches || - !(authdir_mode_any_main(options) || options->FetchV2Networkstatus) || - (networkstatus_v2_list && - smartlist_len(networkstatus_v2_list) > get_n_v2_authorities() / 2); + /* If we have a consensus, we should consider pruning current routers that + * are too old and that nobody recommends. (If we don't have a consensus, + * then we should get one before we decide to kill routers.) */ - if (have_enough_v2 && consensus) { + if (consensus) { cutoff = now - ROUTER_MAX_AGE; /* Remove too-old unrecommended members of routerlist->routers. */ for (i = 0; i < smartlist_len(routerlist->routers); ++i) { @@ -3941,7 +4007,8 @@ router_load_single_router(const char *s, uint8_t purpose, int cache, "@source controller\n" "@purpose %s\n", router_purpose_to_string(purpose)); - if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0, annotation_buf))) { + if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0, + annotation_buf, NULL))) { log_warn(LD_DIR, "Error parsing router descriptor; dropping."); *msg = "Couldn't parse router descriptor."; return -1; @@ -4005,9 +4072,11 @@ router_load_routers_from_string(const char *s, const char *eos, int from_cache = (saved_location != SAVED_NOWHERE); int allow_annotations = (saved_location != SAVED_NOWHERE); int any_changed = 0; + smartlist_t *invalid_digests = smartlist_new(); router_parse_list_from_string(&s, eos, routers, saved_location, 0, - allow_annotations, prepend_annotations); + allow_annotations, prepend_annotations, + invalid_digests); routers_update_status_from_consensus_networkstatus(routers, !from_cache); @@ -4043,7 +4112,7 @@ router_load_routers_from_string(const char *s, const char *eos, smartlist_add(changed, ri); routerlist_descriptors_added(changed, from_cache); smartlist_clear(changed); - } else if (WRA_WAS_REJECTED(r)) { + } else if (WRA_NEVER_DOWNLOADABLE(r)) { download_status_t *dl_status; dl_status = router_get_dl_status_by_descriptor_digest(d); if (dl_status) { @@ -4054,6 +4123,27 @@ router_load_routers_from_string(const char *s, const char *eos, } } SMARTLIST_FOREACH_END(ri); + SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) { + /* This digest is never going to be parseable. */ + base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN); + if (requested_fingerprints && descriptor_digests) { + if (! smartlist_contains_string(requested_fingerprints, fp)) { + /* But we didn't ask for it, so we should assume shennanegans. */ + continue; + } + smartlist_string_remove(requested_fingerprints, fp); + } + download_status_t *dls; + dls = router_get_dl_status_by_descriptor_digest((char*)bad_digest); + if (dls) { + log_info(LD_GENERAL, "Marking router with descriptor %s as unparseable, " + "and therefore undownloadable", fp); + download_status_mark_impossible(dls); + } + } SMARTLIST_FOREACH_END(bad_digest); + SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d)); + smartlist_free(invalid_digests); + routerlist_assert_ok(routerlist); if (any_changed) @@ -4077,13 +4167,16 @@ router_load_extrainfo_from_string(const char *s, const char *eos, smartlist_t *extrainfo_list = smartlist_new(); const char *msg; int from_cache = (saved_location != SAVED_NOWHERE); + smartlist_t *invalid_digests = smartlist_new(); router_parse_list_from_string(&s, eos, extrainfo_list, saved_location, 1, 0, - NULL); + NULL, invalid_digests); log_info(LD_DIR, "%d elements to add", smartlist_len(extrainfo_list)); SMARTLIST_FOREACH_BEGIN(extrainfo_list, extrainfo_t *, ei) { + uint8_t d[DIGEST_LEN]; + memcpy(d, ei->cache_info.signed_descriptor_digest, DIGEST_LEN); was_router_added_t added = router_add_extrainfo_to_routerlist(ei, &msg, from_cache, !from_cache); if (WRA_WAS_ADDED(added) && requested_fingerprints) { @@ -4097,9 +4190,39 @@ router_load_extrainfo_from_string(const char *s, const char *eos, * so long as we would have wanted them anyway. Since we always fetch * all the extrainfos we want, and we never actually act on them * inside Tor, this should be harmless. */ + } else if (WRA_NEVER_DOWNLOADABLE(added)) { + signed_descriptor_t *sd = router_get_by_extrainfo_digest((char*)d); + if (sd) { + log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as " + "unparseable, and therefore undownloadable", + hex_str((char*)d,DIGEST_LEN)); + download_status_mark_impossible(&sd->ei_dl_status); + } } } SMARTLIST_FOREACH_END(ei); + SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) { + /* This digest is never going to be parseable. */ + char fp[HEX_DIGEST_LEN+1]; + base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN); + if (requested_fingerprints) { + if (! smartlist_contains_string(requested_fingerprints, fp)) { + /* But we didn't ask for it, so we should assume shennanegans. */ + continue; + } + smartlist_string_remove(requested_fingerprints, fp); + } + signed_descriptor_t *sd = + router_get_by_extrainfo_digest((char*)bad_digest); + if (sd) { + log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as " + "unparseable, and therefore undownloadable", fp); + download_status_mark_impossible(&sd->ei_dl_status); + } + } SMARTLIST_FOREACH_END(bad_digest); + SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d)); + smartlist_free(invalid_digests); + routerlist_assert_ok(routerlist); router_rebuild_store(0, &router_get_routerlist()->extrainfo_store); @@ -4113,8 +4236,6 @@ signed_desc_digest_is_recognized(signed_descriptor_t *desc) { const routerstatus_t *rs; networkstatus_t *consensus = networkstatus_get_latest_consensus(); - int caches = directory_caches_dir_info(get_options()); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); if (consensus) { rs = networkstatus_vote_find_entry(consensus, desc->identity_digest); @@ -4122,16 +4243,6 @@ signed_desc_digest_is_recognized(signed_descriptor_t *desc) desc->signed_descriptor_digest, DIGEST_LEN)) return 1; } - if (caches && networkstatus_v2_list) { - SMARTLIST_FOREACH(networkstatus_v2_list, networkstatus_v2_t *, ns, - { - if (!(rs = networkstatus_v2_find_entry(ns, desc->identity_digest))) - continue; - if (tor_memeq(rs->descriptor_digest, - desc->signed_descriptor_digest, DIGEST_LEN)) - return 1; - }); - } return 0; } @@ -4147,17 +4258,15 @@ update_all_descriptor_downloads(time_t now) launch_dummy_descriptor_download_as_needed(now, get_options()); } -/** Clear all our timeouts for fetching v2 and v3 directory stuff, and then +/** Clear all our timeouts for fetching v3 directory stuff, and then * give it all a try again. */ void routerlist_retry_directory_downloads(time_t now) { + (void)now; router_reset_status_download_failures(); router_reset_descriptor_download_failures(); - if (get_options()->DisableNetwork) - return; - update_networkstatus_downloads(now); - update_all_descriptor_downloads(now); + reschedule_directory_downloads(); } /** Return true iff <b>router</b> does not permit exit streams. @@ -4169,15 +4278,16 @@ router_exit_policy_rejects_all(const routerinfo_t *router) } /** Create an directory server at <b>address</b>:<b>port</b>, with OR identity - * key <b>digest</b>. If <b>address</b> is NULL, add ourself. If - * <b>is_authority</b>, this is a directory authority. Return the new - * directory server entry on success or NULL on failure. */ + * key <b>digest</b> which has DIGEST_LEN bytes. If <b>address</b> is NULL, + * add ourself. If <b>is_authority</b>, this is a directory authority. Return + * the new directory server entry on success or NULL on failure. */ static dir_server_t * dir_server_new(int is_authority, const char *nickname, const tor_addr_t *addr, const char *hostname, uint16_t dir_port, uint16_t or_port, + const tor_addr_port_t *addrport_ipv6, const char *digest, const char *v3_auth_digest, dirinfo_type_t type, double weight) @@ -4186,13 +4296,15 @@ dir_server_new(int is_authority, uint32_t a; char *hostname_ = NULL; + tor_assert(digest); + if (weight < 0) return NULL; if (tor_addr_family(addr) == AF_INET) a = tor_addr_to_ipv4h(addr); else - return NULL; /*XXXX Support IPv6 */ + return NULL; if (!hostname) hostname_ = tor_dup_addr(addr); @@ -4209,18 +4321,31 @@ dir_server_new(int is_authority, ent->is_authority = is_authority; ent->type = type; ent->weight = weight; + if (addrport_ipv6) { + if (tor_addr_family(&addrport_ipv6->addr) != AF_INET6) { + log_warn(LD_BUG, "Hey, I got a non-ipv6 addr as addrport_ipv6."); + tor_addr_make_unspec(&ent->ipv6_addr); + } else { + tor_addr_copy(&ent->ipv6_addr, &addrport_ipv6->addr); + ent->ipv6_orport = addrport_ipv6->port; + } + } else { + tor_addr_make_unspec(&ent->ipv6_addr); + } + memcpy(ent->digest, digest, DIGEST_LEN); if (v3_auth_digest && (type & V3_DIRINFO)) memcpy(ent->v3_identity_digest, v3_auth_digest, DIGEST_LEN); if (nickname) tor_asprintf(&ent->description, "directory server \"%s\" at %s:%d", - nickname, hostname, (int)dir_port); + nickname, hostname_, (int)dir_port); else tor_asprintf(&ent->description, "directory server at %s:%d", - hostname, (int)dir_port); + hostname_, (int)dir_port); ent->fake_status.addr = ent->addr; + tor_addr_copy(&ent->fake_status.ipv6_addr, &ent->ipv6_addr); memcpy(ent->fake_status.identity_digest, digest, DIGEST_LEN); if (nickname) strlcpy(ent->fake_status.nickname, nickname, @@ -4229,6 +4354,7 @@ dir_server_new(int is_authority, ent->fake_status.nickname[0] = '\0'; ent->fake_status.dir_port = ent->dir_port; ent->fake_status.or_port = ent->or_port; + ent->fake_status.ipv6_orport = ent->ipv6_orport; return ent; } @@ -4240,6 +4366,7 @@ dir_server_new(int is_authority, dir_server_t * trusted_dir_server_new(const char *nickname, const char *address, uint16_t dir_port, uint16_t or_port, + const tor_addr_port_t *ipv6_addrport, const char *digest, const char *v3_auth_digest, dirinfo_type_t type, double weight) { @@ -4270,7 +4397,9 @@ trusted_dir_server_new(const char *nickname, const char *address, tor_addr_from_ipv4h(&addr, a); result = dir_server_new(1, nickname, &addr, hostname, - dir_port, or_port, digest, + dir_port, or_port, + ipv6_addrport, + digest, v3_auth_digest, type, weight); tor_free(hostname); return result; @@ -4282,9 +4411,12 @@ trusted_dir_server_new(const char *nickname, const char *address, dir_server_t * fallback_dir_server_new(const tor_addr_t *addr, uint16_t dir_port, uint16_t or_port, + const tor_addr_port_t *addrport_ipv6, const char *id_digest, double weight) { - return dir_server_new(0, NULL, addr, NULL, dir_port, or_port, id_digest, + return dir_server_new(0, NULL, addr, NULL, dir_port, or_port, + addrport_ipv6, + id_digest, NULL, ALL_DIRINFO, weight); } @@ -4353,11 +4485,11 @@ clear_dir_servers(void) /** For every current directory connection whose purpose is <b>purpose</b>, * and where the resource being downloaded begins with <b>prefix</b>, split * rest of the resource into base16 fingerprints (or base64 fingerprints if - * purpose==DIR_PURPPOSE_FETCH_MICRODESC), decode them, and set the + * purpose==DIR_PURPOSE_FETCH_MICRODESC), decode them, and set the * corresponding elements of <b>result</b> to a nonzero value. */ static void -list_pending_downloads(digestmap_t *result, +list_pending_downloads(digestmap_t *result, digest256map_t *result256, int purpose, const char *prefix) { const size_t p_len = strlen(prefix); @@ -4367,7 +4499,7 @@ list_pending_downloads(digestmap_t *result, if (purpose == DIR_PURPOSE_FETCH_MICRODESC) flags = DSR_DIGEST256|DSR_BASE64; - tor_assert(result); + tor_assert(result || result256); SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->type == CONN_TYPE_DIR && @@ -4380,11 +4512,19 @@ list_pending_downloads(digestmap_t *result, } } SMARTLIST_FOREACH_END(conn); - SMARTLIST_FOREACH(tmp, char *, d, + if (result) { + SMARTLIST_FOREACH(tmp, char *, d, { digestmap_set(result, d, (void*)1); tor_free(d); }); + } else if (result256) { + SMARTLIST_FOREACH(tmp, uint8_t *, d, + { + digest256map_set(result256, d, (void*)1); + tor_free(d); + }); + } smartlist_free(tmp); } @@ -4396,20 +4536,16 @@ list_pending_descriptor_downloads(digestmap_t *result, int extrainfo) { int purpose = extrainfo ? DIR_PURPOSE_FETCH_EXTRAINFO : DIR_PURPOSE_FETCH_SERVERDESC; - list_pending_downloads(result, purpose, "d/"); + list_pending_downloads(result, NULL, purpose, "d/"); } /** For every microdescriptor we are currently downloading by descriptor - * digest, set result[d] to (void*)1. (Note that microdescriptor digests - * are 256-bit, and digestmap_t only holds 160-bit digests, so we're only - * getting the first 20 bytes of each digest here.) - * - * XXXX Let there be a digestmap256_t, and use that instead. + * digest, set result[d] to (void*)1. */ void -list_pending_microdesc_downloads(digestmap_t *result) +list_pending_microdesc_downloads(digest256map_t *result) { - list_pending_downloads(result, DIR_PURPOSE_FETCH_MICRODESC, "d/"); + list_pending_downloads(NULL, result, DIR_PURPOSE_FETCH_MICRODESC, "d/"); } /** For every certificate we are currently downloading by (identity digest, @@ -4452,73 +4588,100 @@ list_pending_fpsk_downloads(fp_pair_map_t *result) * range.) If <b>source</b> is given, download from <b>source</b>; * otherwise, download from an appropriate random directory server. */ -static void -initiate_descriptor_downloads(const routerstatus_t *source, - int purpose, - smartlist_t *digests, - int lo, int hi, int pds_flags) +MOCK_IMPL(STATIC void, initiate_descriptor_downloads, + (const routerstatus_t *source, int purpose, smartlist_t *digests, + int lo, int hi, int pds_flags)) { - int i, n = hi-lo; char *resource, *cp; - size_t r_len; - - int digest_len = DIGEST_LEN, enc_digest_len = HEX_DIGEST_LEN; - char sep = '+'; - int b64_256 = 0; + int digest_len, enc_digest_len; + const char *sep; + int b64_256; + smartlist_t *tmp; if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { /* Microdescriptors are downloaded by "-"-separated base64-encoded * 256-bit digests. */ digest_len = DIGEST256_LEN; - enc_digest_len = BASE64_DIGEST256_LEN; - sep = '-'; + enc_digest_len = BASE64_DIGEST256_LEN + 1; + sep = "-"; b64_256 = 1; + } else { + digest_len = DIGEST_LEN; + enc_digest_len = HEX_DIGEST_LEN + 1; + sep = "+"; + b64_256 = 0; } - if (n <= 0) - return; if (lo < 0) lo = 0; if (hi > smartlist_len(digests)) hi = smartlist_len(digests); - r_len = 8 + (enc_digest_len+1)*n; - cp = resource = tor_malloc(r_len); - memcpy(cp, "d/", 2); - cp += 2; - for (i = lo; i < hi; ++i) { + if (hi-lo <= 0) + return; + + tmp = smartlist_new(); + + for (; lo < hi; ++lo) { + cp = tor_malloc(enc_digest_len); if (b64_256) { - digest256_to_base64(cp, smartlist_get(digests, i)); + digest256_to_base64(cp, smartlist_get(digests, lo)); } else { - base16_encode(cp, r_len-(cp-resource), - smartlist_get(digests,i), digest_len); + base16_encode(cp, enc_digest_len, smartlist_get(digests, lo), + digest_len); } - cp += enc_digest_len; - *cp++ = sep; + smartlist_add(tmp, cp); } - memcpy(cp-1, ".z", 3); + + cp = smartlist_join_strings(tmp, sep, 0, NULL); + tor_asprintf(&resource, "d/%s.z", cp); + + SMARTLIST_FOREACH(tmp, char *, cp1, tor_free(cp1)); + smartlist_free(tmp); + tor_free(cp); if (source) { - /* We know which authority we want. */ + /* We know which authority or directory mirror we want. */ directory_initiate_command_routerstatus(source, purpose, ROUTER_PURPOSE_GENERAL, DIRIND_ONEHOP, resource, NULL, 0, 0); } else { directory_get_from_dirserver(purpose, ROUTER_PURPOSE_GENERAL, resource, - pds_flags); + pds_flags, DL_WANT_ANY_DIRSERVER); } tor_free(resource); } -/** Max amount of hashes to download per request. - * Since squid does not like URLs >= 4096 bytes we limit it to 96. - * 4096 - strlen(http://255.255.255.255/tor/server/d/.z) == 4058 - * 4058/41 (40 for the hash and 1 for the + that separates them) => 98 - * So use 96 because it's a nice number. +/** Return the max number of hashes to put in a URL for a given request. */ -#define MAX_DL_PER_REQUEST 96 -#define MAX_MICRODESC_DL_PER_REQUEST 92 +static int +max_dl_per_request(const or_options_t *options, int purpose) +{ + /* Since squid does not like URLs >= 4096 bytes we limit it to 96. + * 4096 - strlen(http://[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535 + * /tor/server/d/.z) == 4026 + * 4026/41 (40 for the hash and 1 for the + that separates them) => 98 + * So use 96 because it's a nice number. + * + * For microdescriptors, the calculation is + * 4096 - strlen(http://[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff]:65535 + * /tor/micro/d/.z) == 4027 + * 4027/44 (43 for the hash and 1 for the - that separates them) => 91 + * So use 90 because it's a nice number. + */ + int max = 96; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + max = 90; + } + /* If we're going to tunnel our connections, we can ask for a lot more + * in a request. */ + if (directory_must_use_begindir(options)) { + max = 500; + } + return max; +} + /** Don't split our requests so finely that we are requesting fewer than * this number per server. */ #define MIN_DL_PER_REQUEST 4 @@ -4526,12 +4689,8 @@ initiate_descriptor_downloads(const routerstatus_t *source, * try to split our requests into at least this many requests. */ #define MIN_REQUESTS 3 /** If we want fewer than this many descriptors, wait until we - * want more, or until MAX_CLIENT_INTERVAL_WITHOUT_REQUEST has - * passed. */ + * want more, or until TestingClientMaxIntervalWithoutRequest has passed. */ #define MAX_DL_TO_DELAY 16 -/** When directory clients have only a few servers to request, they batch - * them until they have more, or until this amount of time has passed. */ -#define MAX_CLIENT_INTERVAL_WITHOUT_REQUEST (10*60) /** Given a <b>purpose</b> (FETCH_MICRODESC or FETCH_SERVERDESC) and a list of * router descriptor digests or microdescriptor digest256s in @@ -4544,238 +4703,89 @@ launch_descriptor_downloads(int purpose, smartlist_t *downloadable, const routerstatus_t *source, time_t now) { - int should_delay = 0, n_downloadable; const or_options_t *options = get_options(); const char *descname; + const int fetch_microdesc = (purpose == DIR_PURPOSE_FETCH_MICRODESC); + int n_downloadable = smartlist_len(downloadable); + + int i, n_per_request, max_dl_per_req; + const char *req_plural = "", *rtr_plural = ""; + int pds_flags = PDS_RETRY_IF_NO_SERVERS; - tor_assert(purpose == DIR_PURPOSE_FETCH_SERVERDESC || - purpose == DIR_PURPOSE_FETCH_MICRODESC); + tor_assert(fetch_microdesc || purpose == DIR_PURPOSE_FETCH_SERVERDESC); + descname = fetch_microdesc ? "microdesc" : "routerdesc"; - descname = (purpose == DIR_PURPOSE_FETCH_SERVERDESC) ? - "routerdesc" : "microdesc"; + if (!n_downloadable) + return; - n_downloadable = smartlist_len(downloadable); if (!directory_fetches_dir_info_early(options)) { if (n_downloadable >= MAX_DL_TO_DELAY) { log_debug(LD_DIR, "There are enough downloadable %ss to launch requests.", descname); - should_delay = 0; } else { - should_delay = (last_descriptor_download_attempted + - MAX_CLIENT_INTERVAL_WITHOUT_REQUEST) > now; - if (!should_delay && n_downloadable) { - if (last_descriptor_download_attempted) { - log_info(LD_DIR, - "There are not many downloadable %ss, but we've " - "been waiting long enough (%d seconds). Downloading.", - descname, - (int)(now-last_descriptor_download_attempted)); - } else { - log_info(LD_DIR, - "There are not many downloadable %ss, but we haven't " - "tried downloading descriptors recently. Downloading.", - descname); - } - } - } - } - if (! should_delay && n_downloadable) { - int i, n_per_request; - const char *req_plural = "", *rtr_plural = ""; - int pds_flags = PDS_RETRY_IF_NO_SERVERS; - if (! authdir_mode_any_nonhidserv(options)) { - /* If we wind up going to the authorities, we want to only open one - * connection to each authority at a time, so that we don't overload - * them. We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH - * regardless of whether we're a cache or not; it gets ignored if we're - * not calling router_pick_trusteddirserver. - * - * Setting this flag can make initiate_descriptor_downloads() ignore - * requests. We need to make sure that we do in fact call - * update_router_descriptor_downloads() later on, once the connections - * have succeeded or failed. - */ - pds_flags |= (purpose == DIR_PURPOSE_FETCH_MICRODESC) ? - PDS_NO_EXISTING_MICRODESC_FETCH : - PDS_NO_EXISTING_SERVERDESC_FETCH; - } + /* should delay */ + if ((last_descriptor_download_attempted + + options->TestingClientMaxIntervalWithoutRequest) > now) + return; - n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); - if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { - if (n_per_request > MAX_MICRODESC_DL_PER_REQUEST) - n_per_request = MAX_MICRODESC_DL_PER_REQUEST; - } else { - if (n_per_request > MAX_DL_PER_REQUEST) - n_per_request = MAX_DL_PER_REQUEST; - } - if (n_per_request < MIN_DL_PER_REQUEST) - n_per_request = MIN_DL_PER_REQUEST; - - if (n_downloadable > n_per_request) - req_plural = rtr_plural = "s"; - else if (n_downloadable > 1) - rtr_plural = "s"; - - log_info(LD_DIR, - "Launching %d request%s for %d %s%s, %d at a time", - CEIL_DIV(n_downloadable, n_per_request), req_plural, - n_downloadable, descname, rtr_plural, n_per_request); - smartlist_sort_digests(downloadable); - for (i=0; i < n_downloadable; i += n_per_request) { - initiate_descriptor_downloads(source, purpose, - downloadable, i, i+n_per_request, - pds_flags); + if (last_descriptor_download_attempted) { + log_info(LD_DIR, + "There are not many downloadable %ss, but we've " + "been waiting long enough (%d seconds). Downloading.", + descname, + (int)(now-last_descriptor_download_attempted)); + } else { + log_info(LD_DIR, + "There are not many downloadable %ss, but we haven't " + "tried downloading descriptors recently. Downloading.", + descname); + } } - last_descriptor_download_attempted = now; } -} -/** Launch downloads for router status as needed, using the strategy used by - * authorities and caches: based on the v2 networkstatuses we have, download - * every descriptor we don't have but would serve, from a random authority - * that lists it. */ -static void -update_router_descriptor_cache_downloads_v2(time_t now) -{ - smartlist_t **downloadable; /* For each authority, what can we dl from it? */ - smartlist_t **download_from; /* ... and, what will we dl from it? */ - digestmap_t *map; /* Which descs are in progress, or assigned? */ - int i, j, n; - int n_download; - const or_options_t *options = get_options(); - const smartlist_t *networkstatus_v2_list = networkstatus_get_v2_list(); - - if (! directory_fetches_dir_info_early(options)) { - log_warn(LD_BUG, "Called update_router_descriptor_cache_downloads_v2() " - "on a non-dir-mirror?"); + if (!authdir_mode_any_nonhidserv(options)) { + /* If we wind up going to the authorities, we want to only open one + * connection to each authority at a time, so that we don't overload + * them. We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH + * regardless of whether we're a cache or not. + * + * Setting this flag can make initiate_descriptor_downloads() ignore + * requests. We need to make sure that we do in fact call + * update_router_descriptor_downloads() later on, once the connections + * have succeeded or failed. + */ + pds_flags |= fetch_microdesc ? + PDS_NO_EXISTING_MICRODESC_FETCH : + PDS_NO_EXISTING_SERVERDESC_FETCH; } - if (!networkstatus_v2_list || !smartlist_len(networkstatus_v2_list)) - return; - - map = digestmap_new(); - n = smartlist_len(networkstatus_v2_list); + n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); + max_dl_per_req = max_dl_per_request(options, purpose); - downloadable = tor_malloc_zero(sizeof(smartlist_t*) * n); - download_from = tor_malloc_zero(sizeof(smartlist_t*) * n); + if (n_per_request > max_dl_per_req) + n_per_request = max_dl_per_req; - /* Set map[d]=1 for the digest of every descriptor that we are currently - * downloading. */ - list_pending_descriptor_downloads(map, 0); - - /* For the digest of every descriptor that we don't have, and that we aren't - * downloading, add d to downloadable[i] if the i'th networkstatus knows - * about that descriptor, and we haven't already failed to get that - * descriptor from the corresponding authority. - */ - n_download = 0; - SMARTLIST_FOREACH_BEGIN(networkstatus_v2_list, networkstatus_v2_t *, ns) { - dir_server_t *ds; - smartlist_t *dl; - dl = downloadable[ns_sl_idx] = smartlist_new(); - download_from[ns_sl_idx] = smartlist_new(); - if (ns->published_on + MAX_NETWORKSTATUS_AGE+10*60 < now) { - /* Don't download if the networkstatus is almost ancient. */ - /* Actually, I suspect what's happening here is that we ask - * for the descriptor when we have a given networkstatus, - * and then we get a newer networkstatus, and then we receive - * the descriptor. Having a networkstatus actually expire is - * probably a rare event, and we'll probably be happiest if - * we take this clause out. -RD */ - continue; - } + if (n_per_request < MIN_DL_PER_REQUEST) + n_per_request = MIN_DL_PER_REQUEST; - /* Don't try dirservers that we think are down -- we might have - * just tried them and just marked them as down. */ - ds = router_get_trusteddirserver_by_digest(ns->identity_digest); - if (ds && !ds->is_running) - continue; + if (n_downloadable > n_per_request) + req_plural = rtr_plural = "s"; + else if (n_downloadable > 1) + rtr_plural = "s"; - SMARTLIST_FOREACH_BEGIN(ns->entries, routerstatus_t * , rs) { - if (!rs->need_to_mirror) - continue; - if (router_get_by_descriptor_digest(rs->descriptor_digest)) { - log_warn(LD_BUG, - "We have a router descriptor, but need_to_mirror=1."); - rs->need_to_mirror = 0; - continue; - } - if (authdir_mode(options) && dirserv_would_reject_router(rs)) { - rs->need_to_mirror = 0; - continue; - } - if (digestmap_get(map, rs->descriptor_digest)) { - /* We're downloading it already. */ - continue; - } else { - /* We could download it from this guy. */ - smartlist_add(dl, rs->descriptor_digest); - ++n_download; - } - } SMARTLIST_FOREACH_END(rs); - } SMARTLIST_FOREACH_END(ns); - - /* At random, assign descriptors to authorities such that: - * - if d is a member of some downloadable[x], d is a member of some - * download_from[y]. (Everything we want to download, we try to download - * from somebody.) - * - If d is a member of download_from[y], d is a member of downloadable[y]. - * (We only try to download descriptors from authorities who claim to have - * them.) - * - No d is a member of download_from[x] and download_from[y] s.t. x != y. - * (We don't try to download anything from two authorities concurrently.) - */ - while (n_download) { - int which_ns = crypto_rand_int(n); - smartlist_t *dl = downloadable[which_ns]; - int idx; - char *d; - if (!smartlist_len(dl)) - continue; - idx = crypto_rand_int(smartlist_len(dl)); - d = smartlist_get(dl, idx); - if (! digestmap_get(map, d)) { - smartlist_add(download_from[which_ns], d); - digestmap_set(map, d, (void*) 1); - } - smartlist_del(dl, idx); - --n_download; - } - - /* Now, we can actually launch our requests. */ - for (i=0; i<n; ++i) { - networkstatus_v2_t *ns = smartlist_get(networkstatus_v2_list, i); - dir_server_t *ds = - router_get_trusteddirserver_by_digest(ns->identity_digest); - smartlist_t *dl = download_from[i]; - int pds_flags = PDS_RETRY_IF_NO_SERVERS; - if (! authdir_mode_any_nonhidserv(options)) - pds_flags |= PDS_NO_EXISTING_SERVERDESC_FETCH; /* XXXX ignored*/ - - if (!ds) { - log_info(LD_DIR, "Networkstatus with no corresponding authority!"); - continue; - } - if (! smartlist_len(dl)) - continue; - log_info(LD_DIR, "Requesting %d descriptors from authority \"%s\"", - smartlist_len(dl), ds->nickname); - for (j=0; j < smartlist_len(dl); j += MAX_DL_PER_REQUEST) { - initiate_descriptor_downloads(&(ds->fake_status), - DIR_PURPOSE_FETCH_SERVERDESC, dl, j, - j+MAX_DL_PER_REQUEST, pds_flags); - } - } - - for (i=0; i<n; ++i) { - smartlist_free(download_from[i]); - smartlist_free(downloadable[i]); + log_info(LD_DIR, + "Launching %d request%s for %d %s%s, %d at a time", + CEIL_DIV(n_downloadable, n_per_request), req_plural, + n_downloadable, descname, rtr_plural, n_per_request); + smartlist_sort_digests(downloadable); + for (i=0; i < n_downloadable; i += n_per_request) { + initiate_descriptor_downloads(source, purpose, + downloadable, i, i+n_per_request, + pds_flags); } - tor_free(download_from); - tor_free(downloadable); - digestmap_free(map,NULL); + last_descriptor_download_attempted = now; } /** For any descriptor that we want that's currently listed in @@ -4836,7 +4846,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote, continue; /* We have an in-progress download. */ } if (!download_status_is_ready(&rs->dl_status, now, - MAX_ROUTERDESC_DOWNLOAD_FAILURES)) { + options->TestingDescriptorMaxDownloadTries)) { ++n_delayed; /* Not ready for retry. */ continue; } @@ -4927,9 +4937,14 @@ launch_dummy_descriptor_download_as_needed(time_t now, last_descriptor_download_attempted + DUMMY_DOWNLOAD_INTERVAL < now && last_dummy_download + DUMMY_DOWNLOAD_INTERVAL < now) { last_dummy_download = now; + /* XX/teor - do we want an authority here, because they are less likely + * to give us the wrong address? (See #17782) + * I'm leaving the previous behaviour intact, because I don't like + * the idea of some relays contacting an authority every 20 minutes. */ directory_get_from_dirserver(DIR_PURPOSE_FETCH_SERVERDESC, ROUTER_PURPOSE_GENERAL, "authority.z", - PDS_RETRY_IF_NO_SERVERS); + PDS_RETRY_IF_NO_SERVERS, + DL_WANT_ANY_DIRSERVER); } } @@ -4938,13 +4953,10 @@ void update_router_descriptor_downloads(time_t now) { const or_options_t *options = get_options(); - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (!we_fetch_router_descriptors(options)) return; - if (directory_fetches_dir_info_early(options)) { - update_router_descriptor_cache_downloads_v2(now); - } update_consensus_router_descriptor_downloads(now, 0, networkstatus_get_reasonably_live_consensus(now, FLAV_NS)); @@ -4958,11 +4970,11 @@ update_extrainfo_downloads(time_t now) routerlist_t *rl; smartlist_t *wanted; digestmap_t *pending; - int old_routers, i; - int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0; + int old_routers, i, max_dl_per_req; + int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0, n_bogus[2] = {0,0}; if (! options->DownloadExtraInfo) return; - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (!router_have_minimum_dir_info()) return; @@ -4996,7 +5008,7 @@ update_extrainfo_downloads(time_t now) continue; } if (!download_status_is_ready(&sd->ei_dl_status, now, - MAX_ROUTERDESC_DOWNLOAD_FAILURES)) { + options->TestingDescriptorMaxDownloadTries)) { ++n_delay; continue; } @@ -5004,19 +5016,54 @@ update_extrainfo_downloads(time_t now) ++n_pending; continue; } + + const signed_descriptor_t *sd2 = router_get_by_extrainfo_digest(d); + if (sd2 != sd) { + if (sd2 != NULL) { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + char d3[HEX_DIGEST_LEN+1], d4[HEX_DIGEST_LEN+1]; + base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), sd2->identity_digest, DIGEST_LEN); + base16_encode(d3, sizeof(d3), d, DIGEST_LEN); + base16_encode(d4, sizeof(d3), sd2->extra_info_digest, DIGEST_LEN); + + log_info(LD_DIR, "Found an entry in %s with mismatched " + "router_get_by_extrainfo_digest() value. This has ID %s " + "but the entry in the map has ID %s. This has EI digest " + "%s and the entry in the map has EI digest %s.", + old_routers?"old_routers":"routers", + d1, d2, d3, d4); + } else { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), d, DIGEST_LEN); + + log_info(LD_DIR, "Found an entry in %s with NULL " + "router_get_by_extrainfo_digest() value. This has ID %s " + "and EI digest %s.", + old_routers?"old_routers":"routers", + d1, d2); + } + ++n_bogus[old_routers]; + continue; + } smartlist_add(wanted, d); } } digestmap_free(pending, NULL); log_info(LD_DIR, "Extrainfo download status: %d router with no ei, %d " - "with present ei, %d delaying, %d pending, %d downloadable.", - n_no_ei, n_have, n_delay, n_pending, smartlist_len(wanted)); + "with present ei, %d delaying, %d pending, %d downloadable, %d " + "bogus in routers, %d bogus in old_routers", + n_no_ei, n_have, n_delay, n_pending, smartlist_len(wanted), + n_bogus[0], n_bogus[1]); smartlist_shuffle(wanted); - for (i = 0; i < smartlist_len(wanted); i += MAX_DL_PER_REQUEST) { + + max_dl_per_req = max_dl_per_request(options, DIR_PURPOSE_FETCH_EXTRAINFO); + for (i = 0; i < smartlist_len(wanted); i += max_dl_per_req) { initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_EXTRAINFO, - wanted, i, i + MAX_DL_PER_REQUEST, + wanted, i, i+max_dl_per_req, PDS_RETRY_IF_NO_SERVERS|PDS_NO_EXISTING_SERVERDESC_FETCH); } @@ -5068,7 +5115,7 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) } /* If any key fields differ, they're different. */ - if (strcasecmp(r1->address, r2->address) || + if (r1->addr != r2->addr || strcasecmp(r1->nickname, r2->nickname) || r1->or_port != r2->or_port || !tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) || @@ -5083,7 +5130,9 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) (r1->contact_info && r2->contact_info && strcasecmp(r1->contact_info, r2->contact_info)) || r1->is_hibernating != r2->is_hibernating || - cmp_addr_policies(r1->exit_policy, r2->exit_policy)) + cmp_addr_policies(r1->exit_policy, r2->exit_policy) || + (r1->supports_tunnelled_dir_requests != + r2->supports_tunnelled_dir_requests)) return 0; if ((r1->declared_family == NULL) != (r2->declared_family == NULL)) return 0; @@ -5128,25 +5177,32 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) return 1; } -/** Check whether <b>ri</b> (a.k.a. sd) is a router compatible with the - * extrainfo document - * <b>ei</b>. If no router is compatible with <b>ei</b>, <b>ei</b> should be +/** Check whether <b>sd</b> describes a router descriptor compatible with the + * extrainfo document <b>ei</b>. + * + * <b>identity_pkey</b> (which must also be provided) is RSA1024 identity key + * for the router. We use it to check the signature of the extrainfo document, + * if it has not already been checked. + * + * If no router is compatible with <b>ei</b>, <b>ei</b> should be * dropped. Return 0 for "compatible", return 1 for "reject, and inform * whoever uploaded <b>ei</b>, and return -1 for "reject silently.". If * <b>msg</b> is present, set *<b>msg</b> to a description of the * incompatibility (if any). + * + * Set the extrainfo_is_bogus field in <b>sd</b> if the digests matched + * but the extrainfo was nonetheless incompatible. **/ int -routerinfo_incompatible_with_extrainfo(const routerinfo_t *ri, +routerinfo_incompatible_with_extrainfo(const crypto_pk_t *identity_pkey, extrainfo_t *ei, signed_descriptor_t *sd, const char **msg) { - int digest_matches, r=1; - tor_assert(ri); + int digest_matches, digest256_matches, r=1; + tor_assert(identity_pkey); + tor_assert(sd); tor_assert(ei); - if (!sd) - sd = (signed_descriptor_t*)&ri->cache_info; if (ei->bad_sig) { if (msg) *msg = "Extrainfo signature was bad, or signed with wrong key."; @@ -5155,19 +5211,31 @@ routerinfo_incompatible_with_extrainfo(const routerinfo_t *ri, digest_matches = tor_memeq(ei->cache_info.signed_descriptor_digest, sd->extra_info_digest, DIGEST_LEN); + /* Set digest256_matches to 1 if the digest is correct, or if no + * digest256 was in the ri. */ + digest256_matches = tor_memeq(ei->digest256, + sd->extra_info_digest256, DIGEST256_LEN); + digest256_matches |= + tor_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN); /* The identity must match exactly to have been generated at the same time * by the same router. */ - if (tor_memneq(ri->cache_info.identity_digest, + if (tor_memneq(sd->identity_digest, ei->cache_info.identity_digest, DIGEST_LEN)) { if (msg) *msg = "Extrainfo nickname or identity did not match routerinfo"; goto err; /* different servers */ } + if (! tor_cert_opt_eq(sd->signing_key_cert, + ei->cache_info.signing_key_cert)) { + if (msg) *msg = "Extrainfo signing key cert didn't match routerinfo"; + goto err; /* different servers */ + } + if (ei->pending_sig) { char signed_digest[128]; - if (crypto_pk_public_checksig(ri->identity_pkey, + if (crypto_pk_public_checksig(identity_pkey, signed_digest, sizeof(signed_digest), ei->pending_sig, ei->pending_sig_len) != DIGEST_LEN || tor_memneq(signed_digest, ei->cache_info.signed_descriptor_digest, @@ -5178,7 +5246,7 @@ routerinfo_incompatible_with_extrainfo(const routerinfo_t *ri, goto err; /* Bad signature, or no match. */ } - ei->cache_info.send_unencrypted = ri->cache_info.send_unencrypted; + ei->cache_info.send_unencrypted = sd->send_unencrypted; tor_free(ei->pending_sig); } @@ -5191,6 +5259,17 @@ routerinfo_incompatible_with_extrainfo(const routerinfo_t *ri, goto err; } + if (!digest256_matches && !digest_matches) { + if (msg) *msg = "Neither digest256 or digest matched " + "digest from routerdesc"; + goto err; + } + + if (!digest256_matches) { + if (msg) *msg = "Extrainfo digest did not match digest256 from routerdesc"; + goto err; /* Digest doesn't match declared value. */ + } + if (!digest_matches) { if (msg) *msg = "Extrainfo digest did not match value from routerdesc"; goto err; /* Digest doesn't match declared value. */ @@ -5250,7 +5329,7 @@ routerlist_assert_ok(const routerlist_t *rl) } SMARTLIST_FOREACH_END(r); SMARTLIST_FOREACH_BEGIN(rl->old_routers, signed_descriptor_t *, sd) { r2 = rimap_get(rl->identity_map, sd->identity_digest); - tor_assert(sd != &(r2->cache_info)); + tor_assert(!r2 || sd != &(r2->cache_info)); sd2 = sdmap_get(rl->desc_digest_map, sd->signed_descriptor_digest); tor_assert(sd == sd2); tor_assert(sd->routerlist_index == sd_sl_idx); @@ -5358,81 +5437,3 @@ refresh_all_country_info(void) nodelist_refresh_countries(); } -/** Determine the routers that are responsible for <b>id</b> (binary) and - * add pointers to those routers' routerstatus_t to <b>responsible_dirs</b>. - * Return -1 if we're returning an empty smartlist, else return 0. - */ -int -hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, - const char *id) -{ - int start, found, n_added = 0, i; - networkstatus_t *c = networkstatus_get_latest_consensus(); - if (!c || !smartlist_len(c->routerstatus_list)) { - log_warn(LD_REND, "We don't have a consensus, so we can't perform v2 " - "rendezvous operations."); - return -1; - } - tor_assert(id); - start = networkstatus_vote_find_entry_idx(c, id, &found); - if (start == smartlist_len(c->routerstatus_list)) start = 0; - i = start; - do { - routerstatus_t *r = smartlist_get(c->routerstatus_list, i); - if (r->is_hs_dir) { - smartlist_add(responsible_dirs, r); - if (++n_added == REND_NUMBER_OF_CONSECUTIVE_REPLICAS) - return 0; - } - if (++i == smartlist_len(c->routerstatus_list)) - i = 0; - } while (i != start); - - /* Even though we don't have the desired number of hidden service - * directories, be happy if we got any. */ - return smartlist_len(responsible_dirs) ? 0 : -1; -} - -/** Return true if this node is currently acting as hidden service - * directory, false otherwise. */ -int -hid_serv_acting_as_directory(void) -{ - const routerinfo_t *me = router_get_my_routerinfo(); - if (!me) - return 0; - if (!get_options()->HidServDirectoryV2) { - log_info(LD_REND, "We are not acting as hidden service directory, " - "because we have not been configured as such."); - return 0; - } - return 1; -} - -/** Return true if this node is responsible for storing the descriptor ID - * in <b>query</b> and false otherwise. */ -int -hid_serv_responsible_for_desc_id(const char *query) -{ - const routerinfo_t *me; - routerstatus_t *last_rs; - const char *my_id, *last_id; - int result; - smartlist_t *responsible; - if (!hid_serv_acting_as_directory()) - return 0; - if (!(me = router_get_my_routerinfo())) - return 0; /* This is redundant, but let's be paranoid. */ - my_id = me->cache_info.identity_digest; - responsible = smartlist_new(); - if (hid_serv_get_responsible_directories(responsible, query) < 0) { - smartlist_free(responsible); - return 0; - } - last_rs = smartlist_get(responsible, smartlist_len(responsible)-1); - last_id = last_rs->identity_digest; - result = rend_id_is_in_interval(my_id, query, last_id); - smartlist_free(responsible); - return result; -} - diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 505685897f..cb5b42a3b8 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,6 +11,8 @@ #ifndef TOR_ROUTERLIST_H #define TOR_ROUTERLIST_H +#include "testsupport.h" + int get_n_authorities(dirinfo_type_t type); int trusted_dirs_reload_certs(void); @@ -48,15 +50,20 @@ const routerstatus_t *router_pick_directory_server(dirinfo_type_t type, dir_server_t *router_get_trusteddirserver_by_digest(const char *d); dir_server_t *router_get_fallback_dirserver_by_digest( const char *digest); +int router_digest_is_fallback_dir(const char *digest); dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d); const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type, int flags); const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type, int flags); -int router_get_my_share_of_directory_requests(double *v2_share_out, - double *v3_share_out); +int router_get_my_share_of_directory_requests(double *v3_share_out); void router_reset_status_download_failures(void); int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2); +void router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid, + int need_uptime, int need_capacity, + int need_guard, int need_desc, + int pref_addr, int direct_conn); + const routerinfo_t *routerlist_find_my_routerinfo(void); uint32_t router_get_advertised_bandwidth(const routerinfo_t *router); uint32_t router_get_advertised_bandwidth_capped(const routerinfo_t *router); @@ -81,7 +88,8 @@ int hexdigest_to_digest(const char *hexdigest, char *digest); const routerinfo_t *router_get_by_id_digest(const char *digest); routerinfo_t *router_get_mutable_by_digest(const char *digest); signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); -signed_descriptor_t *router_get_by_extrainfo_digest(const char *digest); +MOCK_DECL(signed_descriptor_t *,router_get_by_extrainfo_digest, + (const char *digest)); signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); const char *signed_descriptor_get_body(const signed_descriptor_t *desc); const char *signed_descriptor_get_annotations(const signed_descriptor_t *desc); @@ -98,11 +106,12 @@ void routerlist_reset_warnings(void); static int WRA_WAS_ADDED(was_router_added_t s); static int WRA_WAS_OUTDATED(was_router_added_t s); static int WRA_WAS_REJECTED(was_router_added_t s); +static int WRA_NEVER_DOWNLOADABLE(was_router_added_t s); /** Return true iff the outcome code in <b>s</b> indicates that the descriptor * was added. It might still be necessary to check whether the descriptor * generator should be notified. */ -static INLINE int +static inline int WRA_WAS_ADDED(was_router_added_t s) { return s == ROUTER_ADDED_SUCCESSFULLY || s == ROUTER_ADDED_NOTIFY_GENERATOR; } @@ -111,19 +120,31 @@ WRA_WAS_ADDED(was_router_added_t s) { * - not in the consensus * - neither in the consensus nor in any networkstatus document * - it was outdated. + * - its certificates were expired. */ -static INLINE int WRA_WAS_OUTDATED(was_router_added_t s) +static inline int WRA_WAS_OUTDATED(was_router_added_t s) { - return (s == ROUTER_WAS_NOT_NEW || + return (s == ROUTER_WAS_TOO_OLD || + s == ROUTER_IS_ALREADY_KNOWN || s == ROUTER_NOT_IN_CONSENSUS || - s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS); + s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS || + s == ROUTER_CERTS_EXPIRED); } /** Return true iff the outcome code in <b>s</b> indicates that the descriptor * was flat-out rejected. */ -static INLINE int WRA_WAS_REJECTED(was_router_added_t s) +static inline int WRA_WAS_REJECTED(was_router_added_t s) { return (s == ROUTER_AUTHDIR_REJECTS); } +/** Return true iff the outcome code in <b>s</b> indicates that the descriptor + * was flat-out rejected. */ +static inline int WRA_NEVER_DOWNLOADABLE(was_router_added_t s) +{ + return (s == ROUTER_AUTHDIR_REJECTS || + s == ROUTER_BAD_EI || + s == ROUTER_WAS_TOO_OLD || + s == ROUTER_CERTS_EXPIRED); +} was_router_added_t router_add_to_routerlist(routerinfo_t *router, const char **msg, int from_cache, @@ -151,10 +172,12 @@ int router_exit_policy_rejects_all(const routerinfo_t *router); dir_server_t *trusted_dir_server_new(const char *nickname, const char *address, uint16_t dir_port, uint16_t or_port, + const tor_addr_port_t *addrport_ipv6, const char *digest, const char *v3_auth_digest, dirinfo_type_t type, double weight); dir_server_t *fallback_dir_server_new(const tor_addr_t *addr, uint16_t dir_port, uint16_t or_port, + const tor_addr_port_t *addrport_ipv6, const char *id_digest, double weight); void dir_server_add(dir_server_t *ent); @@ -168,7 +191,7 @@ void update_extrainfo_downloads(time_t now); void router_reset_descriptor_download_failures(void); int router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2); -int routerinfo_incompatible_with_extrainfo(const routerinfo_t *ri, +int routerinfo_incompatible_with_extrainfo(const crypto_pk_t *ri, extrainfo_t *ei, signed_descriptor_t *sd, const char **msg); @@ -179,12 +202,7 @@ void routers_sort_by_identity(smartlist_t *routers); void refresh_all_country_info(void); -int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, - const char *id); -int hid_serv_acting_as_directory(void); -int hid_serv_responsible_for_desc_id(const char *id); - -void list_pending_microdesc_downloads(digestmap_t *result); +void list_pending_microdesc_downloads(digest256map_t *result); void launch_descriptor_downloads(int purpose, smartlist_t *downloadable, const routerstatus_t *source, @@ -207,9 +225,25 @@ typedef union u64_dbl_t { double dbl; } u64_dbl_t; -int choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries); -void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, - uint64_t *total_out); +STATIC int choose_array_element_by_weight(const u64_dbl_t *entries, + int n_entries); +STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, + uint64_t *total_out); +STATIC const routerstatus_t *router_pick_directory_server_impl( + dirinfo_type_t auth, int flags, + int *n_busy_out); + +MOCK_DECL(int, router_descriptor_is_older_than, (const routerinfo_t *router, + int seconds)); +MOCK_DECL(STATIC was_router_added_t, extrainfo_insert, + (routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible)); + +MOCK_DECL(STATIC void, initiate_descriptor_downloads, + (const routerstatus_t *source, int purpose, smartlist_t *digests, + int lo, int hi, int pds_flags)); +STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap, + int serverdesc, int microdesc); + #endif #endif diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 71373ce63e..4db40c8435 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -9,6 +9,8 @@ * \brief Code to parse and validate router descriptors and directories. **/ +#define ROUTERPARSE_PRIVATE + #include "or.h" #include "config.h" #include "circuitstats.h" @@ -22,15 +24,20 @@ #include "microdesc.h" #include "networkstatus.h" #include "rephist.h" +#include "routerkeys.h" #include "routerparse.h" +#include "entrynodes.h" +#include "torcert.h" + #undef log #include <math.h> /****************************************************************************/ /** Enumeration of possible token types. The ones starting with K_ correspond - * to directory 'keywords'. ERR_ is an error in the tokenizing process, EOF_ - * is an end-of-file marker, and NIL_ is used to encode not-a-token. + * to directory 'keywords'. A_ is for an annotation, R or C is related to + * hidden services, ERR_ is an error in the tokenizing process, EOF_ is an + * end-of-file marker, and NIL_ is used to encode not-a-token. */ typedef enum { K_ACCEPT = 0, @@ -66,6 +73,7 @@ typedef enum { K_CLIENT_VERSIONS, K_SERVER_VERSIONS, K_OR_ADDRESS, + K_ID, K_P, K_P6, K_R, @@ -80,6 +88,11 @@ typedef enum { K_HIDDEN_SERVICE_DIR, K_ALLOW_SINGLE_HOP_EXITS, K_IPV6_POLICY, + K_ROUTER_SIG_ED25519, + K_IDENTITY_ED25519, + K_MASTER_KEY_ED25519, + K_ONION_KEY_CROSSCERT, + K_NTOR_ONION_KEY_CROSSCERT, K_DIRREQ_END, K_DIRREQ_V2_IPS, @@ -113,6 +126,7 @@ typedef enum { K_DIR_KEY_CERTIFICATION, K_DIR_KEY_CROSSCERT, K_DIR_ADDRESS, + K_DIR_TUNNELLED, K_VOTE_STATUS, K_VALID_AFTER, @@ -131,6 +145,7 @@ typedef enum { K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, K_DIRECTORY_FOOTER, + K_PACKAGE, A_PURPOSE, A_LAST_LISTED, @@ -289,6 +304,13 @@ static token_rule_t routerdesc_token_table[] = { T01("write-history", K_WRITE_HISTORY, ARGS, NO_OBJ ), T01("extra-info-digest", K_EXTRA_INFO_DIGEST, GE(1), NO_OBJ ), T01("hidden-service-dir", K_HIDDEN_SERVICE_DIR, NO_ARGS, NO_OBJ ), + T01("identity-ed25519", K_IDENTITY_ED25519, NO_ARGS, NEED_OBJ ), + T01("master-key-ed25519", K_MASTER_KEY_ED25519, GE(1), NO_OBJ ), + T01("router-sig-ed25519", K_ROUTER_SIG_ED25519, GE(1), NO_OBJ ), + T01("onion-key-crosscert", K_ONION_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ), + T01("ntor-onion-key-crosscert", K_NTOR_ONION_KEY_CROSSCERT, + EQ(1), NEED_OBJ ), + T01("allow-single-hop-exits",K_ALLOW_SINGLE_HOP_EXITS, NO_ARGS, NO_OBJ ), T01("family", K_FAMILY, ARGS, NO_OBJ ), @@ -298,6 +320,7 @@ static token_rule_t routerdesc_token_table[] = { T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T1( "bandwidth", K_BANDWIDTH, GE(3), NO_OBJ ), A01("@purpose", A_PURPOSE, GE(1), NO_OBJ ), + T01("tunnelled-dir-server",K_DIR_TUNNELLED, NO_ARGS, NO_OBJ ), END_OF_TABLE }; @@ -306,6 +329,8 @@ static token_rule_t routerdesc_token_table[] = { static token_rule_t extrainfo_token_table[] = { T1_END( "router-signature", K_ROUTER_SIGNATURE, NO_ARGS, NEED_OBJ ), T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ ), + T01("identity-ed25519", K_IDENTITY_ED25519, NO_ARGS, NEED_OBJ ), + T01("router-sig-ed25519", K_ROUTER_SIG_ED25519, GE(1), NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), T01("read-history", K_READ_HISTORY, ARGS, NO_OBJ ), T01("write-history", K_WRITE_HISTORY, ARGS, NO_OBJ ), @@ -339,7 +364,7 @@ static token_rule_t extrainfo_token_table[] = { END_OF_TABLE }; -/** List of tokens recognized in the body part of v2 and v3 networkstatus +/** List of tokens recognized in the body part of v3 networkstatus * documents. */ static token_rule_t rtrstatus_token_table[] = { T01("p", K_P, CONCAT_ARGS, NO_OBJ ), @@ -349,35 +374,11 @@ static token_rule_t rtrstatus_token_table[] = { T01("v", K_V, CONCAT_ARGS, NO_OBJ ), T01("w", K_W, ARGS, NO_OBJ ), T0N("m", K_M, CONCAT_ARGS, NO_OBJ ), + T0N("id", K_ID, GE(2), NO_OBJ ), T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), END_OF_TABLE }; -/** List of tokens recognized in the header part of v2 networkstatus documents. - */ -static token_rule_t netstatus_token_table[] = { - T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ ), - T0N("opt", K_OPT, CONCAT_ARGS, OBJ_OK ), - T1( "contact", K_CONTACT, CONCAT_ARGS, NO_OBJ ), - T1( "dir-signing-key", K_DIR_SIGNING_KEY, NO_ARGS, NEED_KEY_1024 ), - T1( "fingerprint", K_FINGERPRINT, CONCAT_ARGS, NO_OBJ ), - T1_START("network-status-version", K_NETWORK_STATUS_VERSION, - GE(1), NO_OBJ ), - T1( "dir-source", K_DIR_SOURCE, GE(3), NO_OBJ ), - T01("dir-options", K_DIR_OPTIONS, ARGS, NO_OBJ ), - T01("client-versions", K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ), - T01("server-versions", K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ), - - END_OF_TABLE -}; - -/** List of tokens recognized in the footer of v1/v2 directory/networkstatus - * footers. */ -static token_rule_t dir_footer_token_table[] = { - T1("directory-signature", K_DIRECTORY_SIGNATURE, EQ(1), NEED_OBJ ), - END_OF_TABLE -}; - /** List of tokens common to V3 authority certificates and V3 consensuses. */ #define CERTIFICATE_MEMBERS \ T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION, \ @@ -386,7 +387,7 @@ static token_rule_t dir_footer_token_table[] = { T1("dir-key-published",K_DIR_KEY_PUBLISHED, CONCAT_ARGS, NO_OBJ), \ T1("dir-key-expires", K_DIR_KEY_EXPIRES, CONCAT_ARGS, NO_OBJ), \ T1("dir-signing-key", K_DIR_SIGNING_KEY, NO_ARGS, NEED_KEY ),\ - T01("dir-key-crosscert", K_DIR_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ),\ + T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT, NO_ARGS, NEED_OBJ ),\ T1("dir-key-certification", K_DIR_KEY_CERTIFICATION, \ NO_ARGS, NEED_OBJ), \ T01("dir-address", K_DIR_ADDRESS, GE(1), NO_OBJ), @@ -445,6 +446,7 @@ static token_rule_t networkstatus_token_table[] = { T1("known-flags", K_KNOWN_FLAGS, ARGS, NO_OBJ ), T01("params", K_PARAMS, ARGS, NO_OBJ ), T( "fingerprint", K_FINGERPRINT, CONCAT_ARGS, NO_OBJ ), + T0N("package", K_PACKAGE, CONCAT_ARGS, NO_OBJ ), CERTIFICATE_MEMBERS @@ -486,8 +488,7 @@ static token_rule_t networkstatus_consensus_token_table[] = { END_OF_TABLE }; -/** List of tokens recognized in the footer of v1/v2 directory/networkstatus - * footers. */ +/** List of tokens recognized in the footer of v1 directory footers. */ static token_rule_t networkstatus_vote_footer_token_table[] = { T01("directory-footer", K_DIRECTORY_FOOTER, NO_ARGS, NO_OBJ ), T01("bandwidth-weights", K_BW_WEIGHTS, ARGS, NO_OBJ ), @@ -511,6 +512,7 @@ static token_rule_t networkstatus_detached_signature_token_table[] = { static token_rule_t microdesc_token_table[] = { T1_START("onion-key", K_ONION_KEY, NO_ARGS, NEED_KEY_1024), T01("ntor-onion-key", K_ONION_KEY_NTOR, GE(1), NO_OBJ ), + T0N("id", K_ID, GE(2), NO_OBJ ), T0N("a", K_A, GE(1), NO_OBJ ), T01("family", K_FAMILY, ARGS, NO_OBJ ), T01("p", K_P, CONCAT_ARGS, NO_OBJ ), @@ -527,12 +529,16 @@ static addr_policy_t *router_parse_addr_policy(directory_token_t *tok, unsigned fmt_flags); static addr_policy_t *router_parse_addr_policy_private(directory_token_t *tok); +static int router_get_hash_impl_helper(const char *s, size_t s_len, + const char *start_str, + const char *end_str, char end_c, + const char **start_out, const char **end_out); static int router_get_hash_impl(const char *s, size_t s_len, char *digest, const char *start_str, const char *end_str, char end_char, digest_algorithm_t alg); static int router_get_hashes_impl(const char *s, size_t s_len, - digests_t *digests, + common_digests_t *digests, const char *start_str, const char *end_str, char end_char); static void token_clear(directory_token_t *tok); @@ -598,7 +604,7 @@ dump_desc(const char *desc, const char *type) char *content = tor_malloc_zero(filelen); tor_snprintf(content, filelen, "Unable to parse descriptor of type " "%s:\n%s", type, desc); - write_str_to_file(debugfile, content, 0); + write_str_to_file(debugfile, content, 1); log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " "unparseable-desc in data directory for details.", type); tor_free(content); @@ -629,32 +635,10 @@ router_get_router_hash(const char *s, size_t s_len, char *digest) DIGEST_SHA1); } -/** Set <b>digest</b> to the SHA-1 digest of the hash of the running-routers - * string in <b>s</b>. Return 0 on success, -1 on failure. - */ -int -router_get_runningrouters_hash(const char *s, char *digest) -{ - return router_get_hash_impl(s, strlen(s), digest, - "network-status","\ndirectory-signature", '\n', - DIGEST_SHA1); -} - -/** Set <b>digest</b> to the SHA-1 digest of the hash of the network-status - * string in <b>s</b>. Return 0 on success, -1 on failure. */ -int -router_get_networkstatus_v2_hash(const char *s, char *digest) -{ - return router_get_hash_impl(s, strlen(s), digest, - "network-status-version","\ndirectory-signature", - '\n', - DIGEST_SHA1); -} - /** Set <b>digests</b> to all the digests of the consensus document in * <b>s</b> */ int -router_get_networkstatus_v3_hashes(const char *s, digests_t *digests) +router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests) { return router_get_hashes_impl(s,strlen(s),digests, "network-status-version", @@ -680,7 +664,7 @@ router_get_extrainfo_hash(const char *s, size_t s_len, char *digest) char * router_get_dirobj_signature(const char *digest, size_t digest_len, - crypto_pk_t *private_key) + const crypto_pk_t *private_key) { char *signature; size_t i, keysize; @@ -707,7 +691,8 @@ router_get_dirobj_signature(const char *digest, goto truncated; i = strlen(buf); - if (base64_encode(buf+i, buf_len-i, signature, siglen) < 0) { + if (base64_encode(buf+i, buf_len-i, signature, siglen, + BASE64_ENCODE_MULTILINE) < 0) { log_warn(LD_BUG,"couldn't base64-encode signature"); goto err; } @@ -728,7 +713,7 @@ router_get_dirobj_signature(const char *digest, /** Helper: used to generate signatures for routers, directories and * network-status objects. Given a digest in <b>digest</b> and a secret - * <b>private_key</b>, generate an PKCS1-padded signature, BASE64-encode it, + * <b>private_key</b>, generate a PKCS1-padded signature, BASE64-encode it, * surround it with -----BEGIN/END----- pairs, and write it to the * <b>buf_len</b>-byte buffer at <b>buf</b>. Return 0 on success, -1 on * failure. @@ -751,6 +736,7 @@ router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, return -1; } memcpy(buf+s_len, sig, sig_len+1); + tor_free(sig); return 0; } @@ -899,8 +885,8 @@ check_signature_token(const char *digest, tor_free(signed_digest); return -1; } -// log_debug(LD_DIR,"Signed %s hash starts %s", doctype, -// hex_str(signed_digest,4)); + // log_debug(LD_DIR,"Signed %s hash starts %s", doctype, + // hex_str(signed_digest,4)); if (tor_memneq(digest, signed_digest, digest_len)) { log_warn(LD_DIR, "Error reading %s: signature does not match.", doctype); tor_free(signed_digest); @@ -958,7 +944,9 @@ find_start_of_next_router_or_extrainfo(const char **s_ptr, * descriptor in the signed_descriptor_body field of each routerinfo_t. If it * isn't SAVED_NOWHERE, remember the offset of each descriptor. * - * Returns 0 on success and -1 on failure. + * Returns 0 on success and -1 on failure. Adds a digest to + * <b>invalid_digests_out</b> for every entry that was unparseable or + * invalid. (This may cause duplicate entries.) */ int router_parse_list_from_string(const char **s, const char *eos, @@ -966,11 +954,12 @@ router_parse_list_from_string(const char **s, const char *eos, saved_location_t saved_location, int want_extrainfo, int allow_annotations, - const char *prepend_annotations) + const char *prepend_annotations, + smartlist_t *invalid_digests_out) { routerinfo_t *router; extrainfo_t *extrainfo; - signed_descriptor_t *signed_desc; + signed_descriptor_t *signed_desc = NULL; void *elt; const char *end, *start; int have_extrainfo; @@ -986,6 +975,9 @@ router_parse_list_from_string(const char **s, const char *eos, tor_assert(eos >= *s); while (1) { + char raw_digest[DIGEST_LEN]; + int have_raw_digest = 0; + int dl_again = 0; if (find_start_of_next_router_or_extrainfo(s, eos, &have_extrainfo) < 0) break; @@ -1002,18 +994,20 @@ router_parse_list_from_string(const char **s, const char *eos, if (have_extrainfo && want_extrainfo) { routerlist_t *rl = router_get_routerlist(); + have_raw_digest = router_get_extrainfo_hash(*s, end-*s, raw_digest) == 0; extrainfo = extrainfo_parse_entry_from_string(*s, end, saved_location != SAVED_IN_CACHE, - rl->identity_map); + rl->identity_map, &dl_again); if (extrainfo) { signed_desc = &extrainfo->cache_info; elt = extrainfo; } } else if (!have_extrainfo && !want_extrainfo) { + have_raw_digest = router_get_router_hash(*s, end-*s, raw_digest) == 0; router = router_parse_entry_from_string(*s, end, saved_location != SAVED_IN_CACHE, allow_annotations, - prepend_annotations); + prepend_annotations, &dl_again); if (router) { log_debug(LD_DIR, "Read router '%s', purpose '%s'", router_describe(router), @@ -1022,11 +1016,15 @@ router_parse_list_from_string(const char **s, const char *eos, elt = router; } } + if (! elt && ! dl_again && have_raw_digest && invalid_digests_out) { + smartlist_add(invalid_digests_out, tor_memdup(raw_digest, DIGEST_LEN)); + } if (!elt) { *s = end; continue; } if (saved_location != SAVED_NOWHERE) { + tor_assert(signed_desc); signed_desc->saved_location = saved_location; signed_desc->saved_offset = *s - start; } @@ -1114,11 +1112,17 @@ find_single_ipv6_orport(const smartlist_t *list, * around when caching the router. * * Only one of allow_annotations and prepend_annotations may be set. + * + * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1 + * if it's okay to try to download a descriptor with this same digest again, + * and 0 if it isn't. (It might not be okay to download it again if part of + * the part covered by the digest is invalid.) */ routerinfo_t * router_parse_entry_from_string(const char *s, const char *end, int cache_copy, int allow_annotations, - const char *prepend_annotations) + const char *prepend_annotations, + int *can_dl_again_out) { routerinfo_t *router = NULL; char digest[128]; @@ -1129,6 +1133,10 @@ router_parse_entry_from_string(const char *s, const char *end, size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0; int ok = 1; memarea_t *area = NULL; + tor_cert_t *ntor_cc_cert = NULL; + /* Do not set this to '1' until we have parsed everything that we intend to + * parse that's covered by the hash. */ + int can_dl_again = 0; tor_assert(!allow_annotations || !prepend_annotations); @@ -1198,9 +1206,11 @@ router_parse_entry_from_string(const char *s, const char *end, } tok = find_by_keyword(tokens, K_ROUTER); + const int router_token_pos = smartlist_pos(tokens, tok); tor_assert(tok->n_args >= 5); router = tor_malloc_zero(sizeof(routerinfo_t)); + router->cert_expiration_time = TIME_MAX; router->cache_info.routerlist_index = -1; router->cache_info.annotations_len = s-start_of_annotations + prepend_len; router->cache_info.signed_descriptor_len = end-s; @@ -1232,8 +1242,7 @@ router_parse_entry_from_string(const char *s, const char *end, log_warn(LD_DIR,"Router nickname is invalid"); goto err; } - router->address = tor_strdup(tok->args[1]); - if (!tor_inet_aton(router->address, &in)) { + if (!tor_inet_aton(tok->args[1], &in)) { log_warn(LD_DIR,"Router address is not an IP address."); goto err; } @@ -1332,6 +1341,173 @@ router_parse_entry_from_string(const char *s, const char *end, log_warn(LD_DIR, "Couldn't calculate key digest"); goto err; } + { + directory_token_t *ed_sig_tok, *ed_cert_tok, *cc_tap_tok, *cc_ntor_tok, + *master_key_tok; + ed_sig_tok = find_opt_by_keyword(tokens, K_ROUTER_SIG_ED25519); + ed_cert_tok = find_opt_by_keyword(tokens, K_IDENTITY_ED25519); + master_key_tok = find_opt_by_keyword(tokens, K_MASTER_KEY_ED25519); + cc_tap_tok = find_opt_by_keyword(tokens, K_ONION_KEY_CROSSCERT); + cc_ntor_tok = find_opt_by_keyword(tokens, K_NTOR_ONION_KEY_CROSSCERT); + int n_ed_toks = !!ed_sig_tok + !!ed_cert_tok + + !!cc_tap_tok + !!cc_ntor_tok; + if ((n_ed_toks != 0 && n_ed_toks != 4) || + (n_ed_toks == 4 && !router->onion_curve25519_pkey)) { + log_warn(LD_DIR, "Router descriptor with only partial ed25519/" + "cross-certification support"); + goto err; + } + if (master_key_tok && !ed_sig_tok) { + log_warn(LD_DIR, "Router descriptor has ed25519 master key but no " + "certificate"); + goto err; + } + if (ed_sig_tok) { + tor_assert(ed_cert_tok && cc_tap_tok && cc_ntor_tok); + const int ed_cert_token_pos = smartlist_pos(tokens, ed_cert_tok); + if (ed_cert_token_pos == -1 || router_token_pos == -1 || + (ed_cert_token_pos != router_token_pos + 1 && + ed_cert_token_pos != router_token_pos - 1)) { + log_warn(LD_DIR, "Ed25519 certificate in wrong position"); + goto err; + } + if (ed_sig_tok != smartlist_get(tokens, smartlist_len(tokens)-2)) { + log_warn(LD_DIR, "Ed25519 signature in wrong position"); + goto err; + } + if (strcmp(ed_cert_tok->object_type, "ED25519 CERT")) { + log_warn(LD_DIR, "Wrong object type on identity-ed25519 in decriptor"); + goto err; + } + if (strcmp(cc_ntor_tok->object_type, "ED25519 CERT")) { + log_warn(LD_DIR, "Wrong object type on ntor-onion-key-crosscert " + "in decriptor"); + goto err; + } + if (strcmp(cc_tap_tok->object_type, "CROSSCERT")) { + log_warn(LD_DIR, "Wrong object type on onion-key-crosscert " + "in decriptor"); + goto err; + } + if (strcmp(cc_ntor_tok->args[0], "0") && + strcmp(cc_ntor_tok->args[0], "1")) { + log_warn(LD_DIR, "Bad sign bit on ntor-onion-key-crosscert"); + goto err; + } + int ntor_cc_sign_bit = !strcmp(cc_ntor_tok->args[0], "1"); + + uint8_t d256[DIGEST256_LEN]; + const char *signed_start, *signed_end; + tor_cert_t *cert = tor_cert_parse( + (const uint8_t*)ed_cert_tok->object_body, + ed_cert_tok->object_size); + if (! cert) { + log_warn(LD_DIR, "Couldn't parse ed25519 cert"); + goto err; + } + /* makes sure it gets freed. */ + router->cache_info.signing_key_cert = cert; + + if (cert->cert_type != CERT_TYPE_ID_SIGNING || + ! cert->signing_key_included) { + log_warn(LD_DIR, "Invalid form for ed25519 cert"); + goto err; + } + + if (master_key_tok) { + /* This token is optional, but if it's present, it must match + * the signature in the signing cert, or supplant it. */ + tor_assert(master_key_tok->n_args >= 1); + ed25519_public_key_t pkey; + if (ed25519_public_from_base64(&pkey, master_key_tok->args[0])<0) { + log_warn(LD_DIR, "Can't parse ed25519 master key"); + goto err; + } + + if (fast_memneq(&cert->signing_key.pubkey, + pkey.pubkey, ED25519_PUBKEY_LEN)) { + log_warn(LD_DIR, "Ed25519 master key does not match " + "key in certificate"); + goto err; + } + } + ntor_cc_cert = tor_cert_parse((const uint8_t*)cc_ntor_tok->object_body, + cc_ntor_tok->object_size); + if (!ntor_cc_cert) { + log_warn(LD_DIR, "Couldn't parse ntor-onion-key-crosscert cert"); + goto err; + } + if (ntor_cc_cert->cert_type != CERT_TYPE_ONION_ID || + ! ed25519_pubkey_eq(&ntor_cc_cert->signed_key, &cert->signing_key)) { + log_warn(LD_DIR, "Invalid contents for ntor-onion-key-crosscert cert"); + goto err; + } + + ed25519_public_key_t ntor_cc_pk; + if (ed25519_public_key_from_curve25519_public_key(&ntor_cc_pk, + router->onion_curve25519_pkey, + ntor_cc_sign_bit)<0) { + log_warn(LD_DIR, "Error converting onion key to ed25519"); + goto err; + } + + if (router_get_hash_impl_helper(s, end-s, "router ", + "\nrouter-sig-ed25519", + ' ', &signed_start, &signed_end) < 0) { + log_warn(LD_DIR, "Can't find ed25519-signed portion of descriptor"); + goto err; + } + crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256); + crypto_digest_add_bytes(d, ED_DESC_SIGNATURE_PREFIX, + strlen(ED_DESC_SIGNATURE_PREFIX)); + crypto_digest_add_bytes(d, signed_start, signed_end-signed_start); + crypto_digest_get_digest(d, (char*)d256, sizeof(d256)); + crypto_digest_free(d); + + ed25519_checkable_t check[3]; + int check_ok[3]; + if (tor_cert_get_checkable_sig(&check[0], cert, NULL) < 0) { + log_err(LD_BUG, "Couldn't create 'checkable' for cert."); + goto err; + } + if (tor_cert_get_checkable_sig(&check[1], + ntor_cc_cert, &ntor_cc_pk) < 0) { + log_err(LD_BUG, "Couldn't create 'checkable' for ntor_cc_cert."); + goto err; + } + + if (ed25519_signature_from_base64(&check[2].signature, + ed_sig_tok->args[0])<0) { + log_warn(LD_DIR, "Couldn't decode ed25519 signature"); + goto err; + } + check[2].pubkey = &cert->signed_key; + check[2].msg = d256; + check[2].len = DIGEST256_LEN; + + if (ed25519_checksig_batch(check_ok, check, 3) < 0) { + log_warn(LD_DIR, "Incorrect ed25519 signature(s)"); + goto err; + } + + if (check_tap_onion_key_crosscert( + (const uint8_t*)cc_tap_tok->object_body, + (int)cc_tap_tok->object_size, + router->onion_pkey, + &cert->signing_key, + (const uint8_t*)router->cache_info.identity_digest)<0) { + log_warn(LD_DIR, "Incorrect TAP cross-verification"); + goto err; + } + + /* We check this before adding it to the routerlist. */ + if (cert->valid_until < ntor_cc_cert->valid_until) + router->cert_expiration_time = cert->valid_until; + else + router->cert_expiration_time = ntor_cc_cert->valid_until; + } + } + if ((tok = find_opt_by_keyword(tokens, K_FINGERPRINT))) { /* If there's a fingerprint line, it must match the identity digest. */ char d[DIGEST_LEN]; @@ -1423,12 +1599,26 @@ router_parse_entry_from_string(const char *s, const char *end, } else { log_warn(LD_DIR, "Invalid extra info digest %s", escaped(tok->args[0])); } + + if (tok->n_args >= 2) { + if (digest256_from_base64(router->cache_info.extra_info_digest256, + tok->args[1]) < 0) { + log_warn(LD_DIR, "Invalid extra info digest256 %s", + escaped(tok->args[1])); + } + } } if (find_opt_by_keyword(tokens, K_HIDDEN_SERVICE_DIR)) { router->wants_to_be_hs_dir = 1; } + /* This router accepts tunnelled directory requests via begindir if it has + * an open dirport or it included "tunnelled-dir-server". */ + if (find_opt_by_keyword(tokens, K_DIR_TUNNELLED) || router->dir_port > 0) { + router->supports_tunnelled_dir_requests = 1; + } + tok = find_by_keyword(tokens, K_ROUTER_SIGNATURE); note_crypto_pk_op(VERIFY_RTR); #ifdef COUNT_DISTINCT_DIGESTS @@ -1436,19 +1626,21 @@ router_parse_entry_from_string(const char *s, const char *end, verified_digests = digestmap_new(); digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1); #endif - if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, - "router descriptor") < 0) - goto err; if (!router->or_port) { log_warn(LD_DIR,"or_port unreadable or 0. Failing."); goto err; } + /* We've checked everything that's covered by the hash. */ + can_dl_again = 1; + if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, + "router descriptor") < 0) + goto err; + if (!router->platform) { router->platform = tor_strdup("<unknown>"); } - goto done; err: @@ -1456,6 +1648,7 @@ router_parse_entry_from_string(const char *s, const char *end, routerinfo_free(router); router = NULL; done: + tor_cert_free(ntor_cc_cert); if (tokens) { SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); smartlist_free(tokens); @@ -1465,6 +1658,8 @@ router_parse_entry_from_string(const char *s, const char *end, DUMP_AREA(area, "routerinfo"); memarea_drop_all(area); } + if (can_dl_again_out) + *can_dl_again_out = can_dl_again; return router; } @@ -1473,10 +1668,16 @@ router_parse_entry_from_string(const char *s, const char *end, * <b>cache_copy</b> is true, make a copy of the extra-info document in the * cache_info fields of the result. If <b>routermap</b> is provided, use it * as a map from router identity to routerinfo_t when looking up signing keys. + * + * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1 + * if it's okay to try to download an extrainfo with this same digest again, + * and 0 if it isn't. (It might not be okay to download it again if part of + * the part covered by the digest is invalid.) */ extrainfo_t * extrainfo_parse_entry_from_string(const char *s, const char *end, - int cache_copy, struct digest_ri_map_t *routermap) + int cache_copy, struct digest_ri_map_t *routermap, + int *can_dl_again_out) { extrainfo_t *extrainfo = NULL; char digest[128]; @@ -1486,6 +1687,9 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, routerinfo_t *router = NULL; memarea_t *area = NULL; const char *s_dup = s; + /* Do not set this to '1' until we have parsed everything that we intend to + * parse that's covered by the hash. */ + int can_dl_again = 0; if (!end) { end = s + strlen(s); @@ -1511,6 +1715,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, goto err; } + /* XXXX Accept this in position 1 too, and ed identity in position 0. */ tok = smartlist_get(tokens,0); if (tok->tp != K_EXTRA_INFO) { log_warn(LD_DIR,"Entry does not start with \"extra-info\""); @@ -1523,6 +1728,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, extrainfo->cache_info.signed_descriptor_body = tor_memdup_nulterm(s,end-s); extrainfo->cache_info.signed_descriptor_len = end-s; memcpy(extrainfo->cache_info.signed_descriptor_digest, digest, DIGEST_LEN); + crypto_digest256((char*)extrainfo->digest256, s, end-s, DIGEST_SHA256); tor_assert(tok->n_args >= 2); if (!is_legal_nickname(tok->args[0])) { @@ -1545,6 +1751,92 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, goto err; } + { + directory_token_t *ed_sig_tok, *ed_cert_tok; + ed_sig_tok = find_opt_by_keyword(tokens, K_ROUTER_SIG_ED25519); + ed_cert_tok = find_opt_by_keyword(tokens, K_IDENTITY_ED25519); + int n_ed_toks = !!ed_sig_tok + !!ed_cert_tok; + if (n_ed_toks != 0 && n_ed_toks != 2) { + log_warn(LD_DIR, "Router descriptor with only partial ed25519/" + "cross-certification support"); + goto err; + } + if (ed_sig_tok) { + tor_assert(ed_cert_tok); + const int ed_cert_token_pos = smartlist_pos(tokens, ed_cert_tok); + if (ed_cert_token_pos != 1) { + /* Accept this in position 0 XXXX */ + log_warn(LD_DIR, "Ed25519 certificate in wrong position"); + goto err; + } + if (ed_sig_tok != smartlist_get(tokens, smartlist_len(tokens)-2)) { + log_warn(LD_DIR, "Ed25519 signature in wrong position"); + goto err; + } + if (strcmp(ed_cert_tok->object_type, "ED25519 CERT")) { + log_warn(LD_DIR, "Wrong object type on identity-ed25519 in decriptor"); + goto err; + } + + uint8_t d256[DIGEST256_LEN]; + const char *signed_start, *signed_end; + tor_cert_t *cert = tor_cert_parse( + (const uint8_t*)ed_cert_tok->object_body, + ed_cert_tok->object_size); + if (! cert) { + log_warn(LD_DIR, "Couldn't parse ed25519 cert"); + goto err; + } + /* makes sure it gets freed. */ + extrainfo->cache_info.signing_key_cert = cert; + + if (cert->cert_type != CERT_TYPE_ID_SIGNING || + ! cert->signing_key_included) { + log_warn(LD_DIR, "Invalid form for ed25519 cert"); + goto err; + } + + if (router_get_hash_impl_helper(s, end-s, "extra-info ", + "\nrouter-sig-ed25519", + ' ', &signed_start, &signed_end) < 0) { + log_warn(LD_DIR, "Can't find ed25519-signed portion of extrainfo"); + goto err; + } + crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA256); + crypto_digest_add_bytes(d, ED_DESC_SIGNATURE_PREFIX, + strlen(ED_DESC_SIGNATURE_PREFIX)); + crypto_digest_add_bytes(d, signed_start, signed_end-signed_start); + crypto_digest_get_digest(d, (char*)d256, sizeof(d256)); + crypto_digest_free(d); + + ed25519_checkable_t check[2]; + int check_ok[2]; + if (tor_cert_get_checkable_sig(&check[0], cert, NULL) < 0) { + log_err(LD_BUG, "Couldn't create 'checkable' for cert."); + goto err; + } + + if (ed25519_signature_from_base64(&check[1].signature, + ed_sig_tok->args[0])<0) { + log_warn(LD_DIR, "Couldn't decode ed25519 signature"); + goto err; + } + check[1].pubkey = &cert->signed_key; + check[1].msg = d256; + check[1].len = DIGEST256_LEN; + + if (ed25519_checksig_batch(check_ok, check, 2) < 0) { + log_warn(LD_DIR, "Incorrect ed25519 signature(s)"); + goto err; + } + /* We don't check the certificate expiration time: checking that it + * matches the cert in the router descriptor is adequate. */ + } + } + + /* We've checked everything that's covered by the hash. */ + can_dl_again = 1; + if (routermap && (router = digestmap_get((digestmap_t*)routermap, extrainfo->cache_info.identity_digest))) { @@ -1587,6 +1879,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, DUMP_AREA(area, "extrainfo"); memarea_drop_all(area); } + if (can_dl_again_out) + *can_dl_again_out = can_dl_again; return extrainfo; } @@ -1728,7 +2022,6 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) log_debug(LD_DIR, "We already checked the signature on this " "certificate; no need to do so again."); found = 1; - cert->is_cross_certified = old_cert->is_cross_certified; } } if (!found) { @@ -1737,18 +2030,14 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) goto err; } - if ((tok = find_opt_by_keyword(tokens, K_DIR_KEY_CROSSCERT))) { - /* XXXX Once all authorities generate cross-certified certificates, - * make this field mandatory. */ - if (check_signature_token(cert->cache_info.identity_digest, - DIGEST_LEN, - tok, - cert->signing_key, - CST_NO_CHECK_OBJTYPE, - "key cross-certification")) { - goto err; - } - cert->is_cross_certified = 1; + tok = find_by_keyword(tokens, K_DIR_KEY_CROSSCERT); + if (check_signature_token(cert->cache_info.identity_digest, + DIGEST_LEN, + tok, + cert->signing_key, + CST_NO_CHECK_OBJTYPE, + "key cross-certification")) { + goto err; } } @@ -1784,7 +2073,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) * object (starting with "r " at the start of a line). If none is found, * return the start of the directory footer, or the next directory signature. * If none is found, return the end of the string. */ -static INLINE const char * +static inline const char * find_start_of_next_routerstatus(const char *s) { const char *eos, *footer, *sig; @@ -1806,6 +2095,63 @@ find_start_of_next_routerstatus(const char *s) return eos; } +/** Parse the GuardFraction string from a consensus or vote. + * + * If <b>vote</b> or <b>vote_rs</b> are set the document getting + * parsed is a vote routerstatus. Otherwise it's a consensus. This is + * the same semantic as in routerstatus_parse_entry_from_string(). */ +STATIC int +routerstatus_parse_guardfraction(const char *guardfraction_str, + networkstatus_t *vote, + vote_routerstatus_t *vote_rs, + routerstatus_t *rs) +{ + int ok; + const char *end_of_header = NULL; + int is_consensus = !vote_rs; + uint32_t guardfraction; + + tor_assert(bool_eq(vote, vote_rs)); + + /* If this info comes from a consensus, but we should't apply + guardfraction, just exit. */ + if (is_consensus && !should_apply_guardfraction(NULL)) { + return 0; + } + + end_of_header = strchr(guardfraction_str, '='); + if (!end_of_header) { + return -1; + } + + guardfraction = (uint32_t)tor_parse_ulong(end_of_header+1, + 10, 0, 100, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Invalid GuardFraction %s", escaped(guardfraction_str)); + return -1; + } + + log_debug(LD_GENERAL, "[*] Parsed %s guardfraction '%s' for '%s'.", + is_consensus ? "consensus" : "vote", + guardfraction_str, rs->nickname); + + if (!is_consensus) { /* We are parsing a vote */ + vote_rs->status.guardfraction_percentage = guardfraction; + vote_rs->status.has_guardfraction = 1; + } else { + /* We are parsing a consensus. Only apply guardfraction to guards. */ + if (rs->is_possible_guard) { + rs->guardfraction_percentage = guardfraction; + rs->has_guardfraction = 1; + } else { + log_warn(LD_BUG, "Got GuardFraction for non-guard %s. " + "This is not supposed to happen. Not applying. ", rs->nickname); + } + } + + return 0; +} + /** Given a string at *<b>s</b>, containing a routerstatus object, and an * empty smartlist at <b>tokens</b>, parse and return the first router status * object in the string, and advance *<b>s</b> to just after the end of the @@ -1948,14 +2294,10 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_named = 1; else if (!strcmp(tok->args[i], "Valid")) rs->is_valid = 1; - else if (!strcmp(tok->args[i], "V2Dir")) - rs->is_v2_dir = 1; else if (!strcmp(tok->args[i], "Guard")) rs->is_possible_guard = 1; else if (!strcmp(tok->args[i], "BadExit")) rs->is_bad_exit = 1; - else if (!strcmp(tok->args[i], "BadDirectory")) - rs->is_bad_directory = 1; else if (!strcmp(tok->args[i], "Authority")) rs->is_authority = 1; else if (!strcmp(tok->args[i], "Unnamed") && @@ -1964,6 +2306,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_unnamed = 1; } else if (!strcmp(tok->args[i], "HSDir")) { rs->is_hs_dir = 1; + } else if (!strcmp(tok->args[i], "V2Dir")) { + rs->is_v2_dir = 1; } } } @@ -1971,13 +2315,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, tor_assert(tok->n_args == 1); rs->version_known = 1; if (strcmpstart(tok->args[0], "Tor ")) { - rs->version_supports_microdesc_cache = 1; - rs->version_supports_optimistic_data = 1; } else { - rs->version_supports_microdesc_cache = - tor_version_supports_microdescriptors(tok->args[0]); - rs->version_supports_optimistic_data = - tor_version_as_new_as(tok->args[0], "0.2.3.1-alpha"); rs->version_supports_extend2_cells = tor_version_as_new_as(tok->args[0], "0.2.4.8-alpha"); } @@ -2015,6 +2353,11 @@ routerstatus_parse_entry_from_string(memarea_t *area, vote->has_measured_bws = 1; } else if (!strcmpstart(tok->args[i], "Unmeasured=1")) { rs->bw_is_unmeasured = 1; + } else if (!strcmpstart(tok->args[i], "GuardFraction=")) { + if (routerstatus_parse_guardfraction(tok->args[i], + vote, vote_rs, rs) < 0) { + goto err; + } } } } @@ -2045,6 +2388,18 @@ routerstatus_parse_entry_from_string(memarea_t *area, line->microdesc_hash_line = tor_strdup(t->args[0]); vote_rs->microdesc = line; } + if (t->tp == K_ID) { + tor_assert(t->n_args >= 2); + if (!strcmp(t->args[0], "ed25519")) { + vote_rs->has_ed25519_listing = 1; + if (strcmp(t->args[1], "none") && + digest256_from_base64((char*)vote_rs->ed25519_id, + t->args[1])<0) { + log_warn(LD_DIR, "Bogus ed25519 key in networkstatus vote"); + goto err; + } + } + } } SMARTLIST_FOREACH_END(t); } else if (flav == FLAV_MICRODESC) { tok = find_opt_by_keyword(tokens, K_M); @@ -2084,14 +2439,6 @@ routerstatus_parse_entry_from_string(memarea_t *area, return rs; } -/** Helper to sort a smartlist of pointers to routerstatus_t */ -int -compare_routerstatus_entries(const void **_a, const void **_b) -{ - const routerstatus_t *a = *_a, *b = *_b; - return fast_memcmp(a->identity_digest, b->identity_digest, DIGEST_LEN); -} - int compare_vote_routerstatus_entries(const void **_a, const void **_b) { @@ -2100,188 +2447,6 @@ compare_vote_routerstatus_entries(const void **_a, const void **_b) DIGEST_LEN); } -/** Helper: used in call to _smartlist_uniq to clear out duplicate entries. */ -static void -free_duplicate_routerstatus_entry_(void *e) -{ - log_warn(LD_DIR, - "Network-status has two entries for the same router. " - "Dropping one."); - routerstatus_free(e); -} - -/** Given a v2 network-status object in <b>s</b>, try to - * parse it and return the result. Return NULL on failure. Check the - * signature of the network status, but do not (yet) check the signing key for - * authority. - */ -networkstatus_v2_t * -networkstatus_v2_parse_from_string(const char *s) -{ - const char *eos, *s_dup = s; - smartlist_t *tokens = smartlist_new(); - smartlist_t *footer_tokens = smartlist_new(); - networkstatus_v2_t *ns = NULL; - char ns_digest[DIGEST_LEN]; - char tmp_digest[DIGEST_LEN]; - struct in_addr in; - directory_token_t *tok; - int i; - memarea_t *area = NULL; - - if (router_get_networkstatus_v2_hash(s, ns_digest)) { - log_warn(LD_DIR, "Unable to compute digest of network-status"); - goto err; - } - - area = memarea_new(); - eos = find_start_of_next_routerstatus(s); - if (tokenize_string(area, s, eos, tokens, netstatus_token_table,0)) { - log_warn(LD_DIR, "Error tokenizing network-status header."); - goto err; - } - ns = tor_malloc_zero(sizeof(networkstatus_v2_t)); - memcpy(ns->networkstatus_digest, ns_digest, DIGEST_LEN); - - tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION); - tor_assert(tok->n_args >= 1); - if (strcmp(tok->args[0], "2")) { - log_warn(LD_BUG, "Got a non-v2 networkstatus. Version was " - "%s", escaped(tok->args[0])); - goto err; - } - - tok = find_by_keyword(tokens, K_DIR_SOURCE); - tor_assert(tok->n_args >= 3); - ns->source_address = tor_strdup(tok->args[0]); - if (tor_inet_aton(tok->args[1], &in) == 0) { - log_warn(LD_DIR, "Error parsing network-status source address %s", - escaped(tok->args[1])); - goto err; - } - ns->source_addr = ntohl(in.s_addr); - ns->source_dirport = - (uint16_t) tor_parse_long(tok->args[2],10,0,65535,NULL,NULL); - if (ns->source_dirport == 0) { - log_warn(LD_DIR, "Directory source without dirport; skipping."); - goto err; - } - - tok = find_by_keyword(tokens, K_FINGERPRINT); - tor_assert(tok->n_args); - if (base16_decode(ns->identity_digest, DIGEST_LEN, tok->args[0], - strlen(tok->args[0]))) { - log_warn(LD_DIR, "Couldn't decode networkstatus fingerprint %s", - escaped(tok->args[0])); - goto err; - } - - if ((tok = find_opt_by_keyword(tokens, K_CONTACT))) { - tor_assert(tok->n_args); - ns->contact = tor_strdup(tok->args[0]); - } - - tok = find_by_keyword(tokens, K_DIR_SIGNING_KEY); - tor_assert(tok->key); - ns->signing_key = tok->key; - tok->key = NULL; - - if (crypto_pk_get_digest(ns->signing_key, tmp_digest)<0) { - log_warn(LD_DIR, "Couldn't compute signing key digest"); - goto err; - } - if (tor_memneq(tmp_digest, ns->identity_digest, DIGEST_LEN)) { - log_warn(LD_DIR, - "network-status fingerprint did not match dir-signing-key"); - goto err; - } - - if ((tok = find_opt_by_keyword(tokens, K_DIR_OPTIONS))) { - for (i=0; i < tok->n_args; ++i) { - if (!strcmp(tok->args[i], "Names")) - ns->binds_names = 1; - if (!strcmp(tok->args[i], "Versions")) - ns->recommends_versions = 1; - if (!strcmp(tok->args[i], "BadExits")) - ns->lists_bad_exits = 1; - if (!strcmp(tok->args[i], "BadDirectories")) - ns->lists_bad_directories = 1; - } - } - - if (ns->recommends_versions) { - if (!(tok = find_opt_by_keyword(tokens, K_CLIENT_VERSIONS))) { - log_warn(LD_DIR, "Missing client-versions on versioning directory"); - goto err; - } - ns->client_versions = tor_strdup(tok->args[0]); - - if (!(tok = find_opt_by_keyword(tokens, K_SERVER_VERSIONS)) || - tok->n_args<1) { - log_warn(LD_DIR, "Missing server-versions on versioning directory"); - goto err; - } - ns->server_versions = tor_strdup(tok->args[0]); - } - - tok = find_by_keyword(tokens, K_PUBLISHED); - tor_assert(tok->n_args == 1); - if (parse_iso_time(tok->args[0], &ns->published_on) < 0) { - goto err; - } - - ns->entries = smartlist_new(); - s = eos; - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); - smartlist_clear(tokens); - memarea_clear(area); - while (!strcmpstart(s, "r ")) { - routerstatus_t *rs; - if ((rs = routerstatus_parse_entry_from_string(area, &s, tokens, - NULL, NULL, 0, 0))) - smartlist_add(ns->entries, rs); - } - smartlist_sort(ns->entries, compare_routerstatus_entries); - smartlist_uniq(ns->entries, compare_routerstatus_entries, - free_duplicate_routerstatus_entry_); - - if (tokenize_string(area,s, NULL, footer_tokens, dir_footer_token_table,0)) { - log_warn(LD_DIR, "Error tokenizing network-status footer."); - goto err; - } - if (smartlist_len(footer_tokens) < 1) { - log_warn(LD_DIR, "Too few items in network-status footer."); - goto err; - } - tok = smartlist_get(footer_tokens, smartlist_len(footer_tokens)-1); - if (tok->tp != K_DIRECTORY_SIGNATURE) { - log_warn(LD_DIR, - "Expected network-status footer to end with a signature."); - goto err; - } - - note_crypto_pk_op(VERIFY_DIR); - if (check_signature_token(ns_digest, DIGEST_LEN, tok, ns->signing_key, 0, - "network-status") < 0) - goto err; - - goto done; - err: - dump_desc(s_dup, "v2 networkstatus"); - networkstatus_v2_free(ns); - ns = NULL; - done: - SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t)); - smartlist_free(tokens); - SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t)); - smartlist_free(footer_tokens); - if (area) { - DUMP_AREA(area, "v2 networkstatus"); - memarea_drop_all(area); - } - return ns; -} - /** Verify the bandwidth weights of a network status document */ int networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method) @@ -2292,6 +2457,7 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method) double Gtotal=0, Mtotal=0, Etotal=0; const char *casename = NULL; int valid = 1; + (void) consensus_method; weight_scale = networkstatus_get_weight_scale_param(ns); Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1); @@ -2371,12 +2537,8 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method) // Then, gather G, M, E, D, T to determine case SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) { int is_exit = 0; - if (consensus_method >= MIN_METHOD_TO_CUT_BADEXIT_WEIGHT) { - /* Bug #2203: Don't count bad exits as exits for balancing */ - is_exit = rs->is_exit && !rs->is_bad_exit; - } else { - is_exit = rs->is_exit; - } + /* Bug #2203: Don't count bad exits as exits for balancing */ + is_exit = rs->is_exit && !rs->is_bad_exit; if (rs->has_bandwidth) { T += rs->bandwidth_kb; if (is_exit && rs->is_possible_guard) { @@ -2688,7 +2850,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, smartlist_t *rs_tokens = NULL, *footer_tokens = NULL; networkstatus_voter_info_t *voter = NULL; networkstatus_t *ns = NULL; - digests_t ns_digests; + common_digests_t ns_digests; const char *cert, *end_of_header, *end_of_footer, *s_dup = s; directory_token_t *tok; int ok; @@ -2714,7 +2876,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, (ns_type == NS_TYPE_CONSENSUS) ? networkstatus_consensus_token_table : networkstatus_token_table, 0)) { - log_warn(LD_DIR, "Error tokenizing network-status vote header"); + log_warn(LD_DIR, "Error tokenizing network-status header"); goto err; } @@ -2812,11 +2974,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, (int) tor_parse_long(tok->args[1], 10, 0, INT_MAX, &ok, NULL); if (!ok) goto err; - if (ns->valid_after + MIN_VOTE_INTERVAL > ns->fresh_until) { + if (ns->valid_after + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) > ns->fresh_until) { log_warn(LD_DIR, "Vote/consensus freshness interval is too short"); goto err; } - if (ns->valid_after + MIN_VOTE_INTERVAL*2 > ns->valid_until) { + if (ns->valid_after + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL)*2 > ns->valid_until) { log_warn(LD_DIR, "Vote/consensus liveness interval is too short"); goto err; } @@ -2836,6 +3002,16 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, ns->server_versions = tor_strdup(tok->args[0]); } + { + smartlist_t *package_lst = find_all_by_keyword(tokens, K_PACKAGE); + ns->package_lines = smartlist_new(); + if (package_lst) { + SMARTLIST_FOREACH(package_lst, directory_token_t *, t, + smartlist_add(ns->package_lines, tor_strdup(t->args[0]))); + } + smartlist_free(package_lst); + } + tok = find_by_keyword(tokens, K_KNOWN_FLAGS); ns->known_flags = smartlist_new(); inorder = 1; @@ -2923,7 +3099,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, base16_decode(voter->identity_digest, sizeof(voter->identity_digest), tok->args[1], HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding identity digest %s in " - "network-status vote.", escaped(tok->args[1])); + "network-status document.", escaped(tok->args[1])); goto err; } if (ns->type != NS_TYPE_CONSENSUS && @@ -2982,7 +3158,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, voter = NULL; } if (smartlist_len(ns->voters) == 0) { - log_warn(LD_DIR, "Missing dir-source elements in a vote networkstatus."); + log_warn(LD_DIR, "Missing dir-source elements in a networkstatus."); goto err; } else if (ns->type != NS_TYPE_CONSENSUS && smartlist_len(ns->voters) != 1) { log_warn(LD_DIR, "Too many dir-source elements in a vote networkstatus."); @@ -3043,11 +3219,27 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } if (fast_memcmp(rs1->identity_digest, rs2->identity_digest, DIGEST_LEN) >= 0) { - log_warn(LD_DIR, "Vote networkstatus entries not sorted by identity " - "digest"); + log_warn(LD_DIR, "Networkstatus entries not sorted by identity digest"); goto err; } } + if (ns_type != NS_TYPE_CONSENSUS) { + digest256map_t *ed_id_map = digest256map_new(); + SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *, + vrs) { + if (! vrs->has_ed25519_listing || + tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN)) + continue; + if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) { + log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not " + "unique"); + digest256map_free(ed_id_map, NULL); + goto err; + } + digest256map_set(ed_id_map, vrs->ed25519_id, (void*)1); + } SMARTLIST_FOREACH_END(vrs); + digest256map_free(ed_id_map, NULL); + } /* Parse footer; check signature. */ footer_tokens = smartlist_new(); @@ -3140,12 +3332,12 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, base16_decode(declared_identity, sizeof(declared_identity), id_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared identity %s in " - "network-status vote.", escaped(id_hexdigest)); + "network-status document.", escaped(id_hexdigest)); goto err; } if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) { - log_warn(LD_DIR, "ID on signature on network-status vote does not match " - "any declared directory source."); + log_warn(LD_DIR, "ID on signature on network-status document does " + "not match any declared directory source."); goto err; } sig = tor_malloc_zero(sizeof(document_signature_t)); @@ -3155,7 +3347,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), sk_hexdigest, HEX_DIGEST_LEN) < 0) { log_warn(LD_DIR, "Error decoding declared signing key digest %s in " - "network-status vote.", escaped(sk_hexdigest)); + "network-status document.", escaped(sk_hexdigest)); tor_free(sig); goto err; } @@ -3174,8 +3366,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, /* We already parsed a vote with this algorithm from this voter. Use the first one. */ log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " - "that contains two votes from the same voter with the same " - "algorithm. Ignoring the second vote."); + "that contains two signatures from the same voter with the same " + "algorithm. Ignoring the second signature."); tor_free(sig); continue; } @@ -3183,7 +3375,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns->type != NS_TYPE_CONSENSUS) { if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN, tok, ns->cert->signing_key, 0, - "network-status vote")) { + "network-status document")) { tor_free(sig); goto err; } @@ -3202,7 +3394,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } SMARTLIST_FOREACH_END(_tok); if (! n_signatures) { - log_warn(LD_DIR, "No signatures on networkstatus vote."); + log_warn(LD_DIR, "No signatures on networkstatus document."); goto err; } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) { log_warn(LD_DIR, "Received more than one signature on a " @@ -3253,15 +3445,16 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, return ns; } -/** Return the digests_t that holds the digests of the +/** Return the common_digests_t that holds the digests of the * <b>flavor_name</b>-flavored networkstatus according to the detached - * signatures document <b>sigs</b>, allocating a new digests_t as neeeded. */ -static digests_t * + * signatures document <b>sigs</b>, allocating a new common_digests_t as + * neeeded. */ +static common_digests_t * detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) { - digests_t *d = strmap_get(sigs->digests, flavor_name); + common_digests_t *d = strmap_get(sigs->digests, flavor_name); if (!d) { - d = tor_malloc_zero(sizeof(digests_t)); + d = tor_malloc_zero(sizeof(common_digests_t)); strmap_set(sigs->digests, flavor_name, d); } return d; @@ -3269,7 +3462,7 @@ detached_get_digests(ns_detached_signatures_t *sigs, const char *flavor_name) /** Return the list of signatures of the <b>flavor_name</b>-flavored * networkstatus according to the detached signatures document <b>sigs</b>, - * allocating a new digests_t as neeeded. */ + * allocating a new common_digests_t as neeeded. */ static smartlist_t * detached_get_signatures(ns_detached_signatures_t *sigs, const char *flavor_name) @@ -3291,7 +3484,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * networkstatus_parse_vote_from_string(). */ directory_token_t *tok; memarea_t *area = NULL; - digests_t *digests; + common_digests_t *digests; smartlist_t *tokens = smartlist_new(); ns_detached_signatures_t *sigs = @@ -3487,24 +3680,43 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * assume_action is nonnegative, then insert its action (ADDR_POLICY_ACCEPT or * ADDR_POLICY_REJECT) for items that specify no action. * + * Returns NULL on policy errors. + * + * Set *<b>malformed_list</b> to true if the entire policy list should be + * discarded. Otherwise, set it to false, and only this item should be ignored + * on error - the rest of the policy list can continue to be processed and + * used. + * * The addr_policy_t returned by this function can have its address set to * AF_UNSPEC for '*'. Use policy_expand_unspec() to turn this into a pair * of AF_INET and AF_INET6 items. */ -addr_policy_t * -router_parse_addr_policy_item_from_string(const char *s, int assume_action) +MOCK_IMPL(addr_policy_t *, +router_parse_addr_policy_item_from_string,(const char *s, int assume_action, + int *malformed_list)) { directory_token_t *tok = NULL; const char *cp, *eos; - /* Longest possible policy is "accept ffff:ffff:..255/ffff:...255:0-65535". + /* Longest possible policy is + * "accept6 [ffff:ffff:..255]/128:10000-65535", + * which contains a max-length IPv6 address, plus 26 characters. * But note that there can be an arbitrary amount of space between the - * accept and the address:mask/port element. */ + * accept and the address:mask/port element. + * We don't need to multiply TOR_ADDR_BUF_LEN by 2, as there is only one + * IPv6 address. But making the buffer shorter might cause valid long lines, + * which parsed in previous versions, to fail to parse in new versions. + * (These lines would have to have excessive amounts of whitespace.) */ char line[TOR_ADDR_BUF_LEN*2 + 32]; addr_policy_t *r; memarea_t *area = NULL; + tor_assert(malformed_list); + *malformed_list = 0; + s = eat_whitespace(s); - if ((*s == '*' || TOR_ISDIGIT(*s)) && assume_action >= 0) { + /* We can only do assume_action on []-quoted IPv6, as "a" (accept) + * and ":" (port separator) are ambiguous */ + if ((*s == '*' || *s == '[' || TOR_ISDIGIT(*s)) && assume_action >= 0) { if (tor_snprintf(line, sizeof(line), "%s %s", assume_action == ADDR_POLICY_ACCEPT?"accept":"reject", s)<0) { log_warn(LD_DIR, "Policy %s is too long.", escaped(s)); @@ -3529,9 +3741,34 @@ router_parse_addr_policy_item_from_string(const char *s, int assume_action) goto err; } + /* Use the extended interpretation of accept/reject *, + * expanding it into an IPv4 wildcard and an IPv6 wildcard. + * Also permit *4 and *6 for IPv4 and IPv6 only wildcards. */ r = router_parse_addr_policy(tok, TAPMP_EXTENDED_STAR); + if (!r) { + goto err; + } + + /* Ensure that accept6/reject6 fields are followed by IPv6 addresses. + * AF_UNSPEC addresses are only permitted on the accept/reject field type. + * Unlike descriptors, torrcs exit policy accept/reject can be followed by + * either an IPv4 or IPv6 address. */ + if ((tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) && + tor_addr_family(&r->addr) != AF_INET6) { + /* This is a non-fatal error, just ignore this one entry. */ + *malformed_list = 0; + log_warn(LD_DIR, "IPv4 address '%s' with accept6/reject6 field type in " + "exit policy. Ignoring, but continuing to parse rules. (Use " + "accept/reject with IPv4 addresses.)", + tok->n_args == 1 ? tok->args[0] : ""); + addr_policy_free(r); + r = NULL; + goto done; + } + goto done; err: + *malformed_list = 1; r = NULL; done: token_clear(tok); @@ -3548,19 +3785,27 @@ static int router_add_exit_policy(routerinfo_t *router, directory_token_t *tok) { addr_policy_t *newe; + /* Use the standard interpretation of accept/reject *, an IPv4 wildcard. */ newe = router_parse_addr_policy(tok, 0); if (!newe) return -1; if (! router->exit_policy) router->exit_policy = smartlist_new(); + /* Ensure that in descriptors, accept/reject fields are followed by + * IPv4 addresses, and accept6/reject6 fields are followed by + * IPv6 addresses. Unlike torrcs, descriptor exit policies do not permit + * accept/reject followed by IPv6. */ if (((tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) && tor_addr_family(&newe->addr) == AF_INET) || ((tok->tp == K_ACCEPT || tok->tp == K_REJECT) && tor_addr_family(&newe->addr) == AF_INET6)) { + /* There's nothing the user can do about other relays' descriptors, + * so we don't provide usage advice here. */ log_warn(LD_DIR, "Mismatch between field type and address type in exit " - "policy"); + "policy '%s'. Discarding entire router descriptor.", + tok->n_args == 1 ? tok->args[0] : ""); addr_policy_free(newe); return -1; } @@ -3570,8 +3815,11 @@ router_add_exit_policy(routerinfo_t *router, directory_token_t *tok) return 0; } -/** Given a K_ACCEPT or K_REJECT token and a router, create and return - * a new exit_policy_t corresponding to the token. */ +/** Given a K_ACCEPT[6] or K_REJECT[6] token and a router, create and return + * a new exit_policy_t corresponding to the token. If TAPMP_EXTENDED_STAR + * is set in fmt_flags, K_ACCEPT6 and K_REJECT6 tokens followed by * + * expand to IPv6-only policies, otherwise they expand to IPv4 and IPv6 + * policies */ static addr_policy_t * router_parse_addr_policy(directory_token_t *tok, unsigned fmt_flags) { @@ -3595,6 +3843,13 @@ router_parse_addr_policy(directory_token_t *tok, unsigned fmt_flags) else newe.policy_type = ADDR_POLICY_ACCEPT; + /* accept6/reject6 * produces an IPv6 wildcard address only. + * (accept/reject * produces rules for IPv4 and IPv6 wildcard addresses.) */ + if ((fmt_flags & TAPMP_EXTENDED_STAR) + && (tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6)) { + fmt_flags |= TAPMP_STAR_IPV6_ONLY; + } + if (tor_addr_parse_mask_ports(arg, fmt_flags, &newe.addr, &newe.maskbits, &newe.prt_min, &newe.prt_max) < 0) { log_warn(LD_DIR,"Couldn't parse line %s. Dropping", escaped(arg)); @@ -3604,9 +3859,12 @@ router_parse_addr_policy(directory_token_t *tok, unsigned fmt_flags) return addr_policy_get_canonical_entry(&newe); } -/** Parse an exit policy line of the format "accept/reject private:...". +/** Parse an exit policy line of the format "accept[6]/reject[6] private:...". * This didn't exist until Tor 0.1.1.15, so nobody should generate it in * router descriptors until earlier versions are obsolete. + * + * accept/reject and accept6/reject6 private all produce rules for both + * IPv4 and IPv6 addresses. */ static addr_policy_t * router_parse_addr_policy_private(directory_token_t *tok) @@ -3636,6 +3894,13 @@ router_parse_addr_policy_private(directory_token_t *tok) result.prt_min = port_min; result.prt_max = port_max; + if (tok->tp == K_ACCEPT6 || tok->tp == K_REJECT6) { + log_warn(LD_GENERAL, + "'%s' expands into rules which apply to all private IPv4 and " + "IPv6 addresses. (Use accept/reject private:* for IPv4 and " + "IPv6.)", tok->n_args == 1 ? tok->args[0] : ""); + } + return addr_policy_get_canonical_entry(&result); } @@ -3682,7 +3947,7 @@ token_clear(directory_token_t *tok) * Return <b>tok</b> on success, or a new ERR_ token if the token didn't * conform to the syntax we wanted. **/ -static INLINE directory_token_t * +static inline directory_token_t * token_check_object(memarea_t *area, const char *kwd, directory_token_t *tok, obj_syntax o_syn) { @@ -3747,7 +4012,7 @@ token_check_object(memarea_t *area, const char *kwd, * number of parsed elements into the n_args field of <b>tok</b>. Allocate * all storage in <b>area</b>. Return the number of arguments parsed, or * return -1 if there was an insanely high number of arguments. */ -static INLINE int +static inline int get_token_arguments(memarea_t *area, directory_token_t *tok, const char *s, const char *eol) { @@ -4182,7 +4447,7 @@ router_get_hash_impl(const char *s, size_t s_len, char *digest, /** As router_get_hash_impl, but compute all hashes. */ static int -router_get_hashes_impl(const char *s, size_t s_len, digests_t *digests, +router_get_hashes_impl(const char *s, size_t s_len, common_digests_t *digests, const char *start_str, const char *end_str, char end_c) { @@ -4191,7 +4456,7 @@ router_get_hashes_impl(const char *s, size_t s_len, digests_t *digests, &start,&end)<0) return -1; - if (crypto_digest_all(digests, start, end-start)) { + if (crypto_common_digests(digests, start, end-start)) { log_warn(LD_BUG,"couldn't compute digests"); return -1; } @@ -4258,12 +4523,15 @@ find_start_of_next_microdesc(const char *s, const char *eos) * If <b>saved_location</b> isn't SAVED_IN_CACHE, make a local copy of each * descriptor in the body field of each microdesc_t. * - * Return all newly - * parsed microdescriptors in a newly allocated smartlist_t. */ + * Return all newly parsed microdescriptors in a newly allocated + * smartlist_t. If <b>invalid_disgests_out</b> is provided, add a SHA256 + * microdesc digest to it for every microdesc that we found to be badly + * formed. (This may cause duplicates) */ smartlist_t * microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, - saved_location_t where) + saved_location_t where, + smartlist_t *invalid_digests_out) { smartlist_t *tokens; smartlist_t *result; @@ -4285,21 +4553,20 @@ microdescs_parse_from_string(const char *s, const char *eos, tokens = smartlist_new(); while (s < eos) { + int okay = 0; + start_of_next_microdesc = find_start_of_next_microdesc(s, eos); if (!start_of_next_microdesc) start_of_next_microdesc = eos; - if (tokenize_string(area, s, start_of_next_microdesc, tokens, - microdesc_token_table, flags)) { - log_warn(LD_DIR, "Unparseable microdescriptor"); - goto next; - } - md = tor_malloc_zero(sizeof(microdesc_t)); { const char *cp = tor_memstr(s, start_of_next_microdesc-s, "onion-key"); - tor_assert(cp); + const int no_onion_key = (cp == NULL); + if (no_onion_key) { + cp = s; /* So that we have *some* junk to put in the body */ + } md->bodylen = start_of_next_microdesc - cp; md->saved_location = where; @@ -4308,6 +4575,17 @@ microdescs_parse_from_string(const char *s, const char *eos, else md->body = (char*)cp; md->off = cp - start; + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + if (no_onion_key) { + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor"); + goto next; + } + } + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor"); + goto next; } if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { @@ -4337,6 +4615,26 @@ microdescs_parse_from_string(const char *s, const char *eos, tor_memdup(&k, sizeof(curve25519_public_key_t)); } + smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID); + if (id_lines) { + SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) { + tor_assert(t->n_args >= 2); + if (!strcmp(t->args[0], "ed25519")) { + if (md->ed25519_identity_pkey) { + log_warn(LD_DIR, "Extra ed25519 key in microdesc"); + goto next; + } + ed25519_public_key_t k; + if (ed25519_public_from_base64(&k, t->args[1])<0) { + log_warn(LD_DIR, "Bogus ed25519 key in microdesc"); + goto next; + } + md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k)); + } + } SMARTLIST_FOREACH_END(t); + smartlist_free(id_lines); + } + { smartlist_t *a_lines = find_all_by_keyword(tokens, K_A); if (a_lines) { @@ -4365,12 +4663,15 @@ microdescs_parse_from_string(const char *s, const char *eos, md->ipv6_exit_policy = parse_short_policy(tok->args[0]); } - crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); - smartlist_add(result, md); + okay = 1; md = NULL; next: + if (! okay && invalid_digests_out) { + smartlist_add(invalid_digests_out, + tor_memdup(md->digest, DIGEST256_LEN)); + } microdesc_free(md); md = NULL; @@ -4387,14 +4688,6 @@ microdescs_parse_from_string(const char *s, const char *eos, return result; } -/** Return true iff this Tor version can answer directory questions - * about microdescriptors. */ -int -tor_version_supports_microdescriptors(const char *platform) -{ - return tor_version_as_new_as(platform, "0.2.3.1-alpha"); -} - /** Parse the Tor version of the platform string <b>platform</b>, * and compare it to the version in <b>cutoff</b>. Return 1 if * the router is at least as new as the cutoff, else return 0. @@ -4451,40 +4744,50 @@ tor_version_parse(const char *s, tor_version_t *out) char *eos=NULL; const char *cp=NULL; /* Format is: - * "Tor " ? NUM dot NUM dot NUM [ ( pre | rc | dot ) NUM [ - tag ] ] + * "Tor " ? NUM dot NUM [ dot NUM [ ( pre | rc | dot ) NUM ] ] [ - tag ] */ tor_assert(s); tor_assert(out); memset(out, 0, sizeof(tor_version_t)); - + out->status = VER_RELEASE; if (!strcasecmpstart(s, "Tor ")) s += 4; - /* Get major. */ - out->major = (int)strtol(s,&eos,10); - if (!eos || eos==s || *eos != '.') return -1; - cp = eos+1; - - /* Get minor */ - out->minor = (int) strtol(cp,&eos,10); - if (!eos || eos==cp || *eos != '.') return -1; - cp = eos+1; - - /* Get micro */ - out->micro = (int) strtol(cp,&eos,10); - if (!eos || eos==cp) return -1; - if (!*eos) { - out->status = VER_RELEASE; - out->patchlevel = 0; + cp = s; + +#define NUMBER(m) \ + do { \ + out->m = (int)strtol(cp, &eos, 10); \ + if (!eos || eos == cp) \ + return -1; \ + cp = eos; \ + } while (0) + +#define DOT() \ + do { \ + if (*cp != '.') \ + return -1; \ + ++cp; \ + } while (0) + + NUMBER(major); + DOT(); + NUMBER(minor); + if (*cp == 0) return 0; - } - cp = eos; + else if (*cp == '-') + goto status_tag; + DOT(); + NUMBER(micro); /* Get status */ - if (*cp == '.') { - out->status = VER_RELEASE; + if (*cp == 0) { + return 0; + } else if (*cp == '.') { ++cp; + } else if (*cp == '-') { + goto status_tag; } else if (0==strncmp(cp, "pre", 3)) { out->status = VER_PRE; cp += 3; @@ -4495,11 +4798,9 @@ tor_version_parse(const char *s, tor_version_t *out) return -1; } - /* Get patchlevel */ - out->patchlevel = (int) strtol(cp,&eos,10); - if (!eos || eos==cp) return -1; - cp = eos; + NUMBER(patchlevel); + status_tag: /* Get status tag. */ if (*cp == '-' || *cp == '.') ++cp; @@ -4535,6 +4836,8 @@ tor_version_parse(const char *s, tor_version_t *out) } return 0; +#undef NUMBER +#undef DOT } /** Compare two tor versions; Return <0 if a < b; 0 if a ==b, >0 if a > @@ -4622,6 +4925,9 @@ sort_version_list(smartlist_t *versions, int remove_duplicates) * to *<b>encoded_size_out</b>, and a pointer to the possibly next * descriptor to *<b>next_out</b>; return 0 for success (including validation) * and -1 for failure. + * + * If <b>as_hsdir</b> is 1, we're parsing this as an HSDir, and we should + * be strict about time formats. */ int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, @@ -4629,7 +4935,8 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, char **intro_points_encrypted_out, size_t *intro_points_encrypted_size_out, size_t *encoded_size_out, - const char **next_out, const char *desc) + const char **next_out, const char *desc, + int as_hsdir) { rend_service_descriptor_t *result = tor_malloc_zero(sizeof(rend_service_descriptor_t)); @@ -4643,6 +4950,8 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, char public_key_hash[DIGEST_LEN]; char test_desc_id[DIGEST_LEN]; memarea_t *area = NULL; + const int strict_time_fmt = as_hsdir; + tor_assert(desc); /* Check if desc starts correctly. */ if (strncmp(desc, "rendezvous-service-descriptor ", @@ -4691,8 +5000,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, tok = find_by_keyword(tokens, R_RENDEZVOUS_SERVICE_DESCRIPTOR); tor_assert(tok == smartlist_get(tokens, 0)); tor_assert(tok->n_args == 1); - if (strlen(tok->args[0]) != REND_DESC_ID_V2_LEN_BASE32 || - strspn(tok->args[0], BASE32_CHARS) != REND_DESC_ID_V2_LEN_BASE32) { + if (!rend_valid_descriptor_id(tok->args[0])) { log_warn(LD_REND, "Invalid descriptor ID: '%s'", tok->args[0]); goto err; } @@ -4737,7 +5045,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, * descriptor. */ tok = find_by_keyword(tokens, R_PUBLICATION_TIME); tor_assert(tok->n_args == 1); - if (parse_iso_time(tok->args[0], &result->timestamp) < 0) { + if (parse_iso_time_(tok->args[0], &result->timestamp, strict_time_fmt) < 0) { log_warn(LD_REND, "Invalid publication time: '%s'", tok->args[0]); goto err; } diff --git a/src/or/routerparse.h b/src/or/routerparse.h index eb2e885cb1..c46eb1c0ae 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,14 +14,13 @@ int router_get_router_hash(const char *s, size_t s_len, char *digest); int router_get_dir_hash(const char *s, char *digest); -int router_get_runningrouters_hash(const char *s, char *digest); -int router_get_networkstatus_v2_hash(const char *s, char *digest); -int router_get_networkstatus_v3_hashes(const char *s, digests_t *digests); +int router_get_networkstatus_v3_hashes(const char *s, + common_digests_t *digests); int router_get_extrainfo_hash(const char *s, size_t s_len, char *digest); #define DIROBJ_MAX_SIG_LEN 256 char *router_get_dirobj_signature(const char *digest, size_t digest_len, - crypto_pk_t *private_key); + const crypto_pk_t *private_key); int router_append_dirobj_signature(char *buf, size_t buf_len, const char *digest, size_t digest_len, @@ -31,19 +30,21 @@ int router_parse_list_from_string(const char **s, const char *eos, saved_location_t saved_location, int is_extrainfo, int allow_annotations, - const char *prepend_annotations); + const char *prepend_annotations, + smartlist_t *invalid_digests_out); routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, int cache_copy, int allow_annotations, - const char *prepend_annotations); + const char *prepend_annotations, + int *can_dl_again_out); extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, - int cache_copy, struct digest_ri_map_t *routermap); -addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, - int assume_action); + int cache_copy, struct digest_ri_map_t *routermap, + int *can_dl_again_out); +MOCK_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string, + (const char *s, int assume_action, int *malformed_list)); version_status_t tor_version_is_obsolete(const char *myversion, const char *versionlist); -int tor_version_supports_microdescriptors(const char *platform); int tor_version_as_new_as(const char *platform, const char *cutoff); int tor_version_parse(const char *s, tor_version_t *out); int tor_version_compare(tor_version_t *a, tor_version_t *b); @@ -52,9 +53,7 @@ void sort_version_list(smartlist_t *lst, int remove_duplicates); void assert_addr_policy_ok(smartlist_t *t); void dump_distinct_digest_count(int severity); -int compare_routerstatus_entries(const void **_a, const void **_b); int compare_vote_routerstatus_entries(const void **_a, const void **_b); -networkstatus_v2_t *networkstatus_v2_parse_from_string(const char *s); int networkstatus_verify_bw_weights(networkstatus_t *ns, int); networkstatus_t *networkstatus_parse_vote_from_string(const char *s, const char **eos_out, @@ -64,7 +63,8 @@ ns_detached_signatures_t *networkstatus_parse_detached_signatures( smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, - saved_location_t where); + saved_location_t where, + smartlist_t *invalid_digests_out); authority_cert_t *authority_cert_parse_from_string(const char *s, const char **end_of_string); @@ -73,7 +73,8 @@ int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, char **intro_points_encrypted_out, size_t *intro_points_encrypted_size_out, size_t *encoded_size_out, - const char **next_out, const char *desc); + const char **next_out, const char *desc, + int as_hsdir); int rend_decrypt_introduction_points(char **ipos_decrypted, size_t *ipos_decrypted_size, const char *descriptor_cookie, @@ -84,5 +85,14 @@ int rend_parse_introduction_points(rend_service_descriptor_t *parsed, size_t intro_points_encoded_size); int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); +#ifdef ROUTERPARSE_PRIVATE +STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str, + networkstatus_t *vote, + vote_routerstatus_t *vote_rs, + routerstatus_t *rs); +#endif + +#define ED_DESC_SIGNATURE_PREFIX "Tor router descriptor signature v1" + #endif diff --git a/src/or/routerset.c b/src/or/routerset.c index 2e41f7f6c4..f260914f4b 100644 --- a/src/or/routerset.c +++ b/src/or/routerset.c @@ -1,9 +1,18 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * \file routerset.c + * + * \brief Functions and structures to handle set-type selection of routers + * by name, ID, address, etc. + */ + +#define ROUTERSET_PRIVATE + #include "or.h" #include "geoip.h" #include "nodelist.h" @@ -12,39 +21,6 @@ #include "routerparse.h" #include "routerset.h" -/** A routerset specifies constraints on a set of possible routerinfos, based - * on their names, identities, or addresses. It is optimized for determining - * whether a router is a member or not, in O(1+P) time, where P is the number - * of address policy constraints. */ -struct routerset_t { - /** A list of strings for the elements of the policy. Each string is either - * a nickname, a hexadecimal identity fingerprint, or an address policy. A - * router belongs to the set if its nickname OR its identity OR its address - * matches an entry here. */ - smartlist_t *list; - /** A map from lowercase nicknames of routers in the set to (void*)1 */ - strmap_t *names; - /** A map from identity digests routers in the set to (void*)1 */ - digestmap_t *digests; - /** An address policy for routers in the set. For implementation reasons, - * a router belongs to the set if it is _rejected_ by this policy. */ - smartlist_t *policies; - - /** A human-readable description of what this routerset is for. Used in - * log messages. */ - char *description; - - /** A list of the country codes in this set. */ - smartlist_t *country_names; - /** Total number of countries we knew about when we built <b>countries</b>.*/ - int n_countries; - /** Bit array mapping the return value of geoip_get_country() to 1 iff the - * country is a member of this routerset. Note that we MUST call - * routerset_refresh_countries() whenever the geoip country list is - * reloaded. */ - bitarray_t *countries; -}; - /** Return a new empty routerset. */ routerset_t * routerset_new(void) @@ -60,7 +36,7 @@ routerset_new(void) /** If <b>c</b> is a country code in the form {cc}, return a newly allocated * string holding the "cc" part. Else, return NULL. */ -static char * +STATIC char * routerset_get_countryname(const char *c) { char *country; @@ -116,10 +92,13 @@ routerset_parse(routerset_t *target, const char *s, const char *description) int added_countries = 0; char *countryname; smartlist_t *list = smartlist_new(); + int malformed_list; smartlist_split_string(list, s, ",", SPLIT_SKIP_SPACE | SPLIT_IGNORE_BLANK, 0); SMARTLIST_FOREACH_BEGIN(list, char *, nick) { addr_policy_t *p; + /* if it doesn't pass our validation, assume it's malformed */ + malformed_list = 1; if (is_legal_hexdigest(nick)) { char d[DIGEST_LEN]; if (*nick == '$') @@ -135,17 +114,25 @@ routerset_parse(routerset_t *target, const char *s, const char *description) description); smartlist_add(target->country_names, countryname); added_countries = 1; - } else if ((strchr(nick,'.') || strchr(nick, '*')) && - (p = router_parse_addr_policy_item_from_string( - nick, ADDR_POLICY_REJECT))) { + } else if ((strchr(nick,'.') || strchr(nick, ':') || strchr(nick, '*')) + && (p = router_parse_addr_policy_item_from_string( + nick, ADDR_POLICY_REJECT, + &malformed_list))) { + /* IPv4 addresses contain '.', IPv6 addresses contain ':', + * and wildcard addresses contain '*'. */ log_debug(LD_CONFIG, "Adding address %s to %s", nick, description); smartlist_add(target->policies, p); - } else { - log_warn(LD_CONFIG, "Entry '%s' in %s is malformed.", nick, - description); + } else if (malformed_list) { + log_warn(LD_CONFIG, "Entry '%s' in %s is malformed. Discarding entire" + " list.", nick, description); r = -1; tor_free(nick); SMARTLIST_DEL_CURRENT(list, nick); + } else { + log_notice(LD_CONFIG, "Entry '%s' in %s is ignored. Using the" + " remainder of the list.", nick, description); + tor_free(nick); + SMARTLIST_DEL_CURRENT(list, nick); } } SMARTLIST_FOREACH_END(nick); policy_expand_unspec(&target->policies); @@ -193,6 +180,17 @@ routerset_is_empty(const routerset_t *set) return !set || smartlist_len(set->list) == 0; } +/** Return the number of entries in <b>set</b>. This does NOT return a + * negative value. */ +int +routerset_len(const routerset_t *set) +{ + if (!set) { + return 0; + } + return smartlist_len(set->list); +} + /** Helper. Return true iff <b>set</b> contains a router based on the other * provided fields. Return higher values for more specific subentries: a * single router is more specific than an address range of routers, which is @@ -200,7 +198,7 @@ routerset_is_empty(const routerset_t *set) * * (If country is -1, then we take the country * from addr.) */ -static int +STATIC int routerset_contains(const routerset_t *set, const tor_addr_t *addr, uint16_t orport, const char *nickname, const char *id_digest, @@ -358,39 +356,6 @@ routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset, } } -#if 0 -/** Add to <b>target</b> every node_t from <b>source</b> except: - * - * 1) Don't add it if <b>include</b> is non-empty and the relay isn't in - * <b>include</b>; and - * 2) Don't add it if <b>exclude</b> is non-empty and the relay is - * excluded in a more specific fashion by <b>exclude</b>. - * 3) If <b>running_only</b>, don't add non-running routers. - */ -void -routersets_get_node_disjunction(smartlist_t *target, - const smartlist_t *source, - const routerset_t *include, - const routerset_t *exclude, int running_only) -{ - SMARTLIST_FOREACH(source, const node_t *, node, { - int include_result; - if (running_only && !node->is_running) - continue; - if (!routerset_is_empty(include)) - include_result = routerset_contains_node(include, node); - else - include_result = 1; - - if (include_result) { - int exclude_result = routerset_contains_node(exclude, node); - if (include_result >= exclude_result) - smartlist_add(target, (void*)node); - } - }); -} -#endif - /** Remove every node_t from <b>lst</b> that is in <b>routerset</b>. */ void routerset_subtract_nodes(smartlist_t *lst, const routerset_t *routerset) diff --git a/src/or/routerset.h b/src/or/routerset.h index bfa0c59ac1..c2f7205c3e 100644 --- a/src/or/routerset.h +++ b/src/or/routerset.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -32,18 +32,53 @@ void routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset, const routerset_t *excludeset, int running_only); int routerset_add_unknown_ccs(routerset_t **setp, int only_if_some_cc_set); -#if 0 -void routersets_get_node_disjunction(smartlist_t *target, - const smartlist_t *source, - const routerset_t *include, - const routerset_t *exclude, int running_only); -#endif void routerset_subtract_nodes(smartlist_t *out, const routerset_t *routerset); char *routerset_to_string(const routerset_t *routerset); int routerset_equal(const routerset_t *old, const routerset_t *new); void routerset_free(routerset_t *routerset); +int routerset_len(const routerset_t *set); + +#ifdef ROUTERSET_PRIVATE +STATIC char * routerset_get_countryname(const char *c); +STATIC int routerset_contains(const routerset_t *set, const tor_addr_t *addr, + uint16_t orport, + const char *nickname, const char *id_digest, + country_t country); + +/** A routerset specifies constraints on a set of possible routerinfos, based + * on their names, identities, or addresses. It is optimized for determining + * whether a router is a member or not, in O(1+P) time, where P is the number + * of address policy constraints. */ +struct routerset_t { + /** A list of strings for the elements of the policy. Each string is either + * a nickname, a hexadecimal identity fingerprint, or an address policy. A + * router belongs to the set if its nickname OR its identity OR its address + * matches an entry here. */ + smartlist_t *list; + /** A map from lowercase nicknames of routers in the set to (void*)1 */ + strmap_t *names; + /** A map from identity digests routers in the set to (void*)1 */ + digestmap_t *digests; + /** An address policy for routers in the set. For implementation reasons, + * a router belongs to the set if it is _rejected_ by this policy. */ + smartlist_t *policies; + /** A human-readable description of what this routerset is for. Used in + * log messages. */ + char *description; + + /** A list of the country codes in this set. */ + smartlist_t *country_names; + /** Total number of countries we knew about when we built <b>countries</b>.*/ + int n_countries; + /** Bit array mapping the return value of geoip_get_country() to 1 iff the + * country is a member of this routerset. Note that we MUST call + * routerset_refresh_countries() whenever the geoip country list is + * reloaded. */ + bitarray_t *countries; +}; +#endif #endif diff --git a/src/or/scheduler.c b/src/or/scheduler.c new file mode 100644 index 0000000000..8e4810b199 --- /dev/null +++ b/src/or/scheduler.c @@ -0,0 +1,711 @@ +/* * Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file scheduler.c + * \brief Relay scheduling system + **/ + +#include "or.h" + +#define TOR_CHANNEL_INTERNAL_ /* For channel_flush_some_cells() */ +#include "channel.h" + +#include "compat_libevent.h" +#define SCHEDULER_PRIVATE_ +#include "scheduler.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +/* + * Scheduler high/low watermarks + */ + +static uint32_t sched_q_low_water = 16384; +static uint32_t sched_q_high_water = 32768; + +/* + * Maximum cells to flush in a single call to channel_flush_some_cells(); + * setting this low means more calls, but too high and we could overshoot + * sched_q_high_water. + */ + +static uint32_t sched_max_flush_cells = 16; + +/* + * Write scheduling works by keeping track of which channels can + * accept cells, and have cells to write. From the scheduler's perspective, + * a channel can be in four possible states: + * + * 1.) Not open for writes, no cells to send + * - Not much to do here, and the channel will have scheduler_state == + * SCHED_CHAN_IDLE + * - Transitions from: + * - Open for writes/has cells by simultaneously draining all circuit + * queues and filling the output buffer. + * - Transitions to: + * - Not open for writes/has cells by arrival of cells on an attached + * circuit (this would be driven from append_cell_to_circuit_queue()) + * - Open for writes/no cells by a channel type specific path; + * driven from connection_or_flushed_some() for channel_tls_t. + * + * 2.) Open for writes, no cells to send + * - Not much here either; this will be the state an idle but open channel + * can be expected to settle in. It will have scheduler_state == + * SCHED_CHAN_WAITING_FOR_CELLS + * - Transitions from: + * - Not open for writes/no cells by flushing some of the output + * buffer. + * - Open for writes/has cells by the scheduler moving cells from + * circuit queues to channel output queue, but not having enough + * to fill the output queue. + * - Transitions to: + * - Open for writes/has cells by arrival of new cells on an attached + * circuit, in append_cell_to_circuit_queue() + * + * 3.) Not open for writes, cells to send + * - This is the state of a busy circuit limited by output bandwidth; + * cells have piled up in the circuit queues waiting to be relayed. + * The channel will have scheduler_state == SCHED_CHAN_WAITING_TO_WRITE. + * - Transitions from: + * - Not open for writes/no cells by arrival of cells on an attached + * circuit + * - Open for writes/has cells by filling an output buffer without + * draining all cells from attached circuits + * - Transitions to: + * - Opens for writes/has cells by draining some of the output buffer + * via the connection_or_flushed_some() path (for channel_tls_t). + * + * 4.) Open for writes, cells to send + * - This connection is ready to relay some cells and waiting for + * the scheduler to choose it. The channel will have scheduler_state == + * SCHED_CHAN_PENDING. + * - Transitions from: + * - Not open for writes/has cells by the connection_or_flushed_some() + * path + * - Open for writes/no cells by the append_cell_to_circuit_queue() + * path + * - Transitions to: + * - Not open for writes/no cells by draining all circuit queues and + * simultaneously filling the output buffer. + * - Not open for writes/has cells by writing enough cells to fill the + * output buffer + * - Open for writes/no cells by draining all attached circuit queues + * without also filling the output buffer + * + * Other event-driven parts of the code move channels between these scheduling + * states by calling scheduler functions; the scheduler only runs on open-for- + * writes/has-cells channels and is the only path for those to transition to + * other states. The scheduler_run() function gives us the opportunity to do + * scheduling work, and is called from other scheduler functions whenever a + * state transition occurs, and periodically from the main event loop. + */ + +/* Scheduler global data structures */ + +/* + * We keep a list of channels that are pending - i.e, have cells to write + * and can accept them to send. The enum scheduler_state in channel_t + * is reserved for our use. + */ + +/* Pqueue of channels that can write and have cells (pending work) */ +STATIC smartlist_t *channels_pending = NULL; + +/* + * This event runs the scheduler from its callback, and is manually + * activated whenever a channel enters open for writes/cells to send. + */ + +STATIC struct event *run_sched_ev = NULL; + +/* + * Queue heuristic; this is not the queue size, but an 'effective queuesize' + * that ages out contributions from stalled channels. + */ + +STATIC uint64_t queue_heuristic = 0; + +/* + * Timestamp for last queue heuristic update + */ + +STATIC time_t queue_heuristic_timestamp = 0; + +/* Scheduler static function declarations */ + +static void scheduler_evt_callback(evutil_socket_t fd, + short events, void *arg); +static int scheduler_more_work(void); +static void scheduler_retrigger(void); +#if 0 +static void scheduler_trigger(void); +#endif + +/* Scheduler function implementations */ + +/** Free everything and shut down the scheduling system */ + +void +scheduler_free_all(void) +{ + log_debug(LD_SCHED, "Shutting down scheduler"); + + if (run_sched_ev) { + if (event_del(run_sched_ev) < 0) { + log_warn(LD_BUG, "Problem deleting run_sched_ev"); + } + tor_event_free(run_sched_ev); + run_sched_ev = NULL; + } + + if (channels_pending) { + smartlist_free(channels_pending); + channels_pending = NULL; + } +} + +/** + * Comparison function to use when sorting pending channels + */ + +MOCK_IMPL(STATIC int, +scheduler_compare_channels, (const void *c1_v, const void *c2_v)) +{ + channel_t *c1 = NULL, *c2 = NULL; + /* These are a workaround for -Wbad-function-cast throwing a fit */ + const circuitmux_policy_t *p1, *p2; + uintptr_t p1_i, p2_i; + + tor_assert(c1_v); + tor_assert(c2_v); + + c1 = (channel_t *)(c1_v); + c2 = (channel_t *)(c2_v); + + tor_assert(c1); + tor_assert(c2); + + if (c1 != c2) { + if (circuitmux_get_policy(c1->cmux) == + circuitmux_get_policy(c2->cmux)) { + /* Same cmux policy, so use the mux comparison */ + return circuitmux_compare_muxes(c1->cmux, c2->cmux); + } else { + /* + * Different policies; not important to get this edge case perfect + * because the current code never actually gives different channels + * different cmux policies anyway. Just use this arbitrary but + * definite choice. + */ + p1 = circuitmux_get_policy(c1->cmux); + p2 = circuitmux_get_policy(c2->cmux); + p1_i = (uintptr_t)p1; + p2_i = (uintptr_t)p2; + + return (p1_i < p2_i) ? -1 : 1; + } + } else { + /* c1 == c2, so always equal */ + return 0; + } +} + +/* + * Scheduler event callback; this should get triggered once per event loop + * if any scheduling work was created during the event loop. + */ + +static void +scheduler_evt_callback(evutil_socket_t fd, short events, void *arg) +{ + (void)fd; + (void)events; + (void)arg; + log_debug(LD_SCHED, "Scheduler event callback called"); + + tor_assert(run_sched_ev); + + /* Run the scheduler */ + scheduler_run(); + + /* Do we have more work to do? */ + if (scheduler_more_work()) scheduler_retrigger(); +} + +/** Mark a channel as no longer ready to accept writes */ + +MOCK_IMPL(void, +scheduler_channel_doesnt_want_writes,(channel_t *chan)) +{ + tor_assert(chan); + + tor_assert(channels_pending); + + /* If it's already in pending, we can put it in waiting_to_write */ + if (chan->scheduler_state == SCHED_CHAN_PENDING) { + /* + * It's in channels_pending, so it shouldn't be in any of + * the other lists. It can't write any more, so it goes to + * channels_waiting_to_write. + */ + smartlist_pqueue_remove(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p went from pending " + "to waiting_to_write", + U64_PRINTF_ARG(chan->global_identifier), chan); + } else { + /* + * It's not in pending, so it can't become waiting_to_write; it's + * either not in any of the lists (nothing to do) or it's already in + * waiting_for_cells (remove it, can't write any more). + */ + if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) { + chan->scheduler_state = SCHED_CHAN_IDLE; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p left waiting_for_cells", + U64_PRINTF_ARG(chan->global_identifier), chan); + } + } +} + +/** Mark a channel as having waiting cells */ + +MOCK_IMPL(void, +scheduler_channel_has_waiting_cells,(channel_t *chan)) +{ + int became_pending = 0; + + tor_assert(chan); + tor_assert(channels_pending); + + /* First, check if this one also writeable */ + if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) { + /* + * It's in channels_waiting_for_cells, so it shouldn't be in any of + * the other lists. It has waiting cells now, so it goes to + * channels_pending. + */ + chan->scheduler_state = SCHED_CHAN_PENDING; + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p went from waiting_for_cells " + "to pending", + U64_PRINTF_ARG(chan->global_identifier), chan); + became_pending = 1; + } else { + /* + * It's not in waiting_for_cells, so it can't become pending; it's + * either not in any of the lists (we add it to waiting_to_write) + * or it's already in waiting_to_write or pending (we do nothing) + */ + if (!(chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE || + chan->scheduler_state == SCHED_CHAN_PENDING)) { + chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p entered waiting_to_write", + U64_PRINTF_ARG(chan->global_identifier), chan); + } + } + + /* + * If we made a channel pending, we potentially have scheduling work + * to do. + */ + if (became_pending) scheduler_retrigger(); +} + +/** Set up the scheduling system */ + +void +scheduler_init(void) +{ + log_debug(LD_SCHED, "Initting scheduler"); + + tor_assert(!run_sched_ev); + run_sched_ev = tor_event_new(tor_libevent_get_base(), -1, + 0, scheduler_evt_callback, NULL); + + channels_pending = smartlist_new(); + queue_heuristic = 0; + queue_heuristic_timestamp = approx_time(); +} + +/** Check if there's more scheduling work */ + +static int +scheduler_more_work(void) +{ + tor_assert(channels_pending); + + return ((scheduler_get_queue_heuristic() < sched_q_low_water) && + ((smartlist_len(channels_pending) > 0))) ? 1 : 0; +} + +/** Retrigger the scheduler in a way safe to use from the callback */ + +static void +scheduler_retrigger(void) +{ + tor_assert(run_sched_ev); + event_active(run_sched_ev, EV_TIMEOUT, 1); +} + +/** Notify the scheduler of a channel being closed */ + +MOCK_IMPL(void, +scheduler_release_channel,(channel_t *chan)) +{ + tor_assert(chan); + tor_assert(channels_pending); + + if (chan->scheduler_state == SCHED_CHAN_PENDING) { + smartlist_pqueue_remove(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + } + + chan->scheduler_state = SCHED_CHAN_IDLE; +} + +/** Run the scheduling algorithm if necessary */ + +MOCK_IMPL(void, +scheduler_run, (void)) +{ + int n_cells, n_chans_before, n_chans_after; + uint64_t q_len_before, q_heur_before, q_len_after, q_heur_after; + ssize_t flushed, flushed_this_time; + smartlist_t *to_readd = NULL; + channel_t *chan = NULL; + + log_debug(LD_SCHED, "We have a chance to run the scheduler"); + + if (scheduler_get_queue_heuristic() < sched_q_low_water) { + n_chans_before = smartlist_len(channels_pending); + q_len_before = channel_get_global_queue_estimate(); + q_heur_before = scheduler_get_queue_heuristic(); + + while (scheduler_get_queue_heuristic() <= sched_q_high_water && + smartlist_len(channels_pending) > 0) { + /* Pop off a channel */ + chan = smartlist_pqueue_pop(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx)); + tor_assert(chan); + + /* Figure out how many cells we can write */ + n_cells = channel_num_cells_writeable(chan); + if (n_cells > 0) { + log_debug(LD_SCHED, + "Scheduler saw pending channel " U64_FORMAT " at %p with " + "%d cells writeable", + U64_PRINTF_ARG(chan->global_identifier), chan, n_cells); + + flushed = 0; + while (flushed < n_cells && + scheduler_get_queue_heuristic() <= sched_q_high_water) { + flushed_this_time = + channel_flush_some_cells(chan, + MIN(sched_max_flush_cells, + (size_t) n_cells - flushed)); + if (flushed_this_time <= 0) break; + flushed += flushed_this_time; + } + + if (flushed < n_cells) { + /* We ran out of cells to flush */ + chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p " + "entered waiting_for_cells from pending", + U64_PRINTF_ARG(chan->global_identifier), + chan); + } else { + /* The channel may still have some cells */ + if (channel_more_to_flush(chan)) { + /* The channel goes to either pending or waiting_to_write */ + if (channel_num_cells_writeable(chan) > 0) { + /* Add it back to pending later */ + if (!to_readd) to_readd = smartlist_new(); + smartlist_add(to_readd, chan); + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p " + "is still pending", + U64_PRINTF_ARG(chan->global_identifier), + chan); + } else { + /* It's waiting to be able to write more */ + chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p " + "entered waiting_to_write from pending", + U64_PRINTF_ARG(chan->global_identifier), + chan); + } + } else { + /* No cells left; it can go to idle or waiting_for_cells */ + if (channel_num_cells_writeable(chan) > 0) { + /* + * It can still accept writes, so it goes to + * waiting_for_cells + */ + chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p " + "entered waiting_for_cells from pending", + U64_PRINTF_ARG(chan->global_identifier), + chan); + } else { + /* + * We exactly filled up the output queue with all available + * cells; go to idle. + */ + chan->scheduler_state = SCHED_CHAN_IDLE; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p " + "become idle from pending", + U64_PRINTF_ARG(chan->global_identifier), + chan); + } + } + } + + log_debug(LD_SCHED, + "Scheduler flushed %d cells onto pending channel " + U64_FORMAT " at %p", + (int)flushed, U64_PRINTF_ARG(chan->global_identifier), + chan); + } else { + log_info(LD_SCHED, + "Scheduler saw pending channel " U64_FORMAT " at %p with " + "no cells writeable", + U64_PRINTF_ARG(chan->global_identifier), chan); + /* Put it back to WAITING_TO_WRITE */ + chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE; + } + } + + /* Readd any channels we need to */ + if (to_readd) { + SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, chan) { + chan->scheduler_state = SCHED_CHAN_PENDING; + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + } SMARTLIST_FOREACH_END(chan); + smartlist_free(to_readd); + } + + n_chans_after = smartlist_len(channels_pending); + q_len_after = channel_get_global_queue_estimate(); + q_heur_after = scheduler_get_queue_heuristic(); + log_debug(LD_SCHED, + "Scheduler handled %d of %d pending channels, queue size from " + U64_FORMAT " to " U64_FORMAT ", queue heuristic from " + U64_FORMAT " to " U64_FORMAT, + n_chans_before - n_chans_after, n_chans_before, + U64_PRINTF_ARG(q_len_before), U64_PRINTF_ARG(q_len_after), + U64_PRINTF_ARG(q_heur_before), U64_PRINTF_ARG(q_heur_after)); + } +} + +/** Trigger the scheduling event so we run the scheduler later */ + +#if 0 +static void +scheduler_trigger(void) +{ + log_debug(LD_SCHED, "Triggering scheduler event"); + + tor_assert(run_sched_ev); + + event_add(run_sched_ev, EV_TIMEOUT, 1); +} +#endif + +/** Mark a channel as ready to accept writes */ + +void +scheduler_channel_wants_writes(channel_t *chan) +{ + int became_pending = 0; + + tor_assert(chan); + tor_assert(channels_pending); + + /* If it's already in waiting_to_write, we can put it in pending */ + if (chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE) { + /* + * It can write now, so it goes to channels_pending. + */ + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + chan->scheduler_state = SCHED_CHAN_PENDING; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p went from waiting_to_write " + "to pending", + U64_PRINTF_ARG(chan->global_identifier), chan); + became_pending = 1; + } else { + /* + * It's not in SCHED_CHAN_WAITING_TO_WRITE, so it can't become pending; + * it's either idle and goes to WAITING_FOR_CELLS, or it's a no-op. + */ + if (!(chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS || + chan->scheduler_state == SCHED_CHAN_PENDING)) { + chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p entered waiting_for_cells", + U64_PRINTF_ARG(chan->global_identifier), chan); + } + } + + /* + * If we made a channel pending, we potentially have scheduling work + * to do. + */ + if (became_pending) scheduler_retrigger(); +} + +/** + * Notify the scheduler that a channel's position in the pqueue may have + * changed + */ + +void +scheduler_touch_channel(channel_t *chan) +{ + tor_assert(chan); + + if (chan->scheduler_state == SCHED_CHAN_PENDING) { + /* Remove and re-add it */ + smartlist_pqueue_remove(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + } + /* else no-op, since it isn't in the queue */ +} + +/** + * Notify the scheduler of a queue size adjustment, to recalculate the + * queue heuristic. + */ + +void +scheduler_adjust_queue_size(channel_t *chan, int dir, uint64_t adj) +{ + time_t now = approx_time(); + + log_debug(LD_SCHED, + "Queue size adjustment by %s" U64_FORMAT " for channel " + U64_FORMAT, + (dir >= 0) ? "+" : "-", + U64_PRINTF_ARG(adj), + U64_PRINTF_ARG(chan->global_identifier)); + + /* Get the queue heuristic up to date */ + scheduler_update_queue_heuristic(now); + + /* Adjust as appropriate */ + if (dir >= 0) { + /* Increasing it */ + queue_heuristic += adj; + } else { + /* Decreasing it */ + if (queue_heuristic > adj) queue_heuristic -= adj; + else queue_heuristic = 0; + } + + log_debug(LD_SCHED, + "Queue heuristic is now " U64_FORMAT, + U64_PRINTF_ARG(queue_heuristic)); +} + +/** + * Query the current value of the queue heuristic + */ + +STATIC uint64_t +scheduler_get_queue_heuristic(void) +{ + time_t now = approx_time(); + + scheduler_update_queue_heuristic(now); + + return queue_heuristic; +} + +/** + * Adjust the queue heuristic value to the present time + */ + +STATIC void +scheduler_update_queue_heuristic(time_t now) +{ + time_t diff; + + if (queue_heuristic_timestamp == 0) { + /* + * Nothing we can sensibly do; must not have been initted properly. + * Oh well. + */ + queue_heuristic_timestamp = now; + } else if (queue_heuristic_timestamp < now) { + diff = now - queue_heuristic_timestamp; + /* + * This is a simple exponential age-out; the other proposed alternative + * was a linear age-out using the bandwidth history in rephist.c; I'm + * going with this out of concern that if an adversary can jam the + * scheduler long enough, it would cause the bandwidth to drop to + * zero and render the aging mechanism ineffective thereafter. + */ + if (0 <= diff && diff < 64) queue_heuristic >>= diff; + else queue_heuristic = 0; + + queue_heuristic_timestamp = now; + + log_debug(LD_SCHED, + "Queue heuristic is now " U64_FORMAT, + U64_PRINTF_ARG(queue_heuristic)); + } + /* else no update needed, or time went backward */ +} + +/** + * Set scheduler watermarks and flush size + */ + +void +scheduler_set_watermarks(uint32_t lo, uint32_t hi, uint32_t max_flush) +{ + /* Sanity assertions - caller should ensure these are true */ + tor_assert(lo > 0); + tor_assert(hi > lo); + tor_assert(max_flush > 0); + + sched_q_low_water = lo; + sched_q_high_water = hi; + sched_max_flush_cells = max_flush; +} + diff --git a/src/or/scheduler.h b/src/or/scheduler.h new file mode 100644 index 0000000000..94a44a0aa3 --- /dev/null +++ b/src/or/scheduler.h @@ -0,0 +1,50 @@ +/* * Copyright (c) 2013-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file scheduler.h + * \brief Header file for scheduler.c + **/ + +#ifndef TOR_SCHEDULER_H +#define TOR_SCHEDULER_H + +#include "or.h" +#include "channel.h" +#include "testsupport.h" + +/* Global-visibility scheduler functions */ + +/* Set up and shut down the scheduler from main.c */ +void scheduler_free_all(void); +void scheduler_init(void); +MOCK_DECL(void, scheduler_run, (void)); + +/* Mark channels as having cells or wanting/not wanting writes */ +MOCK_DECL(void,scheduler_channel_doesnt_want_writes,(channel_t *chan)); +MOCK_DECL(void,scheduler_channel_has_waiting_cells,(channel_t *chan)); +void scheduler_channel_wants_writes(channel_t *chan); + +/* Notify the scheduler of a channel being closed */ +MOCK_DECL(void,scheduler_release_channel,(channel_t *chan)); + +/* Notify scheduler of queue size adjustments */ +void scheduler_adjust_queue_size(channel_t *chan, int dir, uint64_t adj); + +/* Notify scheduler that a channel's queue position may have changed */ +void scheduler_touch_channel(channel_t *chan); + +/* Adjust the watermarks from config file*/ +void scheduler_set_watermarks(uint32_t lo, uint32_t hi, uint32_t max_flush); + +/* Things only scheduler.c and its test suite should see */ + +#ifdef SCHEDULER_PRIVATE_ +MOCK_DECL(STATIC int, scheduler_compare_channels, + (const void *c1_v, const void *c2_v)); +STATIC uint64_t scheduler_get_queue_heuristic(void); +STATIC void scheduler_update_queue_heuristic(time_t now); +#endif + +#endif /* !defined(TOR_SCHEDULER_H) */ + diff --git a/src/or/statefile.c b/src/or/statefile.c index bcb7b07417..9594d9cec3 100644 --- a/src/or/statefile.c +++ b/src/or/statefile.c @@ -1,17 +1,27 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * \file statefile.c + * + * \brief Handles parsing and encoding the persistent 'state' file that carries + * miscellaneous persistent state between Tor invocations. + */ + +#define STATEFILE_PRIVATE #include "or.h" #include "circuitstats.h" #include "config.h" #include "confparse.h" +#include "connection.h" #include "entrynodes.h" #include "hibernate.h" #include "rephist.h" #include "router.h" +#include "sandbox.h" #include "statefile.h" /** A list of state-file "abbreviations," for compatibility. */ @@ -90,8 +100,11 @@ static config_var_t state_vars_[] = { #undef VAR #undef V -static int or_state_validate(or_state_t *old_options, or_state_t *options, - int from_setconf, char **msg); +static int or_state_validate(or_state_t *state, char **msg); + +static int or_state_validate_cb(void *old_options, void *options, + void *default_options, + int from_setconf, char **msg); /** Magic value for or_state_t. */ #define OR_STATE_MAGIC 0x57A73f57 @@ -109,7 +122,7 @@ static const config_format_t state_format = { STRUCT_OFFSET(or_state_t, magic_), state_abbrevs_, state_vars_, - (validate_fn_t)or_state_validate, + or_state_validate_cb, &state_extra_var, }; @@ -117,8 +130,8 @@ static const config_format_t state_format = { static or_state_t *global_state = NULL; /** Return the persistent state struct for this Tor. */ -or_state_t * -get_or_state(void) +MOCK_IMPL(or_state_t *, +get_or_state, (void)) { tor_assert(global_state); return global_state; @@ -194,21 +207,27 @@ validate_transports_in_state(or_state_t *state) return 0; } -/** Return 0 if every setting in <b>state</b> is reasonable, and a - * permissible transition from <b>old_state</b>. Else warn and return -1. - * Should have no side effects, except for normalizing the contents of - * <b>state</b>. - */ -/* XXX from_setconf is here because of bug 238 */ static int -or_state_validate(or_state_t *old_state, or_state_t *state, - int from_setconf, char **msg) +or_state_validate_cb(void *old_state, void *state, void *default_state, + int from_setconf, char **msg) { /* We don't use these; only options do. Still, we need to match that * signature. */ (void) from_setconf; + (void) default_state; (void) old_state; + return or_state_validate(state, msg); +} + +/** Return 0 if every setting in <b>state</b> is reasonable, and a + * permissible transition from <b>old_state</b>. Else warn and return -1. + * Should have no side effects, except for normalizing the contents of + * <b>state</b>. + */ +static int +or_state_validate(or_state_t *state, char **msg) +{ if (entry_guards_parse_state(state, 0, msg)<0) return -1; @@ -237,7 +256,8 @@ or_state_set(or_state_t *new_state) tor_free(err); ret = -1; } - if (circuit_build_times_parse_state(&circ_times, global_state) < 0) { + if (circuit_build_times_parse_state( + get_circuit_build_times_mutable(),global_state) < 0) { ret = -1; } return ret; @@ -249,7 +269,7 @@ or_state_set(or_state_t *new_state) static void or_state_save_broken(char *fname) { - int i; + int i, res; file_status_t status; char *fname2 = NULL; for (i = 0; i < 100; ++i) { @@ -263,12 +283,18 @@ or_state_save_broken(char *fname) log_warn(LD_BUG, "Unable to parse state in \"%s\"; too many saved bad " "state files to move aside. Discarding the old state file.", fname); - unlink(fname); + res = unlink(fname); + if (res != 0) { + log_warn(LD_FS, + "Also couldn't discard old state file \"%s\" because " + "unlink() failed: %s", + fname, strerror(errno)); + } } else { log_warn(LD_BUG, "Unable to parse state in \"%s\". Moving it aside " "to \"%s\". This could be a bug in Tor; please tell " "the developers.", fname, fname2); - if (rename(fname, fname2) < 0) { + if (tor_rename(fname, fname2) < 0) {//XXXX sandbox prohibits log_warn(LD_BUG, "Weirdly, I couldn't even move the state aside. The " "OS gave an error of %s", strerror(errno)); } @@ -276,6 +302,16 @@ or_state_save_broken(char *fname) tor_free(fname2); } +STATIC or_state_t * +or_state_new(void) +{ + or_state_t *new_state = tor_malloc_zero(sizeof(or_state_t)); + new_state->magic_ = OR_STATE_MAGIC; + config_init(&state_format, new_state); + + return new_state; +} + /** Reload the persistent state from disk, generating a new state as needed. * Return 0 on success, less than 0 on failure. */ @@ -295,7 +331,10 @@ or_state_load(void) goto done; } break; + /* treat empty state files as if the file doesn't exist, and generate + * a new state file, overwriting the empty file in or_state_save() */ case FN_NOENT: + case FN_EMPTY: break; case FN_ERROR: case FN_DIR: @@ -303,9 +342,7 @@ or_state_load(void) log_warn(LD_GENERAL,"State file \"%s\" is not a file? Failing.", fname); goto done; } - new_state = tor_malloc_zero(sizeof(or_state_t)); - new_state->magic_ = OR_STATE_MAGIC; - config_init(&state_format, new_state); + new_state = or_state_new(); if (contents) { config_line_t *lines=NULL; int assign_retval; @@ -322,7 +359,7 @@ or_state_load(void) } } - if (!badstate && or_state_validate(NULL, new_state, 1, &errmsg) < 0) + if (!badstate && or_state_validate(new_state, &errmsg) < 0) badstate = 1; if (errmsg) { @@ -340,11 +377,15 @@ or_state_load(void) tor_free(contents); config_free(&state_format, new_state); - new_state = tor_malloc_zero(sizeof(or_state_t)); - new_state->magic_ = OR_STATE_MAGIC; - config_init(&state_format, new_state); + new_state = or_state_new(); } else if (contents) { log_info(LD_GENERAL, "Loaded state from \"%s\"", fname); + /* Warn the user if their clock has been set backwards, + * they could be tricked into using old consensuses */ + time_t apparent_skew = new_state->LastWritten - time(NULL); + if (apparent_skew > 0) + clock_skew_warning(NULL, (long)apparent_skew, 1, LD_GENERAL, + "local state file", fname); } else { log_info(LD_GENERAL, "Initialized state"); } @@ -404,7 +445,7 @@ or_state_save(time_t now) * to avoid redundant writes. */ entry_guards_update_state(global_state); rep_hist_update_state(global_state); - circuit_build_times_update_state(&circ_times, global_state); + circuit_build_times_update_state(get_circuit_build_times(), global_state); if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); @@ -449,7 +490,7 @@ or_state_save(time_t now) /** Return the config line for transport <b>transport</b> in the current state. * Return NULL if there is no config line for <b>transport</b>. */ -static config_line_t * +STATIC config_line_t * get_transport_in_state_by_name(const char *transport) { or_state_t *or_state = get_or_state(); @@ -607,10 +648,19 @@ save_transport_to_state(const char *transport, tor_free(transport_addrport); } +STATIC void +or_state_free(or_state_t *state) +{ + if (!state) + return; + + config_free(&state_format, state); +} + void or_state_free_all(void) { - config_free(&state_format, global_state); + or_state_free(global_state); global_state = NULL; } diff --git a/src/or/statefile.h b/src/or/statefile.h index dcdee6c604..b13743481d 100644 --- a/src/or/statefile.h +++ b/src/or/statefile.h @@ -1,13 +1,13 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_STATEFILE_H #define TOR_STATEFILE_H -or_state_t *get_or_state(void); +MOCK_DECL(or_state_t *,get_or_state,(void)); int did_last_state_file_write_fail(void); int or_state_save(time_t now); @@ -18,5 +18,11 @@ int or_state_load(void); int or_state_loaded(void); void or_state_free_all(void); +#ifdef STATEFILE_PRIVATE +STATIC config_line_t *get_transport_in_state_by_name(const char *transport); +STATIC void or_state_free(or_state_t *state); +STATIC or_state_t *or_state_new(void); +#endif + #endif diff --git a/src/or/status.c b/src/or/status.c index 69f92ed097..749cee4edf 100644 --- a/src/or/status.c +++ b/src/or/status.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2013, The Tor Project, Inc. */ +/* Copyright (c) 2010-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -6,7 +6,10 @@ * \brief Keep status information and log the heartbeat messages. **/ +#define STATUS_PRIVATE + #include "or.h" +#include "circuituse.h" #include "config.h" #include "status.h" #include "nodelist.h" @@ -14,25 +17,24 @@ #include "router.h" #include "circuitlist.h" #include "main.h" +#include "rephist.h" #include "hibernate.h" #include "rephist.h" +#include "statefile.h" + +static void log_accounting(const time_t now, const or_options_t *options); +#include "geoip.h" /** Return the total number of circuits. */ -static int +STATIC int count_circuits(void) { - circuit_t *circ; - int nr=0; - - for (circ = circuit_get_global_list_(); circ; circ = circ->next) - nr++; - - return nr; + return smartlist_len(circuit_get_global_list()); } /** Take seconds <b>secs</b> and return a newly allocated human-readable * uptime string */ -static char * +STATIC char * secs_to_uptime(long secs) { long int days = secs / 86400; @@ -59,7 +61,7 @@ secs_to_uptime(long secs) /** Take <b>bytes</b> and returns a newly allocated human-readable usage * string. */ -static char * +STATIC char * bytes_to_usage(uint64_t bytes) { char *bw_string = NULL; @@ -91,7 +93,6 @@ log_heartbeat(time_t now) const int hibernating = we_are_hibernating(); const or_options_t *options = get_options(); - (void)now; if (public_server_mode(options) && !hibernating) { /* Let's check if we are in the current cached consensus. */ @@ -109,21 +110,46 @@ log_heartbeat(time_t now) log_fn(LOG_NOTICE, LD_HEARTBEAT, "Heartbeat: Tor's uptime is %s, with %d " "circuits open. I've sent %s and received %s.%s", - uptime, count_circuits(),bw_sent,bw_rcvd, + uptime, count_circuits(), bw_sent, bw_rcvd, hibernating?" We are currently hibernating.":""); - if (stats_n_data_cells_packaged && !hibernating) - log_notice(LD_HEARTBEAT, "Average packaged cell fullness: %2.3f%%", - 100*(U64_TO_DBL(stats_n_data_bytes_packaged) / - U64_TO_DBL(stats_n_data_cells_packaged*RELAY_PAYLOAD_SIZE)) ); + if (server_mode(options) && accounting_is_enabled(options) && !hibernating) { + log_accounting(now, options); + } - if (r > 1.0) { - double overhead = ( r - 1.0 ) * 100.0; - log_notice(LD_HEARTBEAT, "TLS write overhead: %.f%%", overhead); + double fullness_pct = 100; + if (stats_n_data_cells_packaged && !hibernating) { + fullness_pct = + 100*(U64_TO_DBL(stats_n_data_bytes_packaged) / + U64_TO_DBL(stats_n_data_cells_packaged*RELAY_PAYLOAD_SIZE)); } + const double overhead_pct = ( r - 1.0 ) * 100.0; - if (public_server_mode(options)) +#define FULLNESS_PCT_THRESHOLD 80 +#define TLS_OVERHEAD_THRESHOLD 15 + + const int severity = (fullness_pct < FULLNESS_PCT_THRESHOLD || + overhead_pct > TLS_OVERHEAD_THRESHOLD) + ? LOG_NOTICE : LOG_INFO; + + log_fn(severity, LD_HEARTBEAT, + "Average packaged cell fullness: %2.3f%%. " + "TLS write overhead: %.f%%", fullness_pct, overhead_pct); + + if (public_server_mode(options)) { rep_hist_log_circuit_handshake_stats(now); + rep_hist_log_link_protocol_counts(); + } + + circuit_log_ancient_one_hop_circuits(1800); + + if (options->BridgeRelay) { + char *msg = NULL; + msg = format_client_stats_heartbeat(now); + if (msg) + log_notice(LD_HEARTBEAT, "%s", msg); + tor_free(msg); + } tor_free(uptime); tor_free(bw_sent); @@ -132,3 +158,45 @@ log_heartbeat(time_t now) return 0; } +static void +log_accounting(const time_t now, const or_options_t *options) +{ + or_state_t *state = get_or_state(); + char *acc_rcvd = bytes_to_usage(state->AccountingBytesReadInInterval); + char *acc_sent = bytes_to_usage(state->AccountingBytesWrittenInInterval); + char *acc_used = bytes_to_usage(get_accounting_bytes()); + uint64_t acc_bytes = options->AccountingMax; + char *acc_max; + time_t interval_end = accounting_get_end_time(); + char end_buf[ISO_TIME_LEN + 1]; + char *remaining = NULL; + acc_max = bytes_to_usage(acc_bytes); + format_local_iso_time(end_buf, interval_end); + remaining = secs_to_uptime(interval_end - now); + + const char *acc_rule; + switch (options->AccountingRule) { + case ACCT_MAX: acc_rule = "max"; + break; + case ACCT_SUM: acc_rule = "sum"; + break; + case ACCT_OUT: acc_rule = "out"; + break; + case ACCT_IN: acc_rule = "in"; + break; + default: acc_rule = "max"; + break; + } + + log_notice(LD_HEARTBEAT, "Heartbeat: Accounting enabled. " + "Sent: %s, Received: %s, Used: %s / %s, Rule: %s. The " + "current accounting interval ends on %s, in %s.", + acc_sent, acc_rcvd, acc_used, acc_max, acc_rule, end_buf, remaining); + + tor_free(acc_rcvd); + tor_free(acc_sent); + tor_free(acc_used); + tor_free(acc_max); + tor_free(remaining); +} + diff --git a/src/or/status.h b/src/or/status.h index 7c3b969c86..b97e835037 100644 --- a/src/or/status.h +++ b/src/or/status.h @@ -1,10 +1,18 @@ -/* Copyright (c) 2010-2013, The Tor Project, Inc. */ +/* Copyright (c) 2010-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_STATUS_H #define TOR_STATUS_H +#include "testsupport.h" + int log_heartbeat(time_t now); +#ifdef STATUS_PRIVATE +STATIC int count_circuits(void); +STATIC char *secs_to_uptime(long secs); +STATIC char *bytes_to_usage(uint64_t bytes); +#endif + #endif diff --git a/src/or/tor_main.c b/src/or/tor_main.c index 05dc0bf0bf..ac32eef559 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** String describing which Tor Git repository version the source was @@ -27,6 +27,10 @@ int tor_main(int argc, char *argv[]); int main(int argc, char *argv[]) { - return tor_main(argc, argv); + int r = tor_main(argc, argv); + if (r < 0 || r > 255) + return 1; + else + return r; } diff --git a/src/or/torcert.c b/src/or/torcert.c new file mode 100644 index 0000000000..a6a33c675a --- /dev/null +++ b/src/or/torcert.c @@ -0,0 +1,297 @@ +/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torcert.c + * + * \brief Implementation for ed25519-signed certificates as used in the Tor + * protocol. + */ + +#include "crypto.h" +#include "torcert.h" +#include "ed25519_cert.h" +#include "torlog.h" +#include "util.h" +#include "compat.h" +#include "link_handshake.h" + +/** Helper for tor_cert_create(): signs any 32 bytes, not just an ed25519 + * key. + */ +static tor_cert_t * +tor_cert_sign_impl(const ed25519_keypair_t *signing_key, + uint8_t cert_type, + uint8_t signed_key_type, + const uint8_t signed_key_info[32], + time_t now, time_t lifetime, + uint32_t flags) +{ + tor_cert_t *torcert = NULL; + + ed25519_cert_t *cert = ed25519_cert_new(); + cert->cert_type = cert_type; + cert->exp_field = (uint32_t) CEIL_DIV(now + lifetime, 3600); + cert->cert_key_type = signed_key_type; + memcpy(cert->certified_key, signed_key_info, 32); + + if (flags & CERT_FLAG_INCLUDE_SIGNING_KEY) { + ed25519_cert_extension_t *ext = ed25519_cert_extension_new(); + ext->ext_type = CERTEXT_SIGNED_WITH_KEY; + memcpy(ext->un_signing_key, signing_key->pubkey.pubkey, 32); + ed25519_cert_add_ext(cert, ext); + ++cert->n_extensions; + } + + const ssize_t alloc_len = ed25519_cert_encoded_len(cert); + tor_assert(alloc_len > 0); + uint8_t *encoded = tor_malloc(alloc_len); + const ssize_t real_len = ed25519_cert_encode(encoded, alloc_len, cert); + if (real_len < 0) + goto err; + tor_assert(real_len == alloc_len); + tor_assert(real_len > ED25519_SIG_LEN); + uint8_t *sig = encoded + (real_len - ED25519_SIG_LEN); + tor_assert(tor_mem_is_zero((char*)sig, ED25519_SIG_LEN)); + + ed25519_signature_t signature; + if (ed25519_sign(&signature, encoded, + real_len-ED25519_SIG_LEN, signing_key)<0) { + log_warn(LD_BUG, "Can't sign certificate"); + goto err; + } + memcpy(sig, signature.sig, ED25519_SIG_LEN); + + torcert = tor_cert_parse(encoded, real_len); + if (! torcert) { + log_warn(LD_BUG, "Generated a certificate we cannot parse"); + goto err; + } + + if (tor_cert_checksig(torcert, &signing_key->pubkey, now) < 0) { + log_warn(LD_BUG, "Generated a certificate whose signature we can't check"); + goto err; + } + + tor_free(encoded); + + goto done; + + err: + tor_cert_free(torcert); + torcert = NULL; + done: + ed25519_cert_free(cert); + tor_free(encoded); + return torcert; +} + +/** + * Create and return a new new certificate of type <b>cert_type</b> to + * authenticate <b>signed_key</b> using the key <b>signing_key</b>. The + * certificate should remain valid for at least <b>lifetime</b> seconds after + * <b>now</b>. + * + * If CERT_FLAG_INCLUDE_SIGNING_KEY is set in <b>flags</b>, embed + * the public part of <b>signing_key</b> in the certificate. + */ +tor_cert_t * +tor_cert_create(const ed25519_keypair_t *signing_key, + uint8_t cert_type, + const ed25519_public_key_t *signed_key, + time_t now, time_t lifetime, + uint32_t flags) +{ + return tor_cert_sign_impl(signing_key, cert_type, + SIGNED_KEY_TYPE_ED25519, signed_key->pubkey, + now, lifetime, flags); +} + +/** Release all storage held for <b>cert</b>. */ +void +tor_cert_free(tor_cert_t *cert) +{ + if (! cert) + return; + + if (cert->encoded) + memwipe(cert->encoded, 0, cert->encoded_len); + tor_free(cert->encoded); + + memwipe(cert, 0, sizeof(tor_cert_t)); + tor_free(cert); +} + +/** Parse a certificate encoded with <b>len</b> bytes in <b>encoded</b>. */ +tor_cert_t * +tor_cert_parse(const uint8_t *encoded, const size_t len) +{ + tor_cert_t *cert = NULL; + ed25519_cert_t *parsed = NULL; + ssize_t got_len = ed25519_cert_parse(&parsed, encoded, len); + if (got_len < 0 || (size_t) got_len != len) + goto err; + + cert = tor_malloc_zero(sizeof(tor_cert_t)); + cert->encoded = tor_memdup(encoded, len); + cert->encoded_len = len; + + memcpy(cert->signed_key.pubkey, parsed->certified_key, 32); + cert->valid_until = parsed->exp_field * 3600; + cert->cert_type = parsed->cert_type; + + for (unsigned i = 0; i < ed25519_cert_getlen_ext(parsed); ++i) { + ed25519_cert_extension_t *ext = ed25519_cert_get_ext(parsed, i); + if (ext->ext_type == CERTEXT_SIGNED_WITH_KEY) { + if (cert->signing_key_included) + goto err; + + cert->signing_key_included = 1; + memcpy(cert->signing_key.pubkey, ext->un_signing_key, 32); + } else if (ext->ext_flags & CERTEXT_FLAG_AFFECTS_VALIDATION) { + /* Unrecognized extension with affects_validation set */ + goto err; + } + } + + goto done; + err: + tor_cert_free(cert); + cert = NULL; + done: + ed25519_cert_free(parsed); + return cert; +} + +/** Fill in <b>checkable_out</b> with the information needed to check + * the signature on <b>cert</b> with <b>pubkey</b>. */ +int +tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out, + const tor_cert_t *cert, + const ed25519_public_key_t *pubkey) +{ + if (! pubkey) { + if (cert->signing_key_included) + pubkey = &cert->signing_key; + else + return -1; + } + + checkable_out->msg = cert->encoded; + checkable_out->pubkey = pubkey; + tor_assert(cert->encoded_len > ED25519_SIG_LEN); + const size_t signed_len = cert->encoded_len - ED25519_SIG_LEN; + checkable_out->len = signed_len; + memcpy(checkable_out->signature.sig, + cert->encoded + signed_len, ED25519_SIG_LEN); + + return 0; +} + +/** Validates the signature on <b>cert</b> with <b>pubkey</b> relative to the + * current time <b>now</b>. (If <b>now</b> is 0, do not check the expiration + * time.) Return 0 on success, -1 on failure. Sets flags in <b>cert</b> as + * appropriate. + */ +int +tor_cert_checksig(tor_cert_t *cert, + const ed25519_public_key_t *pubkey, time_t now) +{ + ed25519_checkable_t checkable; + int okay; + + if (now && now > cert->valid_until) { + cert->cert_expired = 1; + return -1; + } + + if (tor_cert_get_checkable_sig(&checkable, cert, pubkey) < 0) + return -1; + + if (ed25519_checksig_batch(&okay, &checkable, 1) < 0) { + cert->sig_bad = 1; + return -1; + } else { + cert->sig_ok = 1; + /* Only copy the checkable public key when it is different from the signing + * key of the certificate to avoid undefined behavior. */ + if (cert->signing_key.pubkey != checkable.pubkey->pubkey) { + memcpy(cert->signing_key.pubkey, checkable.pubkey->pubkey, 32); + } + cert->cert_valid = 1; + return 0; + } +} + +/** Return a new copy of <b>cert</b> */ +tor_cert_t * +tor_cert_dup(const tor_cert_t *cert) +{ + tor_cert_t *newcert = tor_memdup(cert, sizeof(tor_cert_t)); + if (cert->encoded) + newcert->encoded = tor_memdup(cert->encoded, cert->encoded_len); + return newcert; +} + +/** Return true iff cert1 and cert2 are the same cert. */ +int +tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2) +{ + tor_assert(cert1); + tor_assert(cert2); + return cert1->encoded_len == cert2->encoded_len && + tor_memeq(cert1->encoded, cert2->encoded, cert1->encoded_len); +} + +/** Return true iff cert1 and cert2 are the same cert, or if they are both + * NULL. */ +int +tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2) +{ + if (cert1 == NULL && cert2 == NULL) + return 1; + if (!cert1 || !cert2) + return 0; + return tor_cert_eq(cert1, cert2); +} + +/** Create new cross-certification object to certify <b>ed_key</b> as the + * master ed25519 identity key for the RSA identity key <b>rsa_key</b>. + * Allocates and stores the encoded certificate in *<b>cert</b>, and returns + * the number of bytes stored. Returns negative on error.*/ +ssize_t +tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key, + const crypto_pk_t *rsa_key, + time_t expires, + uint8_t **cert) +{ + uint8_t *res; + + rsa_ed_crosscert_t *cc = rsa_ed_crosscert_new(); + memcpy(cc->ed_key, ed_key->pubkey, ED25519_PUBKEY_LEN); + cc->expiration = (uint32_t) CEIL_DIV(expires, 3600); + cc->sig_len = crypto_pk_keysize(rsa_key); + rsa_ed_crosscert_setlen_sig(cc, crypto_pk_keysize(rsa_key)); + + ssize_t alloc_sz = rsa_ed_crosscert_encoded_len(cc); + tor_assert(alloc_sz > 0); + res = tor_malloc_zero(alloc_sz); + ssize_t sz = rsa_ed_crosscert_encode(res, alloc_sz, cc); + tor_assert(sz > 0 && sz <= alloc_sz); + + const int signed_part_len = 32 + 4; + int siglen = crypto_pk_private_sign(rsa_key, + (char*)rsa_ed_crosscert_getarray_sig(cc), + rsa_ed_crosscert_getlen_sig(cc), + (char*)res, signed_part_len); + tor_assert(siglen > 0 && siglen <= (int)crypto_pk_keysize(rsa_key)); + tor_assert(siglen <= UINT8_MAX); + cc->sig_len = siglen; + rsa_ed_crosscert_setlen_sig(cc, siglen); + + sz = rsa_ed_crosscert_encode(res, alloc_sz, cc); + rsa_ed_crosscert_free(cc); + *cert = res; + return sz; +} + diff --git a/src/or/torcert.h b/src/or/torcert.h new file mode 100644 index 0000000000..9c819c0abb --- /dev/null +++ b/src/or/torcert.h @@ -0,0 +1,76 @@ +/* Copyright (c) 2014-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TORCERT_H_INCLUDED +#define TORCERT_H_INCLUDED + +#include "crypto_ed25519.h" + +#define SIGNED_KEY_TYPE_ED25519 0x01 + +#define CERT_TYPE_ID_SIGNING 0x04 +#define CERT_TYPE_SIGNING_LINK 0x05 +#define CERT_TYPE_SIGNING_AUTH 0x06 +#define CERT_TYPE_ONION_ID 0x0A + +#define CERT_FLAG_INCLUDE_SIGNING_KEY 0x1 + +/** An ed25519-signed certificate as used throughout the Tor protocol. + **/ +typedef struct tor_cert_st { + /** The key authenticated by this certificate */ + ed25519_public_key_t signed_key; + /** The key that signed this certificate. This value may be unset if the + * certificate has never been checked, and didn't include its own key. */ + ed25519_public_key_t signing_key; + /** A time after which this certificate will no longer be valid. */ + time_t valid_until; + + /** The encoded representation of this certificate */ + uint8_t *encoded; + /** The length of <b>encoded</b> */ + size_t encoded_len; + + /** One of CERT_TYPE_... */ + uint8_t cert_type; + /** True iff we received a signing key embedded in this certificate */ + unsigned signing_key_included : 1; + /** True iff we checked the signature and found it bad */ + unsigned sig_bad : 1; + /** True iff we checked the signature and found it correct */ + unsigned sig_ok : 1; + /** True iff we checked the signature and first found that the cert + * had expired */ + unsigned cert_expired : 1; + /** True iff we checked the signature and found the whole cert valid */ + unsigned cert_valid : 1; +} tor_cert_t; + +tor_cert_t *tor_cert_create(const ed25519_keypair_t *signing_key, + uint8_t cert_type, + const ed25519_public_key_t *signed_key, + time_t now, time_t lifetime, + uint32_t flags); + +tor_cert_t *tor_cert_parse(const uint8_t *cert, size_t certlen); + +void tor_cert_free(tor_cert_t *cert); + +int tor_cert_get_checkable_sig(ed25519_checkable_t *checkable_out, + const tor_cert_t *out, + const ed25519_public_key_t *pubkey); + +int tor_cert_checksig(tor_cert_t *cert, + const ed25519_public_key_t *pubkey, time_t now); + +tor_cert_t *tor_cert_dup(const tor_cert_t *cert); +int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2); +int tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2); + +ssize_t tor_make_rsa_ed25519_crosscert(const ed25519_public_key_t *ed_key, + const crypto_pk_t *rsa_key, + time_t expires, + uint8_t **cert); + +#endif + diff --git a/src/or/transports.c b/src/or/transports.c index 3749d6bb21..1b8b1e678c 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2013, The Tor Project, Inc. */ +/* Copyright (c) 2011-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -51,35 +51,37 @@ * logic, because of race conditions that can cause dangling * pointers. ] * - * <b>In even more detail, this is what happens when a SIGHUP - * occurs:</b> + * <b>In even more detail, this is what happens when a config read + * (like a SIGHUP or a SETCONF) occurs:</b> * * We immediately destroy all unconfigured proxies (We shouldn't have - * unconfigured proxies in the first place, except when SIGHUP rings - * immediately after tor is launched.). + * unconfigured proxies in the first place, except when the config + * read happens immediately after tor is launched.). * * We mark all managed proxies and transports to signify that they * must be removed if they don't contribute by the new torrc * (we mark using the <b>marked_for_removal</b> element). * We also mark all managed proxies to signify that they might need to * be restarted so that they end up supporting all the transports the - * new torrc wants them to support (using the <b>got_hup</b> element). + * new torrc wants them to support + * (we mark using the <b>was_around_before_config_read</b> element). * We also clear their <b>transports_to_launch</b> list so that we can * put there the transports we need to launch according to the new * torrc. * * We then start parsing torrc again. * - * Everytime we encounter a transport line using a known pre-SIGHUP - * managed proxy, we cleanse that proxy from the removal mark. - * We also mark it as unconfigured so that on the next scheduled - * events tick, we investigate whether we need to restart the proxy - * so that it also spawns the new transports. - * If the post-SIGHUP <b>transports_to_launch</b> list is identical to - * the pre-SIGHUP one, it means that no changes were introduced to - * this proxy during the SIGHUP and no restart has to take place. + * Everytime we encounter a transport line using a managed proxy that + * was around before the config read, we cleanse that proxy from the + * removal mark. We also toggle the <b>check_if_restarts_needed</b> + * flag, so that on the next <b>pt_configure_remaining_proxies</b> + * tick, we investigate whether we need to restart the proxy so that + * it also spawns the new transports. If the post-config-read + * <b>transports_to_launch</b> list is identical to the pre-config-read + * one, it means that no changes were introduced to this proxy during + * the config read and no restart has to take place. * - * During the post-SIGHUP torrc parsing, we unmark all transports + * During the post-config-read torrc parsing, we unmark all transports * spawned by managed proxies that we find in our torrc. * We do that so that if we don't need to restart a managed proxy, we * can continue using its old transports normally. @@ -95,24 +97,21 @@ #include "util.h" #include "router.h" #include "statefile.h" +#include "entrynodes.h" +#include "connection_or.h" +#include "ext_orport.h" +#include "control.h" static process_environment_t * create_managed_proxy_environment(const managed_proxy_t *mp); -static INLINE int proxy_configuration_finished(const managed_proxy_t *mp); - -static void managed_proxy_destroy(managed_proxy_t *mp, - int also_terminate_process); +static inline int proxy_configuration_finished(const managed_proxy_t *mp); static void handle_finished_proxy(managed_proxy_t *mp); -static int configure_proxy(managed_proxy_t *mp); - static void parse_method_error(const char *line, int is_server_method); #define parse_server_method_error(l) parse_method_error(l, 1) #define parse_client_method_error(l) parse_method_error(l, 0) -static INLINE void free_execve_args(char **arg); - /** Managed proxy protocol strings */ #define PROTO_ENV_ERROR "ENV-ERROR" #define PROTO_NEG_SUCCESS "VERSION" @@ -123,6 +122,8 @@ static INLINE void free_execve_args(char **arg); #define PROTO_SMETHOD_ERROR "SMETHOD-ERROR" #define PROTO_CMETHODS_DONE "CMETHODS DONE" #define PROTO_SMETHODS_DONE "SMETHODS DONE" +#define PROTO_PROXY_DONE "PROXY DONE" +#define PROTO_PROXY_ERROR "PROXY-ERROR" /** The first and only supported - at the moment - configuration protocol version. */ @@ -136,7 +137,8 @@ static smartlist_t *transport_list = NULL; SOCKS version <b>socks_ver</b>. */ static transport_t * transport_new(const tor_addr_t *addr, uint16_t port, - const char *name, int socks_ver) + const char *name, int socks_ver, + const char *extra_info_args) { transport_t *t = tor_malloc_zero(sizeof(transport_t)); @@ -144,6 +146,8 @@ transport_new(const tor_addr_t *addr, uint16_t port, t->port = port; t->name = tor_strdup(name); t->socks_version = socks_ver; + if (extra_info_args) + t->extra_info_args = tor_strdup(extra_info_args); return t; } @@ -156,6 +160,7 @@ transport_free(transport_t *transport) return; tor_free(transport->name); + tor_free(transport->extra_info_args); tor_free(transport); } @@ -319,11 +324,11 @@ transport_add(transport_t *t) /** Remember a new pluggable transport proxy at <b>addr</b>:<b>port</b>. * <b>name</b> is set to the name of the protocol this proxy uses. * <b>socks_ver</b> is set to the SOCKS version of the proxy. */ -int -transport_add_from_config(const tor_addr_t *addr, uint16_t port, - const char *name, int socks_ver) +MOCK_IMPL(int, +transport_add_from_config, (const tor_addr_t *addr, uint16_t port, + const char *name, int socks_ver)) { - transport_t *t = transport_new(addr, port, name, socks_ver); + transport_t *t = transport_new(addr, port, name, socks_ver, NULL); int r = transport_add(t); @@ -434,6 +439,17 @@ add_transport_to_proxy(const char *transport, managed_proxy_t *mp) static int proxy_needs_restart(const managed_proxy_t *mp) { + int ret = 1; + char* proxy_uri; + + /* If the PT proxy config has changed, then all existing pluggable transports + * should be restarted. + */ + + proxy_uri = get_pt_proxy_uri(); + if (strcmp_opt(proxy_uri, mp->proxy_uri) != 0) + goto needs_restart; + /* mp->transport_to_launch is populated with the names of the transports that must be launched *after* the SIGHUP. mp->transports is populated with the transports that were @@ -454,10 +470,10 @@ proxy_needs_restart(const managed_proxy_t *mp) } SMARTLIST_FOREACH_END(t); - return 0; - + ret = 0; needs_restart: - return 1; + tor_free(proxy_uri); + return ret; } /** Managed proxy <b>mp</b> must be restarted. Do all the necessary @@ -488,6 +504,11 @@ proxy_prepare_for_restart(managed_proxy_t *mp) SMARTLIST_FOREACH(mp->transports, transport_t *, t, transport_free(t)); smartlist_clear(mp->transports); + /* Reset the proxy's HTTPS/SOCKS proxy */ + tor_free(mp->proxy_uri); + mp->proxy_uri = get_pt_proxy_uri(); + mp->proxy_supported = 0; + /* flag it as an infant proxy so that it gets launched on next tick */ mp->conf_state = PT_PROTO_INFANT; unconfigured_proxies_n++; @@ -531,8 +552,7 @@ launch_managed_proxy(managed_proxy_t *mp) } /** Check if any of the managed proxies we are currently trying to - * configure have anything new to say. This is called from - * run_scheduled_events(). */ + * configure has anything new to say. */ void pt_configure_remaining_proxies(void) { @@ -549,14 +569,15 @@ pt_configure_remaining_proxies(void) assert_unconfigured_count_ok(); SMARTLIST_FOREACH_BEGIN(tmp, managed_proxy_t *, mp) { - tor_assert(mp->conf_state != PT_PROTO_BROKEN || + tor_assert(mp->conf_state != PT_PROTO_BROKEN && mp->conf_state != PT_PROTO_FAILED_LAUNCH); - if (mp->got_hup) { - mp->got_hup = 0; + if (mp->was_around_before_config_read) { + /* This proxy is marked by a config read. Check whether we need + to restart it. */ + + mp->was_around_before_config_read = 0; - /* This proxy is marked by a SIGHUP. Check whether we need to - restart it. */ if (proxy_needs_restart(mp)) { log_info(LD_GENERAL, "Preparing managed proxy '%s' for restart.", mp->argv[0]); @@ -589,7 +610,7 @@ pt_configure_remaining_proxies(void) * Return 1 if the transport configuration finished, and return 0 * otherwise (if we still have more configuring to do for this * proxy). */ -static int +STATIC int configure_proxy(managed_proxy_t *mp) { int configuration_finished = 0; @@ -657,6 +678,7 @@ register_server_proxy(const managed_proxy_t *mp) save_transport_to_state(t->name, &t->addr, t->port); log_notice(LD_GENERAL, "Registered server transport '%s' at '%s'", t->name, fmt_addrport(&t->addr, t->port)); + control_event_transport_launched("server", t->name, &t->addr, t->port); } SMARTLIST_FOREACH_END(t); } @@ -679,9 +701,11 @@ register_client_proxy(const managed_proxy_t *mp) break; case 0: log_info(LD_GENERAL, "Successfully registered transport %s", t->name); + control_event_transport_launched("client", t->name, &t->addr, t->port); break; case 1: log_info(LD_GENERAL, "Successfully registered transport %s", t->name); + control_event_transport_launched("client", t->name, &t->addr, t->port); transport_free(transport_tmp); break; } @@ -689,7 +713,7 @@ register_client_proxy(const managed_proxy_t *mp) } /** Register the transports of managed proxy <b>mp</b>. */ -static INLINE void +static inline void register_proxy(const managed_proxy_t *mp) { if (mp->is_server) @@ -699,7 +723,7 @@ register_proxy(const managed_proxy_t *mp) } /** Free memory allocated by managed proxy <b>mp</b>. */ -static void +STATIC void managed_proxy_destroy(managed_proxy_t *mp, int also_terminate_process) { @@ -713,17 +737,60 @@ managed_proxy_destroy(managed_proxy_t *mp, smartlist_free(mp->transports_to_launch); /* remove it from the list of managed proxies */ - smartlist_remove(managed_proxy_list, mp); + if (managed_proxy_list) + smartlist_remove(managed_proxy_list, mp); /* free the argv */ free_execve_args(mp->argv); + /* free the outgoing proxy URI */ + tor_free(mp->proxy_uri); + tor_process_handle_destroy(mp->process_handle, also_terminate_process); mp->process_handle = NULL; tor_free(mp); } +/** Convert the tor proxy options to a URI suitable for TOR_PT_PROXY. + * Return a newly allocated string containing the URI, or NULL if no + * proxy is set. */ +STATIC char * +get_pt_proxy_uri(void) +{ + const or_options_t *options = get_options(); + char *uri = NULL; + + if (options->Socks4Proxy || options->Socks5Proxy || options->HTTPSProxy) { + char addr[TOR_ADDR_BUF_LEN+1]; + + if (options->Socks4Proxy) { + tor_addr_to_str(addr, &options->Socks4ProxyAddr, sizeof(addr), 1); + tor_asprintf(&uri, "socks4a://%s:%d", addr, options->Socks4ProxyPort); + } else if (options->Socks5Proxy) { + tor_addr_to_str(addr, &options->Socks5ProxyAddr, sizeof(addr), 1); + if (!options->Socks5ProxyUsername && !options->Socks5ProxyPassword) { + tor_asprintf(&uri, "socks5://%s:%d", addr, options->Socks5ProxyPort); + } else { + tor_asprintf(&uri, "socks5://%s:%s@%s:%d", + options->Socks5ProxyUsername, + options->Socks5ProxyPassword, + addr, options->Socks5ProxyPort); + } + } else if (options->HTTPSProxy) { + tor_addr_to_str(addr, &options->HTTPSProxyAddr, sizeof(addr), 1); + if (!options->HTTPSProxyAuthenticator) { + tor_asprintf(&uri, "http://%s:%d", addr, options->HTTPSProxyPort); + } else { + tor_asprintf(&uri, "http://%s@%s:%d", options->HTTPSProxyAuthenticator, + addr, options->HTTPSProxyPort); + } + } + } + + return uri; +} + /** Handle a configured or broken managed proxy <b>mp</b>. */ static void handle_finished_proxy(managed_proxy_t *mp) @@ -736,6 +803,13 @@ handle_finished_proxy(managed_proxy_t *mp) managed_proxy_destroy(mp, 0); /* destroy it but don't terminate */ break; case PT_PROTO_CONFIGURED: /* if configured correctly: */ + if (mp->proxy_uri && !mp->proxy_supported) { + log_warn(LD_CONFIG, "Managed proxy '%s' did not configure the " + "specified outgoing proxy and will be terminated.", + mp->argv[0]); + managed_proxy_destroy(mp, 1); /* annihilate it. */ + break; + } register_proxy(mp); /* register its transports */ mp->conf_state = PT_PROTO_COMPLETED; /* and mark it as completed. */ break; @@ -750,12 +824,11 @@ handle_finished_proxy(managed_proxy_t *mp) } unconfigured_proxies_n--; - tor_assert(unconfigured_proxies_n >= 0); } /** Return true if the configuration of the managed proxy <b>mp</b> is finished. */ -static INLINE int +static inline int proxy_configuration_finished(const managed_proxy_t *mp) { return (mp->conf_state == PT_PROTO_CONFIGURED || @@ -781,7 +854,7 @@ handle_methods_done(const managed_proxy_t *mp) /** Handle a configuration protocol <b>line</b> received from a * managed proxy <b>mp</b>. */ -void +STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp) { log_info(LD_GENERAL, "Got a line from managed proxy '%s': (%s)", @@ -854,6 +927,22 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) goto err; return; + } else if (!strcmpstart(line, PROTO_PROXY_DONE)) { + if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS) + goto err; + + if (mp->proxy_uri) { + mp->proxy_supported = 1; + return; + } + + /* No proxy was configured, this should log */ + } else if (!strcmpstart(line, PROTO_PROXY_ERROR)) { + if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS) + goto err; + + parse_proxy_error(line); + goto err; } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) { /* managed proxy launch failed: parse error message to learn why. */ int retval, child_state, saved_errno; @@ -882,7 +971,7 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) } /** Parses an ENV-ERROR <b>line</b> and warns the user accordingly. */ -void +STATIC void parse_env_error(const char *line) { /* (Length of the protocol string) plus (a space) and (the first char of @@ -898,7 +987,7 @@ parse_env_error(const char *line) /** Handles a VERSION <b>line</b>. Updates the configuration protocol * version in <b>mp</b>. */ -int +STATIC int parse_version(const char *line, managed_proxy_t *mp) { if (strlen(line) < (strlen(PROTO_NEG_SUCCESS) + 2)) { @@ -939,14 +1028,14 @@ parse_method_error(const char *line, int is_server) /** Parses an SMETHOD <b>line</b> and if well-formed it registers the * new transport in <b>mp</b>. */ -int +STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp) { int r; smartlist_t *items = NULL; char *method_name=NULL; - + char *args_string=NULL; char *addrport=NULL; tor_addr_t tor_addr; char *address=NULL; @@ -963,6 +1052,9 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) goto err; } + /* Example of legit SMETHOD line: + SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */ + tor_assert(!strcmp(smartlist_get(items,0),PROTO_SMETHOD)); method_name = smartlist_get(items,1); @@ -990,13 +1082,25 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) goto err; } - transport = transport_new(&tor_addr, port, method_name, PROXY_NONE); + if (smartlist_len(items) > 3) { + /* Seems like there are also some [options] in the SMETHOD line. + Let's see if we can parse them. */ + char *options_string = smartlist_get(items, 3); + log_debug(LD_CONFIG, "Got options_string: %s", options_string); + if (!strcmpstart(options_string, "ARGS:")) { + args_string = options_string+strlen("ARGS:"); + log_debug(LD_CONFIG, "Got ARGS: %s", args_string); + } + } + + transport = transport_new(&tor_addr, port, method_name, + PROXY_NONE, args_string); if (!transport) goto err; smartlist_add(mp->transports, transport); - /* For now, notify the user so that he knows where the server + /* For now, notify the user so that they know where the server transport is listening. */ log_info(LD_CONFIG, "Server transport %s at %s:%d.", method_name, address, (int)port); @@ -1016,7 +1120,7 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) /** Parses a CMETHOD <b>line</b>, and if well-formed it registers * the new transport in <b>mp</b>. */ -int +STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp) { int r; @@ -1082,7 +1186,7 @@ parse_cmethod_line(const char *line, managed_proxy_t *mp) goto err; } - transport = transport_new(&tor_addr, port, method_name, socks_ver); + transport = transport_new(&tor_addr, port, method_name, socks_ver, NULL); if (!transport) goto err; @@ -1105,6 +1209,65 @@ parse_cmethod_line(const char *line, managed_proxy_t *mp) return r; } +/** Parses an PROXY-ERROR <b>line</b> and warns the user accordingly. */ +STATIC void +parse_proxy_error(const char *line) +{ + /* (Length of the protocol string) plus (a space) and (the first char of + the error message) */ + if (strlen(line) < (strlen(PROTO_PROXY_ERROR) + 2)) + log_notice(LD_CONFIG, "Managed proxy sent us an %s without an error " + "message.", PROTO_PROXY_ERROR); + + log_warn(LD_CONFIG, "Managed proxy failed to configure the " + "pluggable transport's outgoing proxy. (%s)", + line+strlen(PROTO_PROXY_ERROR)+1); +} + +/** Return a newly allocated string that tor should place in + * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server + * manged proxy in <b>mp</b>. Return NULL if no such options are found. */ +STATIC char * +get_transport_options_for_server_proxy(const managed_proxy_t *mp) +{ + char *options_string = NULL; + smartlist_t *string_sl = smartlist_new(); + + tor_assert(mp->is_server); + + /** Loop over the transports of the proxy. If we have options for + any of them, format them appropriately and place them in our + smartlist. Finally, join our smartlist to get the final + string. */ + SMARTLIST_FOREACH_BEGIN(mp->transports_to_launch, const char *, transport) { + smartlist_t *options_tmp_sl = NULL; + options_tmp_sl = get_options_for_server_transport(transport); + if (!options_tmp_sl) + continue; + + /** Loop over the options of this transport, escape them, and + place them in the smartlist. */ + SMARTLIST_FOREACH_BEGIN(options_tmp_sl, const char *, options) { + char *escaped_opts = tor_escape_str_for_pt_args(options, ":;\\"); + smartlist_add_asprintf(string_sl, "%s:%s", + transport, escaped_opts); + tor_free(escaped_opts); + } SMARTLIST_FOREACH_END(options); + + SMARTLIST_FOREACH(options_tmp_sl, char *, c, tor_free(c)); + smartlist_free(options_tmp_sl); + } SMARTLIST_FOREACH_END(transport); + + if (smartlist_len(string_sl)) { + options_string = smartlist_join_strings(string_sl, ";", 0, NULL); + } + + SMARTLIST_FOREACH(string_sl, char *, t, tor_free(t)); + smartlist_free(string_sl); + + return options_string; +} + /** Return the string that tor should place in TOR_PT_SERVER_BINDADDR * while configuring the server managed proxy in <b>mp</b>. The * string is stored in the heap, and it's the the responsibility of @@ -1139,6 +1302,8 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp) static process_environment_t * create_managed_proxy_environment(const managed_proxy_t *mp) { + const or_options_t *options = get_options(); + /* Environment variables to be added to or set in mp's environment. */ smartlist_t *envs = smartlist_new(); /* XXXX The next time someone touches this code, shorten the name of @@ -1176,8 +1341,10 @@ create_managed_proxy_environment(const managed_proxy_t *mp) { char *orport_tmp = get_first_listener_addrport_string(CONN_TYPE_OR_LISTENER); - smartlist_add_asprintf(envs, "TOR_PT_ORPORT=%s", orport_tmp); - tor_free(orport_tmp); + if (orport_tmp) { + smartlist_add_asprintf(envs, "TOR_PT_ORPORT=%s", orport_tmp); + tor_free(orport_tmp); + } } { @@ -1186,13 +1353,54 @@ create_managed_proxy_environment(const managed_proxy_t *mp) tor_free(bindaddr_tmp); } + { + char *server_transport_options = + get_transport_options_for_server_proxy(mp); + if (server_transport_options) { + smartlist_add_asprintf(envs, "TOR_PT_SERVER_TRANSPORT_OPTIONS=%s", + server_transport_options); + tor_free(server_transport_options); + } + } + /* XXX024 Remove the '=' here once versions of obfsproxy which * assert that this env var exists are sufficiently dead. * * (If we remove this line entirely, some joker will stick this * variable in Tor's environment and crash PTs that try to parse * it even when not run in server mode.) */ - smartlist_add(envs, tor_strdup("TOR_PT_EXTENDED_SERVER_PORT=")); + + if (options->ExtORPort_lines) { + char *ext_or_addrport_tmp = + get_first_listener_addrport_string(CONN_TYPE_EXT_OR_LISTENER); + char *cookie_file_loc = get_ext_or_auth_cookie_file_name(); + + if (ext_or_addrport_tmp) { + smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=%s", + ext_or_addrport_tmp); + } + smartlist_add_asprintf(envs, "TOR_PT_AUTH_COOKIE_FILE=%s", + cookie_file_loc); + + tor_free(ext_or_addrport_tmp); + tor_free(cookie_file_loc); + + } else { + smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT="); + } + + /* All new versions of tor will keep stdin open, so PTs can use it + * as a reliable termination detection mechanism. + */ + smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1"); + } else { + /* If ClientTransportPlugin has a HTTPS/SOCKS proxy configured, set the + * TOR_PT_PROXY line. + */ + + if (mp->proxy_uri) { + smartlist_add_asprintf(envs, "TOR_PT_PROXY=%s", mp->proxy_uri); + } } SMARTLIST_FOREACH_BEGIN(envs, const char *, env_var) { @@ -1216,7 +1424,7 @@ create_managed_proxy_environment(const managed_proxy_t *mp) * <b>proxy_argv</b>. * * Requires that proxy_argv have at least one element. */ -static managed_proxy_t * +STATIC managed_proxy_t * managed_proxy_create(const smartlist_t *transport_list, char **proxy_argv, int is_server) { @@ -1225,6 +1433,7 @@ managed_proxy_create(const smartlist_t *transport_list, mp->is_server = is_server; mp->argv = proxy_argv; mp->transports = smartlist_new(); + mp->proxy_uri = get_pt_proxy_uri(); mp->transports_to_launch = smartlist_new(); SMARTLIST_FOREACH(transport_list, const char *, transport, @@ -1250,9 +1459,9 @@ managed_proxy_create(const smartlist_t *transport_list, * Requires that proxy_argv be a NULL-terminated array of command-line * elements, containing at least one element. **/ -void -pt_kickstart_proxy(const smartlist_t *transport_list, - char **proxy_argv, int is_server) +MOCK_IMPL(void, +pt_kickstart_proxy, (const smartlist_t *transport_list, + char **proxy_argv, int is_server)) { managed_proxy_t *mp=NULL; transport_t *old_transport = NULL; @@ -1267,19 +1476,20 @@ pt_kickstart_proxy(const smartlist_t *transport_list, managed_proxy_create(transport_list, proxy_argv, is_server); } else { /* known proxy. add its transport to its transport list */ - if (mp->got_hup) { - /* If the managed proxy we found is marked by a SIGHUP, it means - that it's not useless and should be kept. If it's marked for - removal, unmark it and increase the unconfigured proxies so - that we try to restart it if we need to. Afterwards, check if - a transport_t for 'transport' used to exist before the SIGHUP - and make sure it doesn't get deleted because we might reuse - it. */ + if (mp->was_around_before_config_read) { + /* If this managed proxy was around even before we read the + config this time, it means that it was already enabled before + and is not useless and should be kept. If it's marked for + removal, unmark it and make sure that we check whether it + needs to be restarted. */ if (mp->marked_for_removal) { mp->marked_for_removal = 0; check_if_restarts_needed = 1; } + /* For each new transport, check if the managed proxy used to + support it before the SIGHUP. If that was the case, make sure + it doesn't get removed because we might reuse it. */ SMARTLIST_FOREACH_BEGIN(transport_list, const char *, transport) { old_transport = transport_get_by_name(transport); if (old_transport) @@ -1295,7 +1505,7 @@ pt_kickstart_proxy(const smartlist_t *transport_list, /** Frees the array of pointers in <b>arg</b> used as arguments to execve(2). */ -static INLINE void +STATIC void free_execve_args(char **arg) { char **tmp = arg; @@ -1328,8 +1538,10 @@ pt_prepare_proxy_list_for_config_read(void) tor_assert(mp->conf_state == PT_PROTO_COMPLETED); + /* Mark all proxies for removal, and also note that they have been + here before the config read. */ mp->marked_for_removal = 1; - mp->got_hup = 1; + mp->was_around_before_config_read = 1; SMARTLIST_FOREACH(mp->transports_to_launch, char *, t, tor_free(t)); smartlist_clear(mp->transports_to_launch); } SMARTLIST_FOREACH_END(mp); @@ -1390,6 +1602,8 @@ pt_get_extra_info_descriptor_string(void) tor_assert(mp->transports); SMARTLIST_FOREACH_BEGIN(mp->transports, const transport_t *, t) { + char *transport_args = NULL; + /* If the transport proxy returned "0.0.0.0" as its address, and * we know our external IP address, use it. Otherwise, use the * returned address. */ @@ -1405,9 +1619,16 @@ pt_get_extra_info_descriptor_string(void) addrport = fmt_addrport(&t->addr, t->port); } + /* If this transport has any arguments with it, prepend a space + to them so that we can add them to the transport line. */ + if (t->extra_info_args) + tor_asprintf(&transport_args, " %s", t->extra_info_args); + smartlist_add_asprintf(string_chunks, - "transport %s %s", - t->name, addrport); + "transport %s %s%s", + t->name, addrport, + transport_args ? transport_args : ""); + tor_free(transport_args); } SMARTLIST_FOREACH_END(t); } SMARTLIST_FOREACH_END(mp); @@ -1426,6 +1647,57 @@ pt_get_extra_info_descriptor_string(void) return the_string; } +/** Stringify the SOCKS arguments in <b>socks_args</b> according to + * 180_pluggable_transport.txt. The string is allocated on the heap + * and it's the responsibility of the caller to free it after use. */ +char * +pt_stringify_socks_args(const smartlist_t *socks_args) +{ + /* tmp place to store escaped socks arguments, so that we can + concatenate them up afterwards */ + smartlist_t *sl_tmp = NULL; + char *escaped_string = NULL; + char *new_string = NULL; + + tor_assert(socks_args); + tor_assert(smartlist_len(socks_args) > 0); + + sl_tmp = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(socks_args, const char *, s) { + /* Escape ';' and '\'. */ + escaped_string = tor_escape_str_for_pt_args(s, ";\\"); + if (!escaped_string) + goto done; + + smartlist_add(sl_tmp, escaped_string); + } SMARTLIST_FOREACH_END(s); + + new_string = smartlist_join_strings(sl_tmp, ";", 0, NULL); + + done: + SMARTLIST_FOREACH(sl_tmp, char *, s, tor_free(s)); + smartlist_free(sl_tmp); + + return new_string; +} + +/** Return a string of the SOCKS arguments that we should pass to the + * pluggable transports proxy in <b>addr</b>:<b>port</b> according to + * 180_pluggable_transport.txt. The string is allocated on the heap + * and it's the responsibility of the caller to free it after use. */ +char * +pt_get_socks_args_for_proxy_addrport(const tor_addr_t *addr, uint16_t port) +{ + const smartlist_t *socks_args = NULL; + + socks_args = get_socks_args_by_bridge_addrport(addr, port); + if (!socks_args) + return NULL; + + return pt_stringify_socks_args(socks_args); +} + /** The tor config was read. * Destroy all managed proxies that were marked by a previous call to * prepare_proxy_list_for_config_read() and are not used by the new diff --git a/src/or/transports.h b/src/or/transports.h index 6ee82f4556..7de90dcbec 100644 --- a/src/or/transports.h +++ b/src/or/transports.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -25,18 +25,23 @@ typedef struct transport_t { /** Boolean: We are re-parsing our transport list, and we are going to remove * this one if we don't find it in the list of configured transports. */ unsigned marked_for_removal : 1; + /** Arguments for this transport that must be written to the + extra-info descriptor. */ + char *extra_info_args; } transport_t; void mark_transport_list(void); void sweep_transport_list(void); -int transport_add_from_config(const tor_addr_t *addr, uint16_t port, - const char *name, int socks_ver); +MOCK_DECL(int, transport_add_from_config, + (const tor_addr_t *addr, uint16_t port, + const char *name, int socks_ver)); void transport_free(transport_t *transport); transport_t *transport_get_by_name(const char *name); -void pt_kickstart_proxy(const smartlist_t *transport_list, char **proxy_argv, - int is_server); +MOCK_DECL(void, pt_kickstart_proxy, + (const smartlist_t *transport_list, char **proxy_argv, + int is_server)); #define pt_kickstart_client_proxy(tl, pa) \ pt_kickstart_proxy(tl, pa, 0) @@ -55,6 +60,10 @@ void pt_prepare_proxy_list_for_config_read(void); void sweep_proxy_list(void); smartlist_t *get_transport_proxy_ports(void); +char *pt_stringify_socks_args(const smartlist_t *socks_args); + +char *pt_get_socks_args_for_proxy_addrport(const tor_addr_t *addr, + uint16_t port); #ifdef PT_PRIVATE /** State of the managed proxy configuration protocol. */ @@ -74,6 +83,9 @@ typedef struct { char **argv; /* the cli arguments of this proxy */ int conf_protocol; /* the configuration protocol version used */ + char *proxy_uri; /* the outgoing proxy in TOR_PT_PROXY URI format */ + unsigned int proxy_supported : 1; /* the proxy honors TOR_PT_PROXY */ + int is_server; /* is it a server proxy? */ /* A pointer to the process handle of this managed proxy. */ @@ -90,7 +102,7 @@ typedef struct { * this flag to signify that this proxy might need to be restarted * so that it can listen for other transports according to the new * torrc. */ - unsigned int got_hup : 1; + unsigned int was_around_before_config_read : 1; /* transports to-be-launched by this proxy */ smartlist_t *transports_to_launch; @@ -100,12 +112,26 @@ typedef struct { smartlist_t *transports; } managed_proxy_t; -int parse_cmethod_line(const char *line, managed_proxy_t *mp); -int parse_smethod_line(const char *line, managed_proxy_t *mp); +STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp); +STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp); + +STATIC int parse_version(const char *line, managed_proxy_t *mp); +STATIC void parse_env_error(const char *line); +STATIC void parse_proxy_error(const char *line); +STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); + +STATIC void managed_proxy_destroy(managed_proxy_t *mp, + int also_terminate_process); + +STATIC managed_proxy_t *managed_proxy_create(const smartlist_t *transport_list, + char **proxy_argv, int is_server); + +STATIC int configure_proxy(managed_proxy_t *mp); + +STATIC char* get_pt_proxy_uri(void); -int parse_version(const char *line, managed_proxy_t *mp); -void parse_env_error(const char *line); -void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC void free_execve_args(char **arg); #endif |