diff options
Diffstat (limited to 'src/or')
115 files changed, 8793 insertions, 4754 deletions
diff --git a/src/or/Makefile.nmake b/src/or/Makefile.nmake index 523bf3306b..2ac98cd372 100644 --- a/src/or/Makefile.nmake +++ b/src/or/Makefile.nmake @@ -63,6 +63,7 @@ LIBTOR_OBJECTS = \ routerlist.obj \ routerparse.obj \ routerset.obj \ + scheduler.obj \ statefile.obj \ status.obj \ transports.obj diff --git a/src/or/addressmap.c b/src/or/addressmap.c index d4b7acf274..9c29fb2acb 100644 --- a/src/or/addressmap.c +++ b/src/or/addressmap.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ADDRESSMAP_PRIVATE @@ -94,7 +94,7 @@ addressmap_ent_free(void *_ent) tor_free(ent); } -/** Free storage held by a virtaddress_entry_t* entry in <b>ent</b>. */ +/** Free storage held by a virtaddress_entry_t* entry in <b>_ent</b>. */ static void addressmap_virtaddress_ent_free(void *_ent) { @@ -104,11 +104,13 @@ addressmap_virtaddress_ent_free(void *_ent) ent = _ent; tor_free(ent->ipv4_address); + tor_free(ent->ipv6_address); tor_free(ent->hostname_address); tor_free(ent); } -/** Free storage held by a virtaddress_entry_t* entry in <b>ent</b>. */ +/** Remove <b>address</b> (which must map to <b>ent</b>) from the + * virtual address map. */ static void addressmap_virtaddress_remove(const char *address, addressmap_entry_t *ent) { @@ -120,9 +122,11 @@ addressmap_virtaddress_remove(const char *address, addressmap_entry_t *ent) if (ve) { if (!strcmp(address, ve->ipv4_address)) tor_free(ve->ipv4_address); + if (!strcmp(address, ve->ipv6_address)) + tor_free(ve->ipv6_address); if (!strcmp(address, ve->hostname_address)) tor_free(ve->hostname_address); - if (!ve->ipv4_address && !ve->hostname_address) { + if (!ve->ipv4_address && !ve->ipv6_address && !ve->hostname_address) { tor_free(ve); strmap_remove(virtaddress_reversemap, ent->new_address); } @@ -131,7 +135,7 @@ addressmap_virtaddress_remove(const char *address, addressmap_entry_t *ent) } /** Remove <b>ent</b> (which must be mapped to by <b>address</b>) from the - * client address maps. */ + * client address maps, and then free it. */ static void addressmap_ent_remove(const char *address, addressmap_entry_t *ent) { @@ -226,6 +230,8 @@ addressmap_address_should_automap(const char *address, return 0; SMARTLIST_FOREACH_BEGIN(suffix_list, const char *, suffix) { + if (!strcmp(suffix, ".")) + return 1; if (!strcasecmpend(address, suffix)) return 1; } SMARTLIST_FOREACH_END(suffix); @@ -384,13 +390,35 @@ addressmap_rewrite(char *address, size_t maxlen, goto done; } - if (ent && ent->source == ADDRMAPSRC_DNS) { - sa_family_t f; - tor_addr_t tmp; - f = tor_addr_parse(&tmp, ent->new_address); - if (f == AF_INET && !(flags & AMR_FLAG_USE_IPV4_DNS)) - goto done; - else if (f == AF_INET6 && !(flags & AMR_FLAG_USE_IPV6_DNS)) + switch (ent->source) { + case ADDRMAPSRC_DNS: + { + sa_family_t f; + tor_addr_t tmp; + f = tor_addr_parse(&tmp, ent->new_address); + if (f == AF_INET && !(flags & AMR_FLAG_USE_IPV4_DNS)) + goto done; + else if (f == AF_INET6 && !(flags & AMR_FLAG_USE_IPV6_DNS)) + goto done; + } + break; + case ADDRMAPSRC_CONTROLLER: + case ADDRMAPSRC_TORRC: + if (!(flags & AMR_FLAG_USE_MAPADDRESS)) + goto done; + break; + case ADDRMAPSRC_AUTOMAP: + if (!(flags & AMR_FLAG_USE_AUTOMAP)) + goto done; + break; + case ADDRMAPSRC_TRACKEXIT: + if (!(flags & AMR_FLAG_USE_TRACKEXIT)) + goto done; + break; + case ADDRMAPSRC_NONE: + default: + log_warn(LD_BUG, "Unknown addrmap source value %d. Ignoring it.", + (int) ent->source); goto done; } @@ -425,7 +453,7 @@ addressmap_rewrite(char *address, size_t maxlen, if (exit_source_out) *exit_source_out = exit_source; if (expires_out) - *expires_out = TIME_MAX; + *expires_out = expires; return (rewrites > 0); } @@ -449,6 +477,8 @@ addressmap_rewrite_reverse(char *address, size_t maxlen, unsigned flags, return 0; else if (f == AF_INET6 && !(flags & AMR_FLAG_USE_IPV6_DNS)) return 0; + /* FFFF we should reverse-map virtual addresses even if we haven't + * enabled DNS cacheing. */ } tor_asprintf(&s, "REVERSE[%s]", address); @@ -496,7 +526,7 @@ addressmap_have_mapping(const char *address, int update_expiry) * equal to <b>address</b>, or any address ending with a period followed by * <b>address</b>. If <b>wildcard_addr</b> and <b>wildcard_new_addr</b> are * both true, the mapping will rewrite addresses that end with - * ".<b>address</b>" into ones that end with ".<b>new_address</b>." + * ".<b>address</b>" into ones that end with ".<b>new_address</b>". * * If <b>new_address</b> is NULL, or <b>new_address</b> is equal to * <b>address</b> and <b>wildcard_addr</b> is equal to @@ -535,9 +565,9 @@ addressmap_register(const char *address, char *new_address, time_t expires, if (expires > 1) { log_info(LD_APP,"Temporary addressmap ('%s' to '%s') not performed, " "since it's already mapped to '%s'", - safe_str_client(address), - safe_str_client(new_address), - safe_str_client(ent->new_address)); + safe_str_client(address), + safe_str_client(new_address), + safe_str_client(ent->new_address)); tor_free(new_address); return; } @@ -670,10 +700,10 @@ client_dns_set_addressmap(entry_connection_t *for_conn, return; /* If address was an IP address already, don't add a mapping. */ if (tor_addr_family(val) == AF_INET) { - if (! for_conn->cache_ipv4_answers) + if (! for_conn->entry_cfg.cache_ipv4_answers) return; } else if (tor_addr_family(val) == AF_INET6) { - if (! for_conn->cache_ipv6_answers) + if (! for_conn->entry_cfg.cache_ipv6_answers) return; } @@ -702,8 +732,8 @@ client_dns_set_reverse_addressmap(entry_connection_t *for_conn, { tor_addr_t tmp_addr; sa_family_t f = tor_addr_parse(&tmp_addr, address); - if ((f == AF_INET && ! for_conn->cache_ipv4_answers) || - (f == AF_INET6 && ! for_conn->cache_ipv6_answers)) + if ((f == AF_INET && ! for_conn->entry_cfg.cache_ipv4_answers) || + (f == AF_INET6 && ! for_conn->entry_cfg.cache_ipv6_answers)) return; } tor_asprintf(&s, "REVERSE[%s]", address); @@ -845,8 +875,8 @@ get_random_virtual_addr(const virtual_addr_conf_t *conf, tor_addr_t *addr_out) } /** Return a newly allocated string holding an address of <b>type</b> - * (one of RESOLVED_TYPE_{IPV4|HOSTNAME}) that has not yet been mapped, - * and that is very unlikely to be the address of any real host. + * (one of RESOLVED_TYPE_{IPV4|IPV6|HOSTNAME}) that has not yet been + * mapped, and that is very unlikely to be the address of any real host. * * May return NULL if we have run out of virtual addresses. */ @@ -894,7 +924,7 @@ addressmap_get_virtual_address(int type) /* XXXX This code is to make sure I didn't add an undecorated version * by mistake. I hope it's needless. */ char tmp[TOR_ADDR_BUF_LEN]; - tor_addr_to_str(buf, &addr, sizeof(tmp), 0); + tor_addr_to_str(tmp, &addr, sizeof(tmp), 0); if (strmap_get(addressmap, tmp)) { log_warn(LD_BUG, "%s wasn't in the addressmap, but %s was.", buf, tmp); @@ -975,6 +1005,8 @@ addressmap_register_virtual_address(int type, char *new_address) strmap_set(virtaddress_reversemap, new_address, vent); addressmap_register(*addrp, new_address, 2, ADDRMAPSRC_AUTOMAP, 0, 0); + /* FFFF register corresponding reverse mapping. */ + #if 0 { /* Try to catch possible bugs */ diff --git a/src/or/addressmap.h b/src/or/addressmap.h index 417832b31f..ff108df024 100644 --- a/src/or/addressmap.h +++ b/src/or/addressmap.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ADDRESSMAP_H @@ -16,8 +16,11 @@ void addressmap_clean(time_t now); void addressmap_clear_configured(void); void addressmap_clear_transient(void); void addressmap_free_all(void); -#define AMR_FLAG_USE_IPV4_DNS (1u<<0) -#define AMR_FLAG_USE_IPV6_DNS (1u<<1) +#define AMR_FLAG_USE_IPV4_DNS (1u<<0) +#define AMR_FLAG_USE_IPV6_DNS (1u<<1) +#define AMR_FLAG_USE_MAPADDRESS (1u<<2) +#define AMR_FLAG_USE_AUTOMAP (1u<<3) +#define AMR_FLAG_USE_TRACKEXIT (1u<<4) int addressmap_rewrite(char *address, size_t maxlen, unsigned flags, time_t *expires_out, addressmap_entry_source_t *exit_source_out); diff --git a/src/or/buffers.c b/src/or/buffers.c index a60c7c0f02..74aebccb77 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -55,6 +55,9 @@ * forever. */ +static void socks_request_set_socks5_error(socks_request_t *req, + socks5_reply_status_t reason); + static int parse_socks(const char *data, size_t datalen, socks_request_t *req, int log_sockstype, int safe_socks, ssize_t *drain_out, size_t *want_length_out); @@ -123,115 +126,6 @@ chunk_repack(chunk_t *chunk) /** Keep track of total size of allocated chunks for consistency asserts */ static size_t total_bytes_allocated_in_chunks = 0; - -#if defined(ENABLE_BUF_FREELISTS) || defined(RUNNING_DOXYGEN) -/** A freelist of chunks. */ -typedef struct chunk_freelist_t { - size_t alloc_size; /**< What size chunks does this freelist hold? */ - int max_length; /**< Never allow more than this number of chunks in the - * freelist. */ - int slack; /**< When trimming the freelist, leave this number of extra - * chunks beyond lowest_length.*/ - int cur_length; /**< How many chunks on the freelist now? */ - int lowest_length; /**< What's the smallest value of cur_length since the - * last time we cleaned this freelist? */ - uint64_t n_alloc; - uint64_t n_free; - uint64_t n_hit; - chunk_t *head; /**< First chunk on the freelist. */ -} chunk_freelist_t; - -/** Macro to help define freelists. */ -#define FL(a,m,s) { a, m, s, 0, 0, 0, 0, 0, NULL } - -/** Static array of freelists, sorted by alloc_len, terminated by an entry - * with alloc_size of 0. */ -static chunk_freelist_t freelists[] = { - FL(4096, 256, 8), FL(8192, 128, 4), FL(16384, 64, 4), FL(32768, 32, 2), - FL(0, 0, 0) -}; -#undef FL -/** How many times have we looked for a chunk of a size that no freelist - * could help with? */ -static uint64_t n_freelist_miss = 0; - -static void assert_freelist_ok(chunk_freelist_t *fl); - -/** Return the freelist to hold chunks of size <b>alloc</b>, or NULL if - * no freelist exists for that size. */ -static INLINE chunk_freelist_t * -get_freelist(size_t alloc) -{ - int i; - for (i=0; (freelists[i].alloc_size <= alloc && - freelists[i].alloc_size); ++i ) { - if (freelists[i].alloc_size == alloc) { - return &freelists[i]; - } - } - return NULL; -} - -/** Deallocate a chunk or put it on a freelist */ -static void -chunk_free_unchecked(chunk_t *chunk) -{ - size_t alloc; - chunk_freelist_t *freelist; - - alloc = CHUNK_ALLOC_SIZE(chunk->memlen); - freelist = get_freelist(alloc); - if (freelist && freelist->cur_length < freelist->max_length) { - chunk->next = freelist->head; - freelist->head = chunk; - ++freelist->cur_length; - } else { - if (freelist) - ++freelist->n_free; -#ifdef DEBUG_CHUNK_ALLOC - tor_assert(alloc == chunk->DBG_alloc); -#endif - tor_assert(total_bytes_allocated_in_chunks >= alloc); - total_bytes_allocated_in_chunks -= alloc; - tor_free(chunk); - } -} - -/** Allocate a new chunk with a given allocation size, or get one from the - * freelist. Note that a chunk with allocation size A can actually hold only - * CHUNK_SIZE_WITH_ALLOC(A) bytes in its mem field. */ -static INLINE chunk_t * -chunk_new_with_alloc_size(size_t alloc) -{ - chunk_t *ch; - chunk_freelist_t *freelist; - tor_assert(alloc >= sizeof(chunk_t)); - freelist = get_freelist(alloc); - if (freelist && freelist->head) { - ch = freelist->head; - freelist->head = ch->next; - if (--freelist->cur_length < freelist->lowest_length) - freelist->lowest_length = freelist->cur_length; - ++freelist->n_hit; - } else { - if (freelist) - ++freelist->n_alloc; - else - ++n_freelist_miss; - ch = tor_malloc(alloc); -#ifdef DEBUG_CHUNK_ALLOC - ch->DBG_alloc = alloc; -#endif - total_bytes_allocated_in_chunks += alloc; - } - ch->next = NULL; - ch->datalen = 0; - ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); - ch->data = &ch->mem[0]; - CHUNK_SET_SENTINEL(ch, alloc); - return ch; -} -#else static void chunk_free_unchecked(chunk_t *chunk) { @@ -261,7 +155,6 @@ chunk_new_with_alloc_size(size_t alloc) CHUNK_SET_SENTINEL(ch, alloc); return ch; } -#endif /** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a * new pointer to <b>chunk</b>. Old pointers are no longer valid. */ @@ -306,115 +199,6 @@ preferred_chunk_size(size_t target) return sz; } -/** Remove from the freelists most chunks that have not been used since the - * last call to buf_shrink_freelists(). Return the amount of memory - * freed. */ -size_t -buf_shrink_freelists(int free_all) -{ -#ifdef ENABLE_BUF_FREELISTS - int i; - size_t total_freed = 0; - disable_control_logging(); - for (i = 0; freelists[i].alloc_size; ++i) { - int slack = freelists[i].slack; - assert_freelist_ok(&freelists[i]); - if (free_all || freelists[i].lowest_length > slack) { - int n_to_free = free_all ? freelists[i].cur_length : - (freelists[i].lowest_length - slack); - int n_to_skip = freelists[i].cur_length - n_to_free; - int orig_length = freelists[i].cur_length; - int orig_n_to_free = n_to_free, n_freed=0; - int orig_n_to_skip = n_to_skip; - int new_length = n_to_skip; - chunk_t **chp = &freelists[i].head; - chunk_t *chunk; - while (n_to_skip) { - if (!(*chp) || ! (*chp)->next) { - log_warn(LD_BUG, "I wanted to skip %d chunks in the freelist for " - "%d-byte chunks, but only found %d. (Length %d)", - orig_n_to_skip, (int)freelists[i].alloc_size, - orig_n_to_skip-n_to_skip, freelists[i].cur_length); - assert_freelist_ok(&freelists[i]); - goto done; - } - // tor_assert((*chp)->next); - chp = &(*chp)->next; - --n_to_skip; - } - chunk = *chp; - *chp = NULL; - while (chunk) { - chunk_t *next = chunk->next; -#ifdef DEBUG_CHUNK_ALLOC - tor_assert(chunk->DBG_alloc == CHUNK_ALLOC_SIZE(chunk->memlen)); -#endif - tor_assert(total_bytes_allocated_in_chunks >= - CHUNK_ALLOC_SIZE(chunk->memlen)); - total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); - total_freed += CHUNK_ALLOC_SIZE(chunk->memlen); - tor_free(chunk); - chunk = next; - --n_to_free; - ++n_freed; - ++freelists[i].n_free; - } - if (n_to_free) { - log_warn(LD_BUG, "Freelist length for %d-byte chunks may have been " - "messed up somehow.", (int)freelists[i].alloc_size); - log_warn(LD_BUG, "There were %d chunks at the start. I decided to " - "keep %d. I wanted to free %d. I freed %d. I somehow think " - "I have %d left to free.", - freelists[i].cur_length, n_to_skip, orig_n_to_free, - n_freed, n_to_free); - } - // tor_assert(!n_to_free); - freelists[i].cur_length = new_length; - tor_assert(orig_n_to_skip == new_length); - log_info(LD_MM, "Cleaned freelist for %d-byte chunks: original " - "length %d, kept %d, dropped %d. New length is %d", - (int)freelists[i].alloc_size, orig_length, - orig_n_to_skip, orig_n_to_free, new_length); - } - freelists[i].lowest_length = freelists[i].cur_length; - assert_freelist_ok(&freelists[i]); - } - done: - enable_control_logging(); - return total_freed; -#else - (void) free_all; - return 0; -#endif -} - -/** Describe the current status of the freelists at log level <b>severity</b>. - */ -void -buf_dump_freelist_sizes(int severity) -{ -#ifdef ENABLE_BUF_FREELISTS - int i; - tor_log(severity, LD_MM, "====== Buffer freelists:"); - for (i = 0; freelists[i].alloc_size; ++i) { - uint64_t total = ((uint64_t)freelists[i].cur_length) * - freelists[i].alloc_size; - tor_log(severity, LD_MM, - U64_FORMAT" bytes in %d %d-byte chunks ["U64_FORMAT - " misses; "U64_FORMAT" frees; "U64_FORMAT" hits]", - U64_PRINTF_ARG(total), - freelists[i].cur_length, (int)freelists[i].alloc_size, - U64_PRINTF_ARG(freelists[i].n_alloc), - U64_PRINTF_ARG(freelists[i].n_free), - U64_PRINTF_ARG(freelists[i].n_hit)); - } - tor_log(severity, LD_MM, U64_FORMAT" allocations in non-freelist sizes", - U64_PRINTF_ARG(n_freelist_miss)); -#else - (void)severity; -#endif -} - /** Collapse data from the first N chunks from <b>buf</b> into buf->head, * growing it as necessary, until buf->head has the first <b>bytes</b> bytes * of data from the buffer, or until buf->head has all the data in <b>buf</b>. @@ -510,15 +294,6 @@ buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz) } #endif -/** Resize buf so it won't hold extra memory that we haven't been - * using lately. - */ -void -buf_shrink(buf_t *buf) -{ - (void)buf; -} - /** Remove the first <b>n</b> bytes from buf. */ static INLINE void buf_remove_from_front(buf_t *buf, size_t n) @@ -584,8 +359,8 @@ buf_clear(buf_t *buf) } /** Return the number of bytes stored in <b>buf</b> */ -size_t -buf_datalen(const buf_t *buf) +MOCK_IMPL(size_t, +buf_datalen, (const buf_t *buf)) { return buf->datalen; } @@ -1856,6 +1631,21 @@ fetch_ext_or_command_from_evbuffer(struct evbuffer *buf, ext_or_cmd_t **out) } #endif +/** Create a SOCKS5 reply message with <b>reason</b> in its REP field and + * have Tor send it as error response to <b>req</b>. + */ +static void +socks_request_set_socks5_error(socks_request_t *req, + socks5_reply_status_t reason) +{ + req->replylen = 10; + memset(req->reply,0,10); + + req->reply[0] = 0x05; // VER field. + req->reply[1] = reason; // REP field. + req->reply[3] = 0x01; // ATYP field. +} + /** Implementation helper to implement fetch_from_*_socks. Instead of looking * at a buffer's contents, we look at the <b>datalen</b> bytes of data in * <b>data</b>. Instead of removing data from the buffer, we set @@ -1919,7 +1709,7 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, } *drain_out = 2u + usernamelen + 1u + passlen; req->got_auth = 1; - *want_length_out = 7; /* Minimal socks5 sommand. */ + *want_length_out = 7; /* Minimal socks5 command. */ return 0; } else if (req->auth_type == SOCKS_USER_PASS) { /* unknown version byte */ @@ -1991,6 +1781,8 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, req->command != SOCKS_COMMAND_RESOLVE && req->command != SOCKS_COMMAND_RESOLVE_PTR) { /* not a connect or resolve or a resolve_ptr? we don't support it. */ + socks_request_set_socks5_error(req,SOCKS5_COMMAND_NOT_SUPPORTED); + log_warn(LD_APP,"socks5: command %d not recognized. Rejecting.", req->command); return -1; @@ -2014,6 +1806,7 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, tor_addr_to_str(tmpbuf, &destaddr, sizeof(tmpbuf), 1); if (strlen(tmpbuf)+1 > MAX_SOCKS_ADDR_LEN) { + socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); log_warn(LD_APP, "socks5 IP takes %d bytes, which doesn't fit in %d. " "Rejecting.", @@ -2026,14 +1819,18 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, if (req->command != SOCKS_COMMAND_RESOLVE_PTR && !addressmap_have_mapping(req->address,0)) { log_unsafe_socks_warning(5, req->address, req->port, safe_socks); - if (safe_socks) + if (safe_socks) { + socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED); return -1; + } } return 1; } case 3: /* fqdn */ log_debug(LD_APP,"socks5: fqdn address type"); if (req->command == SOCKS_COMMAND_RESOLVE_PTR) { + socks_request_set_socks5_error(req, + SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED); log_warn(LD_APP, "socks5 received RESOLVE_PTR command with " "hostname type. Rejecting."); return -1; @@ -2044,6 +1841,7 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, return 0; /* not yet */ } if (len+1 > MAX_SOCKS_ADDR_LEN) { + socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); log_warn(LD_APP, "socks5 hostname is %d bytes, which doesn't fit in " "%d. Rejecting.", len+1,MAX_SOCKS_ADDR_LEN); @@ -2053,7 +1851,18 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, req->address[len] = 0; req->port = ntohs(get_uint16(data+5+len)); *drain_out = 5+len+2; - if (!tor_strisprint(req->address) || strchr(req->address,'\"')) { + + if (string_is_valid_ipv4_address(req->address) || + string_is_valid_ipv6_address(req->address)) { + log_unsafe_socks_warning(5,req->address,req->port,safe_socks); + + if (safe_socks) { + socks_request_set_socks5_error(req, SOCKS5_NOT_ALLOWED); + return -1; + } + } else if (!string_is_valid_hostname(req->address)) { + socks_request_set_socks5_error(req, SOCKS5_GENERAL_ERROR); + log_warn(LD_PROTOCOL, "Your application (using socks5 to port %d) gave Tor " "a malformed hostname: %s. Rejecting the connection.", @@ -2067,6 +1876,8 @@ parse_socks(const char *data, size_t datalen, socks_request_t *req, "necessary. This is good.", req->port); return 1; default: /* unsupported */ + socks_request_set_socks5_error(req, + SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED); log_warn(LD_APP,"socks5: unsupported address type %d. Rejecting.", (int) *(data+3)); return -1; @@ -2665,23 +2476,3 @@ assert_buf_ok(buf_t *buf) } } -#ifdef ENABLE_BUF_FREELISTS -/** Log an error and exit if <b>fl</b> is corrupted. - */ -static void -assert_freelist_ok(chunk_freelist_t *fl) -{ - chunk_t *ch; - int n; - tor_assert(fl->alloc_size > 0); - n = 0; - for (ch = fl->head; ch; ch = ch->next) { - tor_assert(CHUNK_ALLOC_SIZE(ch->memlen) == fl->alloc_size); - ++n; - } - tor_assert(n == fl->cur_length); - tor_assert(n >= fl->lowest_length); - tor_assert(n <= fl->max_length); -} -#endif - diff --git a/src/or/buffers.h b/src/or/buffers.h index c90e14750e..6d0c68500b 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,11 +20,8 @@ size_t buf_get_default_chunk_size(const buf_t *buf); void buf_free(buf_t *buf); void buf_clear(buf_t *buf); buf_t *buf_copy(const buf_t *buf); -void buf_shrink(buf_t *buf); -size_t buf_shrink_freelists(int free_all); -void buf_dump_freelist_sizes(int severity); -size_t buf_datalen(const buf_t *buf); +MOCK_DECL(size_t, buf_datalen, (const buf_t *buf)); size_t buf_allocation(const buf_t *buf); size_t buf_slack(const buf_t *buf); @@ -108,9 +105,9 @@ STATIC void buf_pullup(buf_t *buf, size_t bytes, int nulterminate); void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz); #define DEBUG_CHUNK_ALLOC -/** A single chunk on a buffer or in a freelist. */ +/** A single chunk on a buffer. */ typedef struct chunk_t { - struct chunk_t *next; /**< The next chunk on the buffer or freelist. */ + struct chunk_t *next; /**< The next chunk on the buffer. */ size_t datalen; /**< The number of bytes stored in this chunk */ size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ #ifdef DEBUG_CHUNK_ALLOC diff --git a/src/or/channel.c b/src/or/channel.c index b2b670e4fb..bf0387f10e 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,6 +13,9 @@ #define TOR_CHANNEL_INTERNAL_ +/* This one's for stuff only channel.c and the test suite should see */ +#define CHANNEL_PRIVATE_ + #include "or.h" #include "channel.h" #include "channeltls.h" @@ -29,29 +32,7 @@ #include "rephist.h" #include "router.h" #include "routerlist.h" - -/* Cell queue structure */ - -typedef struct cell_queue_entry_s cell_queue_entry_t; -struct cell_queue_entry_s { - TOR_SIMPLEQ_ENTRY(cell_queue_entry_s) next; - enum { - CELL_QUEUE_FIXED, - CELL_QUEUE_VAR, - CELL_QUEUE_PACKED - } type; - union { - struct { - cell_t *cell; - } fixed; - struct { - var_cell_t *var_cell; - } var; - struct { - packed_cell_t *packed_cell; - } packed; - } u; -}; +#include "scheduler.h" /* Global lists of channels */ @@ -75,6 +56,59 @@ static smartlist_t *finished_listeners = NULL; /* Counter for ID numbers */ static uint64_t n_channels_allocated = 0; +/* + * Channel global byte/cell counters, for statistics and for scheduler high + * /low-water marks. + */ + +/* + * Total number of cells ever given to any channel with the + * channel_write_*_cell() functions. + */ + +static uint64_t n_channel_cells_queued = 0; + +/* + * Total number of cells ever passed to a channel lower layer with the + * write_*_cell() methods. + */ + +static uint64_t n_channel_cells_passed_to_lower_layer = 0; + +/* + * Current number of cells in all channel queues; should be + * n_channel_cells_queued - n_channel_cells_passed_to_lower_layer. + */ + +static uint64_t n_channel_cells_in_queues = 0; + +/* + * Total number of bytes for all cells ever queued to a channel and + * counted in n_channel_cells_queued. + */ + +static uint64_t n_channel_bytes_queued = 0; + +/* + * Total number of bytes for all cells ever passed to a channel lower layer + * and counted in n_channel_cells_passed_to_lower_layer. + */ + +static uint64_t n_channel_bytes_passed_to_lower_layer = 0; + +/* + * Current number of bytes in all channel queues; should be + * n_channel_bytes_queued - n_channel_bytes_passed_to_lower_layer. + */ + +static uint64_t n_channel_bytes_in_queues = 0; + +/* + * Current total estimated queue size *including lower layer queues and + * transmit overhead* + */ + +STATIC uint64_t estimated_total_queue_size = 0; /* Digest->channel map * @@ -108,11 +142,10 @@ channel_idmap_eq(const channel_idmap_entry_t *a, HT_PROTOTYPE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, channel_idmap_eq); -HT_GENERATE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, - channel_idmap_eq, 0.5, tor_malloc, tor_realloc, tor_free_); +HT_GENERATE2(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, + channel_idmap_eq, 0.5, tor_reallocarray_, tor_free_); static cell_queue_entry_t * cell_queue_entry_dup(cell_queue_entry_t *q); -static void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off); #if 0 static int cell_queue_entry_is_padding(cell_queue_entry_t *q); #endif @@ -123,6 +156,8 @@ cell_queue_entry_new_var(var_cell_t *var_cell); static int is_destroy_cell(channel_t *chan, const cell_queue_entry_t *q, circid_t *circid_out); +static void channel_assert_counter_consistency(void); + /* Functions to maintain the digest map */ static void channel_add_to_digest_map(channel_t *chan); static void channel_remove_from_digest_map(channel_t *chan); @@ -140,6 +175,8 @@ channel_free_list(smartlist_t *channels, int mark_for_close); static void channel_listener_free_list(smartlist_t *channels, int mark_for_close); static void channel_listener_force_free(channel_listener_t *chan_l); +static size_t channel_get_cell_queue_entry_size(channel_t *chan, + cell_queue_entry_t *q); static void channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q); @@ -378,8 +415,7 @@ channel_register(channel_t *chan) smartlist_add(all_channels, chan); /* Is it finished? */ - if (chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) { + if (CHANNEL_FINISHED(chan)) { /* Put it in the finished list, creating it if necessary */ if (!finished_channels) finished_channels = smartlist_new(); smartlist_add(finished_channels, chan); @@ -388,7 +424,7 @@ channel_register(channel_t *chan) if (!active_channels) active_channels = smartlist_new(); smartlist_add(active_channels, chan); - if (chan->state != CHANNEL_STATE_CLOSING) { + if (!CHANNEL_IS_CLOSING(chan)) { /* It should have a digest set */ if (!tor_digest_is_zero(chan->identity_digest)) { /* Yeah, we're good, add it to the map */ @@ -423,8 +459,7 @@ channel_unregister(channel_t *chan) if (!(chan->registered)) return; /* Is it finished? */ - if (chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) { + if (CHANNEL_FINISHED(chan)) { /* Get it out of the finished list */ if (finished_channels) smartlist_remove(finished_channels, chan); } else { @@ -440,9 +475,7 @@ channel_unregister(channel_t *chan) /* Should it be in the digest map? */ if (!tor_digest_is_zero(chan->identity_digest) && - !(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { + !(CHANNEL_CONDEMNED(chan))) { /* Remove it */ channel_remove_from_digest_map(chan); } @@ -542,9 +575,7 @@ channel_add_to_digest_map(channel_t *chan) tor_assert(chan); /* Assert that the state makes sense */ - tor_assert(!(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)); + tor_assert(!CHANNEL_CONDEMNED(chan)); /* Assert that there is a digest */ tor_assert(!tor_digest_is_zero(chan->identity_digest)); @@ -746,6 +777,9 @@ channel_init(channel_t *chan) /* It hasn't been open yet. */ chan->has_been_open = 0; + + /* Scheduler state is idle */ + chan->scheduler_state = SCHED_CHAN_IDLE; } /** @@ -779,8 +813,8 @@ channel_free(channel_t *chan) if (!chan) return; /* It must be closed or errored */ - tor_assert(chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + tor_assert(CHANNEL_FINISHED(chan)); + /* It must be deregistered */ tor_assert(!(chan->registered)); @@ -788,6 +822,9 @@ channel_free(channel_t *chan) "Freeing channel " U64_FORMAT " at %p", U64_PRINTF_ARG(chan->global_identifier), chan); + /* Get this one out of the scheduler */ + scheduler_release_channel(chan); + /* * Get rid of cmux policy before we do anything, so cmux policies don't * see channels in weird half-freed states. @@ -863,6 +900,9 @@ channel_force_free(channel_t *chan) "Force-freeing channel " U64_FORMAT " at %p", U64_PRINTF_ARG(chan->global_identifier), chan); + /* Get this one out of the scheduler */ + scheduler_release_channel(chan); + /* * Get rid of cmux policy before we do anything, so cmux policies don't * see channels in weird half-freed states. @@ -988,9 +1028,7 @@ channel_get_cell_handler(channel_t *chan) { tor_assert(chan); - if (chan->state == CHANNEL_STATE_OPENING || - chan->state == CHANNEL_STATE_OPEN || - chan->state == CHANNEL_STATE_MAINT) + if (CHANNEL_CAN_HANDLE_CELLS(chan)) return chan->cell_handler; return NULL; @@ -1008,9 +1046,7 @@ channel_get_var_cell_handler(channel_t *chan) { tor_assert(chan); - if (chan->state == CHANNEL_STATE_OPENING || - chan->state == CHANNEL_STATE_OPEN || - chan->state == CHANNEL_STATE_MAINT) + if (CHANNEL_CAN_HANDLE_CELLS(chan)) return chan->var_cell_handler; return NULL; @@ -1033,9 +1069,7 @@ channel_set_cell_handlers(channel_t *chan, int try_again = 0; tor_assert(chan); - tor_assert(chan->state == CHANNEL_STATE_OPENING || - chan->state == CHANNEL_STATE_OPEN || - chan->state == CHANNEL_STATE_MAINT); + tor_assert(CHANNEL_CAN_HANDLE_CELLS(chan)); log_debug(LD_CHANNEL, "Setting cell_handler callback for channel %p to %p", @@ -1089,9 +1123,8 @@ channel_mark_for_close(channel_t *chan) tor_assert(chan->close != NULL); /* If it's already in CLOSING, CLOSED or ERROR, this is a no-op */ - if (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) return; + if (CHANNEL_CONDEMNED(chan)) + return; log_debug(LD_CHANNEL, "Closing channel %p (global ID " U64_FORMAT ") " @@ -1170,9 +1203,8 @@ channel_close_from_lower_layer(channel_t *chan) tor_assert(chan != NULL); /* If it's already in CLOSING, CLOSED or ERROR, this is a no-op */ - if (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) return; + if (CHANNEL_CONDEMNED(chan)) + return; log_debug(LD_CHANNEL, "Closing channel %p (global ID " U64_FORMAT ") " @@ -1230,9 +1262,8 @@ channel_close_for_error(channel_t *chan) tor_assert(chan != NULL); /* If it's already in CLOSING, CLOSED or ERROR, this is a no-op */ - if (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) return; + if (CHANNEL_CONDEMNED(chan)) + return; log_debug(LD_CHANNEL, "Closing channel %p due to lower-layer error", @@ -1288,18 +1319,16 @@ void channel_closed(channel_t *chan) { tor_assert(chan); - tor_assert(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + tor_assert(CHANNEL_CONDEMNED(chan)); /* No-op if already inactive */ - if (chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) return; + if (CHANNEL_FINISHED(chan)) + return; /* Inform any pending (not attached) circs that they should * give up. */ if (! chan->has_been_open) - circuit_n_chan_done(chan, 0); + circuit_n_chan_done(chan, 0, 0); /* Now close all the attached circuits on it. */ circuit_unlink_all_from_channel(chan, END_CIRC_REASON_CHANNEL_CLOSED); @@ -1357,10 +1386,7 @@ channel_clear_identity_digest(channel_t *chan) "global ID " U64_FORMAT, chan, U64_PRINTF_ARG(chan->global_identifier)); - state_not_in_map = - (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + state_not_in_map = CHANNEL_CONDEMNED(chan); if (!state_not_in_map && chan->registered && !tor_digest_is_zero(chan->identity_digest)) @@ -1393,10 +1419,8 @@ channel_set_identity_digest(channel_t *chan, identity_digest ? hex_str(identity_digest, DIGEST_LEN) : "(null)"); - state_not_in_map = - (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + state_not_in_map = CHANNEL_CONDEMNED(chan); + was_in_digest_map = !state_not_in_map && chan->registered && @@ -1446,10 +1470,7 @@ channel_clear_remote_end(channel_t *chan) "global ID " U64_FORMAT, chan, U64_PRINTF_ARG(chan->global_identifier)); - state_not_in_map = - (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + state_not_in_map = CHANNEL_CONDEMNED(chan); if (!state_not_in_map && chan->registered && !tor_digest_is_zero(chan->identity_digest)) @@ -1485,10 +1506,8 @@ channel_set_remote_end(channel_t *chan, identity_digest ? hex_str(identity_digest, DIGEST_LEN) : "(null)"); - state_not_in_map = - (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR); + state_not_in_map = CHANNEL_CONDEMNED(chan); + was_in_digest_map = !state_not_in_map && chan->registered && @@ -1548,7 +1567,7 @@ cell_queue_entry_dup(cell_queue_entry_t *q) * them) or not (we should free). */ -static void +STATIC void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off) { if (!q) return; @@ -1666,6 +1685,36 @@ cell_queue_entry_new_var(var_cell_t *var_cell) } /** + * Ask how big the cell contained in a cell_queue_entry_t is + */ + +static size_t +channel_get_cell_queue_entry_size(channel_t *chan, cell_queue_entry_t *q) +{ + size_t rv = 0; + + tor_assert(chan); + tor_assert(q); + + switch (q->type) { + case CELL_QUEUE_FIXED: + rv = get_cell_network_size(chan->wide_circ_ids); + break; + case CELL_QUEUE_VAR: + rv = get_var_cell_header_size(chan->wide_circ_ids) + + (q->u.var.var_cell ? q->u.var.var_cell->payload_len : 0); + break; + case CELL_QUEUE_PACKED: + rv = get_cell_network_size(chan->wide_circ_ids); + break; + default: + tor_assert(1); + } + + return rv; +} + +/** * Write to a channel based on a cell_queue_entry_t * * Given a cell_queue_entry_t filled out by the caller, try to send the cell @@ -1677,14 +1726,13 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) { int result = 0, sent = 0; cell_queue_entry_t *tmp = NULL; + size_t cell_bytes; tor_assert(chan); tor_assert(q); /* Assert that the state makes sense for a cell write */ - tor_assert(chan->state == CHANNEL_STATE_OPENING || - chan->state == CHANNEL_STATE_OPEN || - chan->state == CHANNEL_STATE_MAINT); + tor_assert(CHANNEL_CAN_HANDLE_CELLS(chan)); { circid_t circ_id; @@ -1693,9 +1741,12 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) } } + /* For statistical purposes, figure out how big this cell is */ + cell_bytes = channel_get_cell_queue_entry_size(chan, q); + /* Can we send it right out? If so, try */ if (TOR_SIMPLEQ_EMPTY(&chan->outgoing_queue) && - chan->state == CHANNEL_STATE_OPEN) { + CHANNEL_IS_OPEN(chan)) { /* Pick the right write function for this cell type and save the result */ switch (q->type) { case CELL_QUEUE_FIXED: @@ -1726,6 +1777,13 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) channel_timestamp_drained(chan); /* Update the counter */ ++(chan->n_cells_xmitted); + chan->n_bytes_xmitted += cell_bytes; + /* Update global counters */ + ++n_channel_cells_queued; + ++n_channel_cells_passed_to_lower_layer; + n_channel_bytes_queued += cell_bytes; + n_channel_bytes_passed_to_lower_layer += cell_bytes; + channel_assert_counter_consistency(); } } @@ -1737,8 +1795,16 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) */ tmp = cell_queue_entry_dup(q); TOR_SIMPLEQ_INSERT_TAIL(&chan->outgoing_queue, tmp, next); + /* Update global counters */ + ++n_channel_cells_queued; + ++n_channel_cells_in_queues; + n_channel_bytes_queued += cell_bytes; + n_channel_bytes_in_queues += cell_bytes; + channel_assert_counter_consistency(); + /* Update channel queue size */ + chan->bytes_in_queue += cell_bytes; /* Try to process the queue? */ - if (chan->state == CHANNEL_STATE_OPEN) channel_flush_cells(chan); + if (CHANNEL_IS_OPEN(chan)) channel_flush_cells(chan); } } @@ -1759,7 +1825,7 @@ channel_write_cell(channel_t *chan, cell_t *cell) tor_assert(chan); tor_assert(cell); - if (chan->state == CHANNEL_STATE_CLOSING) { + if (CHANNEL_IS_CLOSING(chan)) { log_debug(LD_CHANNEL, "Discarding cell_t %p on closing channel %p with " "global ID "U64_FORMAT, cell, chan, U64_PRINTF_ARG(chan->global_identifier)); @@ -1775,6 +1841,9 @@ channel_write_cell(channel_t *chan, cell_t *cell) q.type = CELL_QUEUE_FIXED; q.u.fixed.cell = cell; channel_write_cell_queue_entry(chan, &q); + + /* Update the queue size estimate */ + channel_update_xmit_queue_size(chan); } /** @@ -1793,7 +1862,7 @@ channel_write_packed_cell(channel_t *chan, packed_cell_t *packed_cell) tor_assert(chan); tor_assert(packed_cell); - if (chan->state == CHANNEL_STATE_CLOSING) { + if (CHANNEL_IS_CLOSING(chan)) { log_debug(LD_CHANNEL, "Discarding packed_cell_t %p on closing channel %p " "with global ID "U64_FORMAT, packed_cell, chan, U64_PRINTF_ARG(chan->global_identifier)); @@ -1810,6 +1879,9 @@ channel_write_packed_cell(channel_t *chan, packed_cell_t *packed_cell) q.type = CELL_QUEUE_PACKED; q.u.packed.packed_cell = packed_cell; channel_write_cell_queue_entry(chan, &q); + + /* Update the queue size estimate */ + channel_update_xmit_queue_size(chan); } /** @@ -1829,7 +1901,7 @@ channel_write_var_cell(channel_t *chan, var_cell_t *var_cell) tor_assert(chan); tor_assert(var_cell); - if (chan->state == CHANNEL_STATE_CLOSING) { + if (CHANNEL_IS_CLOSING(chan)) { log_debug(LD_CHANNEL, "Discarding var_cell_t %p on closing channel %p " "with global ID "U64_FORMAT, var_cell, chan, U64_PRINTF_ARG(chan->global_identifier)); @@ -1846,6 +1918,9 @@ channel_write_var_cell(channel_t *chan, var_cell_t *var_cell) q.type = CELL_QUEUE_VAR; q.u.var.var_cell = var_cell; channel_write_cell_queue_entry(chan, &q); + + /* Update the queue size estimate */ + channel_update_xmit_queue_size(chan); } /** @@ -1941,6 +2016,41 @@ channel_change_state(channel_t *chan, channel_state_t to_state) } } + /* + * If we're going to a closed/closing state, we don't need scheduling any + * more; in CHANNEL_STATE_MAINT we can't accept writes. + */ + if (to_state == CHANNEL_STATE_CLOSING || + to_state == CHANNEL_STATE_CLOSED || + to_state == CHANNEL_STATE_ERROR) { + scheduler_release_channel(chan); + } else if (to_state == CHANNEL_STATE_MAINT) { + scheduler_channel_doesnt_want_writes(chan); + } + + /* + * If we're closing, this channel no longer counts toward the global + * estimated queue size; if we're open, it now does. + */ + if ((to_state == CHANNEL_STATE_CLOSING || + to_state == CHANNEL_STATE_CLOSED || + to_state == CHANNEL_STATE_ERROR) && + (from_state == CHANNEL_STATE_OPEN || + from_state == CHANNEL_STATE_MAINT)) { + estimated_total_queue_size -= chan->bytes_in_queue; + } + + /* + * If we're opening, this channel now does count toward the global + * estimated queue size. + */ + if ((to_state == CHANNEL_STATE_OPEN || + to_state == CHANNEL_STATE_MAINT) && + !(from_state == CHANNEL_STATE_OPEN || + from_state == CHANNEL_STATE_MAINT)) { + estimated_total_queue_size += chan->bytes_in_queue; + } + /* Tell circuits if we opened and stuff */ if (to_state == CHANNEL_STATE_OPEN) { channel_do_open_actions(chan); @@ -2056,12 +2166,13 @@ channel_listener_change_state(channel_listener_t *chan_l, #define MAX_CELLS_TO_GET_FROM_CIRCUITS_FOR_UNLIMITED 256 -ssize_t -channel_flush_some_cells(channel_t *chan, ssize_t num_cells) +MOCK_IMPL(ssize_t, +channel_flush_some_cells, (channel_t *chan, ssize_t num_cells)) { unsigned int unlimited = 0; ssize_t flushed = 0; int num_cells_from_circs, clamped_num_cells; + int q_len_before, q_len_after; tor_assert(chan); @@ -2069,7 +2180,7 @@ channel_flush_some_cells(channel_t *chan, ssize_t num_cells) if (!unlimited && num_cells <= flushed) goto done; /* If we aren't in CHANNEL_STATE_OPEN, nothing goes through */ - if (chan->state == CHANNEL_STATE_OPEN) { + if (CHANNEL_IS_OPEN(chan)) { /* Try to flush as much as we can that's already queued */ flushed += channel_flush_some_cells_from_outgoing_queue(chan, (unlimited ? -1 : num_cells - flushed)); @@ -2087,14 +2198,45 @@ channel_flush_some_cells(channel_t *chan, ssize_t num_cells) clamped_num_cells = (int)(num_cells - flushed); } } + + /* + * Keep track of the change in queue size; we have to count cells + * channel_flush_from_first_active_circuit() writes out directly, + * but not double-count ones we might get later in + * channel_flush_some_cells_from_outgoing_queue() + */ + q_len_before = chan_cell_queue_len(&(chan->outgoing_queue)); + /* Try to get more cells from any active circuits */ num_cells_from_circs = channel_flush_from_first_active_circuit( chan, clamped_num_cells); - /* If it claims we got some, process the queue again */ + q_len_after = chan_cell_queue_len(&(chan->outgoing_queue)); + + /* + * If it claims we got some, adjust the flushed counter and consider + * processing the queue again + */ if (num_cells_from_circs > 0) { - flushed += channel_flush_some_cells_from_outgoing_queue(chan, - (unlimited ? -1 : num_cells - flushed)); + /* + * Adjust flushed by the number of cells counted in + * num_cells_from_circs that didn't go to the cell queue. + */ + + if (q_len_after > q_len_before) { + num_cells_from_circs -= (q_len_after - q_len_before); + if (num_cells_from_circs < 0) num_cells_from_circs = 0; + } + + flushed += num_cells_from_circs; + + /* Now process the queue if necessary */ + + if ((q_len_after > q_len_before) && + (unlimited || (flushed < num_cells))) { + flushed += channel_flush_some_cells_from_outgoing_queue(chan, + (unlimited ? -1 : num_cells - flushed)); + } } } } @@ -2117,6 +2259,8 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, unsigned int unlimited = 0; ssize_t flushed = 0; cell_queue_entry_t *q = NULL; + size_t cell_size; + int free_q = 0, handed_off = 0; tor_assert(chan); tor_assert(chan->write_cell); @@ -2127,11 +2271,15 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, if (!unlimited && num_cells <= flushed) return 0; /* If we aren't in CHANNEL_STATE_OPEN, nothing goes through */ - if (chan->state == CHANNEL_STATE_OPEN) { + if (CHANNEL_IS_OPEN(chan)) { while ((unlimited || num_cells > flushed) && NULL != (q = TOR_SIMPLEQ_FIRST(&chan->outgoing_queue))) { + free_q = 0; + handed_off = 0; if (1) { + /* Figure out how big it is for statistical purposes */ + cell_size = channel_get_cell_queue_entry_size(chan, q); /* * Okay, we have a good queue entry, try to give it to the lower * layer. @@ -2144,8 +2292,9 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, ++flushed; channel_timestamp_xmit(chan); ++(chan->n_cells_xmitted); - cell_queue_entry_free(q, 1); - q = NULL; + chan->n_bytes_xmitted += cell_size; + free_q = 1; + handed_off = 1; } /* Else couldn't write it; leave it on the queue */ } else { @@ -2156,8 +2305,8 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, "(global ID " U64_FORMAT ").", chan, U64_PRINTF_ARG(chan->global_identifier)); /* Throw it away */ - cell_queue_entry_free(q, 0); - q = NULL; + free_q = 1; + handed_off = 0; } break; case CELL_QUEUE_PACKED: @@ -2167,8 +2316,9 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, ++flushed; channel_timestamp_xmit(chan); ++(chan->n_cells_xmitted); - cell_queue_entry_free(q, 1); - q = NULL; + chan->n_bytes_xmitted += cell_size; + free_q = 1; + handed_off = 1; } /* Else couldn't write it; leave it on the queue */ } else { @@ -2179,8 +2329,8 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, "(global ID " U64_FORMAT ").", chan, U64_PRINTF_ARG(chan->global_identifier)); /* Throw it away */ - cell_queue_entry_free(q, 0); - q = NULL; + free_q = 1; + handed_off = 0; } break; case CELL_QUEUE_VAR: @@ -2190,8 +2340,9 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, ++flushed; channel_timestamp_xmit(chan); ++(chan->n_cells_xmitted); - cell_queue_entry_free(q, 1); - q = NULL; + chan->n_bytes_xmitted += cell_size; + free_q = 1; + handed_off = 1; } /* Else couldn't write it; leave it on the queue */ } else { @@ -2202,8 +2353,8 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, "(global ID " U64_FORMAT ").", chan, U64_PRINTF_ARG(chan->global_identifier)); /* Throw it away */ - cell_queue_entry_free(q, 0); - q = NULL; + free_q = 1; + handed_off = 0; } break; default: @@ -2213,12 +2364,32 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, "(global ID " U64_FORMAT "; ignoring it." " Someone should fix this.", q->type, chan, U64_PRINTF_ARG(chan->global_identifier)); - cell_queue_entry_free(q, 0); - q = NULL; + free_q = 1; + handed_off = 0; } - /* if q got NULLed out, we used it and should remove the queue entry */ - if (!q) TOR_SIMPLEQ_REMOVE_HEAD(&chan->outgoing_queue, next); + /* + * if free_q is set, we used it and should remove the queue entry; + * we have to do the free down here so TOR_SIMPLEQ_REMOVE_HEAD isn't + * accessing freed memory + */ + if (free_q) { + TOR_SIMPLEQ_REMOVE_HEAD(&chan->outgoing_queue, next); + /* + * ...and we handed a cell off to the lower layer, so we should + * update the counters. + */ + ++n_channel_cells_passed_to_lower_layer; + --n_channel_cells_in_queues; + n_channel_bytes_passed_to_lower_layer += cell_size; + n_channel_bytes_in_queues -= cell_size; + channel_assert_counter_consistency(); + /* Update the channel's queue size too */ + chan->bytes_in_queue -= cell_size; + /* Finally, free q */ + cell_queue_entry_free(q, handed_off); + q = NULL; + } /* No cell removed from list, so we can't go on any further */ else break; } @@ -2230,6 +2401,9 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, channel_timestamp_drained(chan); } + /* Update the estimate queue size */ + channel_update_xmit_queue_size(chan); + return flushed; } @@ -2352,8 +2526,9 @@ void channel_do_open_actions(channel_t *chan) { tor_addr_t remote_addr; - int started_here, not_using = 0; + int started_here; time_t now = time(NULL); + int close_origin_circuits = 0; tor_assert(chan); @@ -2370,8 +2545,7 @@ channel_do_open_actions(channel_t *chan) log_debug(LD_OR, "New entry guard was reachable, but closing this " "connection so we can retry the earlier entry guards."); - circuit_n_chan_done(chan, 0); - not_using = 1; + close_origin_circuits = 1; } router_set_status(chan->identity_digest, 1); } else { @@ -2391,7 +2565,7 @@ channel_do_open_actions(channel_t *chan) } } - if (!not_using) circuit_n_chan_done(chan, 1); + circuit_n_chan_done(chan, 1, close_origin_circuits); } /** @@ -2462,9 +2636,8 @@ channel_process_cells(channel_t *chan) { cell_queue_entry_t *q; tor_assert(chan); - tor_assert(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_MAINT || - chan->state == CHANNEL_STATE_OPEN); + tor_assert(CHANNEL_IS_CLOSING(chan) || CHANNEL_IS_MAINT(chan) || + CHANNEL_IS_OPEN(chan)); log_debug(LD_CHANNEL, "Processing as many incoming cells as we can for channel %p", @@ -2531,7 +2704,7 @@ channel_queue_cell(channel_t *chan, cell_t *cell) tor_assert(chan); tor_assert(cell); - tor_assert(chan->state == CHANNEL_STATE_OPEN); + tor_assert(CHANNEL_IS_OPEN(chan)); /* Do we need to queue it, or can we just call the handler right away? */ if (!(chan->cell_handler)) need_to_queue = 1; @@ -2541,8 +2714,9 @@ channel_queue_cell(channel_t *chan, cell_t *cell) /* Timestamp for receiving */ channel_timestamp_recv(chan); - /* Update the counter */ + /* Update the counters */ ++(chan->n_cells_recved); + chan->n_bytes_recved += get_cell_network_size(chan->wide_circ_ids); /* If we don't need to queue we can just call cell_handler */ if (!need_to_queue) { @@ -2584,7 +2758,7 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) tor_assert(chan); tor_assert(var_cell); - tor_assert(chan->state == CHANNEL_STATE_OPEN); + tor_assert(CHANNEL_IS_OPEN(chan)); /* Do we need to queue it, or can we just call the handler right away? */ if (!(chan->var_cell_handler)) need_to_queue = 1; @@ -2596,6 +2770,8 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) /* Update the counter */ ++(chan->n_cells_recved); + chan->n_bytes_recved += get_var_cell_header_size(chan->wide_circ_ids) + + var_cell->payload_len; /* If we don't need to queue we can just call cell_handler */ if (!need_to_queue) { @@ -2645,6 +2821,19 @@ packed_cell_is_destroy(channel_t *chan, return 0; } +/** + * Assert that the global channel stats counters are internally consistent + */ + +static void +channel_assert_counter_consistency(void) +{ + tor_assert(n_channel_cells_queued == + (n_channel_cells_in_queues + n_channel_cells_passed_to_lower_layer)); + tor_assert(n_channel_bytes_queued == + (n_channel_bytes_in_queues + n_channel_bytes_passed_to_lower_layer)); +} + /** DOCDOC */ static int is_destroy_cell(channel_t *chan, @@ -2692,10 +2881,7 @@ channel_send_destroy(circid_t circ_id, channel_t *chan, int reason) } /* Check to make sure we can send on this channel first */ - if (!(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) && - chan->cmux) { + if (!CHANNEL_CONDEMNED(chan) && chan->cmux) { channel_note_destroy_pending(chan, circ_id); circuitmux_append_destroy_cell(chan, chan->cmux, circ_id, reason); log_debug(LD_OR, @@ -2727,6 +2913,19 @@ channel_dumpstats(int severity) { if (all_channels && smartlist_len(all_channels) > 0) { tor_log(severity, LD_GENERAL, + "Channels have queued " U64_FORMAT " bytes in " U64_FORMAT " cells, " + "and handed " U64_FORMAT " bytes in " U64_FORMAT " cells to the lower" + " layer.", + U64_PRINTF_ARG(n_channel_bytes_queued), + U64_PRINTF_ARG(n_channel_cells_queued), + U64_PRINTF_ARG(n_channel_bytes_passed_to_lower_layer), + U64_PRINTF_ARG(n_channel_cells_passed_to_lower_layer)); + tor_log(severity, LD_GENERAL, + "There are currently " U64_FORMAT " bytes in " U64_FORMAT " cells " + "in channel queues.", + U64_PRINTF_ARG(n_channel_bytes_in_queues), + U64_PRINTF_ARG(n_channel_cells_in_queues)); + tor_log(severity, LD_GENERAL, "Dumping statistics about %d channels:", smartlist_len(all_channels)); tor_log(severity, LD_GENERAL, @@ -2872,9 +3071,7 @@ channel_free_list(smartlist_t *channels, int mark_for_close) } channel_unregister(curr); if (mark_for_close) { - if (!(curr->state == CHANNEL_STATE_CLOSING || - curr->state == CHANNEL_STATE_CLOSED || - curr->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(curr)) { channel_mark_for_close(curr); } channel_force_free(curr); @@ -3088,9 +3285,7 @@ channel_get_for_extend(const char *digest, tor_assert(tor_memeq(chan->identity_digest, digest, DIGEST_LEN)); - if (chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR) + if (CHANNEL_CONDEMNED(chan)) continue; /* Never return a channel on which the other end appears to be @@ -3100,7 +3295,7 @@ channel_get_for_extend(const char *digest, } /* Never return a non-open connection. */ - if (chan->state != CHANNEL_STATE_OPEN) { + if (!CHANNEL_IS_OPEN(chan)) { /* If the address matches, don't launch a new connection for this * circuit. */ if (channel_matches_target_addr_for_extend(chan, target_addr)) @@ -3200,7 +3395,7 @@ channel_listener_describe_transport(channel_listener_t *chan_l) /** * Return the number of entries in <b>queue</b> */ -static int +STATIC int chan_cell_queue_len(const chan_cell_queue_t *queue) { int r = 0; @@ -3216,8 +3411,8 @@ chan_cell_queue_len(const chan_cell_queue_t *queue) * Dump statistics for one channel to the log */ -void -channel_dump_statistics(channel_t *chan, int severity) +MOCK_IMPL(void, +channel_dump_statistics, (channel_t *chan, int severity)) { double avg, interval, age; time_t now = time(NULL); @@ -3369,12 +3564,22 @@ channel_dump_statistics(channel_t *chan, int severity) /* Describe counters and rates */ tor_log(severity, LD_GENERAL, " * Channel " U64_FORMAT " has received " - U64_FORMAT " cells and transmitted " U64_FORMAT, + U64_FORMAT " bytes in " U64_FORMAT " cells and transmitted " + U64_FORMAT " bytes in " U64_FORMAT " cells", U64_PRINTF_ARG(chan->global_identifier), + U64_PRINTF_ARG(chan->n_bytes_recved), U64_PRINTF_ARG(chan->n_cells_recved), + U64_PRINTF_ARG(chan->n_bytes_xmitted), U64_PRINTF_ARG(chan->n_cells_xmitted)); if (now > chan->timestamp_created && chan->timestamp_created > 0) { + if (chan->n_bytes_recved > 0) { + avg = (double)(chan->n_bytes_recved) / age; + tor_log(severity, LD_GENERAL, + " * Channel " U64_FORMAT " has averaged %f " + "bytes received per second", + U64_PRINTF_ARG(chan->global_identifier), avg); + } if (chan->n_cells_recved > 0) { avg = (double)(chan->n_cells_recved) / age; if (avg >= 1.0) { @@ -3390,6 +3595,13 @@ channel_dump_statistics(channel_t *chan, int severity) U64_PRINTF_ARG(chan->global_identifier), interval); } } + if (chan->n_bytes_xmitted > 0) { + avg = (double)(chan->n_bytes_xmitted) / age; + tor_log(severity, LD_GENERAL, + " * Channel " U64_FORMAT " has averaged %f " + "bytes transmitted per second", + U64_PRINTF_ARG(chan->global_identifier), avg); + } if (chan->n_cells_xmitted > 0) { avg = (double)(chan->n_cells_xmitted) / age; if (avg >= 1.0) { @@ -3807,6 +4019,50 @@ channel_mark_outgoing(channel_t *chan) chan->is_incoming = 0; } +/************************ + * Flow control queries * + ***********************/ + +/* + * Get the latest estimate for the total queue size of all open channels + */ + +uint64_t +channel_get_global_queue_estimate(void) +{ + return estimated_total_queue_size; +} + +/* + * Estimate the number of writeable cells + * + * Ask the lower layer for an estimate of how many cells it can accept, and + * then subtract the length of our outgoing_queue, if any, to produce an + * estimate of the number of cells this channel can accept for writes. + */ + +int +channel_num_cells_writeable(channel_t *chan) +{ + int result; + + tor_assert(chan); + tor_assert(chan->num_cells_writeable); + + if (chan->state == CHANNEL_STATE_OPEN) { + /* Query lower layer */ + result = chan->num_cells_writeable(chan); + /* Subtract cell queue length, if any */ + result -= chan_cell_queue_len(&chan->outgoing_queue); + if (result < 0) result = 0; + } else { + /* No cells are writeable in any other state */ + result = 0; + } + + return result; +} + /********************* * Timestamp updates * ********************/ @@ -4209,3 +4465,87 @@ channel_set_circid_type(channel_t *chan, } } +/** + * Update the estimated number of bytes queued to transmit for this channel, + * and notify the scheduler. The estimate includes both the channel queue and + * the queue size reported by the lower layer, and an overhead estimate + * optionally provided by the lower layer. + */ + +void +channel_update_xmit_queue_size(channel_t *chan) +{ + uint64_t queued, adj; + double overhead; + + tor_assert(chan); + tor_assert(chan->num_bytes_queued); + + /* + * First, get the number of bytes we have queued without factoring in + * lower-layer overhead. + */ + queued = chan->num_bytes_queued(chan) + chan->bytes_in_queue; + /* Next, adjust by the overhead factor, if any is available */ + if (chan->get_overhead_estimate) { + overhead = chan->get_overhead_estimate(chan); + if (overhead >= 1.0f) { + queued *= overhead; + } else { + /* Ignore silly overhead factors */ + log_notice(LD_CHANNEL, "Ignoring silly overhead factor %f", overhead); + } + } + + /* Now, compare to the previous estimate */ + if (queued > chan->bytes_queued_for_xmit) { + adj = queued - chan->bytes_queued_for_xmit; + log_debug(LD_CHANNEL, + "Increasing queue size for channel " U64_FORMAT " by " U64_FORMAT + " from " U64_FORMAT " to " U64_FORMAT, + U64_PRINTF_ARG(chan->global_identifier), + U64_PRINTF_ARG(adj), + U64_PRINTF_ARG(chan->bytes_queued_for_xmit), + U64_PRINTF_ARG(queued)); + /* Update the channel's estimate */ + chan->bytes_queued_for_xmit = queued; + + /* Update the global queue size estimate if appropriate */ + if (chan->state == CHANNEL_STATE_OPEN || + chan->state == CHANNEL_STATE_MAINT) { + estimated_total_queue_size += adj; + log_debug(LD_CHANNEL, + "Increasing global queue size by " U64_FORMAT " for channel " + U64_FORMAT ", new size is " U64_FORMAT, + U64_PRINTF_ARG(adj), U64_PRINTF_ARG(chan->global_identifier), + U64_PRINTF_ARG(estimated_total_queue_size)); + /* Tell the scheduler we're increasing the queue size */ + scheduler_adjust_queue_size(chan, 1, adj); + } + } else if (queued < chan->bytes_queued_for_xmit) { + adj = chan->bytes_queued_for_xmit - queued; + log_debug(LD_CHANNEL, + "Decreasing queue size for channel " U64_FORMAT " by " U64_FORMAT + " from " U64_FORMAT " to " U64_FORMAT, + U64_PRINTF_ARG(chan->global_identifier), + U64_PRINTF_ARG(adj), + U64_PRINTF_ARG(chan->bytes_queued_for_xmit), + U64_PRINTF_ARG(queued)); + /* Update the channel's estimate */ + chan->bytes_queued_for_xmit = queued; + + /* Update the global queue size estimate if appropriate */ + if (chan->state == CHANNEL_STATE_OPEN || + chan->state == CHANNEL_STATE_MAINT) { + estimated_total_queue_size -= adj; + log_debug(LD_CHANNEL, + "Decreasing global queue size by " U64_FORMAT " for channel " + U64_FORMAT ", new size is " U64_FORMAT, + U64_PRINTF_ARG(adj), U64_PRINTF_ARG(chan->global_identifier), + U64_PRINTF_ARG(estimated_total_queue_size)); + /* Tell the scheduler we're decreasing the queue size */ + scheduler_adjust_queue_size(chan, -1, adj); + } + } +} + diff --git a/src/or/channel.h b/src/or/channel.h index 148199235a..ecc2a092e4 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -57,6 +57,32 @@ struct channel_s { CHANNEL_CLOSE_FOR_ERROR } reason_for_closing; + /** State variable for use by the scheduler */ + enum { + /* + * The channel is not open, or it has a full output buffer but no queued + * cells. + */ + SCHED_CHAN_IDLE = 0, + /* + * The channel has space on its output buffer to write, but no queued + * cells. + */ + SCHED_CHAN_WAITING_FOR_CELLS, + /* + * The scheduler has queued cells but no output buffer space to write. + */ + SCHED_CHAN_WAITING_TO_WRITE, + /* + * The scheduler has both queued cells and output buffer space, and is + * eligible for the scheduler loop. + */ + SCHED_CHAN_PENDING + } scheduler_state; + + /** Heap index for use by the scheduler */ + int sched_heap_idx; + /** Timestamps for both cell channels and listeners */ time_t timestamp_created; /* Channel created */ time_t timestamp_active; /* Any activity */ @@ -79,6 +105,11 @@ struct channel_s { /* Methods implemented by the lower layer */ /** + * Ask the lower layer for an estimate of the average overhead for + * transmissions on this channel. + */ + double (*get_overhead_estimate)(channel_t *); + /* * Ask the underlying transport what the remote endpoint address is, in * a tor_addr_t. This is optional and subclasses may leave this NULL. * If they implement it, they should write the address out to the @@ -110,7 +141,11 @@ struct channel_s { int (*matches_extend_info)(channel_t *, extend_info_t *); /** Check if this channel matches a target address when extending */ int (*matches_target)(channel_t *, const tor_addr_t *); - /** Write a cell to an open channel */ + /* Ask the lower layer how many bytes it has queued but not yet sent */ + size_t (*num_bytes_queued)(channel_t *); + /* Ask the lower layer how many cells can be written */ + int (*num_cells_writeable)(channel_t *); + /* Write a cell to an open channel */ int (*write_cell)(channel_t *, cell_t *); /** Write a packed cell to an open channel */ int (*write_packed_cell)(channel_t *, packed_cell_t *); @@ -198,8 +233,16 @@ struct channel_s { uint64_t dirreq_id; /** Channel counters for cell channels */ - uint64_t n_cells_recved; - uint64_t n_cells_xmitted; + uint64_t n_cells_recved, n_bytes_recved; + uint64_t n_cells_xmitted, n_bytes_xmitted; + + /** Our current contribution to the scheduler's total xmit queue */ + uint64_t bytes_queued_for_xmit; + + /** Number of bytes in this channel's cell queue; does not include + * lower-layer queueing. + */ + uint64_t bytes_in_queue; }; struct channel_listener_s { @@ -311,6 +354,36 @@ void channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol); #ifdef TOR_CHANNEL_INTERNAL_ +#ifdef CHANNEL_PRIVATE_ +/* Cell queue structure (here rather than channel.c for test suite use) */ + +typedef struct cell_queue_entry_s cell_queue_entry_t; +struct cell_queue_entry_s { + TOR_SIMPLEQ_ENTRY(cell_queue_entry_s) next; + enum { + CELL_QUEUE_FIXED, + CELL_QUEUE_VAR, + CELL_QUEUE_PACKED + } type; + union { + struct { + cell_t *cell; + } fixed; + struct { + var_cell_t *var_cell; + } var; + struct { + packed_cell_t *packed_cell; + } packed; + } u; +}; + +/* Cell queue functions for benefit of test suite */ +STATIC int chan_cell_queue_len(const chan_cell_queue_t *queue); + +STATIC void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off); +#endif + /* Channel operations for subclasses and internal use only */ /* Initialize a newly allocated channel - do this first in subclass @@ -384,7 +457,8 @@ void channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell); void channel_flush_cells(channel_t *chan); /* Request from lower layer for more cells if available */ -ssize_t channel_flush_some_cells(channel_t *chan, ssize_t num_cells); +MOCK_DECL(ssize_t, channel_flush_some_cells, + (channel_t *chan, ssize_t num_cells)); /* Query if data available on this channel */ int channel_more_to_flush(channel_t *chan); @@ -431,11 +505,44 @@ channel_t * channel_find_by_remote_digest(const char *identity_digest); channel_t * channel_next_with_digest(channel_t *chan); /* + * Helper macros to lookup state of given channel. + */ + +#define CHANNEL_IS_CLOSED(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_CLOSED)) +#define CHANNEL_IS_OPENING(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_OPENING)) +#define CHANNEL_IS_OPEN(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_OPEN)) +#define CHANNEL_IS_MAINT(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_MAINT)) +#define CHANNEL_IS_CLOSING(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_CLOSING)) +#define CHANNEL_IS_ERROR(chan) (channel_is_in_state((chan), \ + CHANNEL_STATE_ERROR)) + +#define CHANNEL_FINISHED(chan) (CHANNEL_IS_CLOSED(chan) || \ + CHANNEL_IS_ERROR(chan)) + +#define CHANNEL_CONDEMNED(chan) (CHANNEL_IS_CLOSING(chan) || \ + CHANNEL_FINISHED(chan)) + +#define CHANNEL_CAN_HANDLE_CELLS(chan) (CHANNEL_IS_OPENING(chan) || \ + CHANNEL_IS_OPEN(chan) || \ + CHANNEL_IS_MAINT(chan)) + +static INLINE int +channel_is_in_state(channel_t *chan, channel_state_t state) +{ + return chan->state == state; +} + +/* * Metadata queries/updates */ const char * channel_describe_transport(channel_t *chan); -void channel_dump_statistics(channel_t *chan, int severity); +MOCK_DECL(void, channel_dump_statistics, (channel_t *chan, int severity)); void channel_dump_transport_statistics(channel_t *chan, int severity); const char * channel_get_actual_remote_descr(channel_t *chan); const char * channel_get_actual_remote_address(channel_t *chan); @@ -458,6 +565,7 @@ unsigned int channel_num_circuits(channel_t *chan); void channel_set_circid_type(channel_t *chan, crypto_pk_t *identity_rcvd, int consider_identity); void channel_timestamp_client(channel_t *chan); +void channel_update_xmit_queue_size(channel_t *chan); const char * channel_listener_describe_transport(channel_listener_t *chan_l); void channel_listener_dump_statistics(channel_listener_t *chan_l, @@ -465,6 +573,10 @@ void channel_listener_dump_statistics(channel_listener_t *chan_l, void channel_listener_dump_transport_statistics(channel_listener_t *chan_l, int severity); +/* Flow control queries */ +uint64_t channel_get_global_queue_estimate(void); +int channel_num_cells_writeable(channel_t *chan); + /* Timestamp queries */ time_t channel_when_created(channel_t *chan); time_t channel_when_last_active(channel_t *chan); diff --git a/src/or/channeltls.c b/src/or/channeltls.c index 245e33583b..1cf697ccc5 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -23,8 +23,10 @@ #include "connection_or.h" #include "control.h" #include "relay.h" +#include "rephist.h" #include "router.h" #include "routerlist.h" +#include "scheduler.h" /** How many CELL_PADDING cells have we received, ever? */ uint64_t stats_n_padding_cells_processed = 0; @@ -54,6 +56,7 @@ static void channel_tls_common_init(channel_tls_t *tlschan); static void channel_tls_close_method(channel_t *chan); static const char * channel_tls_describe_transport_method(channel_t *chan); static void channel_tls_free_method(channel_t *chan); +static double channel_tls_get_overhead_estimate_method(channel_t *chan); static int channel_tls_get_remote_addr_method(channel_t *chan, tor_addr_t *addr_out); static int @@ -67,6 +70,8 @@ channel_tls_matches_extend_info_method(channel_t *chan, extend_info_t *extend_info); static int channel_tls_matches_target_method(channel_t *chan, const tor_addr_t *target); +static int channel_tls_num_cells_writeable_method(channel_t *chan); +static size_t channel_tls_num_bytes_queued_method(channel_t *chan); static int channel_tls_write_cell_method(channel_t *chan, cell_t *cell); static int channel_tls_write_packed_cell_method(channel_t *chan, @@ -116,6 +121,7 @@ channel_tls_common_init(channel_tls_t *tlschan) chan->close = channel_tls_close_method; chan->describe_transport = channel_tls_describe_transport_method; chan->free = channel_tls_free_method; + chan->get_overhead_estimate = channel_tls_get_overhead_estimate_method; chan->get_remote_addr = channel_tls_get_remote_addr_method; chan->get_remote_descr = channel_tls_get_remote_descr_method; chan->get_transport_name = channel_tls_get_transport_name_method; @@ -123,6 +129,8 @@ channel_tls_common_init(channel_tls_t *tlschan) chan->is_canonical = channel_tls_is_canonical_method; chan->matches_extend_info = channel_tls_matches_extend_info_method; chan->matches_target = channel_tls_matches_target_method; + chan->num_bytes_queued = channel_tls_num_bytes_queued_method; + chan->num_cells_writeable = channel_tls_num_cells_writeable_method; chan->write_cell = channel_tls_write_cell_method; chan->write_packed_cell = channel_tls_write_packed_cell_method; chan->write_var_cell = channel_tls_write_var_cell_method; @@ -435,6 +443,40 @@ channel_tls_free_method(channel_t *chan) } /** + * Get an estimate of the average TLS overhead for the upper layer + */ + +static double +channel_tls_get_overhead_estimate_method(channel_t *chan) +{ + double overhead = 1.0f; + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(tlschan->conn); + + /* Just return 1.0f if we don't have sensible data */ + if (tlschan->conn->bytes_xmitted > 0 && + tlschan->conn->bytes_xmitted_by_tls >= + tlschan->conn->bytes_xmitted) { + overhead = ((double)(tlschan->conn->bytes_xmitted_by_tls)) / + ((double)(tlschan->conn->bytes_xmitted)); + + /* + * Never estimate more than 2.0; otherwise we get silly large estimates + * at the very start of a new TLS connection. + */ + if (overhead > 2.0f) overhead = 2.0f; + } + + log_debug(LD_CHANNEL, + "Estimated overhead ratio for TLS chan " U64_FORMAT " is %f", + U64_PRINTF_ARG(chan->global_identifier), overhead); + + return overhead; +} + +/** * Get the remote address of a channel_tls_t * * This implements the get_remote_addr method for channel_tls_t; copy the @@ -673,6 +715,53 @@ channel_tls_matches_target_method(channel_t *chan, } /** + * Tell the upper layer how many bytes we have queued and not yet + * sent. + */ + +static size_t +channel_tls_num_bytes_queued_method(channel_t *chan) +{ + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + + tor_assert(tlschan); + tor_assert(tlschan->conn); + + return connection_get_outbuf_len(TO_CONN(tlschan->conn)); +} + +/** + * Tell the upper layer how many cells we can accept to write + * + * This implements the num_cells_writeable method for channel_tls_t; it + * returns an estimate of the number of cells we can accept with + * channel_tls_write_*_cell(). + */ + +static int +channel_tls_num_cells_writeable_method(channel_t *chan) +{ + size_t outbuf_len; + ssize_t n; + channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); + size_t cell_network_size; + + tor_assert(tlschan); + tor_assert(tlschan->conn); + + cell_network_size = get_cell_network_size(tlschan->conn->wide_circ_ids); + outbuf_len = connection_get_outbuf_len(TO_CONN(tlschan->conn)); + /* Get the number of cells */ + n = CEIL_DIV(OR_CONN_HIGHWATER - outbuf_len, cell_network_size); + if (n < 0) n = 0; +#if SIZEOF_SIZE_T > SIZEOF_INT + if (n > INT_MAX) n = INT_MAX; +#endif + + return (int)n; +} + +/** * Write a cell to a channel_tls_t * * This implements the write_cell method for channel_tls_t; given a @@ -847,18 +936,18 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, tor_assert(conn); tor_assert(conn->chan == chan); tor_assert(chan->conn == conn); - /* -Werror appeasement */ - tor_assert(old_state == old_state); + /* Shut the compiler up without triggering -Wtautological-compare */ + (void)old_state; base_chan = TLS_CHAN_TO_BASE(chan); - /* Make sure the base connection state makes sense - shouldn't be error, - * closed or listening. */ + /* Make sure the base connection state makes sense - shouldn't be error + * or closed. */ - tor_assert(base_chan->state == CHANNEL_STATE_OPENING || - base_chan->state == CHANNEL_STATE_OPEN || - base_chan->state == CHANNEL_STATE_MAINT || - base_chan->state == CHANNEL_STATE_CLOSING); + tor_assert(CHANNEL_IS_OPENING(base_chan) || + CHANNEL_IS_OPEN(base_chan) || + CHANNEL_IS_MAINT(base_chan) || + CHANNEL_IS_CLOSING(base_chan)); /* Did we just go to state open? */ if (state == OR_CONN_STATE_OPEN) { @@ -867,69 +956,21 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, * CHANNEL_STATE_MAINT on this. */ channel_change_state(base_chan, CHANNEL_STATE_OPEN); + /* We might have just become writeable; check and tell the scheduler */ + if (connection_or_num_cells_writeable(conn) > 0) { + scheduler_channel_wants_writes(base_chan); + } } else { /* * Not open, so from CHANNEL_STATE_OPEN we go to CHANNEL_STATE_MAINT, * otherwise no change. */ - if (base_chan->state == CHANNEL_STATE_OPEN) { + if (CHANNEL_IS_OPEN(base_chan)) { channel_change_state(base_chan, CHANNEL_STATE_MAINT); } } } -/** - * Flush cells from a channel_tls_t - * - * Try to flush up to about num_cells cells, and return how many we flushed. - */ - -ssize_t -channel_tls_flush_some_cells(channel_tls_t *chan, ssize_t num_cells) -{ - ssize_t flushed = 0; - - tor_assert(chan); - - if (flushed >= num_cells) goto done; - - /* - * If channel_tls_t ever buffers anything below the channel_t layer, flush - * that first here. - */ - - flushed += channel_flush_some_cells(TLS_CHAN_TO_BASE(chan), - num_cells - flushed); - - /* - * If channel_tls_t ever buffers anything below the channel_t layer, check - * how much we actually got and push it on down here. - */ - - done: - return flushed; -} - -/** - * Check if a channel_tls_t has anything to flush - * - * Return true if there is any more to flush on this channel (cells in queue - * or active circuits). - */ - -int -channel_tls_more_to_flush(channel_tls_t *chan) -{ - tor_assert(chan); - - /* - * If channel_tls_t ever buffers anything below channel_t, the - * check for that should go here first. - */ - - return channel_more_to_flush(TLS_CHAN_TO_BASE(chan)); -} - #ifdef KEEP_TIMING_STATS /** @@ -1423,6 +1464,8 @@ channel_tls_process_versions_cell(var_cell_t *cell, channel_tls_t *chan) return; } + rep_hist_note_negotiated_link_proto(highest_supported_version, started_here); + chan->conn->link_proto = highest_supported_version; chan->conn->handshake_state->received_versions = 1; diff --git a/src/or/channeltls.h b/src/or/channeltls.h index c872a09d79..507429420b 100644 --- a/src/or/channeltls.h +++ b/src/or/channeltls.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -40,8 +40,6 @@ channel_t * channel_tls_to_base(channel_tls_t *tlschan); channel_tls_t * channel_tls_from_base(channel_t *chan); /* Things for connection_or.c to call back into */ -ssize_t channel_tls_flush_some_cells(channel_tls_t *chan, ssize_t num_cells); -int channel_tls_more_to_flush(channel_tls_t *chan); void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn); void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, or_connection_t *conn, diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c index 51a75cf502..a0115cc6ec 100644 --- a/src/or/circpathbias.c +++ b/src/or/circpathbias.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" @@ -768,8 +768,8 @@ pathbias_send_usable_probe(circuit_t *circ) /* Can't probe if the channel isn't open */ if (circ->n_chan == NULL || - (circ->n_chan->state != CHANNEL_STATE_OPEN - && circ->n_chan->state != CHANNEL_STATE_MAINT)) { + (!CHANNEL_IS_OPEN(circ->n_chan) + && !CHANNEL_IS_MAINT(circ->n_chan))) { log_info(LD_CIRC, "Skipping pathbias probe for circuit %d: Channel is not open.", ocirc->global_identifier); @@ -1140,11 +1140,10 @@ pathbias_count_circs_in_states(entry_guard_t *guard, path_state_t from, path_state_t to) { - circuit_t *circ; int open_circuits = 0; /* Count currently open circuits. Give them the benefit of the doubt. */ - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { origin_circuit_t *ocirc = NULL; if (!CIRCUIT_IS_ORIGIN(circ) || /* didn't originate here */ circ->marked_for_close) /* already counted */ @@ -1167,6 +1166,7 @@ pathbias_count_circs_in_states(entry_guard_t *guard, open_circuits++; } } + SMARTLIST_FOREACH_END(circ); return open_circuits; } diff --git a/src/or/circpathbias.h b/src/or/circpathbias.h index c95d801a4b..9e973850d5 100644 --- a/src/or/circpathbias.h +++ b/src/or/circpathbias.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 897f90fe4c..946c002735 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,6 +14,7 @@ #include "or.h" #include "channel.h" #include "circpathbias.h" +#define CIRCUITBUILD_PRIVATE #include "circuitbuild.h" #include "circuitlist.h" #include "circuitstats.h" @@ -59,9 +60,7 @@ static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath); static int onion_extend_cpath(origin_circuit_t *circ); static int count_acceptable_nodes(smartlist_t *routers); static int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); -#ifdef CURVE25519_ENABLED static int circuits_can_use_ntor(void); -#endif /** This function tries to get a channel to the specified endpoint, * and then calls command_setup_channel() to give it the right @@ -368,7 +367,6 @@ circuit_rep_hist_note_result(origin_circuit_t *circ) } while (hop!=circ->cpath); } -#ifdef CURVE25519_ENABLED /** Return 1 iff at least one node in circ's cpath supports ntor. */ static int circuit_cpath_supports_ntor(const origin_circuit_t *circ) @@ -388,9 +386,6 @@ circuit_cpath_supports_ntor(const origin_circuit_t *circ) return 0; } -#else -#define circuit_cpath_supports_ntor(circ) 0 -#endif /** Pick all the entries in our cpath. Stop and return 0 when we're * happy, or return -1 if an error occurs. */ @@ -398,11 +393,7 @@ static int onion_populate_cpath(origin_circuit_t *circ) { int n_tries = 0; -#ifdef CURVE25519_ENABLED const int using_ntor = circuits_can_use_ntor(); -#else - const int using_ntor = 0; -#endif #define MAX_POPULATE_ATTEMPTS 32 @@ -560,9 +551,13 @@ circuit_handle_first_hop(origin_circuit_t *circ) * open and get them to send their create cells forward. * * Status is 1 if connect succeeded, or 0 if connect failed. + * + * Close_origin_circuits is 1 if we should close all the origin circuits + * through this channel, or 0 otherwise. (This happens when we want to retry + * an older guard.) */ void -circuit_n_chan_done(channel_t *chan, int status) +circuit_n_chan_done(channel_t *chan, int status, int close_origin_circuits) { smartlist_t *pending_circs; int err_reason = 0; @@ -600,6 +595,11 @@ circuit_n_chan_done(channel_t *chan, int status) circuit_mark_for_close(circ, END_CIRC_REASON_CHANNEL_CLOSED); continue; } + if (close_origin_circuits && CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC,"Channel deprecated for origin circs; closing circ."); + circuit_mark_for_close(circ, END_CIRC_REASON_CHANNEL_CLOSED); + continue; + } log_debug(LD_CIRC, "Found circ, sending create cell."); /* circuit_deliver_create_cell will set n_circ_id and add us to * chan_circuid_circuit_map, so we don't need to call @@ -681,7 +681,7 @@ circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, if (CIRCUIT_IS_ORIGIN(circ)) { /* Update began timestamp for circuits starting their first hop */ if (TO_ORIGIN_CIRCUIT(circ)->cpath->state == CPATH_STATE_CLOSED) { - if (circ->n_chan->state != CHANNEL_STATE_OPEN) { + if (!CHANNEL_IS_OPEN(circ->n_chan)) { log_warn(LD_CIRC, "Got first hop for a circuit without an opened channel. " "State: %s.", channel_state_to_string(circ->n_chan->state)); @@ -772,7 +772,6 @@ circuit_timeout_want_to_count_circ(origin_circuit_t *circ) && circ->build_state->desired_path_len == DEFAULT_ROUTE_LEN; } -#ifdef CURVE25519_ENABLED /** Return true if the ntor handshake is enabled in the configuration, or if * it's been set to "auto" in the configuration and it's enabled in the * consensus. */ @@ -784,7 +783,6 @@ circuits_can_use_ntor(void) return options->UseNTorHandshake; return networkstatus_get_param(NULL, "UseNTorHandshake", 0, 0, 1); } -#endif /** Decide whether to use a TAP or ntor handshake for connecting to <b>ei</b> * directly, and set *<b>cell_type_out</b> and *<b>handshake_type_out</b> @@ -794,7 +792,6 @@ circuit_pick_create_handshake(uint8_t *cell_type_out, uint16_t *handshake_type_out, const extend_info_t *ei) { -#ifdef CURVE25519_ENABLED if (!tor_mem_is_zero((const char*)ei->curve25519_onion_key.public_key, CURVE25519_PUBKEY_LEN) && circuits_can_use_ntor()) { @@ -802,9 +799,6 @@ circuit_pick_create_handshake(uint8_t *cell_type_out, *handshake_type_out = ONION_HANDSHAKE_TYPE_NTOR; return; } -#else - (void) ei; -#endif *cell_type_out = CELL_CREATE; *handshake_type_out = ONION_HANDSHAKE_TYPE_TAP; @@ -959,14 +953,18 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) circuit_rep_hist_note_result(circ); circuit_has_opened(circ); /* do other actions as necessary */ - if (!can_complete_circuit && !circ->build_state->onehop_tunnel) { + if (!have_completed_a_circuit() && !circ->build_state->onehop_tunnel) { const or_options_t *options = get_options(); - can_complete_circuit=1; + note_that_we_completed_a_circuit(); /* FFFF Log a count of known routers here */ log_notice(LD_GENERAL, "Tor has successfully opened a circuit. " "Looks like client functionality is working."); - control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0); + if (control_event_bootstrap(BOOTSTRAP_STATUS_DONE, 0) == 0) { + log_notice(LD_GENERAL, + "Tor has successfully opened a circuit. " + "Looks like client functionality is working."); + } control_event_client_status(LOG_NOTICE, "CIRCUIT_ESTABLISHED"); clear_broken_connection_map(1); if (server_mode(options) && !check_whether_orport_reachable()) { @@ -1049,7 +1047,8 @@ circuit_note_clock_jumped(int seconds_elapsed) seconds_elapsed >=0 ? "forward" : "backward"); control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%d", seconds_elapsed); - can_complete_circuit=0; /* so it'll log when it works again */ + /* so we log when it works again */ + note_that_we_maybe_cant_complete_circuits(); control_event_client_status(severity, "CIRCUIT_NOT_ESTABLISHED REASON=%s", "CLOCK_JUMPED"); circuit_mark_all_unused_circs(); @@ -1256,8 +1255,10 @@ circuit_finish_handshake(origin_circuit_t *circ, crypt_path_t *hop; int rv; - if ((rv = pathbias_count_build_attempt(circ)) < 0) + if ((rv = pathbias_count_build_attempt(circ)) < 0) { + log_warn(LD_CIRC, "pathbias_count_build_attempt failed: %d", rv); return rv; + } if (circ->cpath->state == CPATH_STATE_AWAITING_KEYS) { hop = circ->cpath; @@ -1271,12 +1272,15 @@ circuit_finish_handshake(origin_circuit_t *circ, tor_assert(hop->state == CPATH_STATE_AWAITING_KEYS); { + const char *msg = NULL; if (onion_skin_client_handshake(hop->handshake_state.tag, &hop->handshake_state, reply->reply, reply->handshake_len, (uint8_t*)keys, sizeof(keys), - (uint8_t*)hop->rend_circ_nonce) < 0) { - log_warn(LD_CIRC,"onion_skin_client_handshake failed."); + (uint8_t*)hop->rend_circ_nonce, + &msg) < 0) { + if (msg) + log_warn(LD_CIRC,"onion_skin_client_handshake failed: %s", msg); return -END_CIRC_REASON_TORPROTOCOL; } } @@ -1392,8 +1396,10 @@ onionskin_answer(or_circuit_t *circ, log_debug(LD_CIRC,"Finished sending '%s' cell.", circ->is_first_hop ? "created_fast" : "created"); - if (!channel_is_local(circ->p_chan) && - !channel_is_outgoing(circ->p_chan)) { + /* Ignore the local bit when testing - many test networks run on local + * addresses */ + if ((!channel_is_local(circ->p_chan) || get_options()->TestingTorNetwork) + && !channel_is_outgoing(circ->p_chan)) { /* record that we could process create cells from a non-local conn * that we didn't initiate; presumably this means that create cells * can reach us too. */ @@ -1564,7 +1570,7 @@ choose_good_exit_server_general(int need_uptime, int need_capacity) * -1 means "Don't use this router at all." */ the_nodes = nodelist_get_list(); - n_supported = tor_malloc(sizeof(int)*smartlist_len(the_nodes)); + n_supported = tor_calloc(smartlist_len(the_nodes), sizeof(int)); SMARTLIST_FOREACH_BEGIN(the_nodes, const node_t *, node) { const int i = node_sl_idx; if (router_digest_is_me(node->identity)) { @@ -1735,6 +1741,83 @@ choose_good_exit_server_general(int need_uptime, int need_capacity) return NULL; } +#if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS) +/* The config option Tor2webRendezvousPoints has been set and we need + * to pick an RP out of that set. Make sure that the RP we choose is + * alive, and return it. Return NULL if no usable RP could be found in + * Tor2webRendezvousPoints. */ +STATIC const node_t * +pick_tor2web_rendezvous_node(router_crn_flags_t flags, + const or_options_t *options) +{ + const node_t *rp_node = NULL; + const int allow_invalid = (flags & CRN_ALLOW_INVALID) != 0; + const int need_desc = (flags & CRN_NEED_DESC) != 0; + + smartlist_t *whitelisted_live_rps = smartlist_new(); + smartlist_t *all_live_nodes = smartlist_new(); + + tor_assert(options->Tor2webRendezvousPoints); + + /* Add all running nodes to all_live_nodes */ + router_add_running_nodes_to_smartlist(all_live_nodes, + allow_invalid, + 0, 0, 0, + need_desc); + + /* Filter all_live_nodes to only add live *and* whitelisted RPs to + * the list whitelisted_live_rps. */ + SMARTLIST_FOREACH_BEGIN(all_live_nodes, node_t *, live_node) { + if (routerset_contains_node(options->Tor2webRendezvousPoints, live_node)) { + smartlist_add(whitelisted_live_rps, live_node); + } + } SMARTLIST_FOREACH_END(live_node); + + /* Honor ExcludeNodes */ + if (options->ExcludeNodes) { + routerset_subtract_nodes(whitelisted_live_rps, options->ExcludeNodes); + } + + /* Now pick randomly amongst the whitelisted RPs. No need to waste time + doing bandwidth load balancing, for most use cases + 'whitelisted_live_rps' contains a single OR anyway. */ + rp_node = smartlist_choose(whitelisted_live_rps); + + if (!rp_node) { + log_warn(LD_REND, "Could not find a Rendezvous Point that suits " + "the purposes of Tor2webRendezvousPoints. Choosing random one."); + } + + smartlist_free(whitelisted_live_rps); + smartlist_free(all_live_nodes); + + return rp_node; +} +#endif + +/* Pick a Rendezvous Point for our HS circuits according to <b>flags</b>. */ +static const node_t * +pick_rendezvous_node(router_crn_flags_t flags) +{ + const or_options_t *options = get_options(); + + if (options->AllowInvalid_ & ALLOW_INVALID_RENDEZVOUS) + flags |= CRN_ALLOW_INVALID; + +#ifdef ENABLE_TOR2WEB_MODE + /* The user wants us to pick specific RPs. */ + if (options->Tor2webRendezvousPoints) { + const node_t *tor2web_rp = pick_tor2web_rendezvous_node(flags, options); + if (tor2web_rp) { + return tor2web_rp; + } + /* Else, if no tor2web RP was found, fall back to choosing a random node */ + } +#endif + + return router_choose_random_node(NULL, options->ExcludeNodes, flags); +} + /** Return a pointer to a suitable router to be the exit node for the * circuit of purpose <b>purpose</b> that we're about to build (or NULL * if no router is suitable). @@ -1765,9 +1848,13 @@ choose_good_exit_server(uint8_t purpose, else return choose_good_exit_server_general(need_uptime,need_capacity); case CIRCUIT_PURPOSE_C_ESTABLISH_REND: - if (options->AllowInvalid_ & ALLOW_INVALID_RENDEZVOUS) - flags |= CRN_ALLOW_INVALID; - return router_choose_random_node(NULL, options->ExcludeNodes, flags); + { + /* Pick a new RP */ + const node_t *rendezvous_node = pick_rendezvous_node(flags); + log_info(LD_REND, "Picked new RP: %s", + safe_str_client(node_describe(rendezvous_node))); + return rendezvous_node; + } } log_warn(LD_BUG,"Unhandled purpose %d", purpose); tor_fragile_assert(); @@ -1877,7 +1964,7 @@ onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit) choose_good_exit_server(circ->base_.purpose, state->need_uptime, state->need_capacity, state->is_internal); if (!node) { - log_warn(LD_CIRC,"failed to choose an exit server"); + log_warn(LD_CIRC,"Failed to choose an exit server"); return -1; } exit = extend_info_from_node(node, 0); @@ -2004,7 +2091,8 @@ choose_good_middle_server(uint8_t purpose, tor_assert(CIRCUIT_PURPOSE_MIN_ <= purpose && purpose <= CIRCUIT_PURPOSE_MAX_); - log_debug(LD_CIRC, "Contemplating intermediate hop: random choice."); + log_debug(LD_CIRC, "Contemplating intermediate hop %d: random choice.", + cur_len); excluded = smartlist_new(); if ((r = build_state_get_exit_node(state))) { nodelist_add_node_and_family(excluded, r); @@ -2066,9 +2154,18 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) smartlist_add(excluded, (void*)node); }); } - /* and exclude current entry guards and their families, if applicable */ + /* and exclude current entry guards and their families, + * unless we're in a test network, and excluding guards + * would exclude all nodes (i.e. we're in an incredibly small tor network, + * or we're using TestingAuthVoteGuard *). + * This is an incomplete fix, but is no worse than the previous behaviour, + * and only applies to minimal, testing tor networks + * (so it's no less secure) */ /*XXXX025 use the using_as_guard flag to accomplish this.*/ - if (options->UseEntryGuards) { + if (options->UseEntryGuards + && (!options->TestingTorNetwork || + smartlist_len(nodelist_get_list()) > smartlist_len(get_entry_guards()) + )) { SMARTLIST_FOREACH(get_entry_guards(), const entry_guard_t *, entry, { if ((node = node_get_by_id(entry->identity))) { @@ -2198,13 +2295,9 @@ extend_info_new(const char *nickname, const char *digest, strlcpy(info->nickname, nickname, sizeof(info->nickname)); if (onion_key) info->onion_key = crypto_pk_dup_key(onion_key); -#ifdef CURVE25519_ENABLED if (curve25519_key) memcpy(&info->curve25519_onion_key, curve25519_key, sizeof(curve25519_public_key_t)); -#else - (void)curve25519_key; -#endif tor_addr_copy(&info->addr, addr); info->port = port; return info; diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index 71caea94ed..01563791b7 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,7 +22,8 @@ origin_circuit_t *circuit_establish_circuit(uint8_t purpose, extend_info_t *exit, int flags); int circuit_handle_first_hop(origin_circuit_t *circ); -void circuit_n_chan_done(channel_t *chan, int status); +void circuit_n_chan_done(channel_t *chan, int status, + int close_origin_circuits); int inform_testing_reachability(void); int circuit_timeout_want_to_count_circ(origin_circuit_t *circ); int circuit_send_next_onion_skin(origin_circuit_t *circ); @@ -60,6 +61,11 @@ const node_t *choose_good_entry_server(uint8_t purpose, #ifdef CIRCUITBUILD_PRIVATE STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); +#if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS) +STATIC const node_t *pick_tor2web_rendezvous_node(router_crn_flags_t flags, + const or_options_t *options); +#endif + #endif #endif diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index f3a83503ef..bf7f8daca7 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1,7 +1,7 @@ /* Copyright 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -21,6 +21,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "main.h" #include "networkstatus.h" #include "nodelist.h" #include "onion.h" @@ -38,8 +39,7 @@ /********* START VARIABLES **********/ /** A global list of all circuits at this hop. */ -struct global_circuitlist_s global_circuitlist = - TOR_LIST_HEAD_INITIALIZER(global_circuitlist); +static smartlist_t *global_circuitlist = NULL; /** A list of all the circuits in CIRCUIT_STATE_CHAN_WAIT. */ static smartlist_t *circuits_pending_chans = NULL; @@ -94,9 +94,9 @@ static HT_HEAD(chan_circid_map, chan_circid_circuit_map_t) chan_circid_map = HT_INITIALIZER(); HT_PROTOTYPE(chan_circid_map, chan_circid_circuit_map_t, node, chan_circid_entry_hash_, chan_circid_entries_eq_) -HT_GENERATE(chan_circid_map, chan_circid_circuit_map_t, node, - chan_circid_entry_hash_, chan_circid_entries_eq_, 0.6, - malloc, realloc, free) +HT_GENERATE2(chan_circid_map, chan_circid_circuit_map_t, node, + chan_circid_entry_hash_, chan_circid_entries_eq_, 0.6, + tor_reallocarray_, tor_free_) /** The most recently returned entry from circuit_get_by_circid_chan; * used to improve performance when many cells arrive in a row from the @@ -302,8 +302,8 @@ channel_note_destroy_pending(channel_t *chan, circid_t id) /** Called to indicate that a DESTROY is no longer pending on <b>chan</b> with * circuit ID <b>id</b> -- typically, because it has been sent. */ -void -channel_note_destroy_not_pending(channel_t *chan, circid_t id) +MOCK_IMPL(void, channel_note_destroy_not_pending, + (channel_t *chan, circid_t id)) { circuit_t *circ = circuit_get_by_circid_channel_even_if_marked(id,chan); if (circ) { @@ -451,17 +451,25 @@ circuit_count_pending_on_channel(channel_t *chan) void circuit_close_all_marked(void) { - circuit_t *circ, *tmp; - TOR_LIST_FOREACH_SAFE(circ, &global_circuitlist, head, tmp) - if (circ->marked_for_close) + smartlist_t *lst = circuit_get_global_list(); + SMARTLIST_FOREACH_BEGIN(lst, circuit_t *, circ) { + /* Fix up index if SMARTLIST_DEL_CURRENT just moved this one. */ + circ->global_circuitlist_idx = circ_sl_idx; + if (circ->marked_for_close) { + circ->global_circuitlist_idx = -1; circuit_free(circ); + SMARTLIST_DEL_CURRENT(lst, circ); + } + } SMARTLIST_FOREACH_END(circ); } /** Return the head of the global linked list of circuits. */ -MOCK_IMPL(struct global_circuitlist_s *, +MOCK_IMPL(smartlist_t *, circuit_get_global_list,(void)) { - return &global_circuitlist; + if (NULL == global_circuitlist) + global_circuitlist = smartlist_new(); + return global_circuitlist; } /** Function to make circ-\>state human-readable */ @@ -678,7 +686,8 @@ init_circuit_base(circuit_t *circ) circ->deliver_window = CIRCWINDOW_START; cell_queue_init(&circ->n_chan_cells); - TOR_LIST_INSERT_HEAD(&global_circuitlist, circ, head); + smartlist_add(circuit_get_global_list(), circ); + circ->global_circuitlist_idx = smartlist_len(circuit_get_global_list()) - 1; } /** Allocate space for a new circuit, initializing with <b>p_circ_id</b> @@ -736,6 +745,7 @@ circuit_free(circuit_t *circ) { void *mem; size_t memlen; + int should_free = 1; if (!circ) return; @@ -775,6 +785,8 @@ circuit_free(circuit_t *circ) memlen = sizeof(or_circuit_t); tor_assert(circ->magic == OR_CIRCUIT_MAGIC); + should_free = (ocirc->workqueue_entry == NULL); + crypto_cipher_free(ocirc->p_crypto); crypto_digest_free(ocirc->p_digest); crypto_cipher_free(ocirc->n_crypto); @@ -799,7 +811,16 @@ circuit_free(circuit_t *circ) extend_info_free(circ->n_hop); tor_free(circ->n_chan_create_cell); - TOR_LIST_REMOVE(circ, head); + if (circ->global_circuitlist_idx != -1) { + int idx = circ->global_circuitlist_idx; + circuit_t *c2 = smartlist_get(global_circuitlist, idx); + tor_assert(c2 == circ); + smartlist_del(global_circuitlist, idx); + if (idx < smartlist_len(global_circuitlist)) { + c2 = smartlist_get(global_circuitlist, idx); + c2->global_circuitlist_idx = idx; + } + } /* Remove from map. */ circuit_set_n_circid_chan(circ, 0, NULL); @@ -808,8 +829,18 @@ circuit_free(circuit_t *circ) * "active" checks will be violated. */ cell_queue_clear(&circ->n_chan_cells); - memwipe(mem, 0xAA, memlen); /* poison memory */ - tor_free(mem); + if (should_free) { + memwipe(mem, 0xAA, memlen); /* poison memory */ + tor_free(mem); + } else { + /* If we made it here, this is an or_circuit_t that still has a pending + * cpuworker request which we weren't able to cancel. Instead, set up + * the magic value so that when the reply comes back, we'll know to discard + * the reply and free this structure. + */ + memwipe(mem, 0xAA, memlen); + circ->magic = DEAD_CIRCUIT_MAGIC; + } } /** Deallocate the linked list circ-><b>cpath</b>, and remove the cpath from @@ -841,9 +872,9 @@ circuit_clear_cpath(origin_circuit_t *circ) void circuit_free_all(void) { - circuit_t *tmp, *tmp2; + smartlist_t *lst = circuit_get_global_list(); - TOR_LIST_FOREACH_SAFE(tmp, &global_circuitlist, head, tmp2) { + SMARTLIST_FOREACH_BEGIN(lst, circuit_t *, tmp) { if (! CIRCUIT_IS_ORIGIN(tmp)) { or_circuit_t *or_circ = TO_OR_CIRCUIT(tmp); while (or_circ->resolving_streams) { @@ -853,8 +884,13 @@ circuit_free_all(void) or_circ->resolving_streams = next_conn; } } + tmp->global_circuitlist_idx = -1; circuit_free(tmp); - } + SMARTLIST_DEL_CURRENT(lst, tmp); + } SMARTLIST_FOREACH_END(tmp); + + smartlist_free(lst); + global_circuitlist = NULL; smartlist_free(circuits_pending_chans); circuits_pending_chans = NULL; @@ -932,10 +968,9 @@ circuit_dump_conn_details(int severity, void circuit_dump_by_conn(connection_t *conn, int severity) { - circuit_t *circ; edge_connection_t *tmpconn; - TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { circid_t n_circ_id = circ->n_circ_id, p_circ_id = 0; if (circ->marked_for_close) { @@ -966,6 +1001,7 @@ circuit_dump_by_conn(connection_t *conn, int severity) } } } + SMARTLIST_FOREACH_END(circ); } /** Return the circuit whose global ID is <b>id</b>, or NULL if no @@ -973,8 +1009,7 @@ circuit_dump_by_conn(connection_t *conn, int severity) origin_circuit_t * circuit_get_by_global_id(uint32_t id) { - circuit_t *circ; - TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (CIRCUIT_IS_ORIGIN(circ) && TO_ORIGIN_CIRCUIT(circ)->global_identifier == id) { if (circ->marked_for_close) @@ -983,6 +1018,7 @@ circuit_get_by_global_id(uint32_t id) return TO_ORIGIN_CIRCUIT(circ); } } + SMARTLIST_FOREACH_END(circ); return NULL; } @@ -1151,17 +1187,17 @@ circuit_unlink_all_from_channel(channel_t *chan, int reason) #ifdef DEBUG_CIRCUIT_UNLINK_ALL { - circuit_t *circ; smartlist_t *detached_2 = smartlist_new(); int mismatch = 0, badlen = 0; - TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (circ->n_chan == chan || (!CIRCUIT_IS_ORIGIN(circ) && TO_OR_CIRCUIT(circ)->p_chan == chan)) { smartlist_add(detached_2, circ); } } + SMARTLIST_FOREACH_END(circ); if (smartlist_len(detached) != smartlist_len(detached_2)) { log_warn(LD_BUG, "List of detached circuits had the wrong length! " @@ -1235,8 +1271,7 @@ circuit_unlink_all_from_channel(channel_t *chan, int reason) origin_circuit_t * circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data) { - circuit_t *circ; - TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->marked_for_close && circ->purpose == CIRCUIT_PURPOSE_C_REND_READY) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); @@ -1249,6 +1284,7 @@ circuit_get_ready_rend_circ_by_rend_data(const rend_data_t *rend_data) return ocirc; } } + SMARTLIST_FOREACH_END(circ); return NULL; } @@ -1261,14 +1297,17 @@ origin_circuit_t * circuit_get_next_by_pk_and_purpose(origin_circuit_t *start, const char *digest, uint8_t purpose) { - circuit_t *circ; + int idx; + smartlist_t *lst = circuit_get_global_list(); tor_assert(CIRCUIT_PURPOSE_IS_ORIGIN(purpose)); if (start == NULL) - circ = TOR_LIST_FIRST(&global_circuitlist); + idx = 0; else - circ = TOR_LIST_NEXT(TO_CIRCUIT(start), head); + idx = TO_CIRCUIT(start)->global_circuitlist_idx + 1; + + for ( ; idx < smartlist_len(lst); ++idx) { + circuit_t *circ = smartlist_get(lst, idx); - for ( ; circ; circ = TOR_LIST_NEXT(circ, head)) { if (circ->marked_for_close) continue; if (circ->purpose != purpose) @@ -1469,7 +1508,6 @@ origin_circuit_t * circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int flags) { - circuit_t *circ_; origin_circuit_t *best=NULL; int need_uptime = (flags & CIRCLAUNCH_NEED_UPTIME) != 0; int need_capacity = (flags & CIRCLAUNCH_NEED_CAPACITY) != 0; @@ -1485,7 +1523,7 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, "capacity %d, internal %d", purpose, need_uptime, need_capacity, internal); - TOR_LIST_FOREACH(circ_, &global_circuitlist, head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) { if (CIRCUIT_IS_ORIGIN(circ_) && circ_->state == CIRCUIT_STATE_OPEN && !circ_->marked_for_close && @@ -1507,7 +1545,7 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, do { const node_t *ri2; if (tor_memeq(hop->extend_info->identity_digest, - info->identity_digest, DIGEST_LEN)) + info->identity_digest, DIGEST_LEN)) goto next; if (ri1 && (ri2 = node_get_by_id(hop->extend_info->identity_digest)) @@ -1535,6 +1573,7 @@ circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, } } } + SMARTLIST_FOREACH_END(circ_); return best; } @@ -1574,13 +1613,13 @@ circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum) void circuit_mark_all_unused_circs(void) { - circuit_t *circ; - TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && !circ->timestamp_dirty) circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); } + SMARTLIST_FOREACH_END(circ); } /** Go through the circuitlist; for each circuit that starts at us @@ -1593,14 +1632,14 @@ circuit_mark_all_unused_circs(void) void circuit_mark_all_dirty_circs_as_unusable(void) { - circuit_t *circ; - TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->timestamp_dirty) { mark_circuit_unusable_for_new_conns(TO_ORIGIN_CIRCUIT(circ)); } } + SMARTLIST_FOREACH_END(circ); } /** Mark <b>circ</b> to be closed next time we call @@ -1693,36 +1732,40 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line, tor_assert(circ->state == CIRCUIT_STATE_OPEN); tor_assert(ocirc->build_state->chosen_exit); tor_assert(ocirc->rend_data); - /* treat this like getting a nack from it */ - log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s", - safe_str_client(ocirc->rend_data->onion_address), - safe_str_client(build_state_get_exit_nickname(ocirc->build_state)), - timed_out ? "Recording timeout." : "Removing from descriptor."); - rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit, - ocirc->rend_data, - timed_out ? - INTRO_POINT_FAILURE_TIMEOUT : - INTRO_POINT_FAILURE_GENERIC); + if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT) { + /* treat this like getting a nack from it */ + log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s", + safe_str_client(ocirc->rend_data->onion_address), + safe_str_client(build_state_get_exit_nickname(ocirc->build_state)), + timed_out ? "Recording timeout." : "Removing from descriptor."); + rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit, + ocirc->rend_data, + timed_out ? + INTRO_POINT_FAILURE_TIMEOUT : + INTRO_POINT_FAILURE_GENERIC); + } } else if (circ->purpose == CIRCUIT_PURPOSE_C_INTRODUCING && reason != END_CIRC_REASON_TIMEOUT) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); if (ocirc->build_state->chosen_exit && ocirc->rend_data) { - log_info(LD_REND, "Failed intro circ %s to %s " - "(building circuit to intro point). " - "Marking intro point as possibly unreachable.", - safe_str_client(ocirc->rend_data->onion_address), - safe_str_client(build_state_get_exit_nickname(ocirc->build_state))); - rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit, - ocirc->rend_data, - INTRO_POINT_FAILURE_UNREACHABLE); + if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT) { + log_info(LD_REND, "Failed intro circ %s to %s " + "(building circuit to intro point). " + "Marking intro point as possibly unreachable.", + safe_str_client(ocirc->rend_data->onion_address), + safe_str_client(build_state_get_exit_nickname( + ocirc->build_state))); + rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit, + ocirc->rend_data, + INTRO_POINT_FAILURE_UNREACHABLE); + } } } + if (circ->n_chan) { circuit_clear_cell_queue(circ, circ->n_chan); /* Only send destroy if the channel isn't closing anyway */ - if (!(circ->n_chan->state == CHANNEL_STATE_CLOSING || - circ->n_chan->state == CHANNEL_STATE_CLOSED || - circ->n_chan->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(circ->n_chan)) { channel_send_destroy(circ->n_circ_id, circ->n_chan, reason); } circuitmux_detach_circuit(circ->n_chan->cmux, circ); @@ -1754,9 +1797,7 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line, if (or_circ->p_chan) { circuit_clear_cell_queue(circ, or_circ->p_chan); /* Only send destroy if the channel isn't closing anyway */ - if (!(or_circ->p_chan->state == CHANNEL_STATE_CLOSING || - or_circ->p_chan->state == CHANNEL_STATE_CLOSED || - or_circ->p_chan->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(or_circ->p_chan)) { channel_send_destroy(or_circ->p_circ_id, or_circ->p_chan, reason); } circuitmux_detach_circuit(or_circ->p_chan->cmux, circ); @@ -1799,6 +1840,29 @@ marked_circuit_free_cells(circuit_t *circ) cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_chan_cells); } +static size_t +single_conn_free_bytes(connection_t *conn) +{ + size_t result = 0; + if (conn->inbuf) { + result += buf_allocation(conn->inbuf); + buf_clear(conn->inbuf); + } + if (conn->outbuf) { + result += buf_allocation(conn->outbuf); + buf_clear(conn->outbuf); + } + if (conn->type == CONN_TYPE_DIR) { + dir_connection_t *dir_conn = TO_DIR_CONN(conn); + if (dir_conn->zlib_state) { + result += tor_zlib_state_size(dir_conn->zlib_state); + tor_zlib_free(dir_conn->zlib_state); + dir_conn->zlib_state = NULL; + } + } + return result; +} + /** Aggressively free buffer contents on all the buffers of all streams in the * list starting at <b>stream</b>. Return the number of bytes recovered. */ static size_t @@ -1807,13 +1871,9 @@ marked_circuit_streams_free_bytes(edge_connection_t *stream) size_t result = 0; for ( ; stream; stream = stream->next_stream) { connection_t *conn = TO_CONN(stream); - if (conn->inbuf) { - result += buf_allocation(conn->inbuf); - buf_clear(conn->inbuf); - } - if (conn->outbuf) { - result += buf_allocation(conn->outbuf); - buf_clear(conn->outbuf); + result += single_conn_free_bytes(conn); + if (conn->linked_conn) { + result += single_conn_free_bytes(conn->linked_conn); } } return result; @@ -1871,6 +1931,28 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) return age; } +/** Return the age in milliseconds of the oldest buffer chunk on <b>conn</b>, + * where age is taken in milliseconds before the time <b>now</b> (in truncated + * milliseconds since the epoch). If the connection has no data, treat + * it as having age zero. + **/ +static uint32_t +conn_get_buffer_age(const connection_t *conn, uint32_t now) +{ + uint32_t age = 0, age2; + if (conn->outbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); + if (age2 > age) + age = age2; + } + if (conn->inbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); + if (age2 > age) + age = age2; + } + return age; +} + /** Return the age in milliseconds of the oldest buffer chunk on any stream in * the linked list <b>stream</b>, where age is taken in milliseconds before * the time <b>now</b> (in truncated milliseconds since the epoch). */ @@ -1880,18 +1962,15 @@ circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) uint32_t age = 0, age2; for (; stream; stream = stream->next_stream) { const connection_t *conn = TO_CONN(stream); - if (conn->outbuf) { - age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); - if (age2 > age) - age = age2; - } - if (conn->inbuf) { - age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); + age2 = conn_get_buffer_age(conn, now); + if (age2 > age) + age = age2; + if (conn->linked_conn) { + age2 = conn_get_buffer_age(conn->linked_conn, now); if (age2 > age) age = age2; } } - return age; } @@ -1942,6 +2021,26 @@ circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) return -1; } +static uint32_t now_ms_for_buf_cmp; + +/** Helper to sort a list of circuit_t by age of oldest item, in descending + * order. */ +static int +conns_compare_by_buffer_age_(const void **a_, const void **b_) +{ + const connection_t *a = *a_; + const connection_t *b = *b_; + time_t age_a = conn_get_buffer_age(a, now_ms_for_buf_cmp); + time_t age_b = conn_get_buffer_age(b, now_ms_for_buf_cmp); + + if (age_a < age_b) + return 1; + else if (age_a == age_b) + return 0; + else + return -1; +} + #define FRACTION_OF_DATA_TO_RETAIN_ON_OOM 0.90 /** We're out of memory for cells, having allocated <b>current_allocation</b> @@ -1950,12 +2049,13 @@ circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) void circuits_handle_oom(size_t current_allocation) { - /* Let's hope there's enough slack space for this allocation here... */ - smartlist_t *circlist = smartlist_new(); - circuit_t *circ; + smartlist_t *circlist; + smartlist_t *connection_array = get_connection_array(); + int conn_idx; size_t mem_to_recover; size_t mem_recovered=0; int n_circuits_killed=0; + int n_dirconns_killed=0; struct timeval now; uint32_t now_ms; log_notice(LD_GENERAL, "We're low on memory. Killing circuits with " @@ -1963,17 +2063,6 @@ circuits_handle_oom(size_t current_allocation) "MaxMemInQueues.)"); { - const size_t recovered = buf_shrink_freelists(1); - if (recovered >= current_allocation) { - log_warn(LD_BUG, "We somehow recovered more memory from freelists " - "than we thought we had allocated"); - current_allocation = 0; - } else { - current_allocation -= recovered; - } - } - - { size_t mem_target = (size_t)(get_options()->MaxMemInQueues * FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) @@ -1984,22 +2073,61 @@ circuits_handle_oom(size_t current_allocation) tor_gettimeofday_cached_monotonic(&now); now_ms = (uint32_t)tv_to_msec(&now); - /* This algorithm itself assumes that you've got enough memory slack - * to actually run it. */ - TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + circlist = circuit_get_global_list(); + SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { circ->age_tmp = circuit_max_queued_item_age(circ, now_ms); - smartlist_add(circlist, circ); - } + } SMARTLIST_FOREACH_END(circ); /* This is O(n log n); there are faster algorithms we could use instead. * Let's hope this doesn't happen enough to be in the critical path. */ smartlist_sort(circlist, circuits_compare_by_oldest_queued_item_); - /* Okay, now the worst circuits are at the front of the list. Let's mark - * them, and reclaim their storage aggressively. */ + /* Fix up the indices before we run into trouble */ + SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { + circ->global_circuitlist_idx = circ_sl_idx; + } SMARTLIST_FOREACH_END(circ); + + /* Now sort the connection array ... */ + now_ms_for_buf_cmp = now_ms; + smartlist_sort(connection_array, conns_compare_by_buffer_age_); + now_ms_for_buf_cmp = 0; + + /* Fix up the connection array to its new order. */ + SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) { + conn->conn_array_index = conn_sl_idx; + } SMARTLIST_FOREACH_END(conn); + + /* Okay, now the worst circuits and connections are at the front of their + * respective lists. Let's mark them, and reclaim their storage + * aggressively. */ + conn_idx = 0; SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { - size_t n = n_cells_in_circ_queues(circ); + size_t n; size_t freed; + + /* Free storage in any non-linked directory connections that have buffered + * data older than this circuit. */ + while (conn_idx < smartlist_len(connection_array)) { + connection_t *conn = smartlist_get(connection_array, conn_idx); + uint32_t conn_age = conn_get_buffer_age(conn, now_ms); + if (conn_age < circ->age_tmp) { + break; + } + if (conn->type == CONN_TYPE_DIR && conn->linked_conn == NULL) { + if (!conn->marked_for_close) + connection_mark_for_close(conn); + mem_recovered += single_conn_free_bytes(conn); + + ++n_dirconns_killed; + + if (mem_recovered >= mem_to_recover) + goto done_recovering_mem; + } + ++conn_idx; + } + + /* Now, kill the circuit. */ + n = n_cells_in_circ_queues(circ); if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } @@ -2012,22 +2140,18 @@ circuits_handle_oom(size_t current_allocation) mem_recovered += freed; if (mem_recovered >= mem_to_recover) - break; + goto done_recovering_mem; } SMARTLIST_FOREACH_END(circ); -#ifdef ENABLE_MEMPOOLS - clean_cell_pool(); /* In case this helps. */ -#endif /* ENABLE_MEMPOOLS */ - buf_shrink_freelists(1); /* This is necessary to actually release buffer - chunks. */ + done_recovering_mem: log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; " - "%d circuits remain alive.", + "%d circuits remain alive. Also killed %d non-linked directory " + "connections.", U64_PRINTF_ARG(mem_recovered), n_circuits_killed, - smartlist_len(circlist) - n_circuits_killed); - - smartlist_free(circlist); + smartlist_len(circlist) - n_circuits_killed, + n_dirconns_killed); } /** Verify that cpath layer <b>cp</b> has all of its invariants diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h index d48d7c3963..4e600da57d 100644 --- a/src/or/circuitlist.h +++ b/src/or/circuitlist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,9 +14,7 @@ #include "testsupport.h" -TOR_LIST_HEAD(global_circuitlist_s, circuit_t); - -MOCK_DECL(struct global_circuitlist_s*, circuit_get_global_list, (void)); +MOCK_DECL(smartlist_t *, circuit_get_global_list, (void)); const char *circuit_state_to_string(int state); const char *circuit_purpose_to_controller_string(uint8_t purpose); const char *circuit_purpose_to_controller_hs_state_string(uint8_t purpose); @@ -74,7 +72,8 @@ void circuit_free_all(void); void circuits_handle_oom(size_t current_allocation); void channel_note_destroy_pending(channel_t *chan, circid_t id); -void channel_note_destroy_not_pending(channel_t *chan, circid_t id); +MOCK_DECL(void, channel_note_destroy_not_pending, + (channel_t *chan, circid_t id)); #ifdef CIRCUITLIST_PRIVATE STATIC void circuit_free(circuit_t *circ); diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c index e4571ff944..a77bffac90 100644 --- a/src/or/circuitmux.c +++ b/src/or/circuitmux.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -363,9 +363,9 @@ HT_HEAD(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t); /* Emit a bunch of hash table stuff */ HT_PROTOTYPE(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node, chanid_circid_entry_hash, chanid_circid_entries_eq); -HT_GENERATE(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node, - chanid_circid_entry_hash, chanid_circid_entries_eq, 0.6, - malloc, realloc, free); +HT_GENERATE2(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node, + chanid_circid_entry_hash, chanid_circid_entries_eq, 0.6, + tor_reallocarray_, tor_free_) /* * Circuitmux alloc/free functions @@ -621,8 +621,8 @@ circuitmux_clear_policy(circuitmux_t *cmux) * Return the policy currently installed on a circuitmux_t */ -const circuitmux_policy_t * -circuitmux_get_policy(circuitmux_t *cmux) +MOCK_IMPL(const circuitmux_policy_t *, +circuitmux_get_policy, (circuitmux_t *cmux)) { tor_assert(cmux); @@ -896,8 +896,8 @@ circuitmux_num_cells_for_circuit(circuitmux_t *cmux, circuit_t *circ) * Query total number of available cells on a circuitmux */ -unsigned int -circuitmux_num_cells(circuitmux_t *cmux) +MOCK_IMPL(unsigned int, +circuitmux_num_cells, (circuitmux_t *cmux)) { tor_assert(cmux); @@ -1951,3 +1951,51 @@ circuitmux_count_queued_destroy_cells(const channel_t *chan, return n_destroy_cells; } +/** + * Compare cmuxes to see which is more preferred; return < 0 if + * cmux_1 has higher priority (i.e., cmux_1 < cmux_2 in the scheduler's + * sort order), > 0 if cmux_2 has higher priority, or 0 if they are + * equally preferred. + * + * If the cmuxes have different cmux policies or the policy does not + * support the cmp_cmux method, return 0. + */ + +MOCK_IMPL(int, +circuitmux_compare_muxes, (circuitmux_t *cmux_1, circuitmux_t *cmux_2)) +{ + const circuitmux_policy_t *policy; + + tor_assert(cmux_1); + tor_assert(cmux_2); + + if (cmux_1 == cmux_2) { + /* Equivalent because they're the same cmux */ + return 0; + } + + if (cmux_1->policy && cmux_2->policy) { + if (cmux_1->policy == cmux_2->policy) { + policy = cmux_1->policy; + + if (policy->cmp_cmux) { + /* Okay, we can compare! */ + return policy->cmp_cmux(cmux_1, cmux_1->policy_data, + cmux_2, cmux_2->policy_data); + } else { + /* + * Equivalent because the policy doesn't know how to compare between + * muxes. + */ + return 0; + } + } else { + /* Equivalent because they have different policies */ + return 0; + } + } else { + /* Equivalent because one or both are missing a policy */ + return 0; + } +} + diff --git a/src/or/circuitmux.h b/src/or/circuitmux.h index 2b5fb7e51e..837e3961bf 100644 --- a/src/or/circuitmux.h +++ b/src/or/circuitmux.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -57,6 +57,9 @@ struct circuitmux_policy_s { /* Choose a circuit */ circuit_t * (*pick_active_circuit)(circuitmux_t *cmux, circuitmux_policy_data_t *pol_data); + /* Optional: channel comparator for use by the scheduler */ + int (*cmp_cmux)(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1, + circuitmux_t *cmux_2, circuitmux_policy_data_t *pol_data_2); }; /* @@ -105,7 +108,8 @@ void circuitmux_free(circuitmux_t *cmux); /* Policy control */ void circuitmux_clear_policy(circuitmux_t *cmux); -const circuitmux_policy_t * circuitmux_get_policy(circuitmux_t *cmux); +MOCK_DECL(const circuitmux_policy_t *, + circuitmux_get_policy, (circuitmux_t *cmux)); void circuitmux_set_policy(circuitmux_t *cmux, const circuitmux_policy_t *pol); @@ -117,7 +121,7 @@ int circuitmux_is_circuit_attached(circuitmux_t *cmux, circuit_t *circ); int circuitmux_is_circuit_active(circuitmux_t *cmux, circuit_t *circ); unsigned int circuitmux_num_cells_for_circuit(circuitmux_t *cmux, circuit_t *circ); -unsigned int circuitmux_num_cells(circuitmux_t *cmux); +MOCK_DECL(unsigned int, circuitmux_num_cells, (circuitmux_t *cmux)); unsigned int circuitmux_num_circuits(circuitmux_t *cmux); unsigned int circuitmux_num_active_circuits(circuitmux_t *cmux); @@ -148,5 +152,9 @@ void circuitmux_append_destroy_cell(channel_t *chan, void circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux, channel_t *chan); +/* Optional interchannel comparisons for scheduling */ +MOCK_DECL(int, circuitmux_compare_muxes, + (circuitmux_t *cmux_1, circuitmux_t *cmux_2)); + #endif /* TOR_CIRCUITMUX_H */ diff --git a/src/or/circuitmux_ewma.c b/src/or/circuitmux_ewma.c index 3f37d7b9a0..1c0318de06 100644 --- a/src/or/circuitmux_ewma.c +++ b/src/or/circuitmux_ewma.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -187,6 +187,9 @@ ewma_notify_xmit_cells(circuitmux_t *cmux, static circuit_t * ewma_pick_active_circuit(circuitmux_t *cmux, circuitmux_policy_data_t *pol_data); +static int +ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1, + circuitmux_t *cmux_2, circuitmux_policy_data_t *pol_data_2); /*** EWMA global variables ***/ @@ -209,7 +212,8 @@ circuitmux_policy_t ewma_policy = { /*.notify_circ_inactive =*/ ewma_notify_circ_inactive, /*.notify_set_n_cells =*/ NULL, /* EWMA doesn't need this */ /*.notify_xmit_cells =*/ ewma_notify_xmit_cells, - /*.pick_active_circuit =*/ ewma_pick_active_circuit + /*.pick_active_circuit =*/ ewma_pick_active_circuit, + /*.cmp_cmux =*/ ewma_cmp_cmux }; /*** EWMA method implementations using the below EWMA helper functions ***/ @@ -273,8 +277,8 @@ ewma_alloc_circ_data(circuitmux_t *cmux, tor_assert(circ); tor_assert(direction == CELL_DIRECTION_OUT || direction == CELL_DIRECTION_IN); - /* Shut the compiler up */ - tor_assert(cell_count == cell_count); + /* Shut the compiler up without triggering -Wtautological-compare */ + (void)cell_count; cdata = tor_malloc_zero(sizeof(*cdata)); cdata->base_.magic = EWMA_POL_CIRC_DATA_MAGIC; @@ -453,6 +457,58 @@ ewma_pick_active_circuit(circuitmux_t *cmux, return circ; } +/** + * Compare two EWMA cmuxes, and return -1, 0 or 1 to indicate which should + * be more preferred - see circuitmux_compare_muxes() of circuitmux.c. + */ + +static int +ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1, + circuitmux_t *cmux_2, circuitmux_policy_data_t *pol_data_2) +{ + ewma_policy_data_t *p1 = NULL, *p2 = NULL; + cell_ewma_t *ce1 = NULL, *ce2 = NULL; + + tor_assert(cmux_1); + tor_assert(pol_data_1); + tor_assert(cmux_2); + tor_assert(pol_data_2); + + p1 = TO_EWMA_POL_DATA(pol_data_1); + p2 = TO_EWMA_POL_DATA(pol_data_1); + + if (p1 != p2) { + /* Get the head cell_ewma_t from each queue */ + if (smartlist_len(p1->active_circuit_pqueue) > 0) { + ce1 = smartlist_get(p1->active_circuit_pqueue, 0); + } + + if (smartlist_len(p2->active_circuit_pqueue) > 0) { + ce2 = smartlist_get(p2->active_circuit_pqueue, 0); + } + + /* Got both of them? */ + if (ce1 != NULL && ce2 != NULL) { + /* Pick whichever one has the better best circuit */ + return compare_cell_ewma_counts(ce1, ce2); + } else { + if (ce1 != NULL ) { + /* We only have a circuit on cmux_1, so prefer it */ + return -1; + } else if (ce2 != NULL) { + /* We only have a circuit on cmux_2, so prefer it */ + return 1; + } else { + /* No circuits at all; no preference */ + return 0; + } + } + } else { + /* We got identical params */ + return 0; + } +} + /** Helper for sorting cell_ewma_t values in their priority queue. */ static int compare_cell_ewma_counts(const void *p1, const void *p2) diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h index a512745c77..3feef834dd 100644 --- a/src/or/circuitmux_ewma.h +++ b/src/or/circuitmux_ewma.h @@ -1,4 +1,4 @@ -/* * Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* * Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c index e362b1b49e..7b3ad56537 100644 --- a/src/or/circuitstats.c +++ b/src/or/circuitstats.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define CIRCUITSTATS_PRIVATE @@ -404,7 +404,7 @@ circuit_build_times_new_consensus_params(circuit_build_times_t *cbt, * distress anyway, so memory correctness here is paramount over * doing acrobatics to preserve the array. */ - recent_circs = tor_malloc_zero(sizeof(int8_t)*num); + recent_circs = tor_calloc(num, sizeof(int8_t)); if (cbt->liveness.timeouts_after_firsthop && cbt->liveness.num_recent_circs > 0) { memcpy(recent_circs, cbt->liveness.timeouts_after_firsthop, @@ -508,7 +508,7 @@ circuit_build_times_init(circuit_build_times_t *cbt) cbt->liveness.num_recent_circs = circuit_build_times_recent_circuit_count(NULL); cbt->liveness.timeouts_after_firsthop = - tor_malloc_zero(sizeof(int8_t)*cbt->liveness.num_recent_circs); + tor_calloc(cbt->liveness.num_recent_circs, sizeof(int8_t)); } else { cbt->liveness.num_recent_circs = 0; cbt->liveness.timeouts_after_firsthop = NULL; @@ -649,7 +649,7 @@ circuit_build_times_create_histogram(const circuit_build_times_t *cbt, int i, c; *nbins = 1 + (max_build_time / CBT_BIN_WIDTH); - histogram = tor_malloc_zero(*nbins * sizeof(build_time_t)); + histogram = tor_calloc(*nbins, sizeof(build_time_t)); // calculate histogram for (i = 0; i < CBT_NCIRCUITS_TO_OBSERVE; i++) { @@ -691,7 +691,7 @@ circuit_build_times_get_xm(circuit_build_times_t *cbt) if (cbt->total_build_times < CBT_NCIRCUITS_TO_OBSERVE) num_modes = 1; - nth_max_bin = (build_time_t*)tor_malloc_zero(num_modes*sizeof(build_time_t)); + nth_max_bin = tor_calloc(num_modes, sizeof(build_time_t)); /* Determine the N most common build times */ for (i = 0; i < nbins; i++) { @@ -873,7 +873,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt, } /* build_time_t 0 means uninitialized */ - loaded_times = tor_malloc_zero(sizeof(build_time_t)*state->TotalBuildTimes); + loaded_times = tor_calloc(state->TotalBuildTimes, sizeof(build_time_t)); for (line = state->BuildtimeHistogram; line; line = line->next) { smartlist_t *args = smartlist_new(); @@ -1074,7 +1074,7 @@ circuit_build_times_update_alpha(circuit_build_times_t *cbt) * random_sample_from_Pareto_distribution * That's right. I'll cite wikipedia all day long. * - * Return value is in milliseconds. + * Return value is in milliseconds, clamped to INT32_MAX. */ STATIC double circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, @@ -1085,7 +1085,21 @@ circuit_build_times_calculate_timeout(circuit_build_times_t *cbt, tor_assert(1.0-quantile > 0); tor_assert(cbt->Xm > 0); - ret = cbt->Xm/pow(1.0-quantile,1.0/cbt->alpha); + /* If either alpha or p are 0, we would divide by zero, yielding an + * infinite (double) result; which would be clamped to INT32_MAX. + * Instead, initialise ret to INT32_MAX, and skip over these + * potentially illegal/trapping divides by zero. + */ + ret = INT32_MAX; + + if (cbt->alpha > 0) { + double p; + p = pow(1.0-quantile,1.0/cbt->alpha); + if (p > 0) { + ret = cbt->Xm/p; + } + } + if (ret > INT32_MAX) { ret = INT32_MAX; } @@ -1371,10 +1385,11 @@ circuit_build_times_network_check_changed(circuit_build_times_t *cbt) } cbt->liveness.after_firsthop_idx = 0; +#define MAX_TIMEOUT ((int32_t) (INT32_MAX/2)) /* Check to see if this has happened before. If so, double the timeout * to give people on abysmally bad network connections a shot at access */ if (cbt->timeout_ms >= circuit_build_times_get_initial_timeout()) { - if (cbt->timeout_ms > INT32_MAX/2 || cbt->close_ms > INT32_MAX/2) { + if (cbt->timeout_ms > MAX_TIMEOUT || cbt->close_ms > MAX_TIMEOUT) { log_warn(LD_CIRC, "Insanely large circuit build timeout value. " "(timeout = %fmsec, close = %fmsec)", cbt->timeout_ms, cbt->close_ms); @@ -1386,6 +1401,7 @@ circuit_build_times_network_check_changed(circuit_build_times_t *cbt) cbt->close_ms = cbt->timeout_ms = circuit_build_times_get_initial_timeout(); } +#undef MAX_TIMEOUT cbt_control_event_buildtimeout_set(cbt, BUILDTIMEOUT_SET_EVENT_RESET); diff --git a/src/or/circuitstats.h b/src/or/circuitstats.h index 3343310b8e..fe05a24e97 100644 --- a/src/or/circuitstats.h +++ b/src/or/circuitstats.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 714754a672..3b6f982666 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -200,7 +200,7 @@ circuit_is_better(const origin_circuit_t *oa, const origin_circuit_t *ob, return 1; } else { if (a->timestamp_dirty || - timercmp(&a->timestamp_began, &b->timestamp_began, >)) + timercmp(&a->timestamp_began, &b->timestamp_began, OP_GT)) return 1; if (ob->build_state->is_internal) /* XXX023 what the heck is this internal thing doing here. I @@ -268,7 +268,6 @@ circuit_get_best(const entry_connection_t *conn, int must_be_open, uint8_t purpose, int need_uptime, int need_internal) { - circuit_t *circ; origin_circuit_t *best=NULL; struct timeval now; int intro_going_on_but_too_old = 0; @@ -281,7 +280,7 @@ circuit_get_best(const entry_connection_t *conn, tor_gettimeofday(&now); - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { origin_circuit_t *origin_circ; if (!CIRCUIT_IS_ORIGIN(circ)) continue; @@ -305,6 +304,7 @@ circuit_get_best(const entry_connection_t *conn, if (!best || circuit_is_better(origin_circ,best,conn)) best = origin_circ; } + SMARTLIST_FOREACH_END(circ); if (!best && intro_going_on_but_too_old) log_info(LD_REND|LD_CIRC, "There is an intro circuit being created " @@ -318,11 +318,9 @@ circuit_get_best(const entry_connection_t *conn, static int count_pending_general_client_circuits(void) { - const circuit_t *circ; - int count = 0; - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (circ->marked_for_close || circ->state == CIRCUIT_STATE_OPEN || circ->purpose != CIRCUIT_PURPOSE_C_GENERAL || @@ -331,6 +329,7 @@ count_pending_general_client_circuits(void) ++count; } + SMARTLIST_FOREACH_END(circ); return count; } @@ -370,7 +369,6 @@ circuit_conforms_to_options(const origin_circuit_t *circ, void circuit_expire_building(void) { - circuit_t *victim, *next_circ; /* circ_times.timeout_ms and circ_times.close_ms are from * circuit_build_times_get_initial_timeout() if we haven't computed * custom timeouts yet */ @@ -388,7 +386,7 @@ circuit_expire_building(void) * we want to be more lenient with timeouts, in case the * user has relocated and/or changed network connections. * See bug #3443. */ - TOR_LIST_FOREACH(next_circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, next_circ) { if (!CIRCUIT_IS_ORIGIN(next_circ) || /* didn't originate here */ next_circ->marked_for_close) { /* don't mess with marked circs */ continue; @@ -402,7 +400,7 @@ circuit_expire_building(void) any_opened_circs = 1; break; } - } + } SMARTLIST_FOREACH_END(next_circ); #define SET_CUTOFF(target, msec) do { \ long ms = tor_lround(msec); \ @@ -473,9 +471,8 @@ circuit_expire_building(void) MAX(get_circuit_build_close_time_ms()*2 + 1000, options->SocksTimeout * 1000)); - TOR_LIST_FOREACH(next_circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *,victim) { struct timeval cutoff; - victim = next_circ; if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */ victim->marked_for_close) /* don't mess with marked circs */ continue; @@ -517,7 +514,7 @@ circuit_expire_building(void) if (TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out) cutoff = hs_extremely_old_cutoff; - if (timercmp(&victim->timestamp_began, &cutoff, >)) + if (timercmp(&victim->timestamp_began, &cutoff, OP_GT)) continue; /* it's still young, leave it alone */ /* We need to double-check the opened state here because @@ -527,7 +524,7 @@ circuit_expire_building(void) * aren't either. */ if (!any_opened_circs && victim->state != CIRCUIT_STATE_OPEN) { /* It's still young enough that we wouldn't close it, right? */ - if (timercmp(&victim->timestamp_began, &close_cutoff, >)) { + if (timercmp(&victim->timestamp_began, &close_cutoff, OP_GT)) { if (!TO_ORIGIN_CIRCUIT(victim)->relaxed_timeout) { int first_hop_succeeded = TO_ORIGIN_CIRCUIT(victim)->cpath->state == CPATH_STATE_OPEN; @@ -675,7 +672,7 @@ circuit_expire_building(void) * it off at, we probably had a suspend event along this codepath, * and we should discard the value. */ - if (timercmp(&victim->timestamp_began, &extremely_old_cutoff, <)) { + if (timercmp(&victim->timestamp_began, &extremely_old_cutoff, OP_LT)) { log_notice(LD_CIRC, "Extremely large value for circuit build timeout: %lds. " "Assuming clock jump. Purpose %d (%s)", @@ -780,7 +777,7 @@ circuit_expire_building(void) circuit_mark_for_close(victim, END_CIRC_REASON_TIMEOUT); pathbias_count_timeout(TO_ORIGIN_CIRCUIT(victim)); - } + } SMARTLIST_FOREACH_END(victim); } /** For debugging #8387: track when we last called @@ -800,9 +797,8 @@ circuit_log_ancient_one_hop_circuits(int age) time_t cutoff = now - age; int n_found = 0; smartlist_t *log_these = smartlist_new(); - const circuit_t *circ; - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { const origin_circuit_t *ocirc; if (! CIRCUIT_IS_ORIGIN(circ)) continue; @@ -817,6 +813,7 @@ circuit_log_ancient_one_hop_circuits(int age) smartlist_add(log_these, (origin_circuit_t*) ocirc); } } + SMARTLIST_FOREACH_END(circ); if (n_found == 0) goto done; @@ -831,7 +828,7 @@ circuit_log_ancient_one_hop_circuits(int age) int stream_num; const edge_connection_t *conn; char *dirty = NULL; - circ = TO_CIRCUIT(ocirc); + const circuit_t *circ = TO_CIRCUIT(ocirc); format_local_iso_time(created, (time_t)circ->timestamp_created.tv_sec); @@ -848,12 +845,14 @@ circuit_log_ancient_one_hop_circuits(int age) } log_notice(LD_HEARTBEAT, " #%d created at %s. %s, %s. %s for close. " + "Package window: %d. " "%s for new conns. %s.", ocirc_sl_idx, created, circuit_state_to_string(circ->state), circuit_purpose_to_string(circ->purpose), circ->marked_for_close ? "Marked" : "Not marked", + circ->package_window, ocirc->unusable_for_new_conns ? "Not usable" : "usable", dirty); tor_free(dirty); @@ -869,12 +868,18 @@ circuit_log_ancient_one_hop_circuits(int age) log_notice(LD_HEARTBEAT, " Stream#%d created at %s. " "%s conn in state %s. " + "It is %slinked and %sreading from a linked connection %p. " + "Package window %d. " "%s for close (%s:%d). Hold-open is %sset. " "Has %ssent RELAY_END. %s on circuit.", stream_num, stream_created, conn_type_to_string(c->type), conn_state_to_string(c->type, c->state), + c->linked ? "" : "not ", + c->reading_from_linked_conn ? "": "not", + c->linked_conn, + conn->package_window, c->marked_for_close ? "Marked" : "Not marked", c->marked_for_close_file ? c->marked_for_close_file : "--", c->marked_for_close, @@ -938,7 +943,6 @@ int circuit_stream_is_being_handled(entry_connection_t *conn, uint16_t port, int min) { - circuit_t *circ; const node_t *exitnode; int num=0; time_t now = time(NULL); @@ -946,7 +950,7 @@ circuit_stream_is_being_handled(entry_connection_t *conn, get_options()->LongLivedPorts, conn ? conn->socks_request->port : port); - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (CIRCUIT_IS_ORIGIN(circ) && !circ->marked_for_close && circ->purpose == CIRCUIT_PURPOSE_C_GENERAL && @@ -976,6 +980,7 @@ circuit_stream_is_being_handled(entry_connection_t *conn, } } } + SMARTLIST_FOREACH_END(circ); return 0; } @@ -989,7 +994,6 @@ circuit_stream_is_being_handled(entry_connection_t *conn, static void circuit_predict_and_launch_new(void) { - circuit_t *circ; int num=0, num_internal=0, num_uptime_internal=0; int hidserv_needs_uptime=0, hidserv_needs_capacity=1; int port_needs_uptime=0, port_needs_capacity=1; @@ -997,7 +1001,7 @@ circuit_predict_and_launch_new(void) int flags = 0; /* First, count how many of each type of circuit we have already. */ - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { cpath_build_state_t *build_state; origin_circuit_t *origin_circ; if (!CIRCUIT_IS_ORIGIN(circ)) @@ -1020,6 +1024,7 @@ circuit_predict_and_launch_new(void) if (build_state->need_uptime && build_state->is_internal) num_uptime_internal++; } + SMARTLIST_FOREACH_END(circ); /* If that's enough, then stop now. */ if (num >= MAX_UNUSED_OPEN_CIRCUITS) @@ -1027,9 +1032,11 @@ circuit_predict_and_launch_new(void) /* Second, see if we need any more exit circuits. */ /* check if we know of a port that's been requested recently - * and no circuit is currently available that can handle it. */ + * and no circuit is currently available that can handle it. + * Exits (obviously) require an exit circuit. */ if (!circuit_all_predicted_ports_handled(now, &port_needs_uptime, - &port_needs_capacity)) { + &port_needs_capacity) + && router_have_consensus_path() == CONSENSUS_PATH_EXIT) { if (port_needs_uptime) flags |= CIRCLAUNCH_NEED_UPTIME; if (port_needs_capacity) @@ -1041,8 +1048,10 @@ circuit_predict_and_launch_new(void) return; } - /* Third, see if we need any more hidden service (server) circuits. */ - if (num_rend_services() && num_uptime_internal < 3) { + /* Third, see if we need any more hidden service (server) circuits. + * HS servers only need an internal circuit. */ + if (num_rend_services() && num_uptime_internal < 3 + && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) { flags = (CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL); log_info(LD_CIRC, @@ -1053,11 +1062,13 @@ circuit_predict_and_launch_new(void) return; } - /* Fourth, see if we need any more hidden service (client) circuits. */ + /* Fourth, see if we need any more hidden service (client) circuits. + * HS clients only need an internal circuit. */ if (rep_hist_get_predicted_internal(now, &hidserv_needs_uptime, &hidserv_needs_capacity) && ((num_uptime_internal<2 && hidserv_needs_uptime) || - num_internal<2)) { + num_internal<2) + && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) { if (hidserv_needs_uptime) flags |= CIRCLAUNCH_NEED_UPTIME; if (hidserv_needs_capacity) @@ -1074,15 +1085,23 @@ circuit_predict_and_launch_new(void) /* Finally, check to see if we still need more circuits to learn * a good build timeout. But if we're close to our max number we * want, don't do another -- we want to leave a few slots open so - * we can still build circuits preemptively as needed. */ - if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && - ! circuit_build_times_disabled() && - circuit_build_times_needs_circuits_now(get_circuit_build_times())) { - flags = CIRCLAUNCH_NEED_CAPACITY; - log_info(LD_CIRC, - "Have %d clean circs need another buildtime test circ.", num); - circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags); - return; + * we can still build circuits preemptively as needed. + * XXXX make the assumption that build timeout streams should be + * created whenever we can build internal circuits. */ + if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) { + if (num < MAX_UNUSED_OPEN_CIRCUITS-2 && + ! circuit_build_times_disabled() && + circuit_build_times_needs_circuits_now(get_circuit_build_times())) { + flags = CIRCLAUNCH_NEED_CAPACITY; + /* if there are no exits in the consensus, make timeout + * circuits internal */ + if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL) + flags |= CIRCLAUNCH_IS_INTERNAL; + log_info(LD_CIRC, + "Have %d clean circs need another buildtime test circ.", num); + circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags); + return; + } } } @@ -1099,11 +1118,17 @@ circuit_build_needed_circs(time_t now) { const or_options_t *options = get_options(); - /* launch a new circ for any pending streams that need one */ - connection_ap_attach_pending(); + /* launch a new circ for any pending streams that need one + * XXXX make the assumption that (some) AP streams (i.e. HS clients) + * don't require an exit circuit, review in #13814. + * This allows HSs to function in a consensus without exits. */ + if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) + connection_ap_attach_pending(); - /* make sure any hidden services have enough intro points */ - rend_services_introduce(); + /* make sure any hidden services have enough intro points + * HS intro point streams only require an internal circuit */ + if (router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) + rend_services_introduce(); circuit_expire_old_circs_as_needed(now); @@ -1223,7 +1248,6 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) static void circuit_expire_old_circuits_clientside(void) { - circuit_t *circ; struct timeval cutoff, now; tor_gettimeofday(&now); @@ -1239,7 +1263,7 @@ circuit_expire_old_circuits_clientside(void) cutoff.tv_sec -= get_options()->CircuitIdleTimeout; } - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (circ->marked_for_close || !CIRCUIT_IS_ORIGIN(circ)) continue; /* If the circuit has been dirty for too long, and there are no streams @@ -1259,7 +1283,7 @@ circuit_expire_old_circuits_clientside(void) if (circ->purpose != CIRCUIT_PURPOSE_PATH_BIAS_TESTING) circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); } else if (!circ->timestamp_dirty && circ->state == CIRCUIT_STATE_OPEN) { - if (timercmp(&circ->timestamp_began, &cutoff, <)) { + if (timercmp(&circ->timestamp_began, &cutoff, OP_LT)) { if (circ->purpose == CIRCUIT_PURPOSE_C_GENERAL || circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT || circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || @@ -1291,7 +1315,7 @@ circuit_expire_old_circuits_clientside(void) } } } - } + } SMARTLIST_FOREACH_END(circ); } /** How long do we wait before killing circuits with the properties @@ -1318,11 +1342,10 @@ circuit_expire_old_circuits_clientside(void) void circuit_expire_old_circuits_serverside(time_t now) { - circuit_t *circ; or_circuit_t *or_circ; time_t cutoff = now - IDLE_ONE_HOP_CIRC_TIMEOUT; - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (circ->marked_for_close || CIRCUIT_IS_ORIGIN(circ)) continue; or_circ = TO_OR_CIRCUIT(circ); @@ -1339,6 +1362,7 @@ circuit_expire_old_circuits_serverside(time_t now) circuit_mark_for_close(circ, END_CIRC_REASON_FINISHED); } } + SMARTLIST_FOREACH_END(circ); } /** Number of testing circuits we want open before testing our bandwidth. */ @@ -1363,18 +1387,18 @@ reset_bandwidth_test(void) int circuit_enough_testing_circs(void) { - circuit_t *circ; int num = 0; if (have_performed_bandwidth_test) return 1; - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->marked_for_close && CIRCUIT_IS_ORIGIN(circ) && circ->purpose == CIRCUIT_PURPOSE_TESTING && circ->state == CIRCUIT_STATE_OPEN) num++; } + SMARTLIST_FOREACH_END(circ); return num >= NUM_PARALLEL_TESTING_CIRCS; } @@ -1636,6 +1660,16 @@ circuit_launch(uint8_t purpose, int flags) return circuit_launch_by_extend_info(purpose, NULL, flags); } +/** DOCDOC */ +static int +have_enough_path_info(int need_exit) +{ + if (need_exit) + return router_have_consensus_path() == CONSENSUS_PATH_EXIT; + else + return router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN; +} + /** Launch a new circuit with purpose <b>purpose</b> and exit node * <b>extend_info</b> (or NULL to select a random exit node). If flags * contains CIRCLAUNCH_NEED_UPTIME, choose among routers with high uptime. If @@ -1650,15 +1684,29 @@ circuit_launch_by_extend_info(uint8_t purpose, { origin_circuit_t *circ; int onehop_tunnel = (flags & CIRCLAUNCH_ONEHOP_TUNNEL) != 0; - - if (!onehop_tunnel && !router_have_minimum_dir_info()) { - log_debug(LD_CIRC,"Haven't fetched enough directory info yet; canceling " - "circuit launch."); + int have_path = have_enough_path_info(! (flags & CIRCLAUNCH_IS_INTERNAL) ); + int need_specific_rp = 0; + + if (!onehop_tunnel && (!router_have_minimum_dir_info() || !have_path)) { + log_debug(LD_CIRC,"Haven't %s yet; canceling " + "circuit launch.", + !router_have_minimum_dir_info() ? + "fetched enough directory info" : + "received a consensus with exits"); return NULL; } + /* If Tor2webRendezvousPoints is enabled and we are dealing with an + RP circuit, we want a specific RP node so we shouldn't canibalize + an already existing circuit. */ + if (get_options()->Tor2webRendezvousPoints && + purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND) { + need_specific_rp = 1; + } + if ((extend_info || purpose != CIRCUIT_PURPOSE_C_GENERAL) && - purpose != CIRCUIT_PURPOSE_TESTING && !onehop_tunnel) { + purpose != CIRCUIT_PURPOSE_TESTING && + !onehop_tunnel && !need_specific_rp) { /* see if there are appropriate circs available to cannibalize. */ /* XXX if we're planning to add a hop, perhaps we want to look for * internal circs rather than exit circs? -RD */ @@ -1810,7 +1858,9 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, return 1; /* we're happy */ } - if (!want_onehop && !router_have_minimum_dir_info()) { + int have_path = have_enough_path_info(!need_internal); + + if (!want_onehop && (!router_have_minimum_dir_info() || !have_path)) { if (!connection_get_by_type(CONN_TYPE_DIR)) { int severity = LOG_NOTICE; /* FFFF if this is a tunneled directory fetch, don't yell @@ -1818,14 +1868,20 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, if (entry_list_is_constrained(options) && entries_known_but_down(options)) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we haven't used client functionality " - "lately. Optimistically trying known %s again.", + "Application request when we haven't %s. " + "Optimistically trying known %s again.", + !router_have_minimum_dir_info() ? + "used client functionality lately" : + "received a consensus with exits", options->UseBridges ? "bridges" : "entrynodes"); entries_retry_all(options); } else if (!options->UseBridges || any_bridge_descriptors_known()) { log_fn(severity, LD_APP|LD_DIR, - "Application request when we haven't used client functionality " - "lately. Optimistically trying directory fetches again."); + "Application request when we haven't %s. " + "Optimistically trying directory fetches again.", + !router_have_minimum_dir_info() ? + "used client functionality lately" : + "received a consensus with exits"); routerlist_retry_directory_downloads(time(NULL)); } } @@ -1980,11 +2036,13 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, else new_circ_purpose = desired_circuit_purpose; +#ifdef ENABLE_TOR2WEB_MODE if (options->Tor2webMode && (new_circ_purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND || new_circ_purpose == CIRCUIT_PURPOSE_C_INTRODUCING)) { want_onehop = 1; } +#endif { int flags = CIRCLAUNCH_NEED_CAPACITY; @@ -2016,7 +2074,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, circ->rend_data = rend_data_dup(ENTRY_TO_EDGE_CONN(conn)->rend_data); if (circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND && circ->base_.state == CIRCUIT_STATE_OPEN) - rend_client_rendcirc_has_opened(circ); + circuit_has_opened(circ); } } } /* endif (!circ) */ @@ -2074,7 +2132,7 @@ static void link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ, crypt_path_t *cpath) { - const node_t *exitnode; + const node_t *exitnode = NULL; /* add it into the linked list of streams on this circuit */ log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.", @@ -2108,23 +2166,25 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ, circ->isolation_any_streams_attached = 1; connection_edge_update_circuit_isolation(apconn, circ, 0); + /* Compute the exitnode if possible, for logging below */ + if (cpath->extend_info) + exitnode = node_get_by_id(cpath->extend_info->identity_digest); + /* See if we can use optimistic data on this circuit */ - if (cpath->extend_info && - (exitnode = node_get_by_id(cpath->extend_info->identity_digest)) && - exitnode->rs) { - /* Okay; we know what exit node this is. */ - if (optimistic_data_enabled() && - circ->base_.purpose == CIRCUIT_PURPOSE_C_GENERAL && - exitnode->rs->version_supports_optimistic_data) - apconn->may_use_optimistic_data = 1; - else - apconn->may_use_optimistic_data = 0; - log_info(LD_APP, "Looks like completed circuit to %s %s allow " - "optimistic data for connection to %s", - safe_str_client(node_describe(exitnode)), - apconn->may_use_optimistic_data ? "does" : "doesn't", - safe_str_client(apconn->socks_request->address)); - } + if (optimistic_data_enabled() && + (circ->base_.purpose == CIRCUIT_PURPOSE_C_GENERAL || + circ->base_.purpose == CIRCUIT_PURPOSE_C_REND_JOINED)) + apconn->may_use_optimistic_data = 1; + else + apconn->may_use_optimistic_data = 0; + log_info(LD_APP, "Looks like completed circuit to %s %s allow " + "optimistic data for connection to %s", + circ->base_.purpose == CIRCUIT_PURPOSE_C_GENERAL ? + /* node_describe() does the right thing if exitnode is NULL */ + safe_str_client(node_describe(exitnode)) : + "hidden service", + apconn->may_use_optimistic_data ? "does" : "doesn't", + safe_str_client(apconn->socks_request->address)); } /** Return true iff <b>address</b> is matched by one of the entries in @@ -2326,7 +2386,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn) tor_assert(rendcirc); /* one is already established, attach */ log_info(LD_REND, - "rend joined circ %d already here. attaching. " + "rend joined circ %u already here. attaching. " "(stream %d sec old)", (unsigned)rendcirc->base_.n_circ_id, conn_age); /* Mark rendezvous circuits as 'newly dirty' every time you use @@ -2346,6 +2406,18 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn) return 1; } + /* At this point we need to re-check the state, since it's possible that + * our call to circuit_get_open_circ_or_launch() changed the connection's + * state from "CIRCUIT_WAIT" to "RENDDESC_WAIT" because we decided to + * re-fetch the descriptor. + */ + if (ENTRY_TO_CONN(conn)->state != AP_CONN_STATE_CIRCUIT_WAIT) { + log_info(LD_REND, "This connection is no longer ready to attach; its " + "state changed." + "(We probably have to re-fetch its descriptor.)"); + return 0; + } + if (rendcirc && (rendcirc->base_.purpose == CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED)) { log_info(LD_REND, diff --git a/src/or/circuituse.h b/src/or/circuituse.h index 4c5977bee0..a59f478ac8 100644 --- a/src/or/circuituse.h +++ b/src/or/circuituse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/command.c b/src/or/command.c index 1f6f93a868..719b10736b 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -310,7 +310,7 @@ command_process_create_cell(cell_t *cell, channel_t *chan) /* hand it off to the cpuworkers, and then return. */ if (connection_or_digest_is_known_relay(chan->identity_digest)) rep_hist_note_circuit_handshake_requested(create_cell->handshake_type); - if (assign_onionskin_to_cpuworker(NULL, circ, create_cell) < 0) { + if (assign_onionskin_to_cpuworker(circ, create_cell) < 0) { log_debug(LD_GENERAL,"Failed to hand off onionskin. Closing."); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_RESOURCELIMIT); return; @@ -398,7 +398,6 @@ command_process_created_cell(cell_t *cell, channel_t *chan) log_debug(LD_OR,"at OP. Finishing handshake."); if ((err_reason = circuit_finish_handshake(origin_circ, &extended_cell.created_cell)) < 0) { - log_warn(LD_OR,"circuit_finish_handshake failed."); circuit_mark_for_close(circ, -err_reason); return; } @@ -438,6 +437,7 @@ command_process_created_cell(cell_t *cell, channel_t *chan) static void command_process_relay_cell(cell_t *cell, channel_t *chan) { + const or_options_t *options = get_options(); circuit_t *circ; int reason, direction; @@ -511,6 +511,14 @@ command_process_relay_cell(cell_t *cell, channel_t *chan) direction==CELL_DIRECTION_OUT?"forward":"backward"); circuit_mark_for_close(circ, -reason); } + + /* If this is a cell in an RP circuit, count it as part of the + hidden service stats */ + if (options->HiddenServiceStatistics && + !CIRCUIT_IS_ORIGIN(circ) && + TO_OR_CIRCUIT(circ)->circuit_carries_hs_traffic_stats) { + rep_hist_seen_new_rp_cell(); + } } /** Process a 'destroy' <b>cell</b> that just arrived from diff --git a/src/or/command.h b/src/or/command.h index adea6adeaa..bea96261bb 100644 --- a/src/or/command.h +++ b/src/or/command.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/config.c b/src/or/config.c index c60dd11c4d..233940ec95 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,6 +11,7 @@ #define CONFIG_PRIVATE #include "or.h" +#include "compat.h" #include "addressmap.h" #include "channel.h" #include "circuitbuild.h" @@ -43,6 +44,7 @@ #include "util.h" #include "routerlist.h" #include "routerset.h" +#include "scheduler.h" #include "statefile.h" #include "transports.h" #include "ext_orport.h" @@ -53,9 +55,22 @@ #include "procmon.h" +#ifdef HAVE_SYSTEMD +# if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) +/* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse + * Coverity. Here's a kludge to unconfuse it. + */ +# define __INCLUDE_LEVEL__ 2 +# endif +#include <systemd/sd-daemon.h> +#endif + /* From main.c */ extern int quiet_level; +/* Prefix used to indicate a Unix socket in a FooPort configuration. */ +static const char unix_socket_prefix[] = "unix:"; + /** A list of abbreviations and aliases to map command-line options, obsolete * option names, or alternative option names, to their current values. */ static config_abbrev_t option_abbrevs_[] = { @@ -63,15 +78,16 @@ static config_abbrev_t option_abbrevs_[] = { PLURAL(AuthDirBadExitCC), PLURAL(AuthDirInvalidCC), PLURAL(AuthDirRejectCC), - PLURAL(ExitNode), PLURAL(EntryNode), PLURAL(ExcludeNode), + PLURAL(Tor2webRendezvousPoint), PLURAL(FirewallPort), PLURAL(LongLivedPort), PLURAL(HiddenServiceNode), PLURAL(HiddenServiceExcludeNode), PLURAL(NumCPU), PLURAL(RendNode), + PLURAL(RecommendedPackage), PLURAL(RendExcludeNode), PLURAL(StrictEntryNode), PLURAL(StrictExitNode), @@ -99,8 +115,6 @@ static config_abbrev_t option_abbrevs_[] = { { "PreferTunnelledDirConns", "PreferTunneledDirConns", 0, 0}, { "BridgeAuthoritativeDirectory", "BridgeAuthoritativeDir", 0, 0}, { "HashedControlPassword", "__HashedControlSessionPassword", 1, 0}, - { "StrictEntryNodes", "StrictNodes", 0, 1}, - { "StrictExitNodes", "StrictNodes", 0, 1}, { "VirtualAddrNetwork", "VirtualAddrNetworkIPv4", 0, 0}, { "_UseFilteringSSLBufferevents", "UseFilteringSSLBufferevents", 0, 1}, { NULL, NULL, 0, 0}, @@ -127,8 +141,8 @@ static config_abbrev_t option_abbrevs_[] = { * be chosen first. */ static config_var_t option_vars_[] = { - OBSOLETE("AccountingMaxKB"), V(AccountingMax, MEMUNIT, "0 bytes"), + VAR("AccountingRule", STRING, AccountingRule_option, "max"), V(AccountingStart, STRING, NULL), V(Address, STRING, NULL), V(AllowDotExit, BOOL, "0"), @@ -140,8 +154,8 @@ static config_var_t option_vars_[] = { V(AlternateDirAuthority, LINELIST, NULL), OBSOLETE("AlternateHSAuthority"), V(AssumeReachable, BOOL, "0"), - V(AuthDirBadDir, LINELIST, NULL), - V(AuthDirBadDirCCs, CSV, ""), + OBSOLETE("AuthDirBadDir"), + OBSOLETE("AuthDirBadDirCCs"), V(AuthDirBadExit, LINELIST, NULL), V(AuthDirBadExitCCs, CSV, ""), V(AuthDirInvalid, LINELIST, NULL), @@ -150,8 +164,8 @@ static config_var_t option_vars_[] = { V(AuthDirGuardBWGuarantee, MEMUNIT, "2 MB"), V(AuthDirReject, LINELIST, NULL), V(AuthDirRejectCCs, CSV, ""), - V(AuthDirRejectUnlisted, BOOL, "0"), - V(AuthDirListBadDirs, BOOL, "0"), + OBSOLETE("AuthDirRejectUnlisted"), + OBSOLETE("AuthDirListBadDirs"), V(AuthDirListBadExits, BOOL, "0"), V(AuthDirMaxServersPerAddr, UINT, "2"), V(AuthDirMaxServersPerAuthAddr,UINT, "5"), @@ -191,26 +205,20 @@ static config_var_t option_vars_[] = { V(ControlPortWriteToFile, FILENAME, NULL), V(ControlSocket, LINELIST, NULL), V(ControlSocketsGroupWritable, BOOL, "0"), + V(SocksSocketsGroupWritable, BOOL, "0"), V(CookieAuthentication, BOOL, "0"), V(CookieAuthFileGroupReadable, BOOL, "0"), V(CookieAuthFile, STRING, NULL), V(CountPrivateBandwidth, BOOL, "0"), V(DataDirectory, FILENAME, NULL), - OBSOLETE("DebugLogFile"), V(DisableNetwork, BOOL, "0"), V(DirAllowPrivateAddresses, BOOL, "0"), V(TestingAuthDirTimeToLearnReachability, INTERVAL, "30 minutes"), V(DirListenAddress, LINELIST, NULL), - OBSOLETE("DirFetchPeriod"), V(DirPolicy, LINELIST, NULL), VPORT(DirPort, LINELIST, NULL), V(DirPortFrontPage, FILENAME, NULL), - OBSOLETE("DirPostPeriod"), - OBSOLETE("DirRecordUsageByCountry"), - OBSOLETE("DirRecordUsageGranularity"), - OBSOLETE("DirRecordUsageRetainIPs"), - OBSOLETE("DirRecordUsageSaveInterval"), - V(DirReqStatistics, BOOL, "1"), + VAR("DirReqStatistics", BOOL, DirReqStatistics_option, "1"), VAR("DirAuthority", LINELIST, DirAuthorities, NULL), V(DirAuthorityFallbackRate, DOUBLE, "1.0"), V(DisableAllSwap, BOOL, "0"), @@ -236,6 +244,7 @@ static config_var_t option_vars_[] = { V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPortStatistics, BOOL, "0"), V(ExtendAllowPrivateAddresses, BOOL, "0"), + V(ExitRelay, AUTOBOOL, "auto"), VPORT(ExtORPort, LINELIST, NULL), V(ExtORPortCookieAuthFile, STRING, NULL), V(ExtORPortCookieAuthFileGroupReadable, BOOL, "0"), @@ -262,7 +271,6 @@ static config_var_t option_vars_[] = { V(GeoIPv6File, FILENAME, SHARE_DATADIR PATH_SEPARATOR "tor" PATH_SEPARATOR "geoip6"), #endif - OBSOLETE("GiveGuardFlagTo_CVE_2011_2768_VulnerableRelays"), OBSOLETE("Group"), V(GuardLifetime, INTERVAL, "0 minutes"), V(HardwareAccel, BOOL, "0"), @@ -272,15 +280,14 @@ static config_var_t option_vars_[] = { V(HashedControlPassword, LINELIST, NULL), V(HidServDirectoryV2, BOOL, "1"), VAR("HiddenServiceDir", LINELIST_S, RendConfigLines, NULL), - OBSOLETE("HiddenServiceExcludeNodes"), - OBSOLETE("HiddenServiceNodes"), + VAR("HiddenServiceDirGroupReadable", LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceOptions",LINELIST_V, RendConfigLines, NULL), VAR("HiddenServicePort", LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceVersion",LINELIST_S, RendConfigLines, NULL), VAR("HiddenServiceAuthorizeClient",LINELIST_S,RendConfigLines, NULL), + VAR("HiddenServiceAllowUnknownPorts",LINELIST_S, RendConfigLines, NULL), + V(HiddenServiceStatistics, BOOL, "0"), V(HidServAuth, LINELIST, NULL), - OBSOLETE("HSAuthoritativeDir"), - OBSOLETE("HSAuthorityRecordStats"), V(CloseHSClientCircuitsImmediatelyOnTimeout, BOOL, "0"), V(CloseHSServiceRendCircuitsImmediatelyOnTimeout, BOOL, "0"), V(HTTPProxy, STRING, NULL), @@ -295,14 +302,11 @@ static config_var_t option_vars_[] = { V(Socks5Proxy, STRING, NULL), V(Socks5ProxyUsername, STRING, NULL), V(Socks5ProxyPassword, STRING, NULL), - OBSOLETE("IgnoreVersion"), V(KeepalivePeriod, INTERVAL, "5 minutes"), VAR("Log", LINELIST, Logs, NULL), V(LogMessageDomains, BOOL, "0"), - OBSOLETE("LinkPadding"), - OBSOLETE("LogLevel"), - OBSOLETE("LogFile"), V(LogTimeGranularity, MSEC_INTERVAL, "1 second"), + V(TruncateLogFile, BOOL, "0"), V(LongLivedPorts, CSV, "21,22,706,1863,5050,5190,5222,5223,6523,6667,6697,8300"), VAR("MapAddress", LINELIST, AddressMap, NULL), @@ -313,16 +317,14 @@ static config_var_t option_vars_[] = { OBSOLETE("MaxOnionsPending"), V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), - OBSOLETE("MonthlyAccountingStart"), V(MyFamily, STRING, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), - VAR("NamingAuthoritativeDirectory",BOOL, NamingAuthoritativeDir, "0"), + OBSOLETE("NamingAuthoritativeDirectory"), V(NATDListenAddress, LINELIST, NULL), VPORT(NATDPort, LINELIST, NULL), V(Nickname, STRING, NULL), V(PredictedPortsRelevanceTime, INTERVAL, "1 hour"), V(WarnUnsafeSocks, BOOL, "1"), - OBSOLETE("NoPublish"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), V(NumCPUs, UINT, "0"), V(NumDirectoryGuards, UINT, "0"), @@ -348,7 +350,6 @@ static config_var_t option_vars_[] = { V(PathBiasScaleUseThreshold, INT, "-1"), V(PathsNeededToBuildCircuits, DOUBLE, "-1"), - OBSOLETE("PathlenCoinWeight"), V(PerConnBWBurst, MEMUNIT, "0"), V(PerConnBWRate, MEMUNIT, "0"), V(PidFile, STRING, NULL), @@ -368,18 +369,14 @@ static config_var_t option_vars_[] = { V(RecommendedVersions, LINELIST, NULL), V(RecommendedClientVersions, LINELIST, NULL), V(RecommendedServerVersions, LINELIST, NULL), - OBSOLETE("RedirectExit"), + V(RecommendedPackages, LINELIST, NULL), V(RefuseUnknownExits, AUTOBOOL, "auto"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), V(RelayBandwidthRate, MEMUNIT, "0"), - OBSOLETE("RendExcludeNodes"), - OBSOLETE("RendNodes"), V(RendPostPeriod, INTERVAL, "1 hour"), V(RephistTrackTime, INTERVAL, "24 hours"), - OBSOLETE("RouterFile"), V(RunAsDaemon, BOOL, "0"), -// V(RunTesting, BOOL, "0"), OBSOLETE("RunTesting"), // currently unused V(Sandbox, BOOL, "0"), V(SafeLogging, STRING, "1"), @@ -392,24 +389,26 @@ static config_var_t option_vars_[] = { V(ServerDNSSearchDomains, BOOL, "0"), V(ServerDNSTestAddresses, CSV, "www.google.com,www.mit.edu,www.yahoo.com,www.slashdot.org"), + V(SchedulerLowWaterMark__, MEMUNIT, "100 MB"), + V(SchedulerHighWaterMark__, MEMUNIT, "101 MB"), + V(SchedulerMaxFlushCells__, UINT, "1000"), V(ShutdownWaitLength, INTERVAL, "30 seconds"), V(SocksListenAddress, LINELIST, NULL), V(SocksPolicy, LINELIST, NULL), VPORT(SocksPort, LINELIST, NULL), V(SocksTimeout, INTERVAL, "2 minutes"), V(SSLKeyLifetime, INTERVAL, "0"), - OBSOLETE("StatusFetchPeriod"), + OBSOLETE("StrictEntryNodes"), + OBSOLETE("StrictExitNodes"), V(StrictNodes, BOOL, "0"), - V(Support022HiddenServices, AUTOBOOL, "auto"), - OBSOLETE("SysLog"), + OBSOLETE("Support022HiddenServices"), V(TestSocks, BOOL, "0"), - OBSOLETE("TestVia"), V(TokenBucketRefillInterval, MSEC_INTERVAL, "100 msec"), V(Tor2webMode, BOOL, "0"), + V(Tor2webRendezvousPoints, ROUTERSET, NULL), V(TLSECGroup, STRING, NULL), V(TrackHostExits, CSV, NULL), V(TrackHostExitsExpire, INTERVAL, "30 minutes"), - OBSOLETE("TrafficShaping"), V(TransListenAddress, LINELIST, NULL), VPORT(TransPort, LINELIST, NULL), V(TransProxyType, STRING, "default"), @@ -418,6 +417,7 @@ static config_var_t option_vars_[] = { V(UseBridges, BOOL, "0"), V(UseEntryGuards, BOOL, "1"), V(UseEntryGuardsAsDirGuards, BOOL, "1"), + V(UseGuardFraction, AUTOBOOL, "auto"), V(UseMicrodescriptors, AUTOBOOL, "auto"), V(UseNTorHandshake, AUTOBOOL, "1"), V(User, STRING, NULL), @@ -435,6 +435,7 @@ static config_var_t option_vars_[] = { V(V3AuthNIntervalsValid, UINT, "3"), V(V3AuthUseLegacyKey, BOOL, "0"), V(V3BandwidthsFile, FILENAME, NULL), + V(GuardfractionFile, FILENAME, NULL), VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"), V(VirtualAddrNetworkIPv4, STRING, "127.192.0.0/10"), V(VirtualAddrNetworkIPv6, STRING, "[FE80::]/10"), @@ -447,7 +448,7 @@ static config_var_t option_vars_[] = { VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword, NULL), VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), - V(MinUptimeHidServDirectoryV2, INTERVAL, "25 hours"), + V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"), V(VoteOnHidServDirectoriesV2, BOOL, "1"), V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 60, 60, 120, " "300, 900, 2147483647"), @@ -466,7 +467,9 @@ static config_var_t option_vars_[] = { V(TestingDescriptorMaxDownloadTries, UINT, "8"), V(TestingMicrodescMaxDownloadTries, UINT, "8"), V(TestingCertMaxDownloadTries, UINT, "8"), + V(TestingDirAuthVoteExit, ROUTERSET, NULL), V(TestingDirAuthVoteGuard, ROUTERSET, NULL), + V(TestingDirAuthVoteHSDir, ROUTERSET, NULL), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } @@ -489,7 +492,7 @@ static const config_var_t testing_tor_network_defaults[] = { V(V3AuthVotingInterval, INTERVAL, "5 minutes"), V(V3AuthVoteDelay, INTERVAL, "20 seconds"), V(V3AuthDistDelay, INTERVAL, "20 seconds"), - V(TestingV3AuthInitialVotingInterval, INTERVAL, "5 minutes"), + V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"), V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"), V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"), V(TestingV3AuthVotingStartOffset, INTERVAL, "0"), @@ -515,6 +518,7 @@ static const config_var_t testing_tor_network_defaults[] = { V(TestingEnableCellStatsEvent, BOOL, "1"), V(TestingEnableTbEmptyEvent, BOOL, "1"), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"), + V(RendPostPeriod, INTERVAL, "2 minutes"), { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } }; @@ -536,12 +540,6 @@ static int options_transition_affects_workers( static int options_transition_affects_descriptor( const or_options_t *old_options, const or_options_t *new_options); static int check_nickname_list(char **lst, const char *name, char **msg); - -static int parse_client_transport_line(const or_options_t *options, - const char *line, int validate_only); - -static int parse_server_transport_line(const or_options_t *options, - const char *line, int validate_only); static char *get_bindaddr_from_transport_listen_line(const char *line, const char *transport); static int parse_dir_authority_line(const char *line, @@ -558,7 +556,8 @@ static int check_server_ports(const smartlist_t *ports, static int validate_data_directory(or_options_t *options); static int write_configuration_file(const char *fname, const or_options_t *options); -static int options_init_logs(or_options_t *options, int validate_only); +static int options_init_logs(const or_options_t *old_options, + or_options_t *options, int validate_only); static void init_libevent(const or_options_t *options); static int opt_streq(const char *s1, const char *s2); @@ -843,7 +842,9 @@ escaped_safe_str(const char *address) } /** Add the default directory authorities directly into the trusted dir list, - * but only add them insofar as they share bits with <b>type</b>. */ + * but only add them insofar as they share bits with <b>type</b>. + * Each authority's bits are restricted to the bits shared with <b>type</b>. + * If <b>type</b> is ALL_DIRINFO or NO_DIRINFO (zero), add all authorities. */ static void add_default_trusted_dir_authorities(dirinfo_type_t type) { @@ -852,9 +853,11 @@ add_default_trusted_dir_authorities(dirinfo_type_t type) "moria1 orport=9101 " "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", - "tor26 orport=443 v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 " + "tor26 orport=443 " + "v3ident=14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4 " "86.59.21.38:80 847B 1F85 0344 D787 6491 A548 92F9 0493 4E4E B85D", - "dizum orport=443 v3ident=E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 " + "dizum orport=443 " + "v3ident=E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58 " "194.109.206.212:80 7EA6 EAD6 FD83 083C 538F 4403 8BBF A077 587D D755", "Bifroest orport=443 bridge " "37.218.247.217:80 1D8F 3A91 C37C 5D1C 4C19 B1AD 1D0C FBE8 BF72 D8E1", @@ -983,7 +986,10 @@ consider_adding_dir_servers(const or_options_t *options, type |= BRIDGE_DIRINFO; if (!options->AlternateDirAuthority) type |= V3_DIRINFO | EXTRAINFO_DIRINFO | MICRODESC_DIRINFO; - add_default_trusted_dir_authorities(type); + /* if type == NO_DIRINFO, we don't want to add any of the + * default authorities, because we've replaced them all */ + if (type != NO_DIRINFO) + add_default_trusted_dir_authorities(type); } if (!options->FallbackDir) add_default_fallback_dir_servers(); @@ -1019,7 +1025,7 @@ options_act_reversible(const or_options_t *old_options, char **msg) int running_tor = options->command == CMD_RUN_TOR; int set_conn_limit = 0; int r = -1; - int logs_marked = 0; + int logs_marked = 0, logs_initialized = 0; int old_min_log_level = get_min_log_level(); /* Daemonize _first_, since we only want to open most of this stuff in @@ -1030,6 +1036,11 @@ options_act_reversible(const or_options_t *old_options, char **msg) start_daemon(); } +#ifdef HAVE_SYSTEMD + /* Our PID may have changed, inform supervisor */ + sd_notifyf(0, "MAINPID=%ld\n", (long int)getpid()); +#endif + #ifndef HAVE_SYS_UN_H if (options->ControlSocket || options->ControlSocketsGroupWritable) { *msg = tor_strdup("Unix domain sockets (ControlSocket) not supported " @@ -1065,6 +1076,14 @@ options_act_reversible(const or_options_t *old_options, char **msg) if (running_tor && !libevent_initialized) { init_libevent(options); libevent_initialized = 1; + + /* + * Initialize the scheduler - this has to come after + * options_init_from_torrc() sets up libevent - why yes, that seems + * completely sensible to hide the libevent setup in the option parsing + * code! It also needs to happen before init_keys(), so it needs to + * happen here too. How yucky. */ + scheduler_init(); } /* Adjust the port configuration so we can launch listeners. */ @@ -1094,6 +1113,8 @@ options_act_reversible(const or_options_t *old_options, char **msg) "non-control network connections. Shutting down all existing " "connections."); connection_mark_all_noncontrol_connections(); + /* We can't complete circuits until the network is re-enabled. */ + note_that_we_maybe_cant_complete_circuits(); } } @@ -1144,10 +1165,12 @@ options_act_reversible(const or_options_t *old_options, char **msg) mark_logs_temp(); /* Close current logs once new logs are open. */ logs_marked = 1; - if (options_init_logs(options, 0)<0) { /* Configure the tor_log(s) */ + /* Configure the tor_log(s) */ + if (options_init_logs(old_options, options, 0)<0) { *msg = tor_strdup("Failed to init Log options. See logs for details."); goto rollback; } + logs_initialized = 1; commit: r = 0; @@ -1160,6 +1183,9 @@ options_act_reversible(const or_options_t *old_options, char **msg) tor_free(severity); tor_log_update_sigsafe_err_fds(); } + if (logs_initialized) { + flush_log_messages_from_startup(); + } { const char *badness = NULL; @@ -1237,7 +1263,8 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out) routerset_needs_geoip(options->EntryNodes) || routerset_needs_geoip(options->ExitNodes) || routerset_needs_geoip(options->ExcludeExitNodes) || - routerset_needs_geoip(options->ExcludeNodes); + routerset_needs_geoip(options->ExcludeNodes) || + routerset_needs_geoip(options->Tor2webRendezvousPoints); if (routerset_usage && reason_out) { *reason_out = "We've been configured to use (or avoid) nodes in certain " @@ -1370,7 +1397,7 @@ options_act(const or_options_t *old_options) log_err(LD_CONFIG, "This copy of Tor was not compiled to run in " "'tor2web mode'. It cannot be run with the Tor2webMode torrc " "option enabled. To enable Tor2webMode recompile with the " - "--enable-tor2webmode option."); + "--enable-tor2web-mode option."); return -1; } #endif @@ -1421,26 +1448,35 @@ options_act(const or_options_t *old_options) rep_hist_load_mtbf_data(time(NULL)); } + /* If we have an ExtORPort, initialize its auth cookie. */ + if (running_tor && + init_ext_or_cookie_authentication(!!options->ExtORPort_lines) < 0) { + log_warn(LD_CONFIG,"Error creating Extended ORPort cookie file."); + return -1; + } + mark_transport_list(); pt_prepare_proxy_list_for_config_read(); - if (options->ClientTransportPlugin) { - for (cl = options->ClientTransportPlugin; cl; cl = cl->next) { - if (parse_client_transport_line(options, cl->value, 0)<0) { - log_warn(LD_BUG, - "Previously validated ClientTransportPlugin line " - "could not be added!"); - return -1; + if (!options->DisableNetwork) { + if (options->ClientTransportPlugin) { + for (cl = options->ClientTransportPlugin; cl; cl = cl->next) { + if (parse_transport_line(options, cl->value, 0, 0) < 0) { + log_warn(LD_BUG, + "Previously validated ClientTransportPlugin line " + "could not be added!"); + return -1; + } } } - } - if (options->ServerTransportPlugin && server_mode(options)) { - for (cl = options->ServerTransportPlugin; cl; cl = cl->next) { - if (parse_server_transport_line(options, cl->value, 0)<0) { - log_warn(LD_BUG, - "Previously validated ServerTransportPlugin line " - "could not be added!"); - return -1; + if (options->ServerTransportPlugin && server_mode(options)) { + for (cl = options->ServerTransportPlugin; cl; cl = cl->next) { + if (parse_transport_line(options, cl->value, 0, 1) < 0) { + log_warn(LD_BUG, + "Previously validated ServerTransportPlugin line " + "could not be added!"); + return -1; + } } } } @@ -1523,12 +1559,6 @@ options_act(const or_options_t *old_options) return -1; } - /* If we have an ExtORPort, initialize its auth cookie. */ - if (init_ext_or_cookie_authentication(!!options->ExtORPort_lines) < 0) { - log_warn(LD_CONFIG,"Error creating Extended ORPort cookie file."); - return -1; - } - monitor_owning_controller_process(options->OwningControllerProcess); /* reload keys as needed for rendezvous services. */ @@ -1537,6 +1567,12 @@ options_act(const or_options_t *old_options) return -1; } + /* Set up scheduler thresholds */ + scheduler_set_watermarks((uint32_t)options->SchedulerLowWaterMark__, + (uint32_t)options->SchedulerHighWaterMark__, + (options->SchedulerMaxFlushCells__ > 0) ? + options->SchedulerMaxFlushCells__ : 1000); + /* Set up accounting */ if (accounting_parse_options(options, 0)<0) { log_warn(LD_CONFIG,"Error in accounting options"); @@ -1586,11 +1622,25 @@ options_act(const or_options_t *old_options) } if (parse_outbound_addresses(options, 0, &msg) < 0) { - log_warn(LD_BUG, "Failed parsing oubound bind addresses: %s", msg); + log_warn(LD_BUG, "Failed parsing outbound bind addresses: %s", msg); tor_free(msg); return -1; } + config_maybe_load_geoip_files_(options, old_options); + + if (geoip_is_loaded(AF_INET) && options->GeoIPExcludeUnknown) { + /* ExcludeUnknown is true or "auto" */ + const int is_auto = options->GeoIPExcludeUnknown == -1; + int changed; + + changed = routerset_add_unknown_ccs(&options->ExcludeNodes, is_auto); + changed += routerset_add_unknown_ccs(&options->ExcludeExitNodes, is_auto); + + if (changed) + routerset_add_unknown_ccs(&options->ExcludeExitNodesUnion_, is_auto); + } + /* Check for transitions that need action. */ if (old_options) { int revise_trackexithosts = 0; @@ -1604,6 +1654,8 @@ options_act(const or_options_t *old_options) options->ExcludeExitNodes) || !routerset_equal(old_options->EntryNodes, options->EntryNodes) || !routerset_equal(old_options->ExitNodes, options->ExitNodes) || + !routerset_equal(old_options->Tor2webRendezvousPoints, + options->Tor2webRendezvousPoints) || options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, "Changed to using entry guards or bridges, or changed " @@ -1669,11 +1721,12 @@ options_act(const or_options_t *old_options) "Worker-related options changed. Rotating workers."); if (server_mode(options) && !server_mode(old_options)) { + cpu_init(); ip_address_changed(0); - if (can_complete_circuit || !any_predicted_circuits(time(NULL))) + if (have_completed_a_circuit() || !any_predicted_circuits(time(NULL))) inform_testing_reachability(); } - cpuworkers_rotate(); + cpuworkers_rotate_keyinfo(); if (dns_reset()) return -1; } else { @@ -1686,36 +1739,23 @@ options_act(const or_options_t *old_options) connection_or_update_token_buckets(get_connection_array(), options); } - config_maybe_load_geoip_files_(options, old_options); - - if (geoip_is_loaded(AF_INET) && options->GeoIPExcludeUnknown) { - /* ExcludeUnknown is true or "auto" */ - const int is_auto = options->GeoIPExcludeUnknown == -1; - int changed; - - changed = routerset_add_unknown_ccs(&options->ExcludeNodes, is_auto); - changed += routerset_add_unknown_ccs(&options->ExcludeExitNodes, is_auto); - - if (changed) - routerset_add_unknown_ccs(&options->ExcludeExitNodesUnion_, is_auto); - } + /* Only collect directory-request statistics on relays and bridges. */ + options->DirReqStatistics = options->DirReqStatistics_option && + server_mode(options); if (options->CellStatistics || options->DirReqStatistics || options->EntryStatistics || options->ExitPortStatistics || options->ConnDirectionStatistics || + options->HiddenServiceStatistics || options->BridgeAuthoritativeDir) { time_t now = time(NULL); int print_notice = 0; - /* Only collect directory-request statistics on relays and bridges. */ - if (!server_mode(options)) { - options->DirReqStatistics = 0; - } - /* Only collect other relay-only statistics on relays. */ if (!public_server_mode(options)) { options->CellStatistics = 0; options->EntryStatistics = 0; + options->HiddenServiceStatistics = 0; options->ExitPortStatistics = 0; } @@ -1730,8 +1770,8 @@ options_act(const or_options_t *old_options) geoip_dirreq_stats_init(now); print_notice = 1; } else { + /* disable statistics collection since we have no geoip file */ options->DirReqStatistics = 0; - /* Don't warn Tor clients, they don't use statistics */ if (options->ORPort_set) log_notice(LD_CONFIG, "Configured to measure directory request " "statistics, but no GeoIP database found. " @@ -1761,6 +1801,11 @@ options_act(const or_options_t *old_options) options->ConnDirectionStatistics) { rep_hist_conn_stats_init(now); } + if ((!old_options || !old_options->HiddenServiceStatistics) && + options->HiddenServiceStatistics) { + log_info(LD_CONFIG, "Configured to measure hidden service statistics."); + rep_hist_hs_stats_init(now); + } if ((!old_options || !old_options->BridgeAuthoritativeDir) && options->BridgeAuthoritativeDir) { rep_hist_desc_stats_init(now); @@ -1772,6 +1817,8 @@ options_act(const or_options_t *old_options) "data directory in 24 hours from now."); } + /* If we used to have statistics enabled but we just disabled them, + stop gathering them. */ if (old_options && old_options->CellStatistics && !options->CellStatistics) rep_hist_buffer_stats_term(); @@ -1781,6 +1828,9 @@ options_act(const or_options_t *old_options) if (old_options && old_options->EntryStatistics && !options->EntryStatistics) geoip_entry_stats_term(); + if (old_options && old_options->HiddenServiceStatistics && + !options->HiddenServiceStatistics) + rep_hist_hs_stats_term(); if (old_options && old_options->ExitPortStatistics && !options->ExitPortStatistics) rep_hist_exit_stats_term(); @@ -1813,7 +1863,7 @@ options_act(const or_options_t *old_options) directory_fetches_dir_info_early(old_options)) || !bool_eq(directory_fetches_dir_info_later(options), directory_fetches_dir_info_later(old_options))) { - /* Make sure update_router_have_min_dir_info gets called. */ + /* Make sure update_router_have_minimum_dir_info() gets called. */ router_dir_info_changed(); /* We might need to download a new consensus status later or sooner than * we had expected. */ @@ -2027,7 +2077,7 @@ print_usage(void) printf( "Copyright (c) 2001-2004, Roger Dingledine\n" "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n" -"Copyright (c) 2007-2013, The Tor Project, Inc.\n\n" +"Copyright (c) 2007-2015, The Tor Project, Inc.\n\n" "tor -f <torrc> [args]\n" "See man page for options, or https://www.torproject.org/ for " "documentation.\n"); @@ -2059,8 +2109,41 @@ get_last_resolved_addr(void) return last_resolved_addr; } +/** Reset last_resolved_addr from outside this file. */ +void +reset_last_resolved_addr(void) +{ + last_resolved_addr = 0; +} + /** - * Use <b>options-\>Address</b> to guess our public IP address. + * Attempt getting our non-local (as judged by tor_addr_is_internal() + * function) IP address using following techniques, listed in + * order from best (most desirable, try first) to worst (least + * desirable, try if everything else fails). + * + * First, attempt using <b>options-\>Address</b> to get our + * non-local IP address. + * + * If <b>options-\>Address</b> represents a non-local IP address, + * consider it ours. + * + * If <b>options-\>Address</b> is a DNS name that resolves to + * a non-local IP address, consider this IP address ours. + * + * If <b>options-\>Address</b> is NULL, fall back to getting local + * hostname and using it in above-described ways to try and + * get our IP address. + * + * In case local hostname cannot be resolved to a non-local IP + * address, try getting an IP address of network interface + * in hopes it will be non-local one. + * + * Fail if one or more of the following is true: + * - DNS name in <b>options-\>Address</b> cannot be resolved. + * - <b>options-\>Address</b> is a local host address. + * - Attempt to getting local hostname fails. + * - Attempt to getting network interface address fails. * * Return 0 if all is well, or -1 if we can't find a suitable * public IP address. @@ -2069,6 +2152,11 @@ get_last_resolved_addr(void) * - Put our public IP address (in host order) into *<b>addr_out</b>. * - If <b>method_out</b> is non-NULL, set *<b>method_out</b> to a static * string describing how we arrived at our answer. + * - "CONFIGURED" - parsed from IP address string in + * <b>options-\>Address</b> + * - "RESOLVED" - resolved from DNS name in <b>options-\>Address</b> + * - "GETHOSTNAME" - resolved from a local hostname. + * - "INTERFACE" - retrieved from a network interface. * - If <b>hostname_out</b> is non-NULL, and we resolved a hostname to * get our address, set *<b>hostname_out</b> to a newly allocated string * holding that hostname. (If we didn't get our address by resolving a @@ -2107,7 +2195,7 @@ resolve_my_address(int warn_severity, const or_options_t *options, explicit_ip = 0; /* it's implicit */ explicit_hostname = 0; /* it's implicit */ - if (gethostname(hostname, sizeof(hostname)) < 0) { + if (tor_gethostname(hostname, sizeof(hostname)) < 0) { log_fn(warn_severity, LD_NET,"Error obtaining local hostname"); return -1; } @@ -2274,8 +2362,8 @@ resolve_my_address(int warn_severity, const or_options_t *options, /** Return true iff <b>addr</b> is judged to be on the same network as us, or * on a private network. */ -int -is_local_addr(const tor_addr_t *addr) +MOCK_IMPL(int, +is_local_addr, (const tor_addr_t *addr)) { if (tor_addr_is_internal(addr, 0)) return 1; @@ -2434,6 +2522,7 @@ compute_publishserverdescriptor(or_options_t *options) /** Lowest allowable value for RendPostPeriod; if this is too low, hidden * services can overload the directory system. */ #define MIN_REND_POST_PERIOD (10*60) +#define MIN_REND_POST_PERIOD_TESTING (5) /** Higest allowable value for PredictedPortsRelevanceTime; if this is * too high, our selection of exits will decrease for an extended @@ -2547,7 +2636,8 @@ options_validate(or_options_t *old_options, or_options_t *options, config_line_append(&options->Logs, "Log", "warn stdout"); } - if (options_init_logs(options, 1)<0) /* Validate the tor_log(s) */ + /* Validate the tor_log(s) */ + if (options_init_logs(old_options, options, 1)<0) REJECT("Failed to validate Log options. See logs for details."); if (authdir_mode(options)) { @@ -2557,11 +2647,6 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Failed to resolve/guess local address. See logs for details."); } -#ifndef _WIN32 - if (options->RunAsDaemon && torrc_fname && path_is_relative(torrc_fname)) - REJECT("Can't use a relative path to torrc when RunAsDaemon is set."); -#endif - if (server_mode(options) && options->RendConfigLines) log_warn(LD_CONFIG, "Tor is currently configured as a relay and a hidden service. " @@ -2583,20 +2668,24 @@ options_validate(or_options_t *old_options, or_options_t *options, if (!strcasecmp(options->TransProxyType, "default")) { options->TransProxyType_parsed = TPT_DEFAULT; } else if (!strcasecmp(options->TransProxyType, "pf-divert")) { -#ifndef __OpenBSD__ - REJECT("pf-divert is a OpenBSD-specific feature."); +#if !defined(__OpenBSD__) && !defined( DARWIN ) + /* Later versions of OS X have pf */ + REJECT("pf-divert is a OpenBSD-specific " + "and OS X/Darwin-specific feature."); #else options->TransProxyType_parsed = TPT_PF_DIVERT; #endif } else if (!strcasecmp(options->TransProxyType, "tproxy")) { -#ifndef __linux__ +#if !defined(__linux__) REJECT("TPROXY is a Linux-specific feature."); #else options->TransProxyType_parsed = TPT_TPROXY; #endif } else if (!strcasecmp(options->TransProxyType, "ipfw")) { -#ifndef __FreeBSD__ - REJECT("ipfw is a FreeBSD-specific feature."); +#if !defined(__FreeBSD__) && !defined( DARWIN ) + /* Earlier versions of OS X have ipfw */ + REJECT("ipfw is a FreeBSD-specific" + "and OS X/Darwin-specific feature."); #else options->TransProxyType_parsed = TPT_IPFW; #endif @@ -2627,6 +2716,17 @@ options_validate(or_options_t *old_options, or_options_t *options, routerset_union(options->ExcludeExitNodesUnion_,options->ExcludeNodes); } + if (options->SchedulerLowWaterMark__ == 0 || + options->SchedulerLowWaterMark__ > UINT32_MAX) { + log_warn(LD_GENERAL, "Bad SchedulerLowWaterMark__ option"); + return -1; + } else if (options->SchedulerHighWaterMark__ <= + options->SchedulerLowWaterMark__ || + options->SchedulerHighWaterMark__ > UINT32_MAX) { + log_warn(LD_GENERAL, "Bad SchedulerHighWaterMark option"); + return -1; + } + if (options->NodeFamilies) { options->NodeFamilySets = smartlist_new(); for (cl = options->NodeFamilies; cl; cl = cl->next) { @@ -2651,6 +2751,13 @@ options_validate(or_options_t *old_options, or_options_t *options, "features to be broken in unpredictable ways."); } + for (cl = options->RecommendedPackages; cl; cl = cl->next) { + if (! validate_recommended_package_line(cl->value)) { + log_warn(LD_CONFIG, "Invalid RecommendedPackage line %s will be ignored", + escaped(cl->value)); + } + } + if (options->AuthoritativeDir) { if (!options->ContactInfo && !options->TestingTorNetwork) REJECT("Authoritative directory servers must set ContactInfo"); @@ -2683,6 +2790,10 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->V3BandwidthsFile && !old_options) { dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); } + /* same for guardfraction file */ + if (options->GuardfractionFile && !old_options) { + dirserv_read_guardfraction_file(options->GuardfractionFile, NULL); + } } if (options->AuthoritativeDir && !options->DirPort_set) @@ -2837,6 +2948,7 @@ options_validate(or_options_t *old_options, or_options_t *options, options->MaxMemInQueues = compute_real_max_mem_in_queues(options->MaxMemInQueues_raw, server_mode(options)); + options->MaxMemInQueues_low_threshold = (options->MaxMemInQueues / 4) * 3; options->AllowInvalid_ = 0; @@ -2901,10 +3013,13 @@ options_validate(or_options_t *old_options, or_options_t *options, options->MinUptimeHidServDirectoryV2 = 0; } - if (options->RendPostPeriod < MIN_REND_POST_PERIOD) { + const int min_rendpostperiod = + options->TestingTorNetwork ? + MIN_REND_POST_PERIOD_TESTING : MIN_REND_POST_PERIOD; + if (options->RendPostPeriod < min_rendpostperiod) { log_warn(LD_CONFIG, "RendPostPeriod option is too short; " - "raising to %d seconds.", MIN_REND_POST_PERIOD); - options->RendPostPeriod = MIN_REND_POST_PERIOD; + "raising to %d seconds.", min_rendpostperiod); + options->RendPostPeriod = min_rendpostperiod;; } if (options->RendPostPeriod > MAX_DIR_PERIOD) { @@ -2920,6 +3035,7 @@ options_validate(or_options_t *old_options, or_options_t *options, options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE; } +#ifdef ENABLE_TOR2WEB_MODE if (options->Tor2webMode && options->LearnCircuitBuildTimeout) { /* LearnCircuitBuildTimeout and Tor2webMode are incompatible in * two ways: @@ -2951,6 +3067,11 @@ options_validate(or_options_t *old_options, or_options_t *options, "Tor2WebMode is enabled; disabling UseEntryGuards."); options->UseEntryGuards = 0; } +#endif + + if (options->Tor2webRendezvousPoints && !options->Tor2webMode) { + REJECT("Tor2webRendezvousPoints cannot be set without Tor2webMode."); + } if (!(options->UseEntryGuards) && (options->RendConfigLines != NULL)) { @@ -3075,29 +3196,34 @@ options_validate(or_options_t *old_options, or_options_t *options, options->RelayBandwidthRate = options->RelayBandwidthBurst; if (server_mode(options)) { - if (options->BandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { + const unsigned required_min_bw = + public_server_mode(options) ? + RELAY_REQUIRED_MIN_BANDWIDTH : BRIDGE_REQUIRED_MIN_BANDWIDTH; + const char * const optbridge = + public_server_mode(options) ? "" : "bridge "; + if (options->BandwidthRate < required_min_bw) { tor_asprintf(msg, "BandwidthRate is set to %d bytes/second. " - "For servers, it must be at least %d.", - (int)options->BandwidthRate, - ROUTER_REQUIRED_MIN_BANDWIDTH); + "For %sservers, it must be at least %u.", + (int)options->BandwidthRate, optbridge, + required_min_bw); return -1; } else if (options->MaxAdvertisedBandwidth < - ROUTER_REQUIRED_MIN_BANDWIDTH/2) { + required_min_bw/2) { tor_asprintf(msg, "MaxAdvertisedBandwidth is set to %d bytes/second. " - "For servers, it must be at least %d.", - (int)options->MaxAdvertisedBandwidth, - ROUTER_REQUIRED_MIN_BANDWIDTH/2); + "For %sservers, it must be at least %u.", + (int)options->MaxAdvertisedBandwidth, optbridge, + required_min_bw/2); return -1; } if (options->RelayBandwidthRate && - options->RelayBandwidthRate < ROUTER_REQUIRED_MIN_BANDWIDTH) { + options->RelayBandwidthRate < required_min_bw) { tor_asprintf(msg, "RelayBandwidthRate is set to %d bytes/second. " - "For servers, it must be at least %d.", - (int)options->RelayBandwidthRate, - ROUTER_REQUIRED_MIN_BANDWIDTH); + "For %sservers, it must be at least %u.", + (int)options->RelayBandwidthRate, optbridge, + required_min_bw); return -1; } } @@ -3133,6 +3259,16 @@ options_validate(or_options_t *old_options, or_options_t *options, } } + options->AccountingRule = ACCT_MAX; + if (options->AccountingRule_option) { + if (!strcmp(options->AccountingRule_option, "sum")) + options->AccountingRule = ACCT_SUM; + else if (!strcmp(options->AccountingRule_option, "max")) + options->AccountingRule = ACCT_MAX; + else + REJECT("AccountingRule must be 'sum' or 'max'"); + } + if (options->HTTPProxy) { /* parse it now */ if (tor_addr_port_lookup(options->HTTPProxy, &options->HTTPProxyAddr, &options->HTTPProxyPort) < 0) @@ -3181,11 +3317,11 @@ options_validate(or_options_t *old_options, or_options_t *options, } } - /* Check if more than one proxy type has been enabled. */ + /* Check if more than one exclusive proxy type has been enabled. */ if (!!options->Socks4Proxy + !!options->Socks5Proxy + - !!options->HTTPSProxy + !!options->ClientTransportPlugin > 1) + !!options->HTTPSProxy > 1) REJECT("You have configured more than one proxy type. " - "(Socks4Proxy|Socks5Proxy|HTTPSProxy|ClientTransportPlugin)"); + "(Socks4Proxy|Socks5Proxy|HTTPSProxy)"); /* Check if the proxies will give surprising behavior. */ if (options->HTTPProxy && !(options->Socks4Proxy || @@ -3293,12 +3429,12 @@ options_validate(or_options_t *old_options, or_options_t *options, } for (cl = options->ClientTransportPlugin; cl; cl = cl->next) { - if (parse_client_transport_line(options, cl->value, 1)<0) + if (parse_transport_line(options, cl->value, 1, 0) < 0) REJECT("Invalid client transport line. See logs for details."); } for (cl = options->ServerTransportPlugin; cl; cl = cl->next) { - if (parse_server_transport_line(options, cl->value, 1)<0) + if (parse_transport_line(options, cl->value, 1, 1) < 0) REJECT("Invalid server transport line. See logs for details."); } @@ -3362,19 +3498,68 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->V3AuthVoteDelay + options->V3AuthDistDelay >= options->V3AuthVotingInterval/2) { - REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half " - "V3AuthVotingInterval"); + /* + This doesn't work, but it seems like it should: + what code is preventing the interval being less than twice the lead-up? + if (options->TestingTorNetwork) { + if (options->V3AuthVoteDelay + options->V3AuthDistDelay >= + options->V3AuthVotingInterval) { + REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than " + "V3AuthVotingInterval"); + } else { + COMPLAIN("V3AuthVoteDelay plus V3AuthDistDelay is more than half " + "V3AuthVotingInterval. This may lead to " + "consensus instability, particularly if clocks drift."); + } + } else { + */ + REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half " + "V3AuthVotingInterval"); + /* + } + */ + } + + if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) { + if (options->TestingTorNetwork) { + if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS_TESTING) { + REJECT("V3AuthVoteDelay is way too low."); + } else { + COMPLAIN("V3AuthVoteDelay is very low. " + "This may lead to failure to vote for a consensus."); + } + } else { + REJECT("V3AuthVoteDelay is way too low."); + } + } + + if (options->V3AuthDistDelay < MIN_DIST_SECONDS) { + if (options->TestingTorNetwork) { + if (options->V3AuthDistDelay < MIN_DIST_SECONDS_TESTING) { + REJECT("V3AuthDistDelay is way too low."); + } else { + COMPLAIN("V3AuthDistDelay is very low. " + "This may lead to missing votes in a consensus."); + } + } else { + REJECT("V3AuthDistDelay is way too low."); + } } - if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) - REJECT("V3AuthVoteDelay is way too low."); - if (options->V3AuthDistDelay < MIN_DIST_SECONDS) - REJECT("V3AuthDistDelay is way too low."); if (options->V3AuthNIntervalsValid < 2) REJECT("V3AuthNIntervalsValid must be at least 2."); if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL) { - REJECT("V3AuthVotingInterval is insanely low."); + if (options->TestingTorNetwork) { + if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL_TESTING) { + REJECT("V3AuthVotingInterval is insanely low."); + } else { + COMPLAIN("V3AuthVotingInterval is very low. " + "This may lead to failure to synchronise for a consensus."); + } + } else { + REJECT("V3AuthVotingInterval is insanely low."); + } } else if (options->V3AuthVotingInterval > 24*60*60) { REJECT("V3AuthVotingInterval is insanely high."); } else if (((24*60*60) % options->V3AuthVotingInterval) != 0) { @@ -3396,15 +3581,6 @@ options_validate(or_options_t *old_options, or_options_t *options, AF_INET6, 1, msg)<0) return -1; - if (options->AutomapHostsSuffixes) { - SMARTLIST_FOREACH(options->AutomapHostsSuffixes, char *, suf, - { - size_t len = strlen(suf); - if (len && suf[len-1] == '.') - suf[len-1] = '\0'; - }); - } - if (options->TestingTorNetwork && !(options->DirAuthorities || (options->AlternateDirAuthority && @@ -3449,26 +3625,27 @@ options_validate(or_options_t *old_options, or_options_t *options, CHECK_DEFAULT(TestingCertMaxDownloadTries); #undef CHECK_DEFAULT - if (options->TestingV3AuthInitialVotingInterval < MIN_VOTE_INTERVAL) { + if (options->TestingV3AuthInitialVotingInterval + < MIN_VOTE_INTERVAL_TESTING_INITIAL) { REJECT("TestingV3AuthInitialVotingInterval is insanely low."); } else if (((30*60) % options->TestingV3AuthInitialVotingInterval) != 0) { REJECT("TestingV3AuthInitialVotingInterval does not divide evenly into " "30 minutes."); } - if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS) { + if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS_TESTING) { REJECT("TestingV3AuthInitialVoteDelay is way too low."); } - if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS) { + if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS_TESTING) { REJECT("TestingV3AuthInitialDistDelay is way too low."); } if (options->TestingV3AuthInitialVoteDelay + options->TestingV3AuthInitialDistDelay >= - options->TestingV3AuthInitialVotingInterval/2) { + options->TestingV3AuthInitialVotingInterval) { REJECT("TestingV3AuthInitialVoteDelay plus TestingV3AuthInitialDistDelay " - "must be less than half TestingV3AuthInitialVotingInterval"); + "must be less than TestingV3AuthInitialVotingInterval"); } if (options->TestingV3AuthVotingStartOffset > @@ -3476,6 +3653,8 @@ options_validate(or_options_t *old_options, or_options_t *options, options->V3AuthVotingInterval)) { REJECT("TestingV3AuthVotingStartOffset is higher than the voting " "interval."); + } else if (options->TestingV3AuthVotingStartOffset < 0) { + REJECT("TestingV3AuthVotingStartOffset must be non-negative."); } if (options->TestingAuthDirTimeToLearnReachability < 0) { @@ -3796,6 +3975,7 @@ options_transition_affects_descriptor(const or_options_t *old_options, !opt_streq(old_options->Nickname,new_options->Nickname) || !opt_streq(old_options->Address,new_options->Address) || !config_lines_eq(old_options->ExitPolicy,new_options->ExitPolicy) || + old_options->ExitRelay != new_options->ExitRelay || old_options->ExitPolicyRejectPrivate != new_options->ExitPolicyRejectPrivate || old_options->IPv6Exit != new_options->IPv6Exit || @@ -3884,7 +4064,10 @@ get_windows_conf_root(void) static const char * get_default_conf_file(int defaults_file) { -#ifdef _WIN32 +#ifdef DISABLE_SYSTEM_TORRC + (void) defaults_file; + return NULL; +#elif defined(_WIN32) if (defaults_file) { static char defaults_path[MAX_PATH+1]; tor_snprintf(defaults_path, MAX_PATH, "%s\\torrc-defaults", @@ -4011,27 +4194,45 @@ find_torrc_filename(config_line_t *cmd_arg, if (*using_default_fname) { /* didn't find one, try CONFDIR */ const char *dflt = get_default_conf_file(defaults_file); - if (dflt && file_status(dflt) == FN_FILE) { + file_status_t st = file_status(dflt); + if (dflt && (st == FN_FILE || st == FN_EMPTY)) { fname = tor_strdup(dflt); } else { #ifndef _WIN32 char *fn = NULL; - if (!defaults_file) + if (!defaults_file) { fn = expand_filename("~/.torrc"); - if (fn && file_status(fn) == FN_FILE) { - fname = fn; + } + if (fn) { + file_status_t hmst = file_status(fn); + if (hmst == FN_FILE || hmst == FN_EMPTY || dflt == NULL) { + fname = fn; + } else { + tor_free(fn); + fname = tor_strdup(dflt); + } } else { - tor_free(fn); - fname = tor_strdup(dflt); + fname = dflt ? tor_strdup(dflt) : NULL; } #else - fname = tor_strdup(dflt); + fname = dflt ? tor_strdup(dflt) : NULL; #endif } } return fname; } +/** Read the torrc from standard input and return it as a string. + * Upon failure, return NULL. + */ +static char * +load_torrc_from_stdin(void) +{ + size_t sz_out; + + return read_file_to_str_until_eof(STDIN_FILENO,SIZE_MAX,&sz_out); +} + /** Load a configuration file from disk, setting torrc_fname or * torrc_defaults_fname if successful. * @@ -4048,16 +4249,20 @@ load_torrc_from_disk(config_line_t *cmd_arg, int defaults_file) int ignore_missing_torrc = 0; char **fname_var = defaults_file ? &torrc_defaults_fname : &torrc_fname; - fname = find_torrc_filename(cmd_arg, defaults_file, - &using_default_torrc, &ignore_missing_torrc); - tor_assert(fname); - log_debug(LD_CONFIG, "Opening config file \"%s\"", fname); - - tor_free(*fname_var); - *fname_var = fname; + if (*fname_var == NULL) { + fname = find_torrc_filename(cmd_arg, defaults_file, + &using_default_torrc, &ignore_missing_torrc); + tor_free(*fname_var); + *fname_var = fname; + } else { + fname = *fname_var; + } + log_debug(LD_CONFIG, "Opening config file \"%s\"", fname?fname:"<NULL>"); /* Open config file */ - if (file_status(fname) != FN_FILE || + file_status_t st = fname ? file_status(fname) : FN_EMPTY; + if (fname == NULL || + !(st == FN_FILE || st == FN_EMPTY) || !(cf = read_file_to_str(fname,0,NULL))) { if (using_default_torrc == 1 || ignore_missing_torrc) { if (!defaults_file) @@ -4168,7 +4373,19 @@ options_init_from_torrc(int argc, char **argv) cf = tor_strdup(""); } else { cf_defaults = load_torrc_from_disk(cmdline_only_options, 1); - cf = load_torrc_from_disk(cmdline_only_options, 0); + + const config_line_t *f_line = config_line_find(cmdline_only_options, + "-f"); + + const int read_torrc_from_stdin = + (f_line != NULL && strcmp(f_line->value, "-") == 0); + + if (read_torrc_from_stdin) { + cf = load_torrc_from_stdin(); + } else { + cf = load_torrc_from_disk(cmdline_only_options, 0); + } + if (!cf) { if (config_line_find(cmdline_only_options, "--allow-missing-torrc")) { cf = tor_strdup(""); @@ -4346,7 +4563,7 @@ options_init_from_string(const char *cf_defaults, const char *cf, return err; } -/** Return the location for our configuration file. +/** Return the location for our configuration file. May return NULL. */ const char * get_torrc_fname(int defaults_fname) @@ -4453,7 +4670,8 @@ addressmap_register_auto(const char *from, const char *to, * Initialize the logs based on the configuration file. */ static int -options_init_logs(or_options_t *options, int validate_only) +options_init_logs(const or_options_t *old_options, or_options_t *options, + int validate_only) { config_line_t *opt; int ok; @@ -4546,7 +4764,21 @@ options_init_logs(or_options_t *options, int validate_only) !strcasecmp(smartlist_get(elts,0), "file")) { if (!validate_only) { char *fname = expand_filename(smartlist_get(elts, 1)); - if (add_file_log(severity, fname) < 0) { + /* Truncate if TruncateLogFile is set and we haven't seen this option + line before. */ + int truncate = 0; + if (options->TruncateLogFile) { + truncate = 1; + if (old_options) { + config_line_t *opt2; + for (opt2 = old_options->Logs; opt2; opt2 = opt2->next) + if (!strcmp(opt->value, opt2->value)) { + truncate = 0; + break; + } + } + } + if (add_file_log(severity, fname, truncate) < 0) { log_warn(LD_CONFIG, "Couldn't open file for 'Log %s': %s", opt->value, strerror(errno)); ok = 0; @@ -4739,46 +4971,52 @@ parse_bridge_line(const char *line) return bridge_line; } -/** Read the contents of a ClientTransportPlugin line from - * <b>line</b>. Return 0 if the line is well-formed, and -1 if it - * isn't. +/** Read the contents of a ClientTransportPlugin or ServerTransportPlugin + * line from <b>line</b>, depending on the value of <b>server</b>. Return 0 + * if the line is well-formed, and -1 if it isn't. * - * If <b>validate_only</b> is 0, the line is well-formed, and the - * transport is needed by some bridge: + * If <b>validate_only</b> is 0, the line is well-formed, and the transport is + * needed by some bridge: * - If it's an external proxy line, add the transport described in the line to * our internal transport list. - * - If it's a managed proxy line, launch the managed proxy. */ -static int -parse_client_transport_line(const or_options_t *options, - const char *line, int validate_only) + * - If it's a managed proxy line, launch the managed proxy. + */ + +STATIC int +parse_transport_line(const or_options_t *options, + const char *line, int validate_only, + int server) { + smartlist_t *items = NULL; int r; - char *field2=NULL; - - const char *transports=NULL; - smartlist_t *transport_list=NULL; - char *addrport=NULL; + const char *transports = NULL; + smartlist_t *transport_list = NULL; + char *type = NULL; + char *addrport = NULL; tor_addr_t addr; uint16_t port = 0; - int socks_ver=PROXY_NONE; + int socks_ver = PROXY_NONE; /* managed proxy options */ - int is_managed=0; - char **proxy_argv=NULL; - char **tmp=NULL; + int is_managed = 0; + char **proxy_argv = NULL; + char **tmp = NULL; int proxy_argc, i; - int is_useless_proxy=1; + int is_useless_proxy = 1; int line_length; + /* Split the line into space-separated tokens */ items = smartlist_new(); smartlist_split_string(items, line, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); + line_length = smartlist_len(items); - line_length = smartlist_len(items); if (line_length < 3) { - log_warn(LD_CONFIG, "Too few arguments on ClientTransportPlugin line."); + log_warn(LD_CONFIG, + "Too few arguments on %sTransportPlugin line.", + server ? "Server" : "Client"); goto err; } @@ -4802,64 +5040,97 @@ parse_client_transport_line(const or_options_t *options, is_useless_proxy = 0; } SMARTLIST_FOREACH_END(transport_name); - /* field2 is either a SOCKS version or "exec" */ - field2 = smartlist_get(items, 1); - - if (!strcmp(field2,"socks4")) { + type = smartlist_get(items, 1); + if (!strcmp(type, "exec")) { + is_managed = 1; + } else if (server && !strcmp(type, "proxy")) { + /* 'proxy' syntax only with ServerTransportPlugin */ + is_managed = 0; + } else if (!server && !strcmp(type, "socks4")) { + /* 'socks4' syntax only with ClientTransportPlugin */ + is_managed = 0; socks_ver = PROXY_SOCKS4; - } else if (!strcmp(field2,"socks5")) { + } else if (!server && !strcmp(type, "socks5")) { + /* 'socks5' syntax only with ClientTransportPlugin */ + is_managed = 0; socks_ver = PROXY_SOCKS5; - } else if (!strcmp(field2,"exec")) { - is_managed=1; } else { - log_warn(LD_CONFIG, "Strange ClientTransportPlugin field '%s'.", - field2); + log_warn(LD_CONFIG, + "Strange %sTransportPlugin type '%s'", + server ? "Server" : "Client", type); goto err; } if (is_managed && options->Sandbox) { - log_warn(LD_CONFIG, "Managed proxies are not compatible with Sandbox mode." - "(ClientTransportPlugin line was %s)", escaped(line)); + log_warn(LD_CONFIG, + "Managed proxies are not compatible with Sandbox mode." + "(%sTransportPlugin line was %s)", + server ? "Server" : "Client", escaped(line)); goto err; } - if (is_managed) { /* managed */ - if (!validate_only && is_useless_proxy) { - log_info(LD_GENERAL, "Pluggable transport proxy (%s) does not provide " - "any needed transports and will not be launched.", line); + if (is_managed) { + /* managed */ + + if (!server && !validate_only && is_useless_proxy) { + log_info(LD_GENERAL, + "Pluggable transport proxy (%s) does not provide " + "any needed transports and will not be launched.", + line); } - /* If we are not just validating, use the rest of the line as the - argv of the proxy to be launched. Also, make sure that we are - only launching proxies that contribute useful transports. */ - if (!validate_only && !is_useless_proxy) { - proxy_argc = line_length-2; + /* + * If we are not just validating, use the rest of the line as the + * argv of the proxy to be launched. Also, make sure that we are + * only launching proxies that contribute useful transports. + */ + + if (!validate_only && (server || !is_useless_proxy)) { + proxy_argc = line_length - 2; tor_assert(proxy_argc > 0); - proxy_argv = tor_malloc_zero(sizeof(char*)*(proxy_argc+1)); + proxy_argv = tor_calloc((proxy_argc + 1), sizeof(char *)); tmp = proxy_argv; - for (i=0;i<proxy_argc;i++) { /* store arguments */ + + for (i = 0; i < proxy_argc; i++) { + /* store arguments */ *tmp++ = smartlist_get(items, 2); smartlist_del_keeporder(items, 2); } - *tmp = NULL; /*terminated with NULL, just like execve() likes it*/ + *tmp = NULL; /* terminated with NULL, just like execve() likes it */ /* kickstart the thing */ - pt_kickstart_client_proxy(transport_list, proxy_argv); + if (server) { + pt_kickstart_server_proxy(transport_list, proxy_argv); + } else { + pt_kickstart_client_proxy(transport_list, proxy_argv); + } } - } else { /* external */ + } else { + /* external */ + + /* ClientTransportPlugins connecting through a proxy is managed only. */ + if (!server && (options->Socks4Proxy || options->Socks5Proxy || + options->HTTPSProxy)) { + log_warn(LD_CONFIG, "You have configured an external proxy with another " + "proxy type. (Socks4Proxy|Socks5Proxy|HTTPSProxy)"); + goto err; + } + if (smartlist_len(transport_list) != 1) { - log_warn(LD_CONFIG, "You can't have an external proxy with " - "more than one transports."); + log_warn(LD_CONFIG, + "You can't have an external proxy with more than " + "one transport."); goto err; } addrport = smartlist_get(items, 2); - if (tor_addr_port_lookup(addrport, &addr, &port)<0) { - log_warn(LD_CONFIG, "Error parsing transport " - "address '%s'", addrport); + if (tor_addr_port_lookup(addrport, &addr, &port) < 0) { + log_warn(LD_CONFIG, + "Error parsing transport address '%s'", addrport); goto err; } + if (!port) { log_warn(LD_CONFIG, "Transport address '%s' has no port.", addrport); @@ -4867,11 +5138,15 @@ parse_client_transport_line(const or_options_t *options, } if (!validate_only) { - transport_add_from_config(&addr, port, smartlist_get(transport_list, 0), - socks_ver); - - log_info(LD_DIR, "Transport '%s' found at %s", + log_info(LD_DIR, "%s '%s' at %s.", + server ? "Server transport" : "Transport", transports, fmt_addrport(&addr, port)); + + if (!server) { + transport_add_from_config(&addr, port, + smartlist_get(transport_list, 0), + socks_ver); + } } } @@ -5043,138 +5318,12 @@ get_options_for_server_transport(const char *transport) return NULL; } -/** Read the contents of a ServerTransportPlugin line from - * <b>line</b>. Return 0 if the line is well-formed, and -1 if it - * isn't. - * If <b>validate_only</b> is 0, the line is well-formed, and it's a - * managed proxy line, launch the managed proxy. */ -static int -parse_server_transport_line(const or_options_t *options, - const char *line, int validate_only) -{ - smartlist_t *items = NULL; - int r; - const char *transports=NULL; - smartlist_t *transport_list=NULL; - char *type=NULL; - char *addrport=NULL; - tor_addr_t addr; - uint16_t port = 0; - - /* managed proxy options */ - int is_managed=0; - char **proxy_argv=NULL; - char **tmp=NULL; - int proxy_argc,i; - - int line_length; - - items = smartlist_new(); - smartlist_split_string(items, line, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - - line_length = smartlist_len(items); - if (line_length < 3) { - log_warn(LD_CONFIG, "Too few arguments on ServerTransportPlugin line."); - goto err; - } - - /* Get the first line element, split it to commas into - transport_list (in case it's multiple transports) and validate - the transport names. */ - transports = smartlist_get(items, 0); - transport_list = smartlist_new(); - smartlist_split_string(transport_list, transports, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(transport_list, const char *, transport_name) { - if (!string_is_C_identifier(transport_name)) { - log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).", - transport_name); - goto err; - } - } SMARTLIST_FOREACH_END(transport_name); - - type = smartlist_get(items, 1); - - if (!strcmp(type, "exec")) { - is_managed=1; - } else if (!strcmp(type, "proxy")) { - is_managed=0; - } else { - log_warn(LD_CONFIG, "Strange ServerTransportPlugin type '%s'", type); - goto err; - } - - if (is_managed && options->Sandbox) { - log_warn(LD_CONFIG, "Managed proxies are not compatible with Sandbox mode." - "(ServerTransportPlugin line was %s)", escaped(line)); - goto err; - } - - if (is_managed) { /* managed */ - if (!validate_only) { - proxy_argc = line_length-2; - tor_assert(proxy_argc > 0); - proxy_argv = tor_malloc_zero(sizeof(char*)*(proxy_argc+1)); - tmp = proxy_argv; - - for (i=0;i<proxy_argc;i++) { /* store arguments */ - *tmp++ = smartlist_get(items, 2); - smartlist_del_keeporder(items, 2); - } - *tmp = NULL; /*terminated with NULL, just like execve() likes it*/ - - /* kickstart the thing */ - pt_kickstart_server_proxy(transport_list, proxy_argv); - } - } else { /* external */ - if (smartlist_len(transport_list) != 1) { - log_warn(LD_CONFIG, "You can't have an external proxy with " - "more than one transports."); - goto err; - } - - addrport = smartlist_get(items, 2); - - if (tor_addr_port_lookup(addrport, &addr, &port)<0) { - log_warn(LD_CONFIG, "Error parsing transport " - "address '%s'", addrport); - goto err; - } - if (!port) { - log_warn(LD_CONFIG, - "Transport address '%s' has no port.", addrport); - goto err; - } - - if (!validate_only) { - log_info(LD_DIR, "Server transport '%s' at %s.", - transports, fmt_addrport(&addr, port)); - } - } - - r = 0; - goto done; - - err: - r = -1; - - done: - SMARTLIST_FOREACH(items, char*, s, tor_free(s)); - smartlist_free(items); - if (transport_list) { - SMARTLIST_FOREACH(transport_list, char*, s, tor_free(s)); - smartlist_free(transport_list); - } - - return r; -} - /** Read the contents of a DirAuthority line from <b>line</b>. If * <b>validate_only</b> is 0, and the line is well-formed, and it * shares any bits with <b>required_type</b> or <b>required_type</b> - * is 0, then add the dirserver described in the line (minus whatever - * bits it's missing) as a valid authority. Return 0 on success, + * is NO_DIRINFO (zero), then add the dirserver described in the line + * (minus whatever bits it's missing) as a valid authority. + * Return 0 on success or filtering out by type, * or -1 if the line isn't well-formed or if we can't add it. */ static int parse_dir_authority_line(const char *line, dirinfo_type_t required_type, @@ -5268,14 +5417,6 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, fingerprint, (int)strlen(fingerprint)); goto err; } - if (!strcmp(fingerprint, "E623F7625FBE0C87820F11EC5F6D5377ED816294")) { - /* a known bad fingerprint. refuse to use it. We can remove this - * clause once Tor 0.1.2.17 is obsolete. */ - log_warn(LD_CONFIG, "Dangerous dirserver line. To correct, erase your " - "torrc file (%s), or reinstall Tor and use the default torrc.", - get_torrc_fname(0)); - goto err; - } if (base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN)<0) { log_warn(LD_CONFIG, "Unable to decode DirAuthority key digest."); goto err; @@ -5405,12 +5546,13 @@ parse_dir_fallback_line(const char *line, /** Allocate and return a new port_cfg_t with reasonable defaults. */ static port_cfg_t * -port_cfg_new(void) +port_cfg_new(size_t namelen) { - port_cfg_t *cfg = tor_malloc_zero(sizeof(port_cfg_t)); - cfg->ipv4_traffic = 1; - cfg->cache_ipv4_answers = 1; - cfg->prefer_ipv6_virtaddr = 1; + tor_assert(namelen <= SIZE_T_CEILING - sizeof(port_cfg_t) - 1); + port_cfg_t *cfg = tor_malloc_zero(sizeof(port_cfg_t) + namelen + 1); + cfg->entry_cfg.ipv4_traffic = 1; + cfg->entry_cfg.cache_ipv4_answers = 1; + cfg->entry_cfg.prefer_ipv6_virtaddr = 1; return cfg; } @@ -5517,6 +5659,56 @@ warn_nonlocal_controller_ports(smartlist_t *ports, unsigned forbid) #define CL_PORT_SERVER_OPTIONS (1u<<3) #define CL_PORT_FORBID_NONLOCAL (1u<<4) #define CL_PORT_TAKES_HOSTNAMES (1u<<5) +#define CL_PORT_IS_UNIXSOCKET (1u<<6) + +#ifdef HAVE_SYS_UN_H + +/** Parse the given <b>addrport</b> and set <b>path_out</b> if a Unix socket + * path is found. Return 0 on success. On error, a negative value is + * returned, -ENOENT if no Unix statement found, -EINVAL if the socket path + * is empty and -ENOSYS if AF_UNIX is not supported (see function in the + * #else statement below). */ + +int +config_parse_unix_port(const char *addrport, char **path_out) +{ + tor_assert(path_out); + tor_assert(addrport); + + if (strcmpstart(addrport, unix_socket_prefix)) { + /* Not a Unix socket path. */ + return -ENOENT; + } + + if (strlen(addrport + strlen(unix_socket_prefix)) == 0) { + /* Empty socket path, not very usable. */ + return -EINVAL; + } + + *path_out = tor_strdup(addrport + strlen(unix_socket_prefix)); + return 0; +} + +#else /* defined(HAVE_SYS_UN_H) */ + +int +config_parse_unix_port(const char *addrport, char **path_out) +{ + tor_assert(path_out); + tor_assert(addrport); + + if (strcmpstart(addrport, unix_socket_prefix)) { + /* Not a Unix socket path. */ + return -ENOENT; + } + + log_warn(LD_CONFIG, + "Port configuration %s is for an AF_UNIX socket, but we have no" + "support available on this platform", + escaped(addrport)); + return -ENOSYS; +} +#endif /* defined(HAVE_SYS_UN_H) */ /** * Parse port configuration for a single port type. @@ -5564,7 +5756,7 @@ parse_port_config(smartlist_t *out, int listener_type, const char *defaultaddr, int defaultport, - unsigned flags) + const unsigned flags) { smartlist_t *elts; int retval = -1; @@ -5577,7 +5769,9 @@ parse_port_config(smartlist_t *out, const unsigned allow_spurious_listenaddr = flags & CL_PORT_ALLOW_EXTRA_LISTENADDR; const unsigned takes_hostnames = flags & CL_PORT_TAKES_HOSTNAMES; + const unsigned is_unix_socket = flags & CL_PORT_IS_UNIXSOCKET; int got_zero_port=0, got_nonzero_port=0; + char *unix_socket_path = NULL; /* FooListenAddress is deprecated; let's make it work like it used to work, * though. */ @@ -5613,14 +5807,14 @@ parse_port_config(smartlist_t *out, if (use_server_options && out) { /* Add a no_listen port. */ - port_cfg_t *cfg = port_cfg_new(); + port_cfg_t *cfg = port_cfg_new(0); cfg->type = listener_type; cfg->port = mainport; tor_addr_make_unspec(&cfg->addr); /* Server ports default to 0.0.0.0 */ - cfg->no_listen = 1; - cfg->bind_ipv4_only = 1; - cfg->ipv4_traffic = 1; - cfg->prefer_ipv6_virtaddr = 1; + cfg->server_cfg.no_listen = 1; + cfg->server_cfg.bind_ipv4_only = 1; + cfg->entry_cfg.ipv4_traffic = 1; + cfg->entry_cfg.prefer_ipv6_virtaddr = 1; smartlist_add(out, cfg); } @@ -5633,13 +5827,13 @@ parse_port_config(smartlist_t *out, return -1; } if (out) { - port_cfg_t *cfg = port_cfg_new(); + port_cfg_t *cfg = port_cfg_new(0); cfg->type = listener_type; cfg->port = port ? port : mainport; tor_addr_copy(&cfg->addr, &addr); - cfg->session_group = SESSION_GROUP_UNSET; - cfg->isolation_flags = ISO_DEFAULT; - cfg->no_advertise = 1; + cfg->entry_cfg.session_group = SESSION_GROUP_UNSET; + cfg->entry_cfg.isolation_flags = ISO_DEFAULT; + cfg->server_cfg.no_advertise = 1; smartlist_add(out, cfg); } } @@ -5658,13 +5852,19 @@ parse_port_config(smartlist_t *out, /* No ListenAddress lines. If there's no FooPort, then maybe make a default * one. */ if (! ports) { - if (defaultport && out) { - port_cfg_t *cfg = port_cfg_new(); + if (defaultport && defaultaddr && out) { + port_cfg_t *cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0); cfg->type = listener_type; - cfg->port = defaultport; - tor_addr_parse(&cfg->addr, defaultaddr); - cfg->session_group = SESSION_GROUP_UNSET; - cfg->isolation_flags = ISO_DEFAULT; + if (is_unix_socket) { + tor_addr_make_unspec(&cfg->addr); + memcpy(cfg->unix_addr, defaultaddr, strlen(defaultaddr) + 1); + cfg->is_unix_addr = 1; + } else { + cfg->port = defaultport; + tor_addr_parse(&cfg->addr, defaultaddr); + } + cfg->entry_cfg.session_group = SESSION_GROUP_UNSET; + cfg->entry_cfg.isolation_flags = ISO_DEFAULT; smartlist_add(out, cfg); } return 0; @@ -5676,7 +5876,7 @@ parse_port_config(smartlist_t *out, for (; ports; ports = ports->next) { tor_addr_t addr; - int port; + int port, ret; int sessiongroup = SESSION_GROUP_UNSET; unsigned isolation = ISO_DEFAULT; int prefer_no_auth = 0; @@ -5705,7 +5905,31 @@ parse_port_config(smartlist_t *out, /* Now parse the addr/port value */ addrport = smartlist_get(elts, 0); - if (!strcmp(addrport, "auto")) { + + /* Let's start to check if it's a Unix socket path. */ + ret = config_parse_unix_port(addrport, &unix_socket_path); + if (ret < 0 && ret != -ENOENT) { + if (ret == -EINVAL) { + log_warn(LD_CONFIG, "Empty Unix socket path."); + } + goto err; + } + + if (unix_socket_path && + ! conn_listener_type_supports_af_unix(listener_type)) { + log_warn(LD_CONFIG, "%sPort does not support unix sockets", portname); + goto err; + } + + if (unix_socket_path) { + port = 1; + } else if (is_unix_socket) { + unix_socket_path = tor_strdup(addrport); + if (!strcmp(addrport, "0")) + port = 0; + else + port = 1; + } else if (!strcmp(addrport, "auto")) { port = CFG_AUTO_PORT; tor_addr_parse(&addr, defaultaddr); } else if (!strcasecmpend(addrport, ":auto")) { @@ -5890,28 +6114,36 @@ parse_port_config(smartlist_t *out, } if (out && port) { - port_cfg_t *cfg = port_cfg_new(); - tor_addr_copy(&cfg->addr, &addr); - cfg->port = port; + size_t namelen = unix_socket_path ? strlen(unix_socket_path) : 0; + port_cfg_t *cfg = port_cfg_new(namelen); + if (unix_socket_path) { + tor_addr_make_unspec(&cfg->addr); + memcpy(cfg->unix_addr, unix_socket_path, namelen + 1); + cfg->is_unix_addr = 1; + tor_free(unix_socket_path); + } else { + tor_addr_copy(&cfg->addr, &addr); + cfg->port = port; + } cfg->type = listener_type; - cfg->isolation_flags = isolation; - cfg->session_group = sessiongroup; - cfg->no_advertise = no_advertise; - cfg->no_listen = no_listen; - cfg->all_addrs = all_addrs; - cfg->bind_ipv4_only = bind_ipv4_only; - cfg->bind_ipv6_only = bind_ipv6_only; - cfg->ipv4_traffic = ipv4_traffic; - cfg->ipv6_traffic = ipv6_traffic; - cfg->prefer_ipv6 = prefer_ipv6; - cfg->cache_ipv4_answers = cache_ipv4; - cfg->cache_ipv6_answers = cache_ipv6; - cfg->use_cached_ipv4_answers = use_cached_ipv4; - cfg->use_cached_ipv6_answers = use_cached_ipv6; - cfg->prefer_ipv6_virtaddr = prefer_ipv6_automap; - cfg->socks_prefer_no_auth = prefer_no_auth; + cfg->entry_cfg.isolation_flags = isolation; + cfg->entry_cfg.session_group = sessiongroup; + cfg->server_cfg.no_advertise = no_advertise; + cfg->server_cfg.no_listen = no_listen; + cfg->server_cfg.all_addrs = all_addrs; + cfg->server_cfg.bind_ipv4_only = bind_ipv4_only; + cfg->server_cfg.bind_ipv6_only = bind_ipv6_only; + cfg->entry_cfg.ipv4_traffic = ipv4_traffic; + cfg->entry_cfg.ipv6_traffic = ipv6_traffic; + cfg->entry_cfg.prefer_ipv6 = prefer_ipv6; + cfg->entry_cfg.cache_ipv4_answers = cache_ipv4; + cfg->entry_cfg.cache_ipv6_answers = cache_ipv6; + cfg->entry_cfg.use_cached_ipv4_answers = use_cached_ipv4; + cfg->entry_cfg.use_cached_ipv6_answers = use_cached_ipv6; + cfg->entry_cfg.prefer_ipv6_virtaddr = prefer_ipv6_automap; + cfg->entry_cfg.socks_prefer_no_auth = prefer_no_auth; if (! (isolation & ISO_SOCKSAUTH)) - cfg->socks_prefer_no_auth = 1; + cfg->entry_cfg.socks_prefer_no_auth = 1; smartlist_add(out, cfg); } @@ -5939,32 +6171,10 @@ parse_port_config(smartlist_t *out, err: SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp)); smartlist_free(elts); + tor_free(unix_socket_path); return retval; } -/** Parse a list of config_line_t for an AF_UNIX unix socket listener option - * from <b>cfg</b> and add them to <b>out</b>. No fancy options are - * supported: the line contains nothing but the path to the AF_UNIX socket. */ -static int -parse_unix_socket_config(smartlist_t *out, const config_line_t *cfg, - int listener_type) -{ - - if (!out) - return 0; - - for ( ; cfg; cfg = cfg->next) { - size_t len = strlen(cfg->value); - port_cfg_t *port = tor_malloc_zero(sizeof(port_cfg_t) + len + 1); - port->is_unix_addr = 1; - memcpy(port->unix_addr, cfg->value, len+1); - port->type = listener_type; - smartlist_add(out, port); - } - - return 0; -} - /** Return the number of ports which are actually going to listen with type * <b>listenertype</b>. Do not count no_listen ports. Do not count unix * sockets. */ @@ -5973,7 +6183,7 @@ count_real_listeners(const smartlist_t *ports, int listenertype) { int n = 0; SMARTLIST_FOREACH_BEGIN(ports, port_cfg_t *, port) { - if (port->no_listen || port->is_unix_addr) + if (port->server_cfg.no_listen || port->is_unix_addr) continue; if (port->type != listenertype) continue; @@ -6053,9 +6263,11 @@ parse_ports(or_options_t *options, int validate_only, "configuration"); goto err; } - if (parse_unix_socket_config(ports, - options->ControlSocket, - CONN_TYPE_CONTROL_LISTENER) < 0) { + + if (parse_port_config(ports, options->ControlSocket, NULL, + "ControlSocket", + CONN_TYPE_CONTROL_LISTENER, NULL, 0, + control_port_flags | CL_PORT_IS_UNIXSOCKET) < 0) { *msg = tor_strdup("Invalid ControlSocket configuration"); goto err; } @@ -6149,25 +6361,25 @@ check_server_ports(const smartlist_t *ports, SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) { if (port->type == CONN_TYPE_DIR_LISTENER) { - if (! port->no_advertise) + if (! port->server_cfg.no_advertise) ++n_dirport_advertised; - if (! port->no_listen) + if (! port->server_cfg.no_listen) ++n_dirport_listeners; } else if (port->type == CONN_TYPE_OR_LISTENER) { - if (! port->no_advertise) { + if (! port->server_cfg.no_advertise) { ++n_orport_advertised; if (tor_addr_family(&port->addr) == AF_INET || (tor_addr_family(&port->addr) == AF_UNSPEC && - !port->bind_ipv6_only)) + !port->server_cfg.bind_ipv6_only)) ++n_orport_advertised_ipv4; } - if (! port->no_listen) + if (! port->server_cfg.no_listen) ++n_orport_listeners; } else { continue; } #ifndef _WIN32 - if (!port->no_listen && port->port < 1024) + if (!port->server_cfg.no_listen && port->port < 1024) ++n_low_port; #endif } SMARTLIST_FOREACH_END(port); @@ -6245,7 +6457,7 @@ get_first_listener_addrport_string(int listener_type) return NULL; SMARTLIST_FOREACH_BEGIN(configured_ports, const port_cfg_t *, cfg) { - if (cfg->no_listen) + if (cfg->server_cfg.no_listen) continue; if (cfg->type == listener_type && @@ -6292,12 +6504,12 @@ get_first_advertised_port_by_type_af(int listener_type, int address_family) return 0; SMARTLIST_FOREACH_BEGIN(configured_ports, const port_cfg_t *, cfg) { if (cfg->type == listener_type && - !cfg->no_advertise && + !cfg->server_cfg.no_advertise && (tor_addr_family(&cfg->addr) == address_family || tor_addr_family(&cfg->addr) == AF_UNSPEC)) { if (tor_addr_family(&cfg->addr) != AF_UNSPEC || - (address_family == AF_INET && !cfg->bind_ipv6_only) || - (address_family == AF_INET6 && !cfg->bind_ipv4_only)) { + (address_family == AF_INET && !cfg->server_cfg.bind_ipv6_only) || + (address_family == AF_INET6 && !cfg->server_cfg.bind_ipv4_only)) { return cfg->port; } } @@ -6381,10 +6593,13 @@ write_configuration_file(const char *fname, const or_options_t *options) char *old_val=NULL, *new_val=NULL, *new_conf=NULL; int rename_old = 0, r; - tor_assert(fname); + if (!fname) + return -1; switch (file_status(fname)) { + /* create backups of old config files, even if they're empty */ case FN_FILE: + case FN_EMPTY: old_val = read_file_to_str(fname, 0, NULL); if (!old_val || strcmpstart(old_val, GENERATED_FILE_PREFIX)) { rename_old = 1; diff --git a/src/or/config.h b/src/or/config.h index 8a1919c2ed..b064f05321 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,10 +29,11 @@ setopt_err_t options_trial_assign(config_line_t *list, int use_defaults, int clear_first, char **msg); uint32_t get_last_resolved_addr(void); +void reset_last_resolved_addr(void); int resolve_my_address(int warn_severity, const or_options_t *options, uint32_t *addr_out, const char **method_out, char **hostname_out); -int is_local_addr(const tor_addr_t *addr); +MOCK_DECL(int, is_local_addr, (const tor_addr_t *addr)); void options_init(or_options_t *options); #define OPTIONS_DUMP_MINIMAL 1 @@ -112,6 +113,7 @@ int addressmap_register_auto(const char *from, const char *to, time_t expires, addressmap_entry_source_t addrmap_source, const char **msg); +int config_parse_unix_port(const char *addrport, char **path_out); /** Represents the information stored in a torrc Bridge line. */ typedef struct bridge_line_t { @@ -140,6 +142,9 @@ STATIC int options_validate(or_options_t *old_options, or_options_t *options, or_options_t *default_options, int from_setconf, char **msg); +STATIC int parse_transport_line(const or_options_t *options, + const char *line, int validate_only, + int server); #endif #endif diff --git a/src/or/confparse.c b/src/or/confparse.c index c5400a6512..ac21df25cb 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/or/confparse.h b/src/or/confparse.h index 2cd6c49a2a..83c0f75b52 100644 --- a/src/or/confparse.h +++ b/src/or/confparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONFPARSE_H diff --git a/src/or/connection.c b/src/or/connection.c index 276dca2818..721ee20d27 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,7 +29,6 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" -#include "cpuworker.h" #include "directory.h" #include "dirserv.h" #include "dns.h" @@ -57,6 +56,11 @@ #include <pwd.h> #endif +#ifdef HAVE_SYS_UN_H +#include <sys/socket.h> +#include <sys/un.h> +#endif + static connection_t *connection_listener_new( const struct sockaddr *listensockaddr, socklen_t listensocklen, int type, @@ -130,7 +134,6 @@ conn_type_to_string(int type) case CONN_TYPE_AP: return "Socks"; case CONN_TYPE_DIR_LISTENER: return "Directory listener"; case CONN_TYPE_DIR: return "Directory"; - case CONN_TYPE_CPUWORKER: return "CPU worker"; case CONN_TYPE_CONTROL_LISTENER: return "Control listener"; case CONN_TYPE_CONTROL: return "Control"; case CONN_TYPE_EXT_OR: return "Extended OR"; @@ -213,12 +216,6 @@ conn_state_to_string(int type, int state) case DIR_CONN_STATE_SERVER_WRITING: return "writing"; } break; - case CONN_TYPE_CPUWORKER: - switch (state) { - case CPUWORKER_STATE_IDLE: return "idle"; - case CPUWORKER_STATE_BUSY_ONION: return "busy with onion"; - } - break; case CONN_TYPE_CONTROL: switch (state) { case CONTROL_CONN_STATE_OPEN: return "open (protocol v1)"; @@ -248,7 +245,6 @@ connection_type_uses_bufferevent(connection_t *conn) case CONN_TYPE_CONTROL: case CONN_TYPE_OR: case CONN_TYPE_EXT_OR: - case CONN_TYPE_CPUWORKER: return 1; default: return 0; @@ -305,9 +301,11 @@ entry_connection_new(int type, int socket_family) * in a little while. Otherwise, we're doing this as a linked connection * of some kind, and we should set it up here based on the socket family */ if (socket_family == AF_INET) - entry_conn->ipv4_traffic_ok = 1; + entry_conn->entry_cfg.ipv4_traffic = 1; else if (socket_family == AF_INET6) - entry_conn->ipv6_traffic_ok = 1; + entry_conn->entry_cfg.ipv6_traffic = 1; + else if (socket_family == AF_UNIX) + entry_conn->is_socks_socket = 1; return entry_conn; } @@ -451,6 +449,22 @@ connection_link_connections(connection_t *conn_a, connection_t *conn_b) conn_b->linked_conn = conn_a; } +/** Return true iff the provided connection listener type supports AF_UNIX + * sockets. */ +int +conn_listener_type_supports_af_unix(int type) +{ + /* For now only control ports or SOCKS ports can be Unix domain sockets + * and listeners at the same time */ + switch (type) { + case CONN_TYPE_CONTROL_LISTENER: + case CONN_TYPE_AP_LISTENER: + return 1; + default: + return 0; + } +} + /** Deallocate memory used by <b>conn</b>. Deallocate its buffers if * necessary, close its socket if necessary, and mark the directory as dirty * if <b>conn</b> is an OR or OP connection. @@ -516,9 +530,9 @@ connection_free_(connection_t *conn) buf_free(conn->outbuf); } else { if (conn->socket_family == AF_UNIX) { - /* For now only control ports can be Unix domain sockets + /* For now only control and SOCKS ports can be Unix domain sockets * and listeners at the same time */ - tor_assert(conn->type == CONN_TYPE_CONTROL_LISTENER); + tor_assert(conn_listener_type_supports_af_unix(conn->type)); if (unlink(conn->address) < 0 && errno != ENOENT) { log_warn(LD_NET, "Could not unlink %s: %s", conn->address, @@ -544,8 +558,7 @@ connection_free_(connection_t *conn) or_conn, TLS_CHAN_TO_BASE(or_conn->chan), U64_PRINTF_ARG( TLS_CHAN_TO_BASE(or_conn->chan)->global_identifier)); - if (!(TLS_CHAN_TO_BASE(or_conn->chan)->state == CHANNEL_STATE_CLOSED || - TLS_CHAN_TO_BASE(or_conn->chan)->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_FINISHED(TLS_CHAN_TO_BASE(or_conn->chan))) { channel_close_for_error(TLS_CHAN_TO_BASE(or_conn->chan)); } @@ -575,8 +588,10 @@ connection_free_(connection_t *conn) tor_free(control_conn->incoming_cmd); } - tor_free(conn->read_event); /* Probably already freed by connection_free. */ - tor_free(conn->write_event); /* Probably already freed by connection_free. */ + /* Probably already freed by connection_free. */ + tor_event_free(conn->read_event); + tor_event_free(conn->write_event); + conn->read_event = conn->write_event = NULL; IF_HAS_BUFFEREVENT(conn, { /* This was a workaround to handle bugs in some old versions of libevent * where callbacks can occur after calling bufferevent_free(). Setting @@ -894,9 +909,9 @@ create_unix_sockaddr(const char *listenaddress, char **readable_address, } #endif /* HAVE_SYS_UN_H */ -/** Warn that an accept or a connect has failed because we're running up - * against our ulimit. Rate-limit these warnings so that we don't spam - * the log. */ +/** Warn that an accept or a connect has failed because we're running out of + * TCP sockets we can use on current system. Rate-limit these warnings so + * that we don't spam the log. */ static void warn_too_many_conns(void) { @@ -906,7 +921,7 @@ warn_too_many_conns(void) if ((m = rate_limit_log(&last_warned, approx_time()))) { int n_conns = get_n_open_sockets(); log_warn(LD_NET,"Failing because we have %d connections already. Please " - "raise your ulimit -n.%s", n_conns, m); + "read doc/TUNING for guidance.%s", n_conns, m); tor_free(m); control_event_general_status(LOG_WARN, "TOO_MANY_CONNECTIONS CURRENT=%d", n_conns); @@ -914,13 +929,57 @@ warn_too_many_conns(void) } #ifdef HAVE_SYS_UN_H + +#define UNIX_SOCKET_PURPOSE_CONTROL_SOCKET 0 +#define UNIX_SOCKET_PURPOSE_SOCKS_SOCKET 1 + +/** Check if the purpose isn't one of the ones we know what to do with */ + +static int +is_valid_unix_socket_purpose(int purpose) +{ + int valid = 0; + + switch (purpose) { + case UNIX_SOCKET_PURPOSE_CONTROL_SOCKET: + case UNIX_SOCKET_PURPOSE_SOCKS_SOCKET: + valid = 1; + break; + } + + return valid; +} + +/** Return a string description of a unix socket purpose */ +static const char * +unix_socket_purpose_to_string(int purpose) +{ + const char *s = "unknown-purpose socket"; + + switch (purpose) { + case UNIX_SOCKET_PURPOSE_CONTROL_SOCKET: + s = "control socket"; + break; + case UNIX_SOCKET_PURPOSE_SOCKS_SOCKET: + s = "SOCKS socket"; + break; + } + + return s; +} + /** Check whether we should be willing to open an AF_UNIX socket in * <b>path</b>. Return 0 if we should go ahead and -1 if we shouldn't. */ static int -check_location_for_unix_socket(const or_options_t *options, const char *path) +check_location_for_unix_socket(const or_options_t *options, const char *path, + int purpose) { int r = -1; - char *p = tor_strdup(path); + char *p = NULL; + + tor_assert(is_valid_unix_socket_purpose(purpose)); + + p = tor_strdup(path); cpd_check_t flags = CPD_CHECK_MODE_ONLY; if (get_parent_directory(p)<0 || p[0] != '/') { log_warn(LD_GENERAL, "Bad unix socket address '%s'. Tor does not support " @@ -928,18 +987,23 @@ check_location_for_unix_socket(const or_options_t *options, const char *path) goto done; } - if (options->ControlSocketsGroupWritable) + if ((purpose == UNIX_SOCKET_PURPOSE_CONTROL_SOCKET && + options->ControlSocketsGroupWritable) || + (purpose == UNIX_SOCKET_PURPOSE_SOCKS_SOCKET && + options->SocksSocketsGroupWritable)) { flags |= CPD_GROUP_OK; + } if (check_private_dir(p, flags, options->User) < 0) { char *escpath, *escdir; escpath = esc_for_log(path); escdir = esc_for_log(p); - log_warn(LD_GENERAL, "Before Tor can create a control socket in %s, the " - "directory %s needs to exist, and to be accessible only by the " - "user%s account that is running Tor. (On some Unix systems, " - "anybody who can list a socket can connect to it, so Tor is " - "being careful.)", escpath, escdir, + log_warn(LD_GENERAL, "Before Tor can create a %s in %s, the directory " + "%s needs to exist, and to be accessible only by the user%s " + "account that is running Tor. (On some Unix systems, anybody " + "who can list a socket can connect to it, so Tor is being " + "careful.)", + unix_socket_purpose_to_string(purpose), escpath, escdir, options->ControlSocketsGroupWritable ? " and group" : ""); tor_free(escpath); tor_free(escdir); @@ -1022,15 +1086,15 @@ connection_listener_new(const struct sockaddr *listensockaddr, static int global_next_session_group = SESSION_GROUP_FIRST_AUTO; tor_addr_t addr; - if (get_n_open_sockets() >= get_options()->ConnLimit_-1) { + if (get_n_open_sockets() >= options->ConnLimit_-1) { warn_too_many_conns(); return NULL; } if (listensockaddr->sa_family == AF_INET || listensockaddr->sa_family == AF_INET6) { - int is_tcp = (type != CONN_TYPE_AP_DNS_LISTENER); - if (is_tcp) + int is_stream = (type != CONN_TYPE_AP_DNS_LISTENER); + if (is_stream) start_reading = 1; tor_addr_from_sockaddr(&addr, listensockaddr, &usePort); @@ -1039,10 +1103,10 @@ connection_listener_new(const struct sockaddr *listensockaddr, conn_type_to_string(type), fmt_addrport(&addr, usePort)); s = tor_open_socket_nonblocking(tor_addr_family(&addr), - is_tcp ? SOCK_STREAM : SOCK_DGRAM, - is_tcp ? IPPROTO_TCP: IPPROTO_UDP); + is_stream ? SOCK_STREAM : SOCK_DGRAM, + is_stream ? IPPROTO_TCP: IPPROTO_UDP); if (!SOCKET_OK(s)) { - log_warn(LD_NET,"Socket creation failed: %s", + log_warn(LD_NET, "Socket creation failed: %s", tor_socket_strerror(tor_socket_errno(-1))); goto err; } @@ -1099,7 +1163,7 @@ connection_listener_new(const struct sockaddr *listensockaddr, goto err; } - if (is_tcp) { + if (is_stream) { if (tor_listen(s) < 0) { log_warn(LD_NET, "Could not listen on %s:%u: %s", address, usePort, tor_socket_strerror(tor_socket_errno(s))); @@ -1122,15 +1186,21 @@ connection_listener_new(const struct sockaddr *listensockaddr, tor_addr_from_sockaddr(&addr2, (struct sockaddr*)&ss, &gotPort); } #ifdef HAVE_SYS_UN_H + /* + * AF_UNIX generic setup stuff + */ } else if (listensockaddr->sa_family == AF_UNIX) { + /* We want to start reading for both AF_UNIX cases */ start_reading = 1; - /* For now only control ports can be Unix domain sockets - * and listeners at the same time */ - tor_assert(type == CONN_TYPE_CONTROL_LISTENER); + tor_assert(conn_listener_type_supports_af_unix(type)); - if (check_location_for_unix_socket(options, address) < 0) - goto err; + if (check_location_for_unix_socket(options, address, + (type == CONN_TYPE_CONTROL_LISTENER) ? + UNIX_SOCKET_PURPOSE_CONTROL_SOCKET : + UNIX_SOCKET_PURPOSE_SOCKS_SOCKET) < 0) { + goto err; + } log_notice(LD_NET, "Opening %s on %s", conn_type_to_string(type), address); @@ -1142,17 +1212,20 @@ connection_listener_new(const struct sockaddr *listensockaddr, strerror(errno)); goto err; } + s = tor_open_socket_nonblocking(AF_UNIX, SOCK_STREAM, 0); if (! SOCKET_OK(s)) { log_warn(LD_NET,"Socket creation failed: %s.", strerror(errno)); goto err; } - if (bind(s, listensockaddr, (socklen_t)sizeof(struct sockaddr_un)) == -1) { + if (bind(s, listensockaddr, + (socklen_t)sizeof(struct sockaddr_un)) == -1) { log_warn(LD_NET,"Bind to %s failed: %s.", address, tor_socket_strerror(tor_socket_errno(s))); goto err; } + #ifdef HAVE_PWD_H if (options->User) { pw = tor_getpwnam(options->User); @@ -1167,13 +1240,27 @@ connection_listener_new(const struct sockaddr *listensockaddr, } } #endif - if (options->ControlSocketsGroupWritable) { + + if ((type == CONN_TYPE_CONTROL_LISTENER && + options->ControlSocketsGroupWritable) || + (type == CONN_TYPE_AP_LISTENER && + options->SocksSocketsGroupWritable)) { /* We need to use chmod; fchmod doesn't work on sockets on all * platforms. */ if (chmod(address, 0660) < 0) { log_warn(LD_FS,"Unable to make %s group-writable.", address); goto err; } + } else if ((type == CONN_TYPE_CONTROL_LISTENER && + !(options->ControlSocketsGroupWritable)) || + (type == CONN_TYPE_AP_LISTENER && + !(options->SocksSocketsGroupWritable))) { + /* We need to use chmod; fchmod doesn't work on sockets on all + * platforms. */ + if (chmod(address, 0600) < 0) { + log_warn(LD_FS,"Unable to make %s group-writable.", address); + goto err; + } } if (listen(s, SOMAXCONN) < 0) { @@ -1181,8 +1268,6 @@ connection_listener_new(const struct sockaddr *listensockaddr, tor_socket_strerror(tor_socket_errno(s))); goto err; } -#else - (void)options; #endif /* HAVE_SYS_UN_H */ } else { log_err(LD_BUG, "Got unexpected address family %d.", @@ -1199,10 +1284,12 @@ connection_listener_new(const struct sockaddr *listensockaddr, conn->port = gotPort; tor_addr_copy(&conn->addr, &addr); - if (port_cfg->isolation_flags) { - lis_conn->isolation_flags = port_cfg->isolation_flags; - if (port_cfg->session_group >= 0) { - lis_conn->session_group = port_cfg->session_group; + memcpy(&lis_conn->entry_cfg, &port_cfg->entry_cfg, sizeof(entry_port_cfg_t)); + + if (port_cfg->entry_cfg.isolation_flags) { + lis_conn->entry_cfg.isolation_flags = port_cfg->entry_cfg.isolation_flags; + if (port_cfg->entry_cfg.session_group >= 0) { + lis_conn->entry_cfg.session_group = port_cfg->entry_cfg.session_group; } else { /* This can wrap after around INT_MAX listeners are opened. But I don't * believe that matters, since you would need to open a ridiculous @@ -1210,23 +1297,15 @@ connection_listener_new(const struct sockaddr *listensockaddr, * hit this. An OR with a dozen ports open, for example, would have to * close and re-open its listeners every second for 4 years nonstop. */ - lis_conn->session_group = global_next_session_group--; + lis_conn->entry_cfg.session_group = global_next_session_group--; } } - if (type == CONN_TYPE_AP_LISTENER) { - lis_conn->socks_ipv4_traffic = port_cfg->ipv4_traffic; - lis_conn->socks_ipv6_traffic = port_cfg->ipv6_traffic; - lis_conn->socks_prefer_ipv6 = port_cfg->prefer_ipv6; - } else { - lis_conn->socks_ipv4_traffic = 1; - lis_conn->socks_ipv6_traffic = 1; + + if (type != CONN_TYPE_AP_LISTENER) { + lis_conn->entry_cfg.ipv4_traffic = 1; + lis_conn->entry_cfg.ipv6_traffic = 1; + lis_conn->entry_cfg.prefer_ipv6 = 0; } - lis_conn->cache_ipv4_answers = port_cfg->cache_ipv4_answers; - lis_conn->cache_ipv6_answers = port_cfg->cache_ipv6_answers; - lis_conn->use_cached_ipv4_answers = port_cfg->use_cached_ipv4_answers; - lis_conn->use_cached_ipv6_answers = port_cfg->use_cached_ipv6_answers; - lis_conn->prefer_ipv6_virtaddr = port_cfg->prefer_ipv6_virtaddr; - lis_conn->socks_prefer_no_auth = port_cfg->socks_prefer_no_auth; if (connection_add(conn) < 0) { /* no space, forget it */ log_warn(LD_NET,"connection_add for listener failed. Giving up."); @@ -1293,6 +1372,8 @@ check_sockaddr(const struct sockaddr *sa, int len, int level) "Address for new connection has address/port equal to zero."); ok = 0; } + } else if (sa->sa_family == AF_UNIX) { + ok = 1; } else { ok = 0; } @@ -1377,7 +1458,8 @@ connection_handle_listener_read(connection_t *conn, int new_type) return 0; } - if (conn->socket_family == AF_INET || conn->socket_family == AF_INET6) { + if (conn->socket_family == AF_INET || conn->socket_family == AF_INET6 || + (conn->socket_family == AF_UNIX && new_type == CONN_TYPE_AP)) { tor_addr_t addr; uint16_t port; if (check_sockaddr(remote, remotelen, LOG_INFO)<0) { @@ -1418,18 +1500,21 @@ connection_handle_listener_read(connection_t *conn, int new_type) newconn->port = port; newconn->address = tor_dup_addr(&addr); - if (new_type == CONN_TYPE_AP) { - TO_ENTRY_CONN(newconn)->socks_request->socks_prefer_no_auth = - TO_LISTENER_CONN(conn)->socks_prefer_no_auth; + if (new_type == CONN_TYPE_AP && conn->socket_family != AF_UNIX) { + log_info(LD_NET, "New SOCKS connection opened from %s.", + fmt_and_decorate_addr(&addr)); + } + if (new_type == CONN_TYPE_AP && conn->socket_family == AF_UNIX) { + newconn->port = 0; + newconn->address = tor_strdup(conn->address); + log_info(LD_NET, "New SOCKS AF_UNIX connection opened"); } if (new_type == CONN_TYPE_CONTROL) { log_notice(LD_CONTROL, "New control connection opened from %s.", fmt_and_decorate_addr(&addr)); } - } else if (conn->socket_family == AF_UNIX) { - /* For now only control ports can be Unix domain sockets - * and listeners at the same time */ + } else if (conn->socket_family == AF_UNIX && conn->type != CONN_TYPE_AP) { tor_assert(conn->type == CONN_TYPE_CONTROL_LISTENER); tor_assert(new_type == CONN_TYPE_CONTROL); log_notice(LD_CONTROL, "New control connection opened."); @@ -1484,25 +1569,16 @@ connection_init_accepted_conn(connection_t *conn, return rv; break; case CONN_TYPE_AP: - TO_ENTRY_CONN(conn)->isolation_flags = listener->isolation_flags; - TO_ENTRY_CONN(conn)->session_group = listener->session_group; + memcpy(&TO_ENTRY_CONN(conn)->entry_cfg, &listener->entry_cfg, + sizeof(entry_port_cfg_t)); TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch(); TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type; - TO_ENTRY_CONN(conn)->ipv4_traffic_ok = listener->socks_ipv4_traffic; - TO_ENTRY_CONN(conn)->ipv6_traffic_ok = listener->socks_ipv6_traffic; - TO_ENTRY_CONN(conn)->prefer_ipv6_traffic = listener->socks_prefer_ipv6; - TO_ENTRY_CONN(conn)->cache_ipv4_answers = listener->cache_ipv4_answers; - TO_ENTRY_CONN(conn)->cache_ipv6_answers = listener->cache_ipv6_answers; - TO_ENTRY_CONN(conn)->use_cached_ipv4_answers = - listener->use_cached_ipv4_answers; - TO_ENTRY_CONN(conn)->use_cached_ipv6_answers = - listener->use_cached_ipv6_answers; - TO_ENTRY_CONN(conn)->prefer_ipv6_virtaddr = - listener->prefer_ipv6_virtaddr; switch (TO_CONN(listener)->type) { case CONN_TYPE_AP_LISTENER: conn->state = AP_CONN_STATE_SOCKS_WAIT; + TO_ENTRY_CONN(conn)->socks_request->socks_prefer_no_auth = + listener->entry_cfg.socks_prefer_no_auth; break; case CONN_TYPE_AP_TRANS_LISTENER: TO_ENTRY_CONN(conn)->is_transparent_ap = 1; @@ -1525,26 +1601,21 @@ connection_init_accepted_conn(connection_t *conn, return 0; } -/** Take conn, make a nonblocking socket; try to connect to - * addr:port (they arrive in *host order*). If fail, return -1 and if - * applicable put your best guess about errno into *<b>socket_error</b>. - * Else assign s to conn-\>s: if connected return 1, if EAGAIN return 0. - * - * address is used to make the logs useful. - * - * On success, add conn to the list of polled connections. - */ -int -connection_connect(connection_t *conn, const char *address, - const tor_addr_t *addr, uint16_t port, int *socket_error) +static int +connection_connect_sockaddr(connection_t *conn, + const struct sockaddr *sa, + socklen_t sa_len, + const struct sockaddr *bindaddr, + socklen_t bindaddr_len, + int *socket_error) { tor_socket_t s; int inprogress = 0; - struct sockaddr_storage addrbuf; - struct sockaddr *dest_addr; - int dest_addr_len; const or_options_t *options = get_options(); - int protocol_family; + + tor_assert(conn); + tor_assert(sa); + tor_assert(socket_error); if (get_n_open_sockets() >= get_options()->ConnLimit_-1) { warn_too_many_conns(); @@ -1552,11 +1623,6 @@ connection_connect(connection_t *conn, const char *address, return -1; } - if (tor_addr_family(addr) == AF_INET6) - protocol_family = PF_INET6; - else - protocol_family = PF_INET; - if (get_options()->DisableNetwork) { /* We should never even try to connect anyplace if DisableNetwork is set. * Warn if we do, and refuse to make the connection. */ @@ -1568,7 +1634,11 @@ connection_connect(connection_t *conn, const char *address, return -1; } - s = tor_open_socket_nonblocking(protocol_family,SOCK_STREAM,IPPROTO_TCP); + const int protocol_family = sa->sa_family; + const int proto = (sa->sa_family == AF_INET6 || + sa->sa_family == AF_INET) ? IPPROTO_TCP : 0; + + s = tor_open_socket_nonblocking(protocol_family, SOCK_STREAM, proto); if (! SOCKET_OK(s)) { *socket_error = tor_socket_errno(-1); log_warn(LD_NET,"Error creating network socket: %s", @@ -1581,6 +1651,73 @@ connection_connect(connection_t *conn, const char *address, tor_socket_strerror(errno)); } + if (bindaddr && bind(s, bindaddr, bindaddr_len) < 0) { + *socket_error = tor_socket_errno(s); + log_warn(LD_NET,"Error binding network socket: %s", + tor_socket_strerror(*socket_error)); + tor_close_socket(s); + return -1; + } + + tor_assert(options); + if (options->ConstrainedSockets) + set_constrained_socket_buffers(s, (int)options->ConstrainedSockSize); + + if (connect(s, sa, sa_len) < 0) { + int e = tor_socket_errno(s); + if (!ERRNO_IS_CONN_EINPROGRESS(e)) { + /* yuck. kill it. */ + *socket_error = e; + log_info(LD_NET, + "connect() to socket failed: %s", + tor_socket_strerror(e)); + tor_close_socket(s); + return -1; + } else { + inprogress = 1; + } + } + + /* it succeeded. we're connected. */ + log_fn(inprogress ? LOG_DEBUG : LOG_INFO, LD_NET, + "Connection to socket %s (sock "TOR_SOCKET_T_FORMAT").", + inprogress ? "in progress" : "established", s); + conn->s = s; + if (connection_add_connecting(conn) < 0) { + /* no space, forget it */ + *socket_error = SOCK_ERRNO(ENOBUFS); + return -1; + } + + return inprogress ? 0 : 1; +} + +/** Take conn, make a nonblocking socket; try to connect to + * addr:port (they arrive in *host order*). If fail, return -1 and if + * applicable put your best guess about errno into *<b>socket_error</b>. + * Else assign s to conn-\>s: if connected return 1, if EAGAIN return 0. + * + * address is used to make the logs useful. + * + * On success, add conn to the list of polled connections. + */ +int +connection_connect(connection_t *conn, const char *address, + const tor_addr_t *addr, uint16_t port, int *socket_error) +{ + struct sockaddr_storage addrbuf; + struct sockaddr_storage bind_addr_ss; + struct sockaddr *bind_addr = NULL; + struct sockaddr *dest_addr; + int dest_addr_len, bind_addr_len = 0; + const or_options_t *options = get_options(); + int protocol_family; + + if (tor_addr_family(addr) == AF_INET6) + protocol_family = PF_INET6; + else + protocol_family = PF_INET; + if (!tor_addr_is_loopback(addr)) { const tor_addr_t *ext_addr = NULL; if (protocol_family == AF_INET && @@ -1590,33 +1727,20 @@ connection_connect(connection_t *conn, const char *address, !tor_addr_is_null(&options->OutboundBindAddressIPv6_)) ext_addr = &options->OutboundBindAddressIPv6_; if (ext_addr) { - struct sockaddr_storage ext_addr_sa; - socklen_t ext_addr_len = 0; - memset(&ext_addr_sa, 0, sizeof(ext_addr_sa)); - ext_addr_len = tor_addr_to_sockaddr(ext_addr, 0, - (struct sockaddr *) &ext_addr_sa, - sizeof(ext_addr_sa)); - if (ext_addr_len == 0) { + memset(&bind_addr_ss, 0, sizeof(bind_addr_ss)); + bind_addr_len = tor_addr_to_sockaddr(ext_addr, 0, + (struct sockaddr *) &bind_addr_ss, + sizeof(bind_addr_ss)); + if (bind_addr_len == 0) { log_warn(LD_NET, "Error converting OutboundBindAddress %s into sockaddr. " "Ignoring.", fmt_and_decorate_addr(ext_addr)); } else { - if (bind(s, (struct sockaddr *) &ext_addr_sa, ext_addr_len) < 0) { - *socket_error = tor_socket_errno(s); - log_warn(LD_NET,"Error binding network socket to %s: %s", - fmt_and_decorate_addr(ext_addr), - tor_socket_strerror(*socket_error)); - tor_close_socket(s); - return -1; - } + bind_addr = (struct sockaddr *)&bind_addr_ss; } } } - tor_assert(options); - if (options->ConstrainedSockets) - set_constrained_socket_buffers(s, (int)options->ConstrainedSockSize); - memset(&addrbuf,0,sizeof(addrbuf)); dest_addr = (struct sockaddr*) &addrbuf; dest_addr_len = tor_addr_to_sockaddr(addr, port, dest_addr, sizeof(addrbuf)); @@ -1625,36 +1749,51 @@ connection_connect(connection_t *conn, const char *address, log_debug(LD_NET, "Connecting to %s:%u.", escaped_safe_str_client(address), port); - if (connect(s, dest_addr, (socklen_t)dest_addr_len) < 0) { - int e = tor_socket_errno(s); - if (!ERRNO_IS_CONN_EINPROGRESS(e)) { - /* yuck. kill it. */ - *socket_error = e; - log_info(LD_NET, - "connect() to %s:%u failed: %s", - escaped_safe_str_client(address), - port, tor_socket_strerror(e)); - tor_close_socket(s); - return -1; - } else { - inprogress = 1; - } - } + return connection_connect_sockaddr(conn, dest_addr, dest_addr_len, + bind_addr, bind_addr_len, socket_error); +} - /* it succeeded. we're connected. */ - log_fn(inprogress?LOG_DEBUG:LOG_INFO, LD_NET, - "Connection to %s:%u %s (sock "TOR_SOCKET_T_FORMAT").", - escaped_safe_str_client(address), - port, inprogress?"in progress":"established", s); - conn->s = s; - if (connection_add_connecting(conn) < 0) { - /* no space, forget it */ - *socket_error = SOCK_ERRNO(ENOBUFS); +#ifdef HAVE_SYS_UN_H + +/** Take conn, make a nonblocking socket; try to connect to + * an AF_UNIX socket at socket_path. If fail, return -1 and if applicable + * put your best guess about errno into *<b>socket_error</b>. Else assign s + * to conn-\>s: if connected return 1, if EAGAIN return 0. + * + * On success, add conn to the list of polled connections. + */ +int +connection_connect_unix(connection_t *conn, const char *socket_path, + int *socket_error) +{ + struct sockaddr_un dest_addr; + + tor_assert(socket_path); + + /* Check that we'll be able to fit it into dest_addr later */ + if (strlen(socket_path) + 1 > sizeof(dest_addr.sun_path)) { + log_warn(LD_NET, + "Path %s is too long for an AF_UNIX socket\n", + escaped_safe_str_client(socket_path)); + *socket_error = SOCK_ERRNO(ENAMETOOLONG); return -1; } - return inprogress ? 0 : 1; + + memset(&dest_addr, 0, sizeof(dest_addr)); + dest_addr.sun_family = AF_UNIX; + strlcpy(dest_addr.sun_path, socket_path, sizeof(dest_addr.sun_path)); + + log_debug(LD_NET, + "Connecting to AF_UNIX socket at %s.", + escaped_safe_str_client(socket_path)); + + return connection_connect_sockaddr(conn, + (struct sockaddr *)&dest_addr, sizeof(dest_addr), + NULL, 0, socket_error); } +#endif /* defined(HAVE_SYS_UN_H) */ + /** Convert state number to string representation for logging purposes. */ static const char * @@ -1688,14 +1827,14 @@ get_proxy_type(void) { const or_options_t *options = get_options(); - if (options->HTTPSProxy) + if (options->ClientTransportPlugin) + return PROXY_PLUGGABLE; + else if (options->HTTPSProxy) return PROXY_CONNECT; else if (options->Socks4Proxy) return PROXY_SOCKS4; else if (options->Socks5Proxy) return PROXY_SOCKS5; - else if (options->ClientTransportPlugin) - return PROXY_PLUGGABLE; else return PROXY_NONE; } @@ -2185,7 +2324,7 @@ retry_listener_ports(smartlist_t *old_conns, (conn->socket_family == AF_UNIX && ! wanted->is_unix_addr)) continue; - if (wanted->no_listen) + if (wanted->server_cfg.no_listen) continue; /* We don't want to open a listener for this one */ if (wanted->is_unix_addr) { @@ -2226,7 +2365,7 @@ retry_listener_ports(smartlist_t *old_conns, connection_t *conn; int real_port = port->port == CFG_AUTO_PORT ? 0 : port->port; tor_assert(real_port <= UINT16_MAX); - if (port->no_listen) + if (port->server_cfg.no_listen) continue; if (port->is_unix_addr) { @@ -2348,7 +2487,6 @@ connection_mark_all_noncontrol_connections(void) if (conn->marked_for_close) continue; switch (conn->type) { - case CONN_TYPE_CPUWORKER: case CONN_TYPE_CONTROL_LISTENER: case CONN_TYPE_CONTROL: break; @@ -2391,6 +2529,7 @@ connection_is_rate_limited(connection_t *conn) return 0; /* Internal connection */ else if (! options->CountPrivateBandwidth && (tor_addr_family(&conn->addr) == AF_UNSPEC || /* no address */ + tor_addr_family(&conn->addr) == AF_UNIX || /* no address */ tor_addr_is_internal(&conn->addr, 0))) return 0; /* Internal address */ else @@ -3716,9 +3855,15 @@ connection_handle_write_impl(connection_t *conn, int force) if (connection_state_is_connecting(conn)) { if (getsockopt(conn->s, SOL_SOCKET, SO_ERROR, (void*)&e, &len) < 0) { log_warn(LD_BUG, "getsockopt() syscall failed"); - if (CONN_IS_EDGE(conn)) - connection_edge_end_errno(TO_EDGE_CONN(conn)); - connection_mark_for_close(conn); + if (conn->type == CONN_TYPE_OR) { + or_connection_t *orconn = TO_OR_CONN(conn); + connection_or_close_for_error(orconn, 0); + } else { + if (CONN_IS_EDGE(conn)) { + connection_edge_end_errno(TO_EDGE_CONN(conn)); + } + connection_mark_for_close(conn); + } return -1; } if (e) { @@ -3833,6 +3978,8 @@ connection_handle_write_impl(connection_t *conn, int force) tor_tls_get_n_raw_bytes(or_conn->tls, &n_read, &n_written); log_debug(LD_GENERAL, "After TLS write of %d: %ld read, %ld written", result, (long)n_read, (long)n_written); + or_conn->bytes_xmitted += result; + or_conn->bytes_xmitted_by_tls += n_written; /* So we notice bytes were written even on error */ /* XXXX024 This cast is safe since we can never write INT_MAX bytes in a * single set of TLS operations. But it looks kinda ugly. If we refactor @@ -4380,6 +4527,8 @@ client_check_address_changed(tor_socket_t sock) SMARTLIST_FOREACH(outgoing_addrs, tor_addr_t*, a_ptr, tor_free(a_ptr)); smartlist_clear(outgoing_addrs); smartlist_add(outgoing_addrs, tor_memdup(&out_addr, sizeof(tor_addr_t))); + /* We'll need to resolve ourselves again. */ + reset_last_resolved_addr(); /* Okay, now change our keys. */ ip_address_changed(1); } @@ -4431,8 +4580,6 @@ connection_process_inbuf(connection_t *conn, int package_partial) package_partial); case CONN_TYPE_DIR: return connection_dir_process_inbuf(TO_DIR_CONN(conn)); - case CONN_TYPE_CPUWORKER: - return connection_cpu_process_inbuf(conn); case CONN_TYPE_CONTROL: return connection_control_process_inbuf(TO_CONTROL_CONN(conn)); default: @@ -4492,8 +4639,6 @@ connection_finished_flushing(connection_t *conn) return connection_edge_finished_flushing(TO_EDGE_CONN(conn)); case CONN_TYPE_DIR: return connection_dir_finished_flushing(TO_DIR_CONN(conn)); - case CONN_TYPE_CPUWORKER: - return connection_cpu_finished_flushing(conn); case CONN_TYPE_CONTROL: return connection_control_finished_flushing(TO_CONTROL_CONN(conn)); default: @@ -4549,8 +4694,6 @@ connection_reached_eof(connection_t *conn) return connection_edge_reached_eof(TO_EDGE_CONN(conn)); case CONN_TYPE_DIR: return connection_dir_reached_eof(TO_DIR_CONN(conn)); - case CONN_TYPE_CPUWORKER: - return connection_cpu_reached_eof(conn); case CONN_TYPE_CONTROL: return connection_control_reached_eof(TO_CONTROL_CONN(conn)); default: @@ -4756,10 +4899,6 @@ assert_connection_ok(connection_t *conn, time_t now) tor_assert(conn->purpose >= DIR_PURPOSE_MIN_); tor_assert(conn->purpose <= DIR_PURPOSE_MAX_); break; - case CONN_TYPE_CPUWORKER: - tor_assert(conn->state >= CPUWORKER_STATE_MIN_); - tor_assert(conn->state <= CPUWORKER_STATE_MAX_); - break; case CONN_TYPE_CONTROL: tor_assert(conn->state >= CONTROL_CONN_STATE_MIN_); tor_assert(conn->state <= CONTROL_CONN_STATE_MAX_); @@ -4781,6 +4920,27 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, { const or_options_t *options = get_options(); + /* Client Transport Plugins can use another proxy, but that should be hidden + * from the rest of tor (as the plugin is responsible for dealing with the + * proxy), check it first, then check the rest of the proxy types to allow + * the config to have unused ClientTransportPlugin entries. + */ + if (options->ClientTransportPlugin) { + const transport_t *transport = NULL; + int r; + r = get_transport_by_bridge_addrport(&conn->addr, conn->port, &transport); + if (r<0) + return -1; + if (transport) { /* transport found */ + tor_addr_copy(addr, &transport->addr); + *port = transport->port; + *proxy_type = transport->socks_version; + return 0; + } + + /* Unused ClientTransportPlugin. */ + } + if (options->HTTPSProxy) { tor_addr_copy(addr, &options->HTTPSProxyAddr); *port = options->HTTPSProxyPort; @@ -4796,19 +4956,6 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, *port = options->Socks5ProxyPort; *proxy_type = PROXY_SOCKS5; return 0; - } else if (options->ClientTransportPlugin || - options->Bridges) { - const transport_t *transport = NULL; - int r; - r = get_transport_by_bridge_addrport(&conn->addr, conn->port, &transport); - if (r<0) - return -1; - if (transport) { /* transport found */ - tor_addr_copy(addr, &transport->addr); - *port = transport->port; - *proxy_type = transport->socks_version; - return 0; - } } tor_addr_make_unspec(addr); @@ -4832,7 +4979,7 @@ log_failed_proxy_connection(connection_t *conn) log_warn(LD_NET, "The connection to the %s proxy server at %s just failed. " "Make sure that the proxy server is up and running.", - proxy_type_to_string(get_proxy_type()), + proxy_type_to_string(proxy_type), fmt_addrport(&proxy_addr, proxy_port)); } @@ -4852,9 +4999,7 @@ proxy_type_to_string(int proxy_type) } /** Call connection_free_() on every connection in our array, and release all - * storage held by connection.c. This is used by cpuworkers and dnsworkers - * when they fork, so they don't keep resources held open (especially - * sockets). + * storage held by connection.c. * * Don't do the checks in connection_free(), because they will * fail. diff --git a/src/or/connection.h b/src/or/connection.h index 13dcbcd919..d0a34ece5c 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,6 +17,7 @@ const char *conn_type_to_string(int type); const char *conn_state_to_string(int type, int state); +int conn_listener_type_supports_af_unix(int type); dir_connection_t *dir_connection_new(int socket_family); or_connection_t *or_connection_new(int type, int socket_family); @@ -89,6 +90,13 @@ int connection_connect(connection_t *conn, const char *address, const tor_addr_t *addr, uint16_t port, int *socket_error); +#ifdef HAVE_SYS_UN_H + +int connection_connect_unix(connection_t *conn, const char *socket_path, + int *socket_error); + +#endif /* defined(HAVE_SYS_UN_H) */ + /** Maximum size of information that we can fit into SOCKS5 username or password fields. */ #define MAX_SOCKS5_AUTH_FIELD_SIZE 255 @@ -189,7 +197,8 @@ dir_connection_t *connection_dir_get_by_purpose_and_resource( int any_other_active_or_conns(const or_connection_t *this_conn); -#define connection_speaks_cells(conn) ((conn)->type == CONN_TYPE_OR) +/* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */ +#define connection_speaks_cells(conn) (((conn)->type == CONN_TYPE_OR) || 0) int connection_is_listener(connection_t *conn); int connection_state_is_open(connection_t *conn); int connection_state_is_connecting(connection_t *conn); diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index d210f93fa1..2a1a2f0fd2 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -46,6 +46,19 @@ #ifdef HAVE_LINUX_NETFILTER_IPV4_H #include <linux/netfilter_ipv4.h> #define TRANS_NETFILTER +#define TRANS_NETFILTER_IPV4 +#endif + +#ifdef HAVE_LINUX_IF_H +#include <linux/if.h> +#endif + +#ifdef HAVE_LINUX_NETFILTER_IPV6_IP6_TABLES_H +#include <linux/netfilter_ipv6/ip6_tables.h> +#if defined(IP6T_SO_ORIGINAL_DST) +#define TRANS_NETFILTER +#define TRANS_NETFILTER_IPV6 +#endif #endif #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) @@ -54,6 +67,10 @@ #define TRANS_PF #endif +#ifdef IP_TRANSPARENT +#define TRANS_TPROXY +#endif + #define SOCKS4_GRANTED 90 #define SOCKS4_REJECT 91 @@ -744,8 +761,9 @@ connection_ap_fail_onehop(const char *failed_digest, /* we don't know the digest; have to compare addr:port */ tor_addr_t addr; if (!build_state || !build_state->chosen_exit || - !entry_conn->socks_request) + !entry_conn->socks_request) { continue; + } if (tor_addr_parse(&addr, entry_conn->socks_request->address)<0 || !tor_addr_eq(&build_state->chosen_exit->addr, &addr) || build_state->chosen_exit->port != entry_conn->socks_request->port) @@ -894,78 +912,102 @@ connection_ap_rewrite_and_attach_if_allowed(entry_connection_t *conn, return connection_ap_handshake_rewrite_and_attach(conn, circ, cpath); } -/** Connection <b>conn</b> just finished its socks handshake, or the - * controller asked us to take care of it. If <b>circ</b> is defined, - * then that's where we'll want to attach it. Otherwise we have to - * figure it out ourselves. - * - * First, parse whether it's a .exit address, remap it, and so on. Then - * if it's for a general circuit, try to attach it to a circuit (or launch - * one as needed), else if it's for a rendezvous circuit, fetch a - * rendezvous descriptor first (or attach/launch a circuit if the - * rendezvous descriptor is already here and fresh enough). - * - * The stream will exit from the hop - * indicated by <b>cpath</b>, or from the last hop in circ's cpath if - * <b>cpath</b> is NULL. +/* Try to perform any map-based rewriting of the target address in + * <b>conn</b>, filling in the fields of <b>out</b> as we go, and modifying + * conn->socks_request.address as appropriate. */ -int -connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, - origin_circuit_t *circ, - crypt_path_t *cpath) +STATIC void +connection_ap_handshake_rewrite(entry_connection_t *conn, + rewrite_result_t *out) { socks_request_t *socks = conn->socks_request; - hostname_type_t addresstype; const or_options_t *options = get_options(); tor_addr_t addr_tmp; - /* We set this to true if this is an address we should automatically - * remap to a local address in VirtualAddrNetwork */ - int automap = 0; - char orig_address[MAX_SOCKS_ADDR_LEN]; - time_t map_expires = TIME_MAX; - time_t now = time(NULL); - connection_t *base_conn = ENTRY_TO_CONN(conn); - addressmap_entry_source_t exit_source = ADDRMAPSRC_NONE; - tor_strlower(socks->address); /* normalize it */ - strlcpy(orig_address, socks->address, sizeof(orig_address)); + /* Initialize all the fields of 'out' to reasonable defaults */ + out->automap = 0; + out->exit_source = ADDRMAPSRC_NONE; + out->map_expires = TIME_MAX; + out->end_reason = 0; + out->should_close = 0; + out->orig_address[0] = 0; + + /* We convert all incoming addresses to lowercase. */ + tor_strlower(socks->address); + /* Remember the original address. */ + strlcpy(out->orig_address, socks->address, sizeof(out->orig_address)); log_debug(LD_APP,"Client asked for %s:%d", safe_str_client(socks->address), socks->port); + /* Check for whether this is a .exit address. By default, those are + * disallowed when they're coming straight from the client, but you're + * allowed to have them in MapAddress commands and so forth. */ if (!strcmpend(socks->address, ".exit") && !options->AllowDotExit) { log_warn(LD_APP, "The \".exit\" notation is disabled in Tor due to " "security risks. Set AllowDotExit in your torrc to enable " "it (at your own risk)."); control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); - return -1; + out->end_reason = END_STREAM_REASON_TORPROTOCOL; + out->should_close = 1; + return; } - if (! conn->original_dest_address) + /* Remember the original address so we can tell the user about what + * they actually said, not just what it turned into. */ + if (! conn->original_dest_address) { + /* Is the 'if' necessary here? XXXX */ conn->original_dest_address = tor_strdup(conn->socks_request->address); + } + /* First, apply MapAddress and MAPADDRESS mappings. We need to do + * these only for non-reverse lookups, since they don't exist for those. + * We need to do this before we consider automapping, since we might + * e.g. resolve irc.oftc.net into irconionaddress.onion, at which point + * we'd need to automap it. */ + if (socks->command != SOCKS_COMMAND_RESOLVE_PTR) { + const unsigned rewrite_flags = AMR_FLAG_USE_MAPADDRESS; + if (addressmap_rewrite(socks->address, sizeof(socks->address), + rewrite_flags, &out->map_expires, &out->exit_source)) { + control_event_stream_status(conn, STREAM_EVENT_REMAP, + REMAP_STREAM_SOURCE_CACHE); + } + } + + /* Now, handle automapping. Automapping happens when we're asked to + * resolve a hostname, and AutomapHostsOnResolve is set, and + * the hostname has a suffix listed in AutomapHostsSuffixes. + */ if (socks->command == SOCKS_COMMAND_RESOLVE && tor_addr_parse(&addr_tmp, socks->address)<0 && options->AutomapHostsOnResolve) { - automap = addressmap_address_should_automap(socks->address, options); - if (automap) { + /* Check the suffix... */ + out->automap = addressmap_address_should_automap(socks->address, options); + if (out->automap) { + /* If we get here, then we should apply an automapping for this. */ const char *new_addr; + /* We return an IPv4 address by default, or an IPv6 address if we + * are allowed to do so. */ int addr_type = RESOLVED_TYPE_IPV4; if (conn->socks_request->socks_version != 4) { - if (!conn->ipv4_traffic_ok || - (conn->ipv6_traffic_ok && conn->prefer_ipv6_traffic) || - conn->prefer_ipv6_virtaddr) + if (!conn->entry_cfg.ipv4_traffic || + (conn->entry_cfg.ipv6_traffic && conn->entry_cfg.prefer_ipv6) || + conn->entry_cfg.prefer_ipv6_virtaddr) addr_type = RESOLVED_TYPE_IPV6; } + /* Okay, register the target address as automapped, and find the new + * address we're supposed to give as a resolve answer. (Return a cached + * value if we've looked up this address before. + */ new_addr = addressmap_register_virtual_address( addr_type, tor_strdup(socks->address)); if (! new_addr) { log_warn(LD_APP, "Unable to automap address %s", escaped_safe_str(socks->address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_INTERNAL); - return -1; + out->end_reason = END_STREAM_REASON_INTERNAL; + out->should_close = 1; + return; } log_info(LD_APP, "Automapping %s to %s", escaped_safe_str_client(socks->address), @@ -974,28 +1016,35 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, } } + /* Now handle reverse lookups, if they're in the cache. This doesn't + * happen too often, since client-side DNS caching is off by default. */ if (socks->command == SOCKS_COMMAND_RESOLVE_PTR) { unsigned rewrite_flags = 0; - if (conn->use_cached_ipv4_answers) + if (conn->entry_cfg.use_cached_ipv4_answers) rewrite_flags |= AMR_FLAG_USE_IPV4_DNS; - if (conn->use_cached_ipv6_answers) + if (conn->entry_cfg.use_cached_ipv6_answers) rewrite_flags |= AMR_FLAG_USE_IPV6_DNS; if (addressmap_rewrite_reverse(socks->address, sizeof(socks->address), - rewrite_flags, &map_expires)) { + rewrite_flags, &out->map_expires)) { char *result = tor_strdup(socks->address); /* remember _what_ is supposed to have been resolved. */ tor_snprintf(socks->address, sizeof(socks->address), "REVERSE[%s]", - orig_address); + out->orig_address); connection_ap_handshake_socks_resolved(conn, RESOLVED_TYPE_HOSTNAME, strlen(result), (uint8_t*)result, -1, - map_expires); - connection_mark_unattached_ap(conn, - END_STREAM_REASON_DONE | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return 0; + out->map_expires); + tor_free(result); + out->end_reason = END_STREAM_REASON_DONE | + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED; + out->should_close = 1; + return; } + + /* Hang on, did we find an answer saying that this is a reverse lookup for + * an internal address? If so, we should reject it if we're condigured to + * do so. */ if (options->ClientDNSRejectInternalAddresses) { /* Don't let people try to do a reverse lookup on 10.0.0.1. */ tor_addr_t addr; @@ -1005,43 +1054,108 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, if (ok == 1 && tor_addr_is_internal(&addr, 0)) { connection_ap_handshake_socks_resolved(conn, RESOLVED_TYPE_ERROR, 0, NULL, -1, TIME_MAX); - connection_mark_unattached_ap(conn, - END_STREAM_REASON_SOCKSPROTOCOL | - END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED); - return -1; + out->end_reason = END_STREAM_REASON_SOCKSPROTOCOL | + END_STREAM_REASON_FLAG_ALREADY_SOCKS_REPLIED; + out->should_close = 1; + return; } } - } else if (!automap) { - /* For address map controls, remap the address. */ - unsigned rewrite_flags = 0; - if (conn->use_cached_ipv4_answers) + } + + /* If we didn't automap it before, then this is still the address + * that came straight from the user, mapped according to any + * MapAddress/MAPADDRESS commands. Now other mappings, including + * previously registered Automap entries, TrackHostExits entries, + * and client-side DNS cache entries (not recommended). + */ + if (socks->command != SOCKS_COMMAND_RESOLVE_PTR && + !out->automap) { + unsigned rewrite_flags = AMR_FLAG_USE_AUTOMAP | AMR_FLAG_USE_TRACKEXIT; + addressmap_entry_source_t exit_source2; + if (conn->entry_cfg.use_cached_ipv4_answers) rewrite_flags |= AMR_FLAG_USE_IPV4_DNS; - if (conn->use_cached_ipv6_answers) + if (conn->entry_cfg.use_cached_ipv6_answers) rewrite_flags |= AMR_FLAG_USE_IPV6_DNS; if (addressmap_rewrite(socks->address, sizeof(socks->address), - rewrite_flags, &map_expires, &exit_source)) { + rewrite_flags, &out->map_expires, &exit_source2)) { control_event_stream_status(conn, STREAM_EVENT_REMAP, REMAP_STREAM_SOURCE_CACHE); } + if (out->exit_source == ADDRMAPSRC_NONE) { + /* If it wasn't a .exit before, maybe it turned into a .exit. Remember + * the original source of a .exit. */ + out->exit_source = exit_source2; + } } - if (!automap && address_is_in_virtual_range(socks->address)) { - /* This address was probably handed out by client_dns_get_unmapped_address, - * but the mapping was discarded for some reason. We *don't* want to send - * the address through Tor; that's likely to fail, and may leak - * information. + /* Check to see whether we're about to use an address in the virtual + * range without actually having gotten it from an Automap. */ + if (!out->automap && address_is_in_virtual_range(socks->address)) { + /* This address was probably handed out by + * client_dns_get_unmapped_address, but the mapping was discarded for some + * reason. Or the user typed in a virtual address range manually. We + * *don't* want to send the address through Tor; that's likely to fail, + * and may leak information. */ log_warn(LD_APP,"Missing mapping for virtual address '%s'. Refusing.", safe_str_client(socks->address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_INTERNAL); - return -1; + out->end_reason = END_STREAM_REASON_INTERNAL; + out->should_close = 1; + return; + } +} + +/** Connection <b>conn</b> just finished its socks handshake, or the + * controller asked us to take care of it. If <b>circ</b> is defined, + * then that's where we'll want to attach it. Otherwise we have to + * figure it out ourselves. + * + * First, parse whether it's a .exit address, remap it, and so on. Then + * if it's for a general circuit, try to attach it to a circuit (or launch + * one as needed), else if it's for a rendezvous circuit, fetch a + * rendezvous descriptor first (or attach/launch a circuit if the + * rendezvous descriptor is already here and fresh enough). + * + * The stream will exit from the hop + * indicated by <b>cpath</b>, or from the last hop in circ's cpath if + * <b>cpath</b> is NULL. + */ +int +connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, + origin_circuit_t *circ, + crypt_path_t *cpath) +{ + socks_request_t *socks = conn->socks_request; + const or_options_t *options = get_options(); + connection_t *base_conn = ENTRY_TO_CONN(conn); + time_t now = time(NULL); + rewrite_result_t rr; + + memset(&rr, 0, sizeof(rr)); + connection_ap_handshake_rewrite(conn,&rr); + + if (rr.should_close) { + /* connection_ap_handshake_rewrite told us to close the connection, + * either because it sent back an answer, or because it sent back an + * error */ + connection_mark_unattached_ap(conn, rr.end_reason); + if (END_STREAM_REASON_DONE == (rr.end_reason & END_STREAM_REASON_MASK)) + return 0; + else + return -1; } + const time_t map_expires = rr.map_expires; + const int automap = rr.automap; + const addressmap_entry_source_t exit_source = rr.exit_source; + /* Parse the address provided by SOCKS. Modify it in-place if it * specifies a hidden-service (.onion) or particular exit node (.exit). */ - addresstype = parse_extended_hostname(socks->address); + const hostname_type_t addresstype = parse_extended_hostname(socks->address); + /* Now see whether the hostname is bogus. This could happen because of an + * onion hostname whose format we don't recognize. */ if (addresstype == BAD_HOSTNAME) { control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); @@ -1049,16 +1163,21 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } + /* If this is a .exit hostname, strip off the .name.exit part, and + * see whether we're going to connect there, and otherwise handle it. + * (The ".exit" part got stripped off by "parse_extended_hostname"). + * + * We'll set chosen_exit_name and/or close the connection as appropriate. + */ if (addresstype == EXIT_HOSTNAME) { - /* foo.exit -- modify conn->chosen_exit_node to specify the exit - * node, and conn->address to hold only the address portion. */ - char *s = strrchr(socks->address,'.'); - - /* If StrictNodes is not set, then .exit overrides ExcludeNodes. */ + /* If StrictNodes is not set, then .exit overrides ExcludeNodes but + * not ExcludeExitNodes. */ routerset_t *excludeset = options->StrictNodes ? options->ExcludeExitNodesUnion_ : options->ExcludeExitNodes; - const node_t *node; + const node_t *node = NULL; + /* If this .exit was added by an AUTOMAP, then it came straight from + * a user. Make sure that options->AllowDotExit permits that. */ if (exit_source == ADDRMAPSRC_AUTOMAP && !options->AllowDotExit) { /* Whoops; this one is stale. It must have gotten added earlier, * when AllowDotExit was on. */ @@ -1071,6 +1190,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } + /* Double-check to make sure there are no .exits coming from + * impossible/weird sources. */ if (exit_source == ADDRMAPSRC_DNS || (exit_source == ADDRMAPSRC_NONE && !options->AllowDotExit)) { /* It shouldn't be possible to get a .exit address from any of these @@ -1085,9 +1206,12 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, } tor_assert(!automap); + /* Now, find the character before the .(name) part. */ + char *s = strrchr(socks->address,'.'); if (s) { /* The address was of the form "(stuff).(name).exit */ if (s[1] != '\0') { + /* Looks like a real .exit one. */ conn->chosen_exit_name = tor_strdup(s+1); node = node_get_by_nickname(conn->chosen_exit_name, 1); @@ -1106,7 +1230,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } } else { - /* It looks like they just asked for "foo.exit". */ + /* It looks like they just asked for "foo.exit". That's a special + * form that means (foo's address).foo.exit. */ conn->chosen_exit_name = tor_strdup(socks->address); node = node_get_by_nickname(conn->chosen_exit_name, 1); @@ -1115,6 +1240,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, node_get_address_string(node, socks->address, sizeof(socks->address)); } } + /* Now make sure that the chosen exit exists... */ if (!node) { log_warn(LD_APP, @@ -1136,8 +1262,12 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, implies no. */ } + /* Now, handle everything that isn't a .onion address. */ if (addresstype != ONION_HOSTNAME) { - /* not a hidden-service request (i.e. normal or .exit) */ + /* Not a hidden-service request. It's either a hostname or an IP, + * possibly with a .exit that we stripped off. */ + + /* Check for funny characters in the address. */ if (address_is_invalid_destination(socks->address, 1)) { control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", escaped(socks->address)); @@ -1148,6 +1278,9 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } +#ifdef ENABLE_TOR2WEB_MODE + /* If we're running in Tor2webMode, we don't allow anything BUT .onion + * addresses. */ if (options->Tor2webMode) { log_warn(LD_APP, "Refusing to connect to non-hidden-service hostname %s " "because tor2web mode is enabled.", @@ -1155,13 +1288,17 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY); return -1; } +#endif + /* See if this is a hostname lookup that we can answer immediately. + * (For example, an attempt to look up the IP address for an IP address.) + */ if (socks->command == SOCKS_COMMAND_RESOLVE) { tor_addr_t answer; /* Reply to resolves immediately if we can. */ if (tor_addr_parse(&answer, socks->address) >= 0) {/* is it an IP? */ /* remember _what_ is supposed to have been resolved. */ - strlcpy(socks->address, orig_address, sizeof(socks->address)); + strlcpy(socks->address, rr.orig_address, sizeof(socks->address)); connection_ap_handshake_socks_resolved_addr(conn, &answer, -1, map_expires); connection_mark_unattached_ap(conn, @@ -1172,14 +1309,22 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, tor_assert(!automap); rep_hist_note_used_resolve(now); /* help predict this next time */ } else if (socks->command == SOCKS_COMMAND_CONNECT) { + /* Special handling for attempts to connect */ tor_assert(!automap); + /* Don't allow connections to port 0. */ if (socks->port == 0) { log_notice(LD_APP,"Application asked to connect to port 0. Refusing."); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } + /* You can't make connections to internal addresses, by default. + * Exceptions are begindir requests (where the address is meaningless, + * or cases where you've hand-configured a particular exit, thereby + * making the local address meaningful. */ if (options->ClientRejectInternalAddresses && !conn->use_begindir && !conn->chosen_exit_name && !circ) { + /* If we reach this point then we don't want to allow internal + * addresses. Check if we got one. */ tor_addr_t addr; if (tor_addr_hostname_is_local(socks->address) || (tor_addr_parse(&addr, socks->address) >= 0 && @@ -1214,39 +1359,58 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, connection_mark_unattached_ap(conn, END_STREAM_REASON_PRIVATE_ADDR); return -1; } - } + } /* end "if we should check for internal addresses" */ + /* Okay. We're still doing a CONNECT, and it wasn't a private + * address. Do special handling for literal IP addresses */ { tor_addr_t addr; /* XXX Duplicate call to tor_addr_parse. */ if (tor_addr_parse(&addr, socks->address) >= 0) { + /* If we reach this point, it's an IPv4 or an IPv6 address. */ sa_family_t family = tor_addr_family(&addr); - if ((family == AF_INET && ! conn->ipv4_traffic_ok) || - (family == AF_INET6 && ! conn->ipv4_traffic_ok)) { + + if ((family == AF_INET && ! conn->entry_cfg.ipv4_traffic) || + (family == AF_INET6 && ! conn->entry_cfg.ipv6_traffic)) { + /* You can't do an IPv4 address on a v6-only socks listener, + * or vice versa. */ log_warn(LD_NET, "Rejecting SOCKS request for an IP address " "family that this listener does not support."); connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY); return -1; } else if (family == AF_INET6 && socks->socks_version == 4) { + /* You can't make a socks4 request to an IPv6 address. Socks4 + * doesn't support that. */ log_warn(LD_NET, "Rejecting SOCKS4 request for an IPv6 address."); connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY); return -1; - } else if (socks->socks_version == 4 && !conn->ipv4_traffic_ok) { + } else if (socks->socks_version == 4 && + !conn->entry_cfg.ipv4_traffic) { + /* You can't do any kind of Socks4 request when IPv4 is forbidden. + * + * XXX raise this check outside the enclosing block? */ log_warn(LD_NET, "Rejecting SOCKS4 request on a listener with " "no IPv4 traffic supported."); connection_mark_unattached_ap(conn, END_STREAM_REASON_ENTRYPOLICY); return -1; } else if (family == AF_INET6) { - conn->ipv4_traffic_ok = 0; + /* Tell the exit: we won't accept any ipv4 connection to an IPv6 + * address. */ + conn->entry_cfg.ipv4_traffic = 0; } else if (family == AF_INET) { - conn->ipv6_traffic_ok = 0; + /* Tell the exit: we won't accept any ipv6 connection to an IPv4 + * address. */ + conn->entry_cfg.ipv6_traffic = 0; } } } if (socks->socks_version == 4) - conn->ipv6_traffic_ok = 0; + conn->entry_cfg.ipv6_traffic = 0; + /* Still handling CONNECT. Now, check for exit enclaves. (Which we + * don't do on BEGINDIR, or there is a chosen exit.) + */ if (!conn->use_begindir && !conn->chosen_exit_name && !circ) { /* see if we can find a suitable enclave exit */ const node_t *r = @@ -1263,11 +1427,13 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, } } - /* warn or reject if it's using a dangerous port */ + /* Still handling CONNECT: warn or reject if it's using a dangerous + * port. */ if (!conn->use_begindir && !conn->chosen_exit_name && !circ) if (consider_plaintext_ports(conn, socks->port) < 0) return -1; + /* Remember the port so that we do predicted requests there. */ if (!conn->use_begindir) { /* help predict this next time */ rep_hist_note_used_port(now, socks->port); @@ -1276,25 +1442,41 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, rep_hist_note_used_resolve(now); /* help predict this next time */ /* no extra processing needed */ } else { + /* We should only be doing CONNECT or RESOLVE! */ tor_fragile_assert(); } + + /* Okay. At this point we've set chosen_exit_name if needed, rewritten the + * address, and decided not to reject it for any number of reasons. Now + * mark the connection as waiting for a circuit, and try to attach it! + */ base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT; - if ((circ && connection_ap_handshake_attach_chosen_circuit( - conn, circ, cpath) < 0) || - (!circ && - connection_ap_handshake_attach_circuit(conn) < 0)) { + + /* If we were given a circuit to attach to, try to attach. Otherwise, + * try to find a good one and attach to that. */ + int rv; + if (circ) + rv = connection_ap_handshake_attach_chosen_circuit(conn, circ, cpath); + else + rv = connection_ap_handshake_attach_circuit(conn); + + /* If the above function returned 0 then we're waiting for a circuit. + * if it returned 1, we're attached. Both are okay. But if it returned + * -1, there was an error, so make sure the connection is marked, and + * return -1. */ + if (rv < 0) { if (!base_conn->marked_for_close) connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); return -1; } + return 0; } else { - /* it's a hidden-service request */ - rend_cache_entry_t *entry; - int r; - rend_service_authorization_t *client_auth; - rend_data_t *rend_data; + /* If we get here, it's a request for a .onion address! */ tor_assert(!automap); + + /* Check whether it's RESOLVE or RESOLVE_PTR. We don't handle those + * for hidden service addresses. */ if (SOCKS_COMMAND_IS_RESOLVE(socks->command)) { /* if it's a resolve request, fail it right now, rather than * building all the circuits and then realizing it won't work. */ @@ -1308,6 +1490,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } + /* If we were passed a circuit, then we need to fail. .onion addresses + * only work when we launch our own circuits for now. */ if (circ) { log_warn(LD_CONTROL, "Attachstream to a circuit is not " "supported for .onion addresses currently. Failing."); @@ -1315,15 +1499,22 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return -1; } - ENTRY_TO_EDGE_CONN(conn)->rend_data = rend_data = + /* Fill in the rend_data field so we can start doing a connection to + * a hidden service. */ + rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data = tor_malloc_zero(sizeof(rend_data_t)); strlcpy(rend_data->onion_address, socks->address, sizeof(rend_data->onion_address)); log_info(LD_REND,"Got a hidden service request for ID '%s'", safe_str_client(rend_data->onion_address)); - /* see if we already have it cached */ - r = rend_cache_lookup_entry(rend_data->onion_address, -1, &entry); - if (r<0) { + + /* see if we already have a hidden service descriptor cached for this + * address. */ + rend_cache_entry_t *entry = NULL; + const int rend_cache_lookup_result = + rend_cache_lookup_entry(rend_data->onion_address, -1, &entry); + if (rend_cache_lookup_result < 0) { + /* We should already have rejected this address! */ log_warn(LD_BUG,"Invalid service name '%s'", safe_str_client(rend_data->onion_address)); connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); @@ -1334,8 +1525,10 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, * a stable circuit yet, but we know we'll need *something*. */ rep_hist_note_used_internal(now, 0, 1); - /* Look up if we have client authorization for it. */ - client_auth = rend_client_lookup_service_authorization( + /* Look up if we have client authorization configured for this hidden + * service. If we do, associate it with the rend_data. */ + rend_service_authorization_t *client_auth = + rend_client_lookup_service_authorization( rend_data->onion_address); if (client_auth) { log_info(LD_REND, "Using previously configured client authorization " @@ -1344,12 +1537,16 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, client_auth->descriptor_cookie, REND_DESC_COOKIE_LEN); rend_data->auth_type = client_auth->auth_type; } - if (r==0) { + + /* Now, we either launch an attempt to connect to the hidden service, + * or we launch an attempt to look up its descriptor, depending on + * whether we had the descriptor. */ + if (rend_cache_lookup_result == 0) { base_conn->state = AP_CONN_STATE_RENDDESC_WAIT; log_info(LD_REND, "Unknown descriptor %s. Fetching.", safe_str_client(rend_data->onion_address)); rend_client_refetch_v2_renddesc(rend_data); - } else { /* r > 0 */ + } else { /* rend_cache_lookup_result > 0 */ base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT; log_info(LD_REND, "Descriptor is here. Great."); if (connection_ap_handshake_attach_circuit(conn) < 0) { @@ -1360,6 +1557,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, } return 0; } + return 0; /* unreached but keeps the compiler happy */ } @@ -1391,7 +1589,7 @@ get_pf_socket(void) } #endif -#if defined(TRANS_NETFILTER) || defined(TRANS_PF) +#if defined(TRANS_NETFILTER) || defined(TRANS_PF) || defined(TRANS_TPROXY) /** Try fill in the address of <b>req</b> from the socket configured * with <b>conn</b>. */ static int @@ -1401,13 +1599,45 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req) socklen_t orig_dst_len = sizeof(orig_dst); tor_addr_t addr; +#ifdef TRANS_TRPOXY + if (options->TransProxyType_parsed == TPT_TPROXY) { + if (getsockname(ENTRY_TO_CONN(conn)->s, (struct sockaddr*)&orig_dst, + &orig_dst_len) < 0) { + int e = tor_socket_errno(ENTRY_TO_CONN(conn)->s); + log_warn(LD_NET, "getsockname() failed: %s", tor_socket_strerror(e)); + return -1; + } + goto done; + } +#endif + #ifdef TRANS_NETFILTER - if (getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IP, SO_ORIGINAL_DST, - (struct sockaddr*)&orig_dst, &orig_dst_len) < 0) { + int rv = -1; + switch (ENTRY_TO_CONN(conn)->socket_family) { +#ifdef TRANS_NETFILTER_IPV4 + case AF_INET: + rv = getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IP, SO_ORIGINAL_DST, + (struct sockaddr*)&orig_dst, &orig_dst_len); + break; +#endif +#ifdef TRANS_NETFILTER_IPV6 + case AF_INET6: + rv = getsockopt(ENTRY_TO_CONN(conn)->s, SOL_IPV6, IP6T_SO_ORIGINAL_DST, + (struct sockaddr*)&orig_dst, &orig_dst_len); + break; +#endif + default: + log_warn(LD_BUG, + "Received transparent data from an unsuported socket family %d", + ENTRY_TO_CONN(conn)->socket_family); + return -1; + } + if (rv < 0) { int e = tor_socket_errno(ENTRY_TO_CONN(conn)->s); log_warn(LD_NET, "getsockopt() failed: %s", tor_socket_strerror(e)); return -1; } + goto done; #elif defined(TRANS_PF) if (getsockname(ENTRY_TO_CONN(conn)->s, (struct sockaddr*)&orig_dst, &orig_dst_len) < 0) { @@ -1415,6 +1645,7 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req) log_warn(LD_NET, "getsockname() failed: %s", tor_socket_strerror(e)); return -1; } + goto done; #else (void)conn; (void)req; @@ -1422,6 +1653,7 @@ destination_from_socket(entry_connection_t *conn, socks_request_t *req) return -1; #endif + done: tor_addr_from_sockaddr(&addr, (struct sockaddr*)&orig_dst, &req->port); tor_addr_to_str(req->address, &addr, sizeof(req->address), 1); @@ -1531,7 +1763,8 @@ connection_ap_get_original_destination(entry_connection_t *conn, if (options->TransProxyType_parsed == TPT_PF_DIVERT) return destination_from_socket(conn, req); - if (options->TransProxyType_parsed == TPT_DEFAULT) + if (options->TransProxyType_parsed == TPT_DEFAULT || + options->TransProxyType_parsed == TPT_IPFW) return destination_from_pf(conn, req); (void)conn; @@ -1767,7 +2000,8 @@ connection_ap_supports_optimistic_data(const entry_connection_t *conn) general circuit. */ if (edge_conn->on_circuit == NULL || edge_conn->on_circuit->state != CIRCUIT_STATE_OPEN || - edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_GENERAL) + (edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_GENERAL && + edge_conn->on_circuit->purpose != CIRCUIT_PURPOSE_C_REND_JOINED)) return 0; return conn->may_use_optimistic_data; @@ -1792,19 +2026,19 @@ connection_ap_get_begincell_flags(entry_connection_t *ap_conn) return 0; /* If only IPv4 is supported, no flags */ - if (ap_conn->ipv4_traffic_ok && !ap_conn->ipv6_traffic_ok) + if (ap_conn->entry_cfg.ipv4_traffic && !ap_conn->entry_cfg.ipv6_traffic) return 0; if (! cpath_layer || ! cpath_layer->extend_info) return 0; - if (!ap_conn->ipv4_traffic_ok) + if (!ap_conn->entry_cfg.ipv4_traffic) flags |= BEGIN_FLAG_IPV4_NOT_OK; exitnode = node_get_by_id(cpath_layer->extend_info->identity_digest); - if (ap_conn->ipv6_traffic_ok && exitnode) { + if (ap_conn->entry_cfg.ipv6_traffic && exitnode) { tor_addr_t a; tor_addr_make_null(&a, AF_INET6); if (compare_tor_addr_to_node_policy(&a, ap_conn->socks_request->port, @@ -1819,7 +2053,7 @@ connection_ap_get_begincell_flags(entry_connection_t *ap_conn) if (flags == BEGIN_FLAG_IPV6_OK) { /* When IPv4 and IPv6 are both allowed, consider whether to say we * prefer IPv6. Otherwise there's no point in declaring a preference */ - if (ap_conn->prefer_ipv6_traffic) + if (ap_conn->entry_cfg.prefer_ipv6) flags |= BEGIN_FLAG_IPV6_PREFERRED; } @@ -2056,8 +2290,8 @@ connection_ap_make_link(connection_t *partner, /* Populate isolation fields. */ conn->socks_request->listener_type = CONN_TYPE_DIR_LISTENER; conn->original_dest_address = tor_strdup(address); - conn->session_group = session_group; - conn->isolation_flags = isolation_flags; + conn->entry_cfg.session_group = session_group; + conn->entry_cfg.isolation_flags = isolation_flags; base_conn->address = tor_strdup("(Tor_internal)"); tor_addr_make_unspec(&base_conn->addr); @@ -2460,7 +2694,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) relay_header_unpack(&rh, cell->payload); if (rh.length > RELAY_PAYLOAD_SIZE) - return -1; + return -END_CIRC_REASON_TORPROTOCOL; /* Note: we have to use relay_send_command_from_edge here, not * connection_edge_end or connection_edge_send_command, since those require @@ -2478,7 +2712,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) r = begin_cell_parse(cell, &bcell, &end_reason); if (r < -1) { - return -1; + return -END_CIRC_REASON_TORPROTOCOL; } else if (r == -1) { tor_free(bcell.address); relay_send_end_cell_from_edge(rh.stream_id, circ, end_reason, NULL); @@ -2576,15 +2810,31 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) n_stream->rend_data = rend_data_dup(origin_circ->rend_data); tor_assert(connection_edge_is_rendezvous_stream(n_stream)); assert_circuit_ok(circ); - if (rend_service_set_connection_addr_port(n_stream, origin_circ) < 0) { + + const int r = rend_service_set_connection_addr_port(n_stream, origin_circ); + if (r < 0) { log_info(LD_REND,"Didn't find rendezvous service (port %d)", n_stream->base_.port); + /* Send back reason DONE because we want to make hidden service port + * scanning harder thus instead of returning that the exit policy + * didn't match, which makes it obvious that the port is closed, + * return DONE and kill the circuit. That way, a user (malicious or + * not) needs one circuit per bad port unless it matches the policy of + * the hidden service. */ relay_send_end_cell_from_edge(rh.stream_id, circ, - END_STREAM_REASON_EXITPOLICY, + END_STREAM_REASON_DONE, origin_circ->cpath->prev); connection_free(TO_CONN(n_stream)); tor_free(address); - return 0; + + /* Drop the circuit here since it might be someone deliberately + * scanning the hidden service ports. Note that this mitigates port + * scanning by adding more work on the attacker side to successfully + * scan but does not fully solve it. */ + if (r < -1) + return END_CIRC_AT_ORIGIN; + else + return 0; } assert_circuit_ok(circ); log_debug(LD_REND,"Finished assigning addr/port"); @@ -2712,7 +2962,7 @@ connection_exit_connect(edge_connection_t *edge_conn) const tor_addr_t *addr; uint16_t port; connection_t *conn = TO_CONN(edge_conn); - int socket_error = 0; + int socket_error = 0, result; if ( (!connection_edge_is_rendezvous_stream(edge_conn) && router_compare_to_my_exit_policy(&edge_conn->base_.addr, @@ -2727,14 +2977,36 @@ connection_exit_connect(edge_connection_t *edge_conn) return; } - addr = &conn->addr; - port = conn->port; +#ifdef HAVE_SYS_UN_H + if (conn->socket_family != AF_UNIX) { +#else + { +#endif /* defined(HAVE_SYS_UN_H) */ + addr = &conn->addr; + port = conn->port; + + if (tor_addr_family(addr) == AF_INET6) + conn->socket_family = AF_INET6; + + log_debug(LD_EXIT, "about to try connecting"); + result = connection_connect(conn, conn->address, + addr, port, &socket_error); +#ifdef HAVE_SYS_UN_H + } else { + /* + * In the AF_UNIX case, we expect to have already had conn->port = 1, + * tor_addr_make_unspec(conn->addr) (cf. the way we mark in the incoming + * case in connection_handle_listener_read()), and conn->address should + * have the socket path to connect to. + */ + tor_assert(conn->address && strlen(conn->address) > 0); - if (tor_addr_family(addr) == AF_INET6) - conn->socket_family = AF_INET6; + log_debug(LD_EXIT, "about to try connecting"); + result = connection_connect_unix(conn, conn->address, &socket_error); +#endif /* defined(HAVE_SYS_UN_H) */ + } - log_debug(LD_EXIT,"about to try connecting"); - switch (connection_connect(conn, conn->address, addr, port, &socket_error)) { + switch (result) { case -1: { int reason = errno_to_stream_end_reason(socket_error); connection_edge_end(edge_conn, reason); @@ -2764,7 +3036,6 @@ connection_exit_connect(edge_connection_t *edge_conn) /* also, deliver a 'connected' cell back through the circuit. */ if (connection_edge_is_rendezvous_stream(edge_conn)) { - /* rendezvous stream */ /* don't send an address back! */ connection_edge_send_command(edge_conn, RELAY_COMMAND_CONNECTED, @@ -2903,10 +3174,10 @@ connection_ap_can_use_exit(const entry_connection_t *conn, const node_t *exit) addr_policy_result_t r; if (0 == tor_addr_parse(&addr, conn->socks_request->address)) { addrp = &addr; - } else if (!conn->ipv4_traffic_ok && conn->ipv6_traffic_ok) { + } else if (!conn->entry_cfg.ipv4_traffic && conn->entry_cfg.ipv6_traffic) { tor_addr_make_null(&addr, AF_INET6); addrp = &addr; - } else if (conn->ipv4_traffic_ok && !conn->ipv6_traffic_ok) { + } else if (conn->entry_cfg.ipv4_traffic && !conn->entry_cfg.ipv6_traffic) { tor_addr_make_null(&addr, AF_INET); addrp = &addr; } @@ -3012,7 +3283,7 @@ int connection_edge_compatible_with_circuit(const entry_connection_t *conn, const origin_circuit_t *circ) { - const uint8_t iso = conn->isolation_flags; + const uint8_t iso = conn->entry_cfg.isolation_flags; const socks_request_t *sr = conn->socks_request; /* If circ has never been used for an isolated connection, we can @@ -3061,7 +3332,8 @@ connection_edge_compatible_with_circuit(const entry_connection_t *conn, if ((iso & ISO_CLIENTADDR) && !tor_addr_eq(&ENTRY_TO_CONN(conn)->addr, &circ->client_addr)) return 0; - if ((iso & ISO_SESSIONGRP) && conn->session_group != circ->session_group) + if ((iso & ISO_SESSIONGRP) && + conn->entry_cfg.session_group != circ->session_group) return 0; if ((iso & ISO_NYM_EPOCH) && conn->nym_epoch != circ->nym_epoch) return 0; @@ -3100,7 +3372,7 @@ connection_edge_update_circuit_isolation(const entry_connection_t *conn, circ->client_proto_type = conn->socks_request->listener_type; circ->client_proto_socksver = conn->socks_request->socks_version; tor_addr_copy(&circ->client_addr, &ENTRY_TO_CONN(conn)->addr); - circ->session_group = conn->session_group; + circ->session_group = conn->entry_cfg.session_group; circ->nym_epoch = conn->nym_epoch; circ->socks_username = sr->username ? tor_memdup(sr->username, sr->usernamelen) : NULL; @@ -3127,7 +3399,7 @@ connection_edge_update_circuit_isolation(const entry_connection_t *conn, mixed |= ISO_CLIENTPROTO; if (!tor_addr_eq(&ENTRY_TO_CONN(conn)->addr, &circ->client_addr)) mixed |= ISO_CLIENTADDR; - if (conn->session_group != circ->session_group) + if (conn->entry_cfg.session_group != circ->session_group) mixed |= ISO_SESSIONGRP; if (conn->nym_epoch != circ->nym_epoch) mixed |= ISO_NYM_EPOCH; @@ -3135,7 +3407,7 @@ connection_edge_update_circuit_isolation(const entry_connection_t *conn, if (dry_run) return mixed; - if ((mixed & conn->isolation_flags) != 0) { + if ((mixed & conn->entry_cfg.isolation_flags) != 0) { log_warn(LD_BUG, "Updating a circuit with seemingly incompatible " "isolation flags."); } diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index 3c0e30a973..7c0b9c0767 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -143,6 +143,30 @@ STATIC int begin_cell_parse(const cell_t *cell, begin_cell_t *bcell, STATIC int connected_cell_format_payload(uint8_t *payload_out, const tor_addr_t *addr, uint32_t ttl); + +typedef struct { + /** Original address, after we lowercased it but before we started + * mapping it. + */ + char orig_address[MAX_SOCKS_ADDR_LEN]; + /** True iff the address has been automatically remapped to a local + * address in VirtualAddrNetwork. (Only set true when we do a resolve + * and get a virtual address; not when we connect to the address.) */ + int automap; + /** If this connection has a .exit address, who put it there? */ + addressmap_entry_source_t exit_source; + /** If we've rewritten the address, when does this map expire? */ + time_t map_expires; + /** If we should close the connection, this is the end_reason to pass + * to connection_mark_unattached_ap */ + int end_reason; + /** True iff we should close the connection, either because of error or + * because of successful early RESOLVED reply. */ + int should_close; +} rewrite_result_t; + +STATIC void connection_ap_handshake_rewrite(entry_connection_t *conn, + rewrite_result_t *out); #endif #endif diff --git a/src/or/connection_or.c b/src/or/connection_or.c index c372270b4c..e0dff1c915 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -38,6 +38,8 @@ #include "router.h" #include "routerlist.h" #include "ext_orport.h" +#include "scheduler.h" + #ifdef USE_BUFFEREVENTS #include <event2/bufferevent_ssl.h> #endif @@ -574,48 +576,51 @@ connection_or_process_inbuf(or_connection_t *conn) return ret; } -/** When adding cells to an OR connection's outbuf, keep adding until the - * outbuf is at least this long, or we run out of cells. */ -#define OR_CONN_HIGHWATER (32*1024) - -/** Add cells to an OR connection's outbuf whenever the outbuf's data length - * drops below this size. */ -#define OR_CONN_LOWWATER (16*1024) - /** Called whenever we have flushed some data on an or_conn: add more data * from active circuits. */ int connection_or_flushed_some(or_connection_t *conn) { - size_t datalen, temp; - ssize_t n, flushed; - size_t cell_network_size = get_cell_network_size(conn->wide_circ_ids); + size_t datalen; + + /* The channel will want to update its estimated queue size */ + channel_update_xmit_queue_size(TLS_CHAN_TO_BASE(conn->chan)); /* If we're under the low water mark, add cells until we're just over the * high water mark. */ datalen = connection_get_outbuf_len(TO_CONN(conn)); if (datalen < OR_CONN_LOWWATER) { - while ((conn->chan) && channel_tls_more_to_flush(conn->chan)) { - /* Compute how many more cells we want at most */ - n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, cell_network_size); - /* Bail out if we don't want any more */ - if (n <= 0) break; - /* We're still here; try to flush some more cells */ - flushed = channel_tls_flush_some_cells(conn->chan, n); - /* Bail out if it says it didn't flush anything */ - if (flushed <= 0) break; - /* How much in the outbuf now? */ - temp = connection_get_outbuf_len(TO_CONN(conn)); - /* Bail out if we didn't actually increase the outbuf size */ - if (temp <= datalen) break; - /* Update datalen for the next iteration */ - datalen = temp; - } + /* Let the scheduler know */ + scheduler_channel_wants_writes(TLS_CHAN_TO_BASE(conn->chan)); } return 0; } +/** This is for channeltls.c to ask how many cells we could accept if + * they were available. */ +ssize_t +connection_or_num_cells_writeable(or_connection_t *conn) +{ + size_t datalen, cell_network_size; + ssize_t n = 0; + + tor_assert(conn); + + /* + * If we're under the high water mark, we're potentially + * writeable; note this is different from the calculation above + * used to trigger when to start writing after we've stopped. + */ + datalen = connection_get_outbuf_len(TO_CONN(conn)); + if (datalen < OR_CONN_HIGHWATER) { + cell_network_size = get_cell_network_size(conn->wide_circ_ids); + n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, cell_network_size); + } + + return n; +} + /** Connection <b>conn</b> has finished writing and has no bytes left on * its outbuf. * @@ -908,18 +913,11 @@ connection_or_init_conn_from_address(or_connection_t *conn, tor_free(conn->base_.address); conn->base_.address = tor_dup_addr(&node_ap.addr); } else { - const char *n; - /* If we're an authoritative directory server, we may know a - * nickname for this router. */ - n = dirserv_get_nickname_by_digest(id_digest); - if (n) { - conn->nickname = tor_strdup(n); - } else { - conn->nickname = tor_malloc(HEX_DIGEST_LEN+2); - conn->nickname[0] = '$'; - base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1, - conn->identity_digest, DIGEST_LEN); - } + conn->nickname = tor_malloc(HEX_DIGEST_LEN+2); + conn->nickname[0] = '$'; + base16_encode(conn->nickname+1, HEX_DIGEST_LEN+1, + conn->identity_digest, DIGEST_LEN); + tor_free(conn->base_.address); conn->base_.address = tor_dup_addr(addr); } @@ -1151,9 +1149,7 @@ connection_or_notify_error(or_connection_t *conn, if (conn->chan) { chan = TLS_CHAN_TO_BASE(conn->chan); /* Don't transition if we're already in closing, closed or error */ - if (!(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(chan)) { channel_close_for_error(chan); } } @@ -1176,10 +1172,10 @@ connection_or_notify_error(or_connection_t *conn, * * Return the launched conn, or NULL if it failed. */ -or_connection_t * -connection_or_connect(const tor_addr_t *_addr, uint16_t port, - const char *id_digest, - channel_tls_t *chan) + +MOCK_IMPL(or_connection_t *, +connection_or_connect, (const tor_addr_t *_addr, uint16_t port, + const char *id_digest, channel_tls_t *chan)) { or_connection_t *conn; const or_options_t *options = get_options(); @@ -1312,9 +1308,7 @@ connection_or_close_normally(or_connection_t *orconn, int flush) if (orconn->chan) { chan = TLS_CHAN_TO_BASE(orconn->chan); /* Don't transition if we're already in closing, closed or error */ - if (!(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(chan)) { channel_close_from_lower_layer(chan); } } @@ -1335,9 +1329,7 @@ connection_or_close_for_error(or_connection_t *orconn, int flush) if (orconn->chan) { chan = TLS_CHAN_TO_BASE(orconn->chan); /* Don't transition if we're already in closing, closed or error */ - if (!(chan->state == CHANNEL_STATE_CLOSING || - chan->state == CHANNEL_STATE_CLOSED || - chan->state == CHANNEL_STATE_ERROR)) { + if (!CHANNEL_CONDEMNED(chan)) { channel_close_for_error(chan); } } @@ -1827,6 +1819,7 @@ connection_tls_finish_handshake(or_connection_t *conn) conn->base_.port, digest_rcvd, 0); } tor_tls_block_renegotiation(conn->tls); + rep_hist_note_negotiated_link_proto(1, started_here); return connection_or_set_state_open(conn); } else { connection_or_change_state(conn, OR_CONN_STATE_OR_HANDSHAKING_V2); diff --git a/src/or/connection_or.h b/src/or/connection_or.h index 143540edd9..fc261c6bac 100644 --- a/src/or/connection_or.h +++ b/src/or/connection_or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -24,6 +24,7 @@ void connection_or_set_bad_connections(const char *digest, int force); void connection_or_block_renegotiation(or_connection_t *conn); int connection_or_reached_eof(or_connection_t *conn); int connection_or_process_inbuf(or_connection_t *conn); +ssize_t connection_or_num_cells_writeable(or_connection_t *conn); int connection_or_flushed_some(or_connection_t *conn); int connection_or_finished_flushing(or_connection_t *conn); int connection_or_finished_connecting(or_connection_t *conn); @@ -36,9 +37,10 @@ void connection_or_connect_failed(or_connection_t *conn, int reason, const char *msg); void connection_or_notify_error(or_connection_t *conn, int reason, const char *msg); -or_connection_t *connection_or_connect(const tor_addr_t *addr, uint16_t port, - const char *id_digest, - channel_tls_t *chan); +MOCK_DECL(or_connection_t *, + connection_or_connect, + (const tor_addr_t *addr, uint16_t port, + const char *id_digest, channel_tls_t *chan)); void connection_or_close_normally(or_connection_t *orconn, int flush); void connection_or_close_for_error(or_connection_t *orconn, int flush); diff --git a/src/or/control.c b/src/or/control.c index 2ff1cc8442..e25c3b2954 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -47,6 +47,7 @@ #include <sys/resource.h> #endif +#include "crypto_s2k.h" #include "procmon.h" /** Yield true iff <b>s</b> is the state of a control_connection_t that has @@ -194,14 +195,14 @@ log_severity_to_event(int severity) static void clear_circ_bw_fields(void) { - circuit_t *circ; origin_circuit_t *ocirc; - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!CIRCUIT_IS_ORIGIN(circ)) continue; ocirc = TO_ORIGIN_CIRCUIT(circ); ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; } + SMARTLIST_FOREACH_END(circ); } /** Set <b>global_event_mask*</b> to the bitwise OR of each live control @@ -949,7 +950,7 @@ static int handle_control_setevents(control_connection_t *conn, uint32_t len, const char *body) { - int event_code = -1; + int event_code; event_mask_t event_mask = 0; smartlist_t *events = smartlist_new(); @@ -963,6 +964,8 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, continue; } else { int i; + event_code = -1; + for (i = 0; control_event_table[i].event_name != NULL; ++i) { if (!strcasecmp(ev, control_event_table[i].event_name)) { event_code = control_event_table[i].event_code; @@ -993,7 +996,8 @@ handle_control_setevents(control_connection_t *conn, uint32_t len, /** Decode the hashed, base64'd passwords stored in <b>passwords</b>. * Return a smartlist of acceptable passwords (unterminated strings of - * length S2K_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on failure. + * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on + * failure. */ smartlist_t * decode_hashed_passwords(config_line_t *passwords) @@ -1009,16 +1013,17 @@ decode_hashed_passwords(config_line_t *passwords) if (!strcmpstart(hashed, "16:")) { if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))<0 - || strlen(hashed+3) != (S2K_SPECIFIER_LEN+DIGEST_LEN)*2) { + || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) { goto err; } } else { if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed)) - != S2K_SPECIFIER_LEN+DIGEST_LEN) { + != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) { goto err; } } - smartlist_add(sl, tor_memdup(decoded, S2K_SPECIFIER_LEN+DIGEST_LEN)); + smartlist_add(sl, + tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)); } return sl; @@ -1039,7 +1044,7 @@ handle_control_authenticate(control_connection_t *conn, uint32_t len, { int used_quoted_string = 0; const or_options_t *options = get_options(); - const char *errstr = NULL; + const char *errstr = "Unknown error"; char *password; size_t password_len; const char *cp; @@ -1160,22 +1165,27 @@ handle_control_authenticate(control_connection_t *conn, uint32_t len, } if (bad) { if (!also_cookie) { - log_warn(LD_CONTROL, + log_warn(LD_BUG, "Couldn't decode HashedControlPassword: invalid base16"); errstr="Couldn't decode HashedControlPassword value in configuration."; + goto err; } bad_password = 1; SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); smartlist_free(sl); + sl = NULL; } else { SMARTLIST_FOREACH(sl, char *, expected, { - secret_to_key(received,DIGEST_LEN,password,password_len,expected); - if (tor_memeq(expected+S2K_SPECIFIER_LEN, received, DIGEST_LEN)) + secret_to_key_rfc2440(received,DIGEST_LEN, + password,password_len,expected); + if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN, + received, DIGEST_LEN)) goto ok; }); SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); smartlist_free(sl); + sl = NULL; if (used_quoted_string) errstr = "Password did not match HashedControlPassword value from " @@ -1198,9 +1208,12 @@ handle_control_authenticate(control_connection_t *conn, uint32_t len, err: tor_free(password); - connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", - errstr ? errstr : "Unknown reason."); + connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", errstr); connection_mark_for_close(TO_CONN(conn)); + if (sl) { /* clean up */ + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + } return 0; ok: log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT @@ -1250,6 +1263,7 @@ static const struct signal_t signal_table[] = { { SIGTERM, "INT" }, { SIGNEWNYM, "NEWNYM" }, { SIGCLEARDNSCACHE, "CLEARDNSCACHE"}, + { SIGHEARTBEAT, "HEARTBEAT"}, { 0, NULL }, }; @@ -1427,9 +1441,13 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, } else if (!strcmp(question, "bw-event-cache")) { *answer = get_bw_samples(); } else if (!strcmp(question, "config-file")) { - *answer = tor_strdup(get_torrc_fname(0)); + const char *a = get_torrc_fname(0); + if (a) + *answer = tor_strdup(a); } else if (!strcmp(question, "config-defaults-file")) { - *answer = tor_strdup(get_torrc_fname(1)); + const char *a = get_torrc_fname(1); + if (a) + *answer = tor_strdup(a); } else if (!strcmp(question, "config-text")) { *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL); } else if (!strcmp(question, "info/names")) { @@ -1864,6 +1882,22 @@ circuit_describe_status_for_controller(origin_circuit_t *circ) smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf); } + // Show username and/or password if available. + if (circ->socks_username_len > 0) { + char* socks_username_escaped = esc_for_log_len(circ->socks_username, + (size_t) circ->socks_username_len); + smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s", + socks_username_escaped); + tor_free(socks_username_escaped); + } + if (circ->socks_password_len > 0) { + char* socks_password_escaped = esc_for_log_len(circ->socks_password, + (size_t) circ->socks_password_len); + smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s", + socks_password_escaped); + tor_free(socks_password_escaped); + } + rv = smartlist_join_strings(descparts, " ", 0, NULL); SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp)); @@ -1881,9 +1915,8 @@ getinfo_helper_events(control_connection_t *control_conn, { (void) control_conn; if (!strcmp(question, "circuit-status")) { - circuit_t *circ_; smartlist_t *status = smartlist_new(); - TOR_LIST_FOREACH(circ_, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) { origin_circuit_t *circ; char *circdesc; const char *state; @@ -1905,6 +1938,7 @@ getinfo_helper_events(control_connection_t *control_conn, state, *circdesc ? " " : "", circdesc); tor_free(circdesc); } + SMARTLIST_FOREACH_END(circ_); *answer = smartlist_join_strings(status, "\r\n", 0, NULL); SMARTLIST_FOREACH(status, char *, cp, tor_free(cp)); smartlist_free(status); @@ -2004,7 +2038,7 @@ getinfo_helper_events(control_connection_t *control_conn, /* Note that status/ is not a catch-all for events; there's only supposed * to be a status GETINFO if there's a corresponding STATUS event. */ if (!strcmp(question, "status/circuit-established")) { - *answer = tor_strdup(can_complete_circuit ? "1" : "0"); + *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0"); } else if (!strcmp(question, "status/enough-dir-info")) { *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0"); } else if (!strcmp(question, "status/good-server-descriptor") || @@ -2151,6 +2185,8 @@ static const getinfo_item_t getinfo_items[] = { "Brief summary of router status by nickname (v2 directory format)."), PREFIX("ns/purpose/", networkstatus, "Brief summary of router status by purpose (v2 directory format)."), + PREFIX("consensus/", networkstatus, + "Information about and from the ns consensus."), ITEM("network-status", dir, "Brief summary of router status (v1 directory format)"), ITEM("circuit-status", events, "List of current circuits originating here."), @@ -2454,6 +2490,14 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, goto done; } + if (smartlist_len(args) < 2) { + connection_printf_to_buf(conn, + "512 syntax error: not enough arguments.\r\n"); + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + goto done; + } + smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0); SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); @@ -3912,12 +3956,11 @@ control_event_stream_bandwidth_used(void) int control_event_circ_bandwidth_used(void) { - circuit_t *circ; origin_circuit_t *ocirc; if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED)) return 0; - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!CIRCUIT_IS_ORIGIN(circ)) continue; ocirc = TO_ORIGIN_CIRCUIT(circ); @@ -3930,6 +3973,7 @@ control_event_circ_bandwidth_used(void) (unsigned long)ocirc->n_written_circ_bw); ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; } + SMARTLIST_FOREACH_END(circ); return 0; } @@ -4094,14 +4138,13 @@ format_cell_stats(char **event_string, circuit_t *circ, int control_event_circuit_cell_stats(void) { - circuit_t *circ; cell_stats_t *cell_stats; char *event_string; if (!get_options()->TestingEnableCellStatsEvent || !EVENT_IS_INTERESTING(EVENT_CELL_STATS)) return 0; cell_stats = tor_malloc(sizeof(cell_stats_t));; - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->testing_cell_stats) continue; sum_up_cell_stats_by_command(circ, cell_stats); @@ -4110,6 +4153,7 @@ control_event_circuit_cell_stats(void) "650 CELL_STATS %s\r\n", event_string); tor_free(event_string); } + SMARTLIST_FOREACH_END(circ); tor_free(cell_stats); return 0; } @@ -4175,12 +4219,12 @@ get_bw_samples(void) int i; int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements) % N_BW_EVENTS_TO_CACHE; - smartlist_t *elements = smartlist_new(); tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE); + smartlist_t *elements = smartlist_new(); + for (i = 0; i < n_measurements; ++i) { tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE); - { const struct cached_bw_event_s *bwe = &cached_bw_events[idx]; smartlist_add_asprintf(elements, "%u,%u", @@ -4188,17 +4232,14 @@ get_bw_samples(void) (unsigned)bwe->n_written); idx = (idx + 1) % N_BW_EVENTS_TO_CACHE; - } } - { char *result = smartlist_join_strings(elements, " ", 0, NULL); SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); smartlist_free(elements); return result; - } } /** Called when we are sending a log message to the controllers: suspend @@ -4494,6 +4535,9 @@ control_event_signal(uintptr_t signal) case SIGCLEARDNSCACHE: signal_string = "CLEARDNSCACHE"; break; + case SIGHEARTBEAT: + signal_string = "HEARTBEAT"; + break; default: log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal", (unsigned long)signal); @@ -4843,23 +4887,43 @@ bootstrap_status_to_string(bootstrap_status_t s, const char **tag, break; case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: *tag = "requesting_descriptors"; - *summary = "Asking for relay descriptors"; + /* XXXX this appears to incorrectly report internal on most loads */ + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Asking for relay descriptors for internal paths" : + "Asking for relay descriptors"; break; + /* If we're sure there are no exits in the consensus, + * inform the controller by adding "internal" + * to the status summaries. + * (We only check this while loading descriptors, + * so we may not know in the earlier stages.) + * But if there are exits, we can't be sure whether + * we're creating internal or exit paths/circuits. + * XXXX Or should be use different tags or statuses + * for internal and exit/all? */ case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: *tag = "loading_descriptors"; - *summary = "Loading relay descriptors"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Loading relay descriptors for internal paths" : + "Loading relay descriptors"; break; case BOOTSTRAP_STATUS_CONN_OR: *tag = "conn_or"; - *summary = "Connecting to the Tor network"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Connecting to the Tor network internally" : + "Connecting to the Tor network"; break; case BOOTSTRAP_STATUS_HANDSHAKE_OR: *tag = "handshake_or"; - *summary = "Finishing handshake with first hop"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Finishing handshake with first hop of internal circuit" : + "Finishing handshake with first hop"; break; case BOOTSTRAP_STATUS_CIRCUIT_CREATE: *tag = "circuit_create"; - *summary = "Establishing a Tor circuit"; + *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? + "Establishing an internal Tor circuit" : + "Establishing a Tor circuit"; break; case BOOTSTRAP_STATUS_DONE: *tag = "done"; @@ -4907,15 +4971,18 @@ static int bootstrap_problems = 0; * * <b>status</b> is the new status, that is, what task we will be doing * next. <b>progress</b> is zero if we just started this task, else it - * represents progress on the task. */ -void + * represents progress on the task. + * + * Return true if we logged a message at level NOTICE, and false otherwise. + */ +int control_event_bootstrap(bootstrap_status_t status, int progress) { const char *tag, *summary; char buf[BOOTSTRAP_MSG_LEN]; if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) - return; /* already bootstrapped; nothing to be done here. */ + return 0; /* already bootstrapped; nothing to be done here. */ /* special case for handshaking status, since our TLS handshaking code * can't distinguish what the connection is going to be for. */ @@ -4962,7 +5029,10 @@ control_event_bootstrap(bootstrap_status_t status, int progress) /* Remember that we gave a notice at this level. */ notice_bootstrap_percent = bootstrap_percent; } + return loglevel == LOG_NOTICE; } + + return 0; } /** Called when Tor has failed to make bootstrapping progress in a way @@ -5016,19 +5086,26 @@ MOCK_IMPL(void, log_fn(severity, LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; " - "count %d; recommendation %s)", + "count %d; recommendation %s; host %s at %s:%d)", status, summary, warn, orconn_end_reason_to_control_string(reason), - bootstrap_problems, recommendation); + bootstrap_problems, recommendation, + hex_str(or_conn->identity_digest, DIGEST_LEN), + or_conn->base_.address, + or_conn->base_.port); connection_or_report_broken_states(severity, LD_HANDSHAKE); tor_snprintf(buf, sizeof(buf), "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " - "COUNT=%d RECOMMENDATION=%s", + "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s:%d\"", bootstrap_percent, tag, summary, warn, orconn_end_reason_to_control_string(reason), bootstrap_problems, - recommendation); + recommendation, + hex_str(or_conn->identity_digest, DIGEST_LEN), + or_conn->base_.address, + (int)or_conn->base_.port); + tor_snprintf(last_sent_bootstrap_message, sizeof(last_sent_bootstrap_message), "WARN %s", buf); @@ -5136,20 +5213,30 @@ control_event_hs_descriptor_requested(const rend_data_t *rend_query, void control_event_hs_descriptor_receive_end(const char *action, const rend_data_t *rend_query, - const char *id_digest) + const char *id_digest, + const char *reason) { + char *reason_field = NULL; + if (!action || !rend_query || !id_digest) { log_warn(LD_BUG, "Called with action==%p, rend_query==%p, " "id_digest==%p", action, rend_query, id_digest); return; } + if (reason) { + tor_asprintf(&reason_field, " REASON=%s", reason); + } + send_control_event(EVENT_HS_DESC, ALL_FORMATS, - "650 HS_DESC %s %s %s %s\r\n", + "650 HS_DESC %s %s %s %s%s\r\n", action, rend_query->onion_address, rend_auth_type_to_string(rend_query->auth_type), - node_describe_longname_by_id(id_digest)); + node_describe_longname_by_id(id_digest), + reason_field ? reason_field : ""); + + tor_free(reason_field); } /** send HS_DESC RECEIVED event @@ -5165,23 +5252,27 @@ control_event_hs_descriptor_received(const rend_data_t *rend_query, rend_query, id_digest); return; } - control_event_hs_descriptor_receive_end("RECEIVED", rend_query, id_digest); + control_event_hs_descriptor_receive_end("RECEIVED", rend_query, + id_digest, NULL); } -/** send HS_DESC FAILED event - * - * called when request for hidden service descriptor returned failure. +/** Send HS_DESC event to inform controller that query <b>rend_query</b> + * failed to retrieve hidden service descriptor identified by + * <b>id_digest</b>. If <b>reason</b> is not NULL, add it to REASON= + * field. */ void control_event_hs_descriptor_failed(const rend_data_t *rend_query, - const char *id_digest) + const char *id_digest, + const char *reason) { if (!rend_query || !id_digest) { log_warn(LD_BUG, "Called with rend_query==%p, id_digest==%p", rend_query, id_digest); return; } - control_event_hs_descriptor_receive_end("FAILED", rend_query, id_digest); + control_event_hs_descriptor_receive_end("FAILED", rend_query, + id_digest, reason); } /** Free any leftover allocated memory of the control.c subsystem. */ diff --git a/src/or/control.h b/src/or/control.h index 8697262176..47a601817a 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -92,7 +92,7 @@ void enable_control_logging(void); void monitor_owning_controller_process(const char *process_spec); -void control_event_bootstrap(bootstrap_status_t status, int progress); +int control_event_bootstrap(bootstrap_status_t status, int progress); MOCK_DECL(void, control_event_bootstrap_problem,(const char *warn, int reason, or_connection_t *or_conn)); @@ -108,11 +108,13 @@ void control_event_hs_descriptor_requested(const rend_data_t *rend_query, const char *hs_dir); void control_event_hs_descriptor_receive_end(const char *action, const rend_data_t *rend_query, - const char *hs_dir); + const char *hs_dir, + const char *reason); void control_event_hs_descriptor_received(const rend_data_t *rend_query, const char *hs_dir); void control_event_hs_descriptor_failed(const rend_data_t *rend_query, - const char *hs_dir); + const char *hs_dir, + const char *reason); void control_free_all(void); diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 61b2c29b38..d511ecf84c 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -1,88 +1,103 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** * \file cpuworker.c - * \brief Implements a farm of 'CPU worker' processes to perform - * CPU-intensive tasks in another thread or process, to not - * interrupt the main thread. + * \brief Uses the workqueue/threadpool code to farm CPU-intensive activities + * out to subprocesses. * * Right now, we only use this for processing onionskins. **/ #include "or.h" -#include "buffers.h" #include "channel.h" -#include "channeltls.h" #include "circuitbuild.h" #include "circuitlist.h" -#include "config.h" -#include "connection.h" #include "connection_or.h" +#include "config.h" #include "cpuworker.h" #include "main.h" #include "onion.h" #include "rephist.h" #include "router.h" +#include "workqueue.h" -/** The maximum number of cpuworker processes we will keep around. */ -#define MAX_CPUWORKERS 16 -/** The minimum number of cpuworker processes we will keep around. */ -#define MIN_CPUWORKERS 1 - -/** The tag specifies which circuit this onionskin was from. */ -#define TAG_LEN 12 - -/** How many cpuworkers we have running right now. */ -static int num_cpuworkers=0; -/** How many of the running cpuworkers have an assigned task right now. */ -static int num_cpuworkers_busy=0; -/** We need to spawn new cpuworkers whenever we rotate the onion keys - * on platforms where execution contexts==processes. This variable stores - * the last time we got a key rotation event. */ -static time_t last_rotation_time=0; - -static void cpuworker_main(void *data) ATTR_NORETURN; -static int spawn_cpuworker(void); -static void spawn_enough_cpuworkers(void); -static void process_pending_task(connection_t *cpuworker); - -/** Initialize the cpuworker subsystem. - */ -void -cpu_init(void) +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +static void queue_pending_tasks(void); + +typedef struct worker_state_s { + int generation; + server_onion_keys_t *onion_keys; +} worker_state_t; + +static void * +worker_state_new(void *arg) { - cpuworkers_rotate(); + worker_state_t *ws; + (void)arg; + ws = tor_malloc_zero(sizeof(worker_state_t)); + ws->onion_keys = server_onion_keys_new(); + return ws; } - -/** Called when we're done sending a request to a cpuworker. */ -int -connection_cpu_finished_flushing(connection_t *conn) +static void +worker_state_free(void *arg) { - tor_assert(conn); - tor_assert(conn->type == CONN_TYPE_CPUWORKER); - return 0; + worker_state_t *ws = arg; + server_onion_keys_free(ws->onion_keys); + tor_free(ws); } -/** Pack global_id and circ_id; set *tag to the result. (See note on - * cpuworker_main for wire format.) */ +static replyqueue_t *replyqueue = NULL; +static threadpool_t *threadpool = NULL; +static struct event *reply_event = NULL; + +static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT; + +static int total_pending_tasks = 0; +static int max_pending_tasks = 128; + static void -tag_pack(uint8_t *tag, uint64_t chan_id, circid_t circ_id) +replyqueue_process_cb(evutil_socket_t sock, short events, void *arg) { - /*XXXX RETHINK THIS WHOLE MESS !!!! !NM NM NM NM*/ - /*XXXX DOUBLEPLUSTHIS!!!! AS AS AS AS*/ - set_uint64(tag, chan_id); - set_uint32(tag+8, circ_id); + replyqueue_t *rq = arg; + (void) sock; + (void) events; + replyqueue_process(rq); } -/** Unpack <b>tag</b> into addr, port, and circ_id. +/** Initialize the cpuworker subsystem. It is OK to call this more than once + * during Tor's lifetime. */ -static void -tag_unpack(const uint8_t *tag, uint64_t *chan_id, circid_t *circ_id) +void +cpu_init(void) { - *chan_id = get_uint64(tag); - *circ_id = get_uint32(tag+8); + if (!replyqueue) { + replyqueue = replyqueue_new(0); + } + if (!reply_event) { + reply_event = tor_event_new(tor_libevent_get_base(), + replyqueue_get_socket(replyqueue), + EV_READ|EV_PERSIST, + replyqueue_process_cb, + replyqueue); + event_add(reply_event, NULL); + } + if (!threadpool) { + threadpool = threadpool_new(get_num_cpus(get_options()), + replyqueue, + worker_state_new, + worker_state_free, + NULL); + } + /* Total voodoo. Can we make this more sensible? */ + max_pending_tasks = get_num_cpus(get_options()) * 64; + crypto_seed_weak_rng(&request_sample_rng); } /** Magic numbers to make sure our cpuworker_requests don't grow any @@ -94,10 +109,6 @@ tag_unpack(const uint8_t *tag, uint64_t *chan_id, circid_t *circ_id) typedef struct cpuworker_request_t { /** Magic number; must be CPUWORKER_REQUEST_MAGIC. */ uint32_t magic; - /** Opaque tag to identify the job */ - uint8_t tag[TAG_LEN]; - /** Task code. Must be one of CPUWORKER_TASK_* */ - uint8_t task; /** Flag: Are we timing this request? */ unsigned timed : 1; @@ -114,8 +125,7 @@ typedef struct cpuworker_request_t { typedef struct cpuworker_reply_t { /** Magic number; must be CPUWORKER_REPLY_MAGIC. */ uint32_t magic; - /** Opaque tag to identify the job; matches the request's tag.*/ - uint8_t tag[TAG_LEN]; + /** True iff we got a successful request. */ uint8_t success; @@ -142,42 +152,45 @@ typedef struct cpuworker_reply_t { uint8_t rend_auth_material[DIGEST_LEN]; } cpuworker_reply_t; -/** Called when the onion key has changed and we need to spawn new - * cpuworkers. Close all currently idle cpuworkers, and mark the last - * rotation time as now. - */ -void -cpuworkers_rotate(void) +typedef struct cpuworker_job_u { + or_circuit_t *circ; + union { + cpuworker_request_t request; + cpuworker_reply_t reply; + } u; +} cpuworker_job_t; + +static int +update_state_threadfn(void *state_, void *work_) { - connection_t *cpuworker; - while ((cpuworker = connection_get_by_type_state(CONN_TYPE_CPUWORKER, - CPUWORKER_STATE_IDLE))) { - connection_mark_for_close(cpuworker); - --num_cpuworkers; - } - last_rotation_time = time(NULL); - if (server_mode(get_options())) - spawn_enough_cpuworkers(); + worker_state_t *state = state_; + worker_state_t *update = work_; + server_onion_keys_free(state->onion_keys); + state->onion_keys = update->onion_keys; + update->onion_keys = NULL; + ++state->generation; + return WQ_RPL_REPLY; } -/** If the cpuworker closes the connection, - * mark it as closed and spawn a new one as needed. */ -int -connection_cpu_reached_eof(connection_t *conn) +/** Called when the onion key has changed so update all CPU worker(s) with + * new function pointers with which a new state will be generated. + */ +void +cpuworkers_rotate_keyinfo(void) { - log_warn(LD_GENERAL,"Read eof. CPU worker died unexpectedly."); - if (conn->state != CPUWORKER_STATE_IDLE) { - /* the circ associated with this cpuworker will have to wait until - * it gets culled in run_connection_housekeeping(), since we have - * no way to find out which circ it was. */ - log_warn(LD_GENERAL,"...and it left a circuit queued; abandoning circ."); - num_cpuworkers_busy--; + if (!threadpool) { + /* If we're a client, then we won't have cpuworkers, and we won't need + * to tell them to rotate their state. + */ + return; + } + if (threadpool_queue_update(threadpool, + worker_state_new, + update_state_threadfn, + worker_state_free, + NULL)) { + log_warn(LD_OR, "Failed to queue key update for worker threads."); } - num_cpuworkers--; - spawn_enough_cpuworkers(); /* try to regrow. hope we don't end up - spinning. */ - connection_mark_for_close(conn); - return 0; } /** Indexed by handshake type: how many onionskins have we processed and @@ -197,8 +210,6 @@ static uint64_t onionskins_usec_roundtrip[MAX_ONION_HANDSHAKE_TYPE+1]; * time. (microseconds) */ #define MAX_BELIEVABLE_ONIONSKIN_DELAY (2*1000*1000) -static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT; - /** Return true iff we'd like to measure a handshake of type * <b>onionskin_type</b>. Call only from the main thread. */ static int @@ -286,438 +297,275 @@ cpuworker_log_onionskin_overhead(int severity, int onionskin_type, onionskin_type_name, (unsigned)overhead, relative_overhead*100); } -/** Called when we get data from a cpuworker. If the answer is not complete, - * wait for a complete answer. If the answer is complete, - * process it as appropriate. - */ -int -connection_cpu_process_inbuf(connection_t *conn) -{ - uint64_t chan_id; - circid_t circ_id; - channel_t *p_chan = NULL; - circuit_t *circ; - - tor_assert(conn); - tor_assert(conn->type == CONN_TYPE_CPUWORKER); - - if (!connection_get_inbuf_len(conn)) - return 0; - - if (conn->state == CPUWORKER_STATE_BUSY_ONION) { - cpuworker_reply_t rpl; - if (connection_get_inbuf_len(conn) < sizeof(cpuworker_reply_t)) - return 0; /* not yet */ - tor_assert(connection_get_inbuf_len(conn) == sizeof(cpuworker_reply_t)); - - connection_fetch_from_buf((void*)&rpl,sizeof(cpuworker_reply_t),conn); - - tor_assert(rpl.magic == CPUWORKER_REPLY_MAGIC); - - if (rpl.timed && rpl.success && - rpl.handshake_type <= MAX_ONION_HANDSHAKE_TYPE) { - /* Time how long this request took. The handshake_type check should be - needless, but let's leave it in to be safe. */ - struct timeval tv_end, tv_diff; - int64_t usec_roundtrip; - tor_gettimeofday(&tv_end); - timersub(&tv_end, &rpl.started_at, &tv_diff); - usec_roundtrip = ((int64_t)tv_diff.tv_sec)*1000000 + tv_diff.tv_usec; - if (usec_roundtrip >= 0 && - usec_roundtrip < MAX_BELIEVABLE_ONIONSKIN_DELAY) { - ++onionskins_n_processed[rpl.handshake_type]; - onionskins_usec_internal[rpl.handshake_type] += rpl.n_usec; - onionskins_usec_roundtrip[rpl.handshake_type] += usec_roundtrip; - if (onionskins_n_processed[rpl.handshake_type] >= 500000) { - /* Scale down every 500000 handshakes. On a busy server, that's - * less impressive than it sounds. */ - onionskins_n_processed[rpl.handshake_type] /= 2; - onionskins_usec_internal[rpl.handshake_type] /= 2; - onionskins_usec_roundtrip[rpl.handshake_type] /= 2; - } - } - } - /* parse out the circ it was talking about */ - tag_unpack(rpl.tag, &chan_id, &circ_id); - circ = NULL; - log_debug(LD_OR, - "Unpacking cpuworker reply, chan_id is " U64_FORMAT - ", circ_id is %u", - U64_PRINTF_ARG(chan_id), (unsigned)circ_id); - p_chan = channel_find_by_global_id(chan_id); - - if (p_chan) - circ = circuit_get_by_circid_channel(circ_id, p_chan); - - if (rpl.success == 0) { - log_debug(LD_OR, - "decoding onionskin failed. " - "(Old key or bad software.) Closing."); - if (circ) - circuit_mark_for_close(circ, END_CIRC_REASON_TORPROTOCOL); - goto done_processing; - } - if (!circ) { - /* This happens because somebody sends us a destroy cell and the - * circuit goes away, while the cpuworker is working. This is also - * why our tag doesn't include a pointer to the circ, because we'd - * never know if it's still valid. - */ - log_debug(LD_OR,"processed onion for a circ that's gone. Dropping."); - goto done_processing; - } - tor_assert(! CIRCUIT_IS_ORIGIN(circ)); - if (onionskin_answer(TO_OR_CIRCUIT(circ), - &rpl.created_cell, - (const char*)rpl.keys, - rpl.rend_auth_material) < 0) { - log_warn(LD_OR,"onionskin_answer failed. Closing."); - circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL); - goto done_processing; - } - log_debug(LD_OR,"onionskin_answer succeeded. Yay."); - } else { - tor_assert(0); /* don't ask me to do handshakes yet */ - } - - done_processing: - conn->state = CPUWORKER_STATE_IDLE; - num_cpuworkers_busy--; - if (conn->timestamp_created < last_rotation_time) { - connection_mark_for_close(conn); - num_cpuworkers--; - spawn_enough_cpuworkers(); - } else { - process_pending_task(conn); - } - return 0; -} - -/** Implement a cpuworker. 'data' is an fdarray as returned by socketpair. - * Read and writes from fdarray[1]. Reads requests, writes answers. - * - * Request format: - * cpuworker_request_t. - * Response format: - * cpuworker_reply_t - */ +/** Handle a reply from the worker threads. */ static void -cpuworker_main(void *data) +cpuworker_onion_handshake_replyfn(void *work_) { - /* For talking to the parent thread/process */ - tor_socket_t *fdarray = data; - tor_socket_t fd; - - /* variables for onion processing */ - server_onion_keys_t onion_keys; - cpuworker_request_t req; + cpuworker_job_t *job = work_; cpuworker_reply_t rpl; - - fd = fdarray[1]; /* this side is ours */ -#ifndef TOR_IS_MULTITHREADED - tor_close_socket(fdarray[0]); /* this is the side of the socketpair the - * parent uses */ - tor_free_all(1); /* so the child doesn't hold the parent's fd's open */ - handle_signals(0); /* ignore interrupts from the keyboard, etc */ -#endif - tor_free(data); - - setup_server_onion_keys(&onion_keys); - - for (;;) { - if (read_all(fd, (void *)&req, sizeof(req), 1) != sizeof(req)) { - log_info(LD_OR, "read request failed. Exiting."); - goto end; - } - tor_assert(req.magic == CPUWORKER_REQUEST_MAGIC); - - memset(&rpl, 0, sizeof(rpl)); - - if (req.task == CPUWORKER_TASK_ONION) { - const create_cell_t *cc = &req.create_cell; - created_cell_t *cell_out = &rpl.created_cell; - struct timeval tv_start = {0,0}, tv_end; - int n; - rpl.timed = req.timed; - rpl.started_at = req.started_at; - rpl.handshake_type = cc->handshake_type; - if (req.timed) - tor_gettimeofday(&tv_start); - n = onion_skin_server_handshake(cc->handshake_type, - cc->onionskin, cc->handshake_len, - &onion_keys, - cell_out->reply, - rpl.keys, CPATH_KEY_MATERIAL_LEN, - rpl.rend_auth_material); - if (n < 0) { - /* failure */ - log_debug(LD_OR,"onion_skin_server_handshake failed."); - memset(&rpl, 0, sizeof(rpl)); - memcpy(rpl.tag, req.tag, TAG_LEN); - rpl.success = 0; - } else { - /* success */ - log_debug(LD_OR,"onion_skin_server_handshake succeeded."); - memcpy(rpl.tag, req.tag, TAG_LEN); - cell_out->handshake_len = n; - switch (cc->cell_type) { - case CELL_CREATE: - cell_out->cell_type = CELL_CREATED; break; - case CELL_CREATE2: - cell_out->cell_type = CELL_CREATED2; break; - case CELL_CREATE_FAST: - cell_out->cell_type = CELL_CREATED_FAST; break; - default: - tor_assert(0); - goto end; - } - rpl.success = 1; - } - rpl.magic = CPUWORKER_REPLY_MAGIC; - if (req.timed) { - struct timeval tv_diff; - int64_t usec; - tor_gettimeofday(&tv_end); - timersub(&tv_end, &tv_start, &tv_diff); - usec = ((int64_t)tv_diff.tv_sec)*1000000 + tv_diff.tv_usec; - if (usec < 0 || usec > MAX_BELIEVABLE_ONIONSKIN_DELAY) - rpl.n_usec = MAX_BELIEVABLE_ONIONSKIN_DELAY; - else - rpl.n_usec = (uint32_t) usec; + or_circuit_t *circ = NULL; + + tor_assert(total_pending_tasks > 0); + --total_pending_tasks; + + /* Could avoid this, but doesn't matter. */ + memcpy(&rpl, &job->u.reply, sizeof(rpl)); + + tor_assert(rpl.magic == CPUWORKER_REPLY_MAGIC); + + if (rpl.timed && rpl.success && + rpl.handshake_type <= MAX_ONION_HANDSHAKE_TYPE) { + /* Time how long this request took. The handshake_type check should be + needless, but let's leave it in to be safe. */ + struct timeval tv_end, tv_diff; + int64_t usec_roundtrip; + tor_gettimeofday(&tv_end); + timersub(&tv_end, &rpl.started_at, &tv_diff); + usec_roundtrip = ((int64_t)tv_diff.tv_sec)*1000000 + tv_diff.tv_usec; + if (usec_roundtrip >= 0 && + usec_roundtrip < MAX_BELIEVABLE_ONIONSKIN_DELAY) { + ++onionskins_n_processed[rpl.handshake_type]; + onionskins_usec_internal[rpl.handshake_type] += rpl.n_usec; + onionskins_usec_roundtrip[rpl.handshake_type] += usec_roundtrip; + if (onionskins_n_processed[rpl.handshake_type] >= 500000) { + /* Scale down every 500000 handshakes. On a busy server, that's + * less impressive than it sounds. */ + onionskins_n_processed[rpl.handshake_type] /= 2; + onionskins_usec_internal[rpl.handshake_type] /= 2; + onionskins_usec_roundtrip[rpl.handshake_type] /= 2; } - if (write_all(fd, (void*)&rpl, sizeof(rpl), 1) != sizeof(rpl)) { - log_err(LD_BUG,"writing response buf failed. Exiting."); - goto end; - } - log_debug(LD_OR,"finished writing response."); - } else if (req.task == CPUWORKER_TASK_SHUTDOWN) { - log_info(LD_OR,"Clean shutdown: exiting"); - goto end; } - memwipe(&req, 0, sizeof(req)); - memwipe(&rpl, 0, sizeof(req)); } - end: - memwipe(&req, 0, sizeof(req)); - memwipe(&rpl, 0, sizeof(req)); - release_server_onion_keys(&onion_keys); - tor_close_socket(fd); - crypto_thread_cleanup(); - spawn_exit(); -} -/** Launch a new cpuworker. Return 0 if we're happy, -1 if we failed. - */ -static int -spawn_cpuworker(void) -{ - tor_socket_t *fdarray; - tor_socket_t fd; - connection_t *conn; - int err; - - fdarray = tor_malloc(sizeof(tor_socket_t)*2); - if ((err = tor_socketpair(AF_UNIX, SOCK_STREAM, 0, fdarray)) < 0) { - log_warn(LD_NET, "Couldn't construct socketpair for cpuworker: %s", - tor_socket_strerror(-err)); - tor_free(fdarray); - return -1; - } + circ = job->circ; - tor_assert(SOCKET_OK(fdarray[0])); - tor_assert(SOCKET_OK(fdarray[1])); + log_debug(LD_OR, + "Unpacking cpuworker reply %p, circ=%p, success=%d", + job, circ, rpl.success); - fd = fdarray[0]; - if (spawn_func(cpuworker_main, (void*)fdarray) < 0) { - tor_close_socket(fdarray[0]); - tor_close_socket(fdarray[1]); - tor_free(fdarray); - return -1; + if (circ->base_.magic == DEAD_CIRCUIT_MAGIC) { + /* The circuit was supposed to get freed while the reply was + * pending. Instead, it got left for us to free so that we wouldn't freak + * out when the job->circ field wound up pointing to nothing. */ + log_debug(LD_OR, "Circuit died while reply was pending. Freeing memory."); + circ->base_.magic = 0; + tor_free(circ); + goto done_processing; } - log_debug(LD_OR,"just spawned a cpu worker."); -#ifndef TOR_IS_MULTITHREADED - tor_close_socket(fdarray[1]); /* don't need the worker's side of the pipe */ - tor_free(fdarray); -#endif - - conn = connection_new(CONN_TYPE_CPUWORKER, AF_UNIX); - /* set up conn so it's got all the data we need to remember */ - conn->s = fd; - conn->address = tor_strdup("localhost"); - tor_addr_make_unspec(&conn->addr); + circ->workqueue_entry = NULL; - if (set_socket_nonblocking(fd) == -1) { - connection_free(conn); /* this closes fd */ - return -1; + if (TO_CIRCUIT(circ)->marked_for_close) { + /* We already marked this circuit; we can't call it open. */ + log_debug(LD_OR,"circuit is already marked."); + goto done_processing; } - if (connection_add(conn) < 0) { /* no space, forget it */ - log_warn(LD_NET,"connection_add for cpuworker failed. Giving up."); - connection_free(conn); /* this closes fd */ - return -1; + if (rpl.success == 0) { + log_debug(LD_OR, + "decoding onionskin failed. " + "(Old key or bad software.) Closing."); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_TORPROTOCOL); + goto done_processing; } - conn->state = CPUWORKER_STATE_IDLE; - connection_start_reading(conn); + if (onionskin_answer(circ, + &rpl.created_cell, + (const char*)rpl.keys, + rpl.rend_auth_material) < 0) { + log_warn(LD_OR,"onionskin_answer failed. Closing."); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); + goto done_processing; + } + log_debug(LD_OR,"onionskin_answer succeeded. Yay."); - return 0; /* success */ + done_processing: + memwipe(&rpl, 0, sizeof(rpl)); + memwipe(job, 0, sizeof(*job)); + tor_free(job); + queue_pending_tasks(); } -/** If we have too few or too many active cpuworkers, try to spawn new ones - * or kill idle ones. - */ -static void -spawn_enough_cpuworkers(void) +/** Implementation function for onion handshake requests. */ +static int +cpuworker_onion_handshake_threadfn(void *state_, void *work_) { - int num_cpuworkers_needed = get_num_cpus(get_options()); - int reseed = 0; + worker_state_t *state = state_; + cpuworker_job_t *job = work_; - if (num_cpuworkers_needed < MIN_CPUWORKERS) - num_cpuworkers_needed = MIN_CPUWORKERS; - if (num_cpuworkers_needed > MAX_CPUWORKERS) - num_cpuworkers_needed = MAX_CPUWORKERS; + /* variables for onion processing */ + server_onion_keys_t *onion_keys = state->onion_keys; + cpuworker_request_t req; + cpuworker_reply_t rpl; - while (num_cpuworkers < num_cpuworkers_needed) { - if (spawn_cpuworker() < 0) { - log_warn(LD_GENERAL,"Cpuworker spawn failed. Will try again later."); - return; + memcpy(&req, &job->u.request, sizeof(req)); + + tor_assert(req.magic == CPUWORKER_REQUEST_MAGIC); + memset(&rpl, 0, sizeof(rpl)); + + const create_cell_t *cc = &req.create_cell; + created_cell_t *cell_out = &rpl.created_cell; + struct timeval tv_start = {0,0}, tv_end; + int n; + rpl.timed = req.timed; + rpl.started_at = req.started_at; + rpl.handshake_type = cc->handshake_type; + if (req.timed) + tor_gettimeofday(&tv_start); + n = onion_skin_server_handshake(cc->handshake_type, + cc->onionskin, cc->handshake_len, + onion_keys, + cell_out->reply, + rpl.keys, CPATH_KEY_MATERIAL_LEN, + rpl.rend_auth_material); + if (n < 0) { + /* failure */ + log_debug(LD_OR,"onion_skin_server_handshake failed."); + memset(&rpl, 0, sizeof(rpl)); + rpl.success = 0; + } else { + /* success */ + log_debug(LD_OR,"onion_skin_server_handshake succeeded."); + cell_out->handshake_len = n; + switch (cc->cell_type) { + case CELL_CREATE: + cell_out->cell_type = CELL_CREATED; break; + case CELL_CREATE2: + cell_out->cell_type = CELL_CREATED2; break; + case CELL_CREATE_FAST: + cell_out->cell_type = CELL_CREATED_FAST; break; + default: + tor_assert(0); + return WQ_RPL_SHUTDOWN; } - num_cpuworkers++; - reseed++; + rpl.success = 1; + } + rpl.magic = CPUWORKER_REPLY_MAGIC; + if (req.timed) { + struct timeval tv_diff; + int64_t usec; + tor_gettimeofday(&tv_end); + timersub(&tv_end, &tv_start, &tv_diff); + usec = ((int64_t)tv_diff.tv_sec)*1000000 + tv_diff.tv_usec; + if (usec < 0 || usec > MAX_BELIEVABLE_ONIONSKIN_DELAY) + rpl.n_usec = MAX_BELIEVABLE_ONIONSKIN_DELAY; + else + rpl.n_usec = (uint32_t) usec; } - if (reseed) - crypto_seed_weak_rng(&request_sample_rng); + memcpy(&job->u.reply, &rpl, sizeof(rpl)); + + memwipe(&req, 0, sizeof(req)); + memwipe(&rpl, 0, sizeof(req)); + return WQ_RPL_REPLY; } -/** Take a pending task from the queue and assign it to 'cpuworker'. */ +/** Take pending tasks from the queue and assign them to cpuworkers. */ static void -process_pending_task(connection_t *cpuworker) +queue_pending_tasks(void) { or_circuit_t *circ; create_cell_t *onionskin = NULL; - tor_assert(cpuworker); - - /* for now only process onion tasks */ - - circ = onion_next_task(&onionskin); - if (!circ) - return; - if (assign_onionskin_to_cpuworker(cpuworker, circ, onionskin)) - log_warn(LD_OR,"assign_to_cpuworker failed. Ignoring."); -} + while (total_pending_tasks < max_pending_tasks) { + circ = onion_next_task(&onionskin); -/** How long should we let a cpuworker stay busy before we give - * up on it and decide that we have a bug or infinite loop? - * This value is high because some servers with low memory/cpu - * sometimes spend an hour or more swapping, and Tor starves. */ -#define CPUWORKER_BUSY_TIMEOUT (60*60*12) + if (!circ) + return; -/** We have a bug that I can't find. Sometimes, very rarely, cpuworkers get - * stuck in the 'busy' state, even though the cpuworker process thinks of - * itself as idle. I don't know why. But here's a workaround to kill any - * cpuworker that's been busy for more than CPUWORKER_BUSY_TIMEOUT. - */ -static void -cull_wedged_cpuworkers(void) -{ - time_t now = time(NULL); - smartlist_t *conns = get_connection_array(); - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { - if (!conn->marked_for_close && - conn->type == CONN_TYPE_CPUWORKER && - conn->state == CPUWORKER_STATE_BUSY_ONION && - conn->timestamp_lastwritten + CPUWORKER_BUSY_TIMEOUT < now) { - log_notice(LD_BUG, - "closing wedged cpuworker. Can somebody find the bug?"); - num_cpuworkers_busy--; - num_cpuworkers--; - connection_mark_for_close(conn); - } - } SMARTLIST_FOREACH_END(conn); + if (assign_onionskin_to_cpuworker(circ, onionskin)) + log_warn(LD_OR,"assign_to_cpuworker failed. Ignoring."); + } } /** Try to tell a cpuworker to perform the public key operations necessary to * respond to <b>onionskin</b> for the circuit <b>circ</b>. * - * If <b>cpuworker</b> is defined, assert that he's idle, and use him. Else, - * look for an idle cpuworker and use him. If none idle, queue task onto the - * pending onion list and return. Return 0 if we successfully assign the - * task, or -1 on failure. + * Return 0 if we successfully assign the task, or -1 on failure. */ int -assign_onionskin_to_cpuworker(connection_t *cpuworker, - or_circuit_t *circ, +assign_onionskin_to_cpuworker(or_circuit_t *circ, create_cell_t *onionskin) { + workqueue_entry_t *queue_entry; + cpuworker_job_t *job; cpuworker_request_t req; - time_t now = approx_time(); - static time_t last_culled_cpuworkers = 0; int should_time; - /* Checking for wedged cpuworkers requires a linear search over all - * connections, so let's do it only once a minute. - */ -#define CULL_CPUWORKERS_INTERVAL 60 + tor_assert(threadpool); - if (last_culled_cpuworkers + CULL_CPUWORKERS_INTERVAL <= now) { - cull_wedged_cpuworkers(); - spawn_enough_cpuworkers(); - last_culled_cpuworkers = now; + if (!circ->p_chan) { + log_info(LD_OR,"circ->p_chan gone. Failing circ."); + tor_free(onionskin); + return -1; } - if (1) { - if (num_cpuworkers_busy == num_cpuworkers) { - log_debug(LD_OR,"No idle cpuworkers. Queuing."); - if (onion_pending_add(circ, onionskin) < 0) { - tor_free(onionskin); - return -1; - } - return 0; - } - - if (!cpuworker) - cpuworker = connection_get_by_type_state(CONN_TYPE_CPUWORKER, - CPUWORKER_STATE_IDLE); - - tor_assert(cpuworker); - - if (!circ->p_chan) { - log_info(LD_OR,"circ->p_chan gone. Failing circ."); + if (total_pending_tasks >= max_pending_tasks) { + log_debug(LD_OR,"No idle cpuworkers. Queuing."); + if (onion_pending_add(circ, onionskin) < 0) { tor_free(onionskin); return -1; } + return 0; + } - if (connection_or_digest_is_known_relay(circ->p_chan->identity_digest)) - rep_hist_note_circuit_handshake_assigned(onionskin->handshake_type); + if (connection_or_digest_is_known_relay(circ->p_chan->identity_digest)) + rep_hist_note_circuit_handshake_assigned(onionskin->handshake_type); - should_time = should_time_request(onionskin->handshake_type); - memset(&req, 0, sizeof(req)); - req.magic = CPUWORKER_REQUEST_MAGIC; - tag_pack(req.tag, circ->p_chan->global_identifier, - circ->p_circ_id); - req.timed = should_time; + should_time = should_time_request(onionskin->handshake_type); + memset(&req, 0, sizeof(req)); + req.magic = CPUWORKER_REQUEST_MAGIC; + req.timed = should_time; - cpuworker->state = CPUWORKER_STATE_BUSY_ONION; - /* touch the lastwritten timestamp, since that's how we check to - * see how long it's been since we asked the question, and sometimes - * we check before the first call to connection_handle_write(). */ - cpuworker->timestamp_lastwritten = now; - num_cpuworkers_busy++; + memcpy(&req.create_cell, onionskin, sizeof(create_cell_t)); - req.task = CPUWORKER_TASK_ONION; - memcpy(&req.create_cell, onionskin, sizeof(create_cell_t)); + tor_free(onionskin); - tor_free(onionskin); + if (should_time) + tor_gettimeofday(&req.started_at); - if (should_time) - tor_gettimeofday(&req.started_at); + job = tor_malloc_zero(sizeof(cpuworker_job_t)); + job->circ = circ; + memcpy(&job->u.request, &req, sizeof(req)); + memwipe(&req, 0, sizeof(req)); - connection_write_to_buf((void*)&req, sizeof(req), cpuworker); - memwipe(&req, 0, sizeof(req)); + ++total_pending_tasks; + queue_entry = threadpool_queue_work(threadpool, + cpuworker_onion_handshake_threadfn, + cpuworker_onion_handshake_replyfn, + job); + if (!queue_entry) { + log_warn(LD_BUG, "Couldn't queue work on threadpool"); + tor_free(job); + return -1; } + + log_debug(LD_OR, "Queued task %p (qe=%p, circ=%p)", + job, queue_entry, job->circ); + + circ->workqueue_entry = queue_entry; + return 0; } +/** If <b>circ</b> has a pending handshake that hasn't been processed yet, + * remove it from the worker queue. */ +void +cpuworker_cancel_circ_handshake(or_circuit_t *circ) +{ + cpuworker_job_t *job; + if (circ->workqueue_entry == NULL) + return; + + job = workqueue_entry_cancel(circ->workqueue_entry); + if (job) { + /* It successfully cancelled. */ + memwipe(job, 0xe0, sizeof(*job)); + tor_free(job); + tor_assert(total_pending_tasks > 0); + --total_pending_tasks; + /* if (!job), this is done in cpuworker_onion_handshake_replyfn. */ + circ->workqueue_entry = NULL; + } +} + diff --git a/src/or/cpuworker.h b/src/or/cpuworker.h index 317cef43ba..70a595e472 100644 --- a/src/or/cpuworker.h +++ b/src/or/cpuworker.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,19 +13,17 @@ #define TOR_CPUWORKER_H void cpu_init(void); -void cpuworkers_rotate(void); -int connection_cpu_finished_flushing(connection_t *conn); -int connection_cpu_reached_eof(connection_t *conn); -int connection_cpu_process_inbuf(connection_t *conn); +void cpuworkers_rotate_keyinfo(void); + struct create_cell_t; -int assign_onionskin_to_cpuworker(connection_t *cpuworker, - or_circuit_t *circ, +int assign_onionskin_to_cpuworker(or_circuit_t *circ, struct create_cell_t *onionskin); uint64_t estimated_usec_for_onionskins(uint32_t n_requests, uint16_t onionskin_type); void cpuworker_log_onionskin_overhead(int severity, int onionskin_type, const char *onionskin_type_name); +void cpuworker_cancel_circ_handshake(or_circuit_t *circ); #endif diff --git a/src/or/directory.c b/src/or/directory.c index 50863d0c7e..d2b6b86f6d 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" @@ -20,6 +20,7 @@ #include "networkstatus.h" #include "nodelist.h" #include "policies.h" +#include "relay.h" #include "rendclient.h" #include "rendcommon.h" #include "rephist.h" @@ -63,8 +64,6 @@ static void directory_send_command(dir_connection_t *conn, time_t if_modified_since); static int directory_handle_command(dir_connection_t *conn); static int body_is_plausible(const char *body, size_t body_len, int purpose); -static int purpose_needs_anonymity(uint8_t dir_purpose, - uint8_t router_purpose); static char *http_get_header(const char *headers, const char *which); static void http_set_address_origin(const char *headers, connection_t *conn); static void connection_dir_download_routerdesc_failed(dir_connection_t *conn); @@ -119,7 +118,7 @@ static void directory_initiate_command_rend(const tor_addr_t *addr, /** Return true iff the directory purpose <b>dir_purpose</b> (and if it's * fetching descriptors, it's fetching them for <b>router_purpose</b>) * must use an anonymous connection to a directory. */ -static int +STATIC int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) { if (get_options()->AllDirActionsPrivate) @@ -197,9 +196,49 @@ dir_conn_purpose_to_string(int purpose) return "(unknown)"; } -/** Return true iff <b>identity_digest</b> is the digest of a router we - * believe to support extrainfo downloads. (If <b>is_authority</b> we do - * additional checking that's only valid for authorities.) */ +/** Return the requisite directory information types. */ +STATIC dirinfo_type_t +dir_fetch_type(int dir_purpose, int router_purpose, const char *resource) +{ + dirinfo_type_t type; + switch (dir_purpose) { + case DIR_PURPOSE_FETCH_EXTRAINFO: + type = EXTRAINFO_DIRINFO; + if (router_purpose == ROUTER_PURPOSE_BRIDGE) + type |= BRIDGE_DIRINFO; + else + type |= V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_SERVERDESC: + if (router_purpose == ROUTER_PURPOSE_BRIDGE) + type = BRIDGE_DIRINFO; + else + type = V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_CERTIFICATE: + type = V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_CONSENSUS: + type = V3_DIRINFO; + if (resource && !strcmp(resource, "microdesc")) + type |= MICRODESC_DIRINFO; + break; + case DIR_PURPOSE_FETCH_MICRODESC: + type = MICRODESC_DIRINFO; + break; + default: + log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose); + type = NO_DIRINFO; + break; + } + return type; +} + +/** Return true iff <b>identity_digest</b> is the digest of a router which + * says that it caches extrainfos. (If <b>is_authority</b> we always + * believe that to be true.) */ int router_supports_extrainfo(const char *identity_digest, int is_authority) { @@ -385,47 +424,21 @@ directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags, * Use <b>pds_flags</b> as arguments to router_pick_directory_server() * or router_pick_trusteddirserver(). */ -void -directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource, int pds_flags) +MOCK_IMPL(void, directory_get_from_dirserver, (uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource, + int pds_flags)) { const routerstatus_t *rs = NULL; const or_options_t *options = get_options(); int prefer_authority = directory_fetches_from_authorities(options); int require_authority = 0; int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose); - dirinfo_type_t type; + dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource); time_t if_modified_since = 0; - /* FFFF we could break this switch into its own function, and call - * it elsewhere in directory.c. -RD */ - switch (dir_purpose) { - case DIR_PURPOSE_FETCH_EXTRAINFO: - type = EXTRAINFO_DIRINFO | - (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO : - V3_DIRINFO); - break; - case DIR_PURPOSE_FETCH_SERVERDESC: - type = (router_purpose == ROUTER_PURPOSE_BRIDGE ? BRIDGE_DIRINFO : - V3_DIRINFO); - break; - case DIR_PURPOSE_FETCH_STATUS_VOTE: - case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: - case DIR_PURPOSE_FETCH_CERTIFICATE: - type = V3_DIRINFO; - break; - case DIR_PURPOSE_FETCH_CONSENSUS: - type = V3_DIRINFO; - if (resource && !strcmp(resource,"microdesc")) - type |= MICRODESC_DIRINFO; - break; - case DIR_PURPOSE_FETCH_MICRODESC: - type = MICRODESC_DIRINFO; - break; - default: - log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose); - return; - } + if (type == NO_DIRINFO) + return; if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS) { int flav = FLAV_NS; @@ -433,18 +446,33 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, if (resource) flav = networkstatus_parse_flavor_name(resource); + /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus + * period of 1 hour. + */ +#define DEFAULT_IF_MODIFIED_SINCE_DELAY (180) if (flav != -1) { /* IF we have a parsed consensus of this type, we can do an * if-modified-time based on it. */ v = networkstatus_get_latest_consensus_by_flavor(flav); - if (v) - if_modified_since = v->valid_after + 180; + if (v) { + /* In networks with particularly short V3AuthVotingIntervals, + * ask for the consensus if it's been modified since half the + * V3AuthVotingInterval of the most recent consensus. */ + time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY; + if (v->fresh_until > v->valid_after + && ims_delay > (v->fresh_until - v->valid_after)/2) { + ims_delay = (v->fresh_until - v->valid_after)/2; + } + if_modified_since = v->valid_after + ims_delay; + } } else { /* Otherwise it might be a consensus we don't parse, but which we * do cache. Look at the cached copy, perhaps. */ cached_dir_t *cd = dirserv_get_consensus(resource); + /* We have no method of determining the voting interval from an + * unparsed consensus, so we use the default. */ if (cd) - if_modified_since = cd->published + 180; + if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY; } } @@ -452,7 +480,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, return; if (!get_via_tor) { - if (options->UseBridges && type != BRIDGE_DIRINFO) { + if (options->UseBridges && !(type & BRIDGE_DIRINFO)) { /* We want to ask a running bridge for which we have a descriptor. * * When we ask choose_random_entry() for a bridge, we specify what @@ -479,7 +507,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, "nodes are available yet."); return; } else { - if (prefer_authority || type == BRIDGE_DIRINFO) { + if (prefer_authority || (type & BRIDGE_DIRINFO)) { /* only ask authdirservers, and don't ask myself */ rs = router_pick_trusteddirserver(type, pds_flags); if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| @@ -506,29 +534,25 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, return; } } - if (!rs && type != BRIDGE_DIRINFO) { + if (!rs && !(type & BRIDGE_DIRINFO)) { /* */ rs = directory_pick_generic_dirserver(type, pds_flags, dir_purpose); - if (!rs) { - /*XXXX024 I'm pretty sure this can never do any good, since - * rs isn't set. */ + if (!rs) get_via_tor = 1; /* last resort: try routing it via Tor */ - } } } - } else { /* get_via_tor */ + } + + if (get_via_tor) { /* Never use fascistfirewall; we're going via Tor. */ - if (1) { - /* anybody with a non-zero dirport will do. Disregard firewalls. */ - pds_flags |= PDS_IGNORE_FASCISTFIREWALL; - rs = router_pick_directory_server(type, pds_flags); - /* If we have any hope of building an indirect conn, we know some router - * descriptors. If (rs==NULL), we can't build circuits anyway, so - * there's no point in falling back to the authorities in this case. */ - } + pds_flags |= PDS_IGNORE_FASCISTFIREWALL; + rs = router_pick_directory_server(type, pds_flags); } + /* If we have any hope of building an indirect conn, we know some router + * descriptors. If (rs==NULL), we can't build circuits anyway, so + * there's no point in falling back to the authorities in this case. */ if (rs) { const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP; @@ -1255,7 +1279,8 @@ directory_send_command(dir_connection_t *conn, return; } - if (strlen(proxystring) + strlen(url) >= 4096) { + /* warn in the non-tunneled case */ + if (direct && (strlen(proxystring) + strlen(url) >= 4096)) { log_warn(LD_BUG, "Squid does not like URLs longer than 4095 bytes, and this " "one is %d bytes long: %s%s", @@ -2073,23 +2098,25 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } if (conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2) { - #define SEND_HS_DESC_FAILED_EVENT() ( \ + #define SEND_HS_DESC_FAILED_EVENT(reason) ( \ control_event_hs_descriptor_failed(conn->rend_data, \ - conn->identity_digest) ) + conn->identity_digest, \ + reason) ) tor_assert(conn->rend_data); log_info(LD_REND,"Received rendezvous descriptor (size %d, status %d " "(%s))", (int)body_len, status_code, escaped(reason)); switch (status_code) { case 200: - switch (rend_cache_store_v2_desc_as_client(body, conn->rend_data)) { + switch (rend_cache_store_v2_desc_as_client(body, + conn->requested_resource, conn->rend_data)) { case RCS_BADDESC: case RCS_NOTDIR: /* Impossible */ log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. " "Retrying at another directory."); /* We'll retry when connection_about_to_close_connection() * cleans this dir conn up. */ - SEND_HS_DESC_FAILED_EVENT(); + SEND_HS_DESC_FAILED_EVENT("BAD_DESC"); break; case RCS_OKAY: default: @@ -2108,14 +2135,14 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * connection_about_to_close_connection() cleans this conn up. */ log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: " "Retrying at another directory."); - SEND_HS_DESC_FAILED_EVENT(); + SEND_HS_DESC_FAILED_EVENT("NOT_FOUND"); break; case 400: log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " "http status 400 (%s). Dirserver didn't like our " "v2 rendezvous query? Retrying at another directory.", escaped(reason)); - SEND_HS_DESC_FAILED_EVENT(); + SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED"); break; default: log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: " @@ -2124,7 +2151,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) "Retrying at another directory.", status_code, escaped(reason), conn->base_.address, conn->base_.port); - SEND_HS_DESC_FAILED_EVENT(); + SEND_HS_DESC_FAILED_EVENT("UNEXPECTED"); break; } } @@ -2215,8 +2242,10 @@ connection_dir_process_inbuf(dir_connection_t *conn) MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE; if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) { - log_warn(LD_HTTP, "Too much data received from directory connection: " - "denial of service attempt, or you need to upgrade?"); + log_warn(LD_HTTP, + "Too much data received from directory connection (%s): " + "denial of service attempt, or you need to upgrade?", + conn->base_.address); connection_mark_for_close(TO_CONN(conn)); return -1; } @@ -2261,6 +2290,7 @@ write_http_status_line(dir_connection_t *conn, int status, log_warn(LD_BUG,"status line too long."); return; } + log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase); connection_write_to_buf(buf, strlen(buf), TO_CONN(conn)); } @@ -2526,6 +2556,24 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) return (have >= need_at_least); } +/** Return the compression level we should use for sending a compressed + * response of size <b>n_bytes</b>. */ +static zlib_compression_level_t +choose_compression_level(ssize_t n_bytes) +{ + if (! have_been_under_memory_pressure()) { + return HIGH_COMPRESSION; /* we have plenty of RAM. */ + } else if (n_bytes < 0) { + return HIGH_COMPRESSION; /* unknown; might be big. */ + } else if (n_bytes < 1024) { + return LOW_COMPRESSION; + } else if (n_bytes < 2048) { + return MEDIUM_COMPRESSION; + } else { + return HIGH_COMPRESSION; + } +} + /** Helper function: called when a dirserver gets a complete HTTP GET * request. Look for a request for a directory or for a rendezvous * service descriptor. On finding one, write a response into @@ -2557,8 +2605,11 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if ((header = http_get_header(headers, "If-Modified-Since: "))) { struct tm tm; if (parse_http_time(header, &tm) == 0) { - if (tor_timegm(&tm, &if_modified_since)<0) + if (tor_timegm(&tm, &if_modified_since)<0) { if_modified_since = 0; + } else { + log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header)); + } } /* The correct behavior on a malformed If-Modified-Since header is to * act as if no If-Modified-Since header had been given. */ @@ -2708,7 +2759,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, smartlist_len(dir_fps) == 1 ? lifetime : 0); conn->fingerprint_stack = dir_fps; if (! compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); /* Prime the connection with some data. */ conn->dir_spool_src = DIR_SPOOL_NETWORKSTATUS; @@ -2796,7 +2847,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, if (smartlist_len(items)) { if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf_zlib(c, strlen(c), conn, 0)); connection_write_to_buf_zlib("", 0, conn, 1); @@ -2845,7 +2897,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, conn->fingerprint_stack = fps; if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(dlen)); connection_dirserv_flushed_some(conn); goto done; @@ -2913,7 +2966,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } write_http_response_header(conn, -1, compressed, cache_lifetime); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(dlen)); /* Prime the connection with some data. */ connection_dirserv_flushed_some(conn); } @@ -2988,7 +3042,8 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_response_header(conn, compressed?-1:len, compressed, 60*60); if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, c->cache_info.signed_descriptor_len, @@ -3449,6 +3504,9 @@ download_status_increment_failure(download_status_t *dls, int status_code, void download_status_reset(download_status_t *dls) { + if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD) + return; /* Don't reset this. */ + const smartlist_t *schedule = find_dl_schedule_and_len( dls, get_options()->DirPort_set); diff --git a/src/or/directory.h b/src/or/directory.h index bc200797d4..4899eb5c8c 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,9 +16,10 @@ int directories_have_accepted_server_descriptor(void); void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose, dirinfo_type_t type, const char *payload, size_t payload_len, size_t extrainfo_len); -void directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, - const char *resource, - int pds_flags); +MOCK_DECL(void, directory_get_from_dirserver, (uint8_t dir_purpose, + uint8_t router_purpose, + const char *resource, + int pds_flags)); void directory_get_from_all_authorities(uint8_t dir_purpose, uint8_t router_purpose, const char *resource); @@ -120,7 +121,12 @@ int download_status_get_n_failures(const download_status_t *dls); #ifdef TOR_UNIT_TESTS /* Used only by directory.c and test_dir.c */ + STATIC int parse_http_url(const char *headers, char **url); +STATIC int purpose_needs_anonymity(uint8_t dir_purpose, + uint8_t router_purpose); +STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose, + const char *resource); #endif #endif diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 03b32cb2f3..65bfafba6c 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRSERV_PRIVATE @@ -56,13 +56,11 @@ static int routers_with_measured_bw = 0; static void directory_remove_invalid(void); static char *format_versions_list(config_line_t *ln); struct authdir_config_t; -static int add_fingerprint_to_dir(const char *nickname, const char *fp, - struct authdir_config_t *list); static uint32_t dirserv_get_status_impl(const char *fp, const char *nickname, uint32_t addr, uint16_t or_port, - const char *platform, const char *contact, - const char **msg, int should_log); + const char *platform, const char **msg, + int should_log); static void clear_cached_dir(cached_dir_t *d); static const signed_descriptor_t *get_signed_descriptor_by_fp( const char *fp, @@ -75,19 +73,19 @@ static uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri); /************** Fingerprint handling code ************/ -#define FP_NAMED 1 /**< Listed in fingerprint file. */ +/* 1 Historically used to indicate Named */ #define FP_INVALID 2 /**< Believed invalid. */ #define FP_REJECT 4 /**< We will not publish this router. */ -#define FP_BADDIR 8 /**< We'll tell clients to avoid using this as a dir. */ +/* 8 Historically used to avoid using this as a dir. */ #define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ -#define FP_UNNAMED 32 /**< Another router has this name in fingerprint file. */ +/* 32 Historically used to indicade Unnamed */ -/** Encapsulate a nickname and an FP_* status; target of status_by_digest - * map. */ -typedef struct router_status_t { - char nickname[MAX_NICKNAME_LEN+1]; - uint32_t status; -} router_status_t; +/** Target of status_by_digest map. */ +typedef uint32_t router_status_t; + +static void add_fingerprint_to_dir(const char *fp, + struct authdir_config_t *list, + router_status_t add_status); /** List of nickname-\>identity fingerprint mappings for all the routers * that we name. Used to prevent router impersonation. */ @@ -109,18 +107,17 @@ authdir_config_new(void) return list; } -/** Add the fingerprint <b>fp</b> for <b>nickname</b> to - * the smartlist of fingerprint_entry_t's <b>list</b>. Return 0 if it's - * new, or 1 if we replaced the old value. +/** Add the fingerprint <b>fp</b> to the smartlist of fingerprint_entry_t's + * <b>list</b>, or-ing the currently set status flags with + * <b>add_status</b>. */ -/* static */ int -add_fingerprint_to_dir(const char *nickname, const char *fp, - authdir_config_t *list) +/* static */ void +add_fingerprint_to_dir(const char *fp, authdir_config_t *list, + router_status_t add_status) { char *fingerprint; char d[DIGEST_LEN]; router_status_t *status; - tor_assert(nickname); tor_assert(fp); tor_assert(list); @@ -130,14 +127,7 @@ add_fingerprint_to_dir(const char *nickname, const char *fp, log_warn(LD_DIRSERV, "Couldn't decode fingerprint \"%s\"", escaped(fp)); tor_free(fingerprint); - return 0; - } - - if (!strcasecmp(nickname, UNNAMED_ROUTER_NICKNAME)) { - log_warn(LD_DIRSERV, "Tried to add a mapping for reserved nickname %s", - UNNAMED_ROUTER_NICKNAME); - tor_free(fingerprint); - return 0; + return; } status = digestmap_get(list->status_by_digest, d); @@ -146,35 +136,15 @@ add_fingerprint_to_dir(const char *nickname, const char *fp, digestmap_set(list->status_by_digest, d, status); } - if (nickname[0] != '!') { - char *old_fp = strmap_get_lc(list->fp_by_name, nickname); - if (old_fp && !strcasecmp(fingerprint, old_fp)) { - tor_free(fingerprint); - } else { - tor_free(old_fp); - strmap_set_lc(list->fp_by_name, nickname, fingerprint); - } - status->status |= FP_NAMED; - strlcpy(status->nickname, nickname, sizeof(status->nickname)); - } else { - tor_free(fingerprint); - if (!strcasecmp(nickname, "!reject")) { - status->status |= FP_REJECT; - } else if (!strcasecmp(nickname, "!invalid")) { - status->status |= FP_INVALID; - } else if (!strcasecmp(nickname, "!baddir")) { - status->status |= FP_BADDIR; - } else if (!strcasecmp(nickname, "!badexit")) { - status->status |= FP_BADEXIT; - } - } - return 0; + tor_free(fingerprint); + *status |= add_status; + return; } -/** Add the nickname and fingerprint for this OR to the - * global list of recognized identity key fingerprints. */ +/** Add the fingerprint for this OR to the global list of recognized + * identity key fingerprints. */ int -dirserv_add_own_fingerprint(const char *nickname, crypto_pk_t *pk) +dirserv_add_own_fingerprint(crypto_pk_t *pk) { char fp[FINGERPRINT_LEN+1]; if (crypto_pk_get_fingerprint(pk, fp, 0)<0) { @@ -183,7 +153,7 @@ dirserv_add_own_fingerprint(const char *nickname, crypto_pk_t *pk) } if (!fingerprint_list) fingerprint_list = authdir_config_new(); - add_fingerprint_to_dir(nickname, fp, fingerprint_list); + add_fingerprint_to_dir(fp, fingerprint_list, 0); return 0; } @@ -201,7 +171,6 @@ dirserv_load_fingerprint_file(void) authdir_config_t *fingerprint_list_new; int result; config_line_t *front=NULL, *list; - const or_options_t *options = get_options(); fname = get_datadir_fname("approved-routers"); log_info(LD_GENERAL, @@ -209,15 +178,9 @@ dirserv_load_fingerprint_file(void) cf = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); if (!cf) { - if (options->NamingAuthoritativeDir) { - log_warn(LD_FS, "Cannot open fingerprint file '%s'. Failing.", fname); - tor_free(fname); - return -1; - } else { - log_info(LD_FS, "Cannot open fingerprint file '%s'. That's ok.", fname); - tor_free(fname); - return 0; - } + log_warn(LD_FS, "Cannot open fingerprint file '%s'. That's ok.", fname); + tor_free(fname); + return 0; } tor_free(fname); @@ -232,22 +195,8 @@ dirserv_load_fingerprint_file(void) for (list=front; list; list=list->next) { char digest_tmp[DIGEST_LEN]; + router_status_t add_status = 0; nickname = list->key; fingerprint = list->value; - if (strlen(nickname) > MAX_NICKNAME_LEN) { - log_notice(LD_CONFIG, - "Nickname '%s' too long in fingerprint file. Skipping.", - nickname); - continue; - } - if (!is_legal_nickname(nickname) && - strcasecmp(nickname, "!reject") && - strcasecmp(nickname, "!invalid") && - strcasecmp(nickname, "!badexit")) { - log_notice(LD_CONFIG, - "Invalid nickname '%s' in fingerprint file. Skipping.", - nickname); - continue; - } tor_strstrip(fingerprint, " "); /* remove spaces */ if (strlen(fingerprint) != HEX_DIGEST_LEN || base16_decode(digest_tmp, sizeof(digest_tmp), @@ -258,26 +207,14 @@ dirserv_load_fingerprint_file(void) nickname, fingerprint); continue; } - if (0==strcasecmp(nickname, DEFAULT_CLIENT_NICKNAME)) { - /* If you approved an OR called "client", then clients who use - * the default nickname could all be rejected. That's no good. */ - log_notice(LD_CONFIG, - "Authorizing nickname '%s' would break " - "many clients; skipping.", - DEFAULT_CLIENT_NICKNAME); - continue; - } - if (0==strcasecmp(nickname, UNNAMED_ROUTER_NICKNAME)) { - /* If you approved an OR called "unnamed", then clients will be - * confused. */ - log_notice(LD_CONFIG, - "Authorizing nickname '%s' is not allowed; skipping.", - UNNAMED_ROUTER_NICKNAME); - continue; + if (!strcasecmp(nickname, "!reject")) { + add_status = FP_REJECT; + } else if (!strcasecmp(nickname, "!badexit")) { + add_status = FP_BADEXIT; + } else if (!strcasecmp(nickname, "!invalid")) { + add_status = FP_INVALID; } - if (add_fingerprint_to_dir(nickname, fingerprint, fingerprint_list_new) - != 0) - log_notice(LD_CONFIG, "Duplicate nickname '%s'.", nickname); + add_fingerprint_to_dir(fingerprint, fingerprint_list_new, add_status); } config_free_lines(front); @@ -308,8 +245,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg) return dirserv_get_status_impl(d, router->nickname, router->addr, router->or_port, - router->platform, router->contact_info, - msg, 1); + router->platform, msg, 1); } /** Return true if there is no point in downloading the router described by @@ -321,37 +257,14 @@ dirserv_would_reject_router(const routerstatus_t *rs) res = dirserv_get_status_impl(rs->identity_digest, rs->nickname, rs->addr, rs->or_port, - NULL, NULL, - NULL, 0); + NULL, NULL, 0); return (res & FP_REJECT) != 0; } -/** Helper: Based only on the ID/Nickname combination, - * return FP_UNNAMED (unnamed), FP_NAMED (named), or 0 (neither). - */ -static uint32_t -dirserv_get_name_status(const char *id_digest, const char *nickname) -{ - char fp[HEX_DIGEST_LEN+1]; - char *fp_by_name; - - base16_encode(fp, sizeof(fp), id_digest, DIGEST_LEN); - - if ((fp_by_name = - strmap_get_lc(fingerprint_list->fp_by_name, nickname))) { - if (!strcasecmp(fp, fp_by_name)) { - return FP_NAMED; - } else { - return FP_UNNAMED; /* Wrong fingerprint. */ - } - } - return 0; -} - /** Helper: As dirserv_router_get_status, but takes the router fingerprint * (hex, no spaces), nickname, address (used for logging only), IP address, OR - * port, platform (logging only) and contact info (logging only) as arguments. + * port and platform (logging only) as arguments. * * If should_log is false, do not log messages. (There's not much point in * logging that we're rejecting servers we'll not download.) @@ -359,11 +272,9 @@ dirserv_get_name_status(const char *id_digest, const char *nickname) static uint32_t dirserv_get_status_impl(const char *id_digest, const char *nickname, uint32_t addr, uint16_t or_port, - const char *platform, const char *contact, - const char **msg, int should_log) + const char *platform, const char **msg, int should_log) { - int reject_unlisted = get_options()->AuthDirRejectUnlisted; - uint32_t result; + uint32_t result = 0; router_status_t *status_by_digest; if (!fingerprint_list) @@ -374,50 +285,18 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, strmap_size(fingerprint_list->fp_by_name), digestmap_size(fingerprint_list->status_by_digest)); - /* Versions before Tor 0.2.3.16-alpha are too old to support, and are + /* Versions before Tor 0.2.4.18-rc are too old to support, and are * missing some important security fixes too. Disable them. */ - if (platform && !tor_version_as_new_as(platform,"0.2.3.16-alpha")) { + if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) { if (msg) *msg = "Tor version is insecure or unsupported. Please upgrade!"; return FP_REJECT; } -#if 0 - else if (platform && tor_version_as_new_as(platform,"0.2.3.0-alpha")) { - /* Versions from 0.2.3-alpha...0.2.3.9-alpha have known security - * issues that make them unusable for the current network */ - if (!tor_version_as_new_as(platform, "0.2.3.10-alpha")) { - if (msg) - *msg = "Tor version is insecure or unsupported. Please upgrade!"; - return FP_REJECT; - } - } -#endif - - result = dirserv_get_name_status(id_digest, nickname); - if (result & FP_NAMED) { - if (should_log) - log_debug(LD_DIRSERV,"Good fingerprint for '%s'",nickname); - } - if (result & FP_UNNAMED) { - if (should_log) { - char *esc_contact = esc_for_log(contact); - log_info(LD_DIRSERV, - "Mismatched fingerprint for '%s'. " - "ContactInfo '%s', platform '%s'.)", - nickname, - esc_contact, - platform ? escaped(platform) : ""); - tor_free(esc_contact); - } - if (msg) - *msg = "Rejected: There is already a named server with this nickname " - "and a different fingerprint."; - } status_by_digest = digestmap_get(fingerprint_list->status_by_digest, id_digest); if (status_by_digest) - result |= (status_by_digest->status & ~FP_NAMED); + result |= *status_by_digest; if (result & FP_REJECT) { if (msg) @@ -428,14 +307,6 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, *msg = "Fingerprint is marked invalid"; } - if (authdir_policy_baddir_address(addr, or_port)) { - if (should_log) - log_info(LD_DIRSERV, - "Marking '%s' as bad directory because of address '%s'", - nickname, fmt_addr32(addr)); - result |= FP_BADDIR; - } - if (authdir_policy_badexit_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Marking '%s' as bad exit because of address '%s'", @@ -443,46 +314,24 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, result |= FP_BADEXIT; } - if (!(result & FP_NAMED)) { - if (!authdir_policy_permits_address(addr, or_port)) { - if (should_log) - log_info(LD_DIRSERV, "Rejecting '%s' because of address '%s'", - nickname, fmt_addr32(addr)); - if (msg) - *msg = "Authdir is rejecting routers in this range."; - return FP_REJECT; - } - if (!authdir_policy_valid_address(addr, or_port)) { - if (should_log) - log_info(LD_DIRSERV, "Not marking '%s' valid because of address '%s'", - nickname, fmt_addr32(addr)); - result |= FP_INVALID; - } - if (reject_unlisted) { - if (msg) - *msg = "Authdir rejects unknown routers."; - return FP_REJECT; - } + if (!authdir_policy_permits_address(addr, or_port)) { + if (should_log) + log_info(LD_DIRSERV, "Rejecting '%s' because of address '%s'", + nickname, fmt_addr32(addr)); + if (msg) + *msg = "Authdir is rejecting routers in this range."; + return FP_REJECT; + } + if (!authdir_policy_valid_address(addr, or_port)) { + if (should_log) + log_info(LD_DIRSERV, "Not marking '%s' valid because of address '%s'", + nickname, fmt_addr32(addr)); + result |= FP_INVALID; } return result; } -/** If we are an authoritative dirserver, and the list of approved - * servers contains one whose identity key digest is <b>digest</b>, - * return that router's nickname. Otherwise return NULL. */ -const char * -dirserv_get_nickname_by_digest(const char *digest) -{ - router_status_t *status; - if (!fingerprint_list) - return NULL; - tor_assert(digest); - - status = digestmap_get(fingerprint_list->status_by_digest, digest); - return status ? status->nickname : NULL; -} - /** Clear the current fingerprint list. */ void dirserv_free_fingerprint_list(void) @@ -519,7 +368,7 @@ dirserv_router_has_valid_address(routerinfo_t *ri) } /** Check whether we, as a directory server, want to accept <b>ri</b>. If so, - * set its is_valid,named,running fields and return 0. Otherwise, return -1. + * set its is_valid,running fields and return 0. Otherwise, return -1. * * If the router is rejected, set *<b>msg</b> to an explanation of why. * @@ -584,7 +433,6 @@ dirserv_set_node_flags_from_authoritative_status(node_t *node, uint32_t authstatus) { node->is_valid = (authstatus & FP_INVALID) ? 0 : 1; - node->is_bad_directory = (authstatus & FP_BADDIR) ? 1 : 0; node->is_bad_exit = (authstatus & FP_BADEXIT) ? 1 : 0; } @@ -630,7 +478,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, s = desc; list = smartlist_new(); if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0, - annotation_buf)) { + annotation_buf, NULL)) { SMARTLIST_FOREACH(list, routerinfo_t *, ri, { msg_out = NULL; tor_assert(ri->purpose == purpose); @@ -646,7 +494,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, s = desc; if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0, - NULL)) { + NULL, NULL)) { SMARTLIST_FOREACH(list, extrainfo_t *, ei, { msg_out = NULL; @@ -664,7 +512,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, if (!n_parsed) { *msg = "No descriptors found in your POST."; if (WRA_WAS_ADDED(r)) - r = ROUTER_WAS_NOT_NEW; + r = ROUTER_IS_ALREADY_KNOWN; } else { *msg = "(no message)"; } @@ -689,7 +537,8 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) was_router_added_t r; routerinfo_t *ri_old; char *desc, *nickname; - size_t desclen = 0; + const size_t desclen = ri->cache_info.signed_descriptor_len + + ri->cache_info.annotations_len; *msg = NULL; /* If it's too big, refuse it now. Otherwise we'll cache it all over the @@ -703,7 +552,7 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) *msg = "Router descriptor was too large."; control_event_or_authdir_new_descriptor("REJECTED", ri->cache_info.signed_descriptor_body, - ri->cache_info.signed_descriptor_len, *msg); + desclen, *msg); routerinfo_free(ri); return ROUTER_AUTHDIR_REJECTS; } @@ -724,14 +573,13 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source) "the last one with this identity."; control_event_or_authdir_new_descriptor("DROPPED", ri->cache_info.signed_descriptor_body, - ri->cache_info.signed_descriptor_len, *msg); + desclen, *msg); routerinfo_free(ri); - return ROUTER_WAS_NOT_NEW; + return ROUTER_IS_ALREADY_KNOWN; } /* Make a copy of desc, since router_add_to_routerlist might free * ri and its associated signed_descriptor_t. */ - desclen = ri->cache_info.signed_descriptor_len; desc = tor_strndup(ri->cache_info.signed_descriptor_body, desclen); nickname = tor_strdup(ri->nickname); @@ -798,7 +646,7 @@ dirserv_add_extrainfo(extrainfo_t *ei, const char **msg) if ((r = routerinfo_incompatible_with_extrainfo(ri, ei, NULL, msg))) { extrainfo_free(ei); - return r < 0 ? ROUTER_WAS_NOT_NEW : ROUTER_BAD_EI; + return r < 0 ? ROUTER_IS_ALREADY_KNOWN : ROUTER_BAD_EI; } router_add_extrainfo_to_routerlist(ei, msg, 0, 0); return ROUTER_ADDED_SUCCESSFULLY; @@ -816,7 +664,7 @@ directory_remove_invalid(void) smartlist_add_all(nodes, nodelist_get_list()); SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) { - const char *msg; + const char *msg = NULL; routerinfo_t *ent = node->ri; char description[NODE_DESC_BUF_LEN]; uint32_t r; @@ -830,30 +678,11 @@ directory_remove_invalid(void) routerlist_remove(rl, ent, 0, time(NULL)); continue; } -#if 0 - if (bool_neq((r & FP_NAMED), ent->auth_says_is_named)) { - log_info(LD_DIRSERV, - "Router %s is now %snamed.", description, - (r&FP_NAMED)?"":"un"); - ent->is_named = (r&FP_NAMED)?1:0; - } - if (bool_neq((r & FP_UNNAMED), ent->auth_says_is_unnamed)) { - log_info(LD_DIRSERV, - "Router '%s' is now %snamed. (FP_UNNAMED)", description, - (r&FP_NAMED)?"":"un"); - ent->is_named = (r&FP_NUNAMED)?0:1; - } -#endif if (bool_neq((r & FP_INVALID), !node->is_valid)) { log_info(LD_DIRSERV, "Router '%s' is now %svalid.", description, (r&FP_INVALID) ? "in" : ""); node->is_valid = (r&FP_INVALID)?0:1; } - if (bool_neq((r & FP_BADDIR), node->is_bad_directory)) { - log_info(LD_DIRSERV, "Router '%s' is now a %s directory", description, - (r & FP_BADDIR) ? "bad" : "good"); - node->is_bad_directory = (r&FP_BADDIR) ? 1: 0; - } if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) { log_info(LD_DIRSERV, "Router '%s' is now a %s exit", description, (r & FP_BADEXIT) ? "bad" : "good"); @@ -904,7 +733,7 @@ running_long_enough_to_decide_unreachable(void) } /** Each server needs to have passed a reachability test no more - * than this number of seconds ago, or he is listed as down in + * than this number of seconds ago, or it is listed as down in * the directory. */ #define REACHABLE_TIMEOUT (45*60) @@ -1051,16 +880,33 @@ format_versions_list(config_line_t *ln) } /** Return 1 if <b>ri</b>'s descriptor is "active" -- running, valid, - * not hibernating, and not too old. Else return 0. + * not hibernating, having observed bw greater 0, and not too old. Else + * return 0. */ static int router_is_active(const routerinfo_t *ri, const node_t *node, time_t now) { time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - if (ri->cache_info.published_on < cutoff) + if (ri->cache_info.published_on < cutoff) { return 0; - if (!node->is_running || !node->is_valid || ri->is_hibernating) + } + if (!node->is_running || !node->is_valid || ri->is_hibernating) { return 0; + } + /* Only require bandwith capacity in non-test networks, or + * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */ + if (!ri->bandwidthcapacity) { + if (get_options()->TestingTorNetwork) { + if (get_options()->TestingMinExitFlagThreshold > 0) { + /* If we're in a TestingTorNetwork, and TestingMinExitFlagThreshold is, + * then require bandwidthcapacity */ + return 0; + } + } else { + /* If we're not in a TestingTorNetwork, then require bandwidthcapacity */ + return 0; + } + } return 1; } @@ -1205,7 +1051,7 @@ directory_fetches_dir_info_later(const or_options_t *options) } /** Return true iff we want to fetch and keep certificates for authorities - * that we don't acknowledge as aurthorities ourself. + * that we don't acknowledge as authorities ourself. */ int directory_caches_unknown_auth_certs(const or_options_t *options) @@ -1432,8 +1278,9 @@ dirserv_thinks_router_is_unreliable(time_t now, } /** Return true iff <b>router</b> should be assigned the "HSDir" flag. - * Right now this means it advertises support for it, it has a high - * uptime, it has a DirPort open, and it's currently considered Running. + * Right now this means it advertises support for it, it has a high uptime, + * it has a DirPort open, it has the Stable flag and it's currently + * considered Running. * * This function needs to be called after router-\>is_running has * been set. @@ -1459,16 +1306,10 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router, else uptime = real_uptime(router, now); - /* XXX We shouldn't need to check dir_port, but we do because of - * bug 1693. In the future, once relays set wants_to_be_hs_dir - * correctly, we can revert to only checking dir_port if router's - * version is too old. */ - /* XXX Unfortunately, we need to keep checking dir_port until all - * *clients* suffering from bug 2722 are obsolete. The first version - * to fix the bug was 0.2.2.25-alpha. */ return (router->wants_to_be_hs_dir && router->dir_port && + node->is_stable && uptime >= get_options()->MinUptimeHidServDirectoryV2 && - node->is_running); + router_is_active(router, node, now)); } /** Don't consider routers with less bandwidth than this when computing @@ -1537,18 +1378,18 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, * sort them and use that to compute thresholds. */ n_active = n_active_nonexit = 0; /* Uptime for every active router. */ - uptimes = tor_malloc(sizeof(uint32_t)*smartlist_len(rl->routers)); + uptimes = tor_calloc(smartlist_len(rl->routers), sizeof(uint32_t)); /* Bandwidth for every active router. */ - bandwidths_kb = tor_malloc(sizeof(uint32_t)*smartlist_len(rl->routers)); + bandwidths_kb = tor_calloc(smartlist_len(rl->routers), sizeof(uint32_t)); /* Bandwidth for every active non-exit router. */ bandwidths_excluding_exits_kb = - tor_malloc(sizeof(uint32_t)*smartlist_len(rl->routers)); + tor_calloc(smartlist_len(rl->routers), sizeof(uint32_t)); /* Weighted mean time between failure for each active router. */ - mtbfs = tor_malloc(sizeof(double)*smartlist_len(rl->routers)); + mtbfs = tor_calloc(smartlist_len(rl->routers), sizeof(double)); /* Time-known for each active router. */ - tks = tor_malloc(sizeof(long)*smartlist_len(rl->routers)); + tks = tor_calloc(smartlist_len(rl->routers), sizeof(long)); /* Weighted fractional uptime for each active router. */ - wfus = tor_malloc(sizeof(double)*smartlist_len(rl->routers)); + wfus = tor_calloc(smartlist_len(rl->routers), sizeof(double)); nodelist_assert_ok(); @@ -1563,6 +1404,8 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, routerinfo_t *ri = node->ri; const char *id = node->identity; uint32_t bw_kb; + /* resolve spurious clang shallow analysis null pointer errors */ + tor_assert(ri); node->is_exit = (!router_exit_policy_rejects_all(ri) && exit_policy_is_general_exit(ri->exit_policy)); uptimes[n_active] = (uint32_t)real_uptime(ri, now); @@ -1586,9 +1429,10 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, /* The 12.5th percentile bandwidth is fast. */ fast_bandwidth_kb = find_nth_uint32(bandwidths_kb, n_active, n_active/8); /* (Now bandwidths is sorted.) */ - if (fast_bandwidth_kb < ROUTER_REQUIRED_MIN_BANDWIDTH/(2 * 1000)) + if (fast_bandwidth_kb < RELAY_REQUIRED_MIN_BANDWIDTH/(2 * 1000)) fast_bandwidth_kb = bandwidths_kb[n_active/4]; - guard_bandwidth_including_exits_kb = bandwidths_kb[n_active*3/4]; + guard_bandwidth_including_exits_kb = + third_quartile_uint32(bandwidths_kb, n_active); guard_tk = find_nth_long(tks, n_active, n_active/8); } @@ -1663,7 +1507,7 @@ dirserv_compute_performance_thresholds(routerlist_t *rl, (unsigned long)guard_tk, (unsigned long)guard_bandwidth_including_exits_kb, (unsigned long)guard_bandwidth_excluding_exits_kb, - enough_mtbf_info ? "" : " don't "); + enough_mtbf_info ? "" : " don't"); tor_free(uptimes); tor_free(mtbfs); @@ -1959,13 +1803,12 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, char published[ISO_TIME_LEN+1]; char identity64[BASE64_DIGEST_LEN+1]; char digest64[BASE64_DIGEST_LEN+1]; - smartlist_t *chunks = NULL; + smartlist_t *chunks = smartlist_new(); format_iso_time(published, rs->published_on); digest_to_base64(identity64, rs->identity_digest); digest_to_base64(digest64, rs->descriptor_digest); - chunks = smartlist_new(); smartlist_add_asprintf(chunks, "r %s %s %s%s%s %s %d %d\n", rs->nickname, @@ -1996,19 +1839,16 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, goto done; smartlist_add_asprintf(chunks, - "s%s%s%s%s%s%s%s%s%s%s%s%s%s\n", + "s%s%s%s%s%s%s%s%s%s%s\n", /* These must stay in alphabetical order. */ rs->is_authority?" Authority":"", - rs->is_bad_directory?" BadDirectory":"", rs->is_bad_exit?" BadExit":"", rs->is_exit?" Exit":"", rs->is_fast?" Fast":"", rs->is_possible_guard?" Guard":"", rs->is_hs_dir?" HSDir":"", - rs->is_named?" Named":"", rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", - rs->is_unnamed?" Unnamed":"", (rs->dir_port!=0)?" V2Dir":"", rs->is_valid?" Valid":""); @@ -2077,6 +1917,13 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, smartlist_add_asprintf(chunks, " Measured=%d", vrs->measured_bw_kb); } + /* Write down guardfraction information if we have it. */ + if (format == NS_V3_VOTE && vrs && vrs->status.has_guardfraction) { + smartlist_add_asprintf(chunks, + " GuardFraction=%d", + vrs->status.guardfraction_percentage); + } + smartlist_add(chunks, tor_strdup("\n")); if (desc) { @@ -2090,10 +1937,8 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, result = smartlist_join_strings(chunks, "", 0, NULL); err: - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); - smartlist_free(chunks); - } + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); return result; } @@ -2199,78 +2044,8 @@ get_possible_sybil_list(const smartlist_t *routers) return omit_as_sybil; } -/** Return non-zero iff a relay running the Tor version specified in - * <b>platform</b> is suitable for use as a potential entry guard. */ -static int -is_router_version_good_for_possible_guard(const char *platform) -{ - static int parsed_versions_initialized = 0; - static tor_version_t first_good_0_2_1_guard_version; - static tor_version_t first_good_0_2_2_guard_version; - static tor_version_t first_good_later_guard_version; - - tor_version_t router_version; - - /* XXX024 This block should be extracted into its own function. */ - /* XXXX Begin code copied from tor_version_as_new_as (in routerparse.c) */ - { - char *s, *s2, *start; - char tmp[128]; - - tor_assert(platform); - - /* nonstandard Tor; be safe and say yes */ - if (strcmpstart(platform,"Tor ")) - return 1; - - start = (char *)eat_whitespace(platform+3); - if (!*start) return 0; - s = (char *)find_whitespace(start); /* also finds '\0', which is fine */ - s2 = (char*)eat_whitespace(s); - if (!strcmpstart(s2, "(r") || !strcmpstart(s2, "(git-")) - s = (char*)find_whitespace(s2); - - if ((size_t)(s-start+1) >= sizeof(tmp)) /* too big, no */ - return 0; - strlcpy(tmp, start, s-start+1); - - if (tor_version_parse(tmp, &router_version)<0) { - log_info(LD_DIR,"Router version '%s' unparseable.",tmp); - return 1; /* be safe and say yes */ - } - } - /* XXXX End code copied from tor_version_as_new_as (in routerparse.c) */ - - if (!parsed_versions_initialized) { - /* CVE-2011-2769 was fixed on the relay side in Tor versions - * 0.2.1.31, 0.2.2.34, and 0.2.3.6-alpha. */ - tor_assert(tor_version_parse("0.2.1.31", - &first_good_0_2_1_guard_version)>=0); - tor_assert(tor_version_parse("0.2.2.34", - &first_good_0_2_2_guard_version)>=0); - tor_assert(tor_version_parse("0.2.3.6-alpha", - &first_good_later_guard_version)>=0); - - /* Don't parse these constant version strings once for every relay - * for every vote. */ - parsed_versions_initialized = 1; - } - - return ((tor_version_same_series(&first_good_0_2_1_guard_version, - &router_version) && - tor_version_compare(&first_good_0_2_1_guard_version, - &router_version) <= 0) || - (tor_version_same_series(&first_good_0_2_2_guard_version, - &router_version) && - tor_version_compare(&first_good_0_2_2_guard_version, - &router_version) <= 0) || - (tor_version_compare(&first_good_later_guard_version, - &router_version) <= 0)); -} - /** Extract status information from <b>ri</b> and from other authority - * functions and store it in <b>rs</b>>. If <b>naming</b>, consider setting - * the named flag in <b>rs</b>. + * functions and store it in <b>rs</b>>. * * We assume that ri-\>is_running has already been set, e.g. by * dirserv_set_router_is_running(ri, now); @@ -2280,8 +2055,8 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, routerinfo_t *ri, time_t now, - int naming, int listbadexits, - int listbaddirs, int vote_on_hsdirs) + int listbadexits, + int vote_on_hsdirs) { const or_options_t *options = get_options(); uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri); @@ -2301,20 +2076,13 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, !dirserv_thinks_router_is_unreliable(now, ri, 0, 1); rs->is_flagged_running = node->is_running; /* computed above */ - if (naming) { - uint32_t name_status = dirserv_get_name_status( - node->identity, ri->nickname); - rs->is_named = (naming && (name_status & FP_NAMED)) ? 1 : 0; - rs->is_unnamed = (naming && (name_status & FP_UNNAMED)) ? 1 : 0; - } rs->is_valid = node->is_valid; if (node->is_fast && ((options->AuthDirGuardBWGuarantee && routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) || routerbw_kb >= MIN(guard_bandwidth_including_exits_kb, - guard_bandwidth_excluding_exits_kb)) && - is_router_version_good_for_possible_guard(ri->platform)) { + guard_bandwidth_excluding_exits_kb))) { long tk = rep_hist_get_weighted_time_known( node->identity, now); double wfu = rep_hist_get_weighted_fractional_uptime( @@ -2323,19 +2091,12 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, } else { rs->is_possible_guard = 0; } - if (options->TestingTorNetwork && - routerset_contains_routerstatus(options->TestingDirAuthVoteGuard, - rs, 0)) { - rs->is_possible_guard = 1; - } - rs->is_bad_directory = listbaddirs && node->is_bad_directory; rs->is_bad_exit = listbadexits && node->is_bad_exit; node->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, node, now); rs->is_hs_dir = vote_on_hsdirs && node->is_hs_dir; - if (!strcasecmp(ri->nickname, UNNAMED_ROUTER_NICKNAME)) - rs->is_named = rs->is_unnamed = 0; + rs->is_named = rs->is_unnamed = 0; rs->published_on = ri->cache_info.published_on; memcpy(rs->identity_digest, node->identity, DIGEST_LEN); @@ -2353,6 +2114,28 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr); rs->ipv6_orport = ri->ipv6_orport; } + + /* Iff we are in a testing network, use TestingDirAuthVoteExit, + TestingDirAuthVoteGuard, and TestingDirAuthVoteHSDir to + give out the Exit, Guard, and HSDir flags, respectively. + But don't set the corresponding node flags. */ + if (options->TestingTorNetwork) { + if (routerset_contains_routerstatus(options->TestingDirAuthVoteExit, + rs, 0)) { + rs->is_exit = 1; + } + + if (routerset_contains_routerstatus(options->TestingDirAuthVoteGuard, + rs, 0)) { + rs->is_possible_guard = 1; + } + + if (routerset_contains_routerstatus(options->TestingDirAuthVoteHSDir, + rs, 0)) { + /* TestingDirAuthVoteHSDir respects VoteOnHidServDirectoriesV2 */ + rs->is_hs_dir = vote_on_hsdirs; + } + } } /** Routerstatus <b>rs</b> is part of a group of routers that are on @@ -2364,13 +2147,325 @@ clear_status_flags_on_sybil(routerstatus_t *rs) { rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = rs->is_flagged_running = rs->is_named = rs->is_valid = - rs->is_hs_dir = rs->is_possible_guard = rs->is_bad_exit = - rs->is_bad_directory = 0; + rs->is_hs_dir = rs->is_possible_guard = rs->is_bad_exit = 0; /* FFFF we might want some mechanism to check later on if we * missed zeroing any flags: it's easy to add a new flag but * forget to add it to this clause. */ } +/** The guardfraction of the guard with identity fingerprint <b>guard_id</b> + * is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for + * this guard in <b>vote_routerstatuses</b>, and if we do, register the + * information to it. + * + * Return 1 if we applied the information and 0 if we couldn't find a + * matching guard. + * + * Requires that <b>vote_routerstatuses</b> be sorted. + */ +static int +guardfraction_line_apply(const char *guard_id, + uint32_t guardfraction_percentage, + smartlist_t *vote_routerstatuses) +{ + vote_routerstatus_t *vrs = NULL; + + tor_assert(vote_routerstatuses); + + vrs = smartlist_bsearch(vote_routerstatuses, guard_id, + compare_digest_to_vote_routerstatus_entry); + + if (!vrs) { + return 0; + } + + vrs->status.has_guardfraction = 1; + vrs->status.guardfraction_percentage = guardfraction_percentage; + + return 1; +} + +/* Given a guard line from a guardfraction file, parse it and register + * its information to <b>vote_routerstatuses</b>. + * + * Return: + * * 1 if the line was proper and its information got registered. + * * 0 if the line was proper but no currently active guard was found + * to register the guardfraction information to. + * * -1 if the line could not be parsed and set <b>err_msg</b> to a + newly allocated string containing the error message. + */ +static int +guardfraction_file_parse_guard_line(const char *guard_line, + smartlist_t *vote_routerstatuses, + char **err_msg) +{ + char guard_id[DIGEST_LEN]; + uint32_t guardfraction; + char *inputs_tmp = NULL; + int num_ok = 1; + + smartlist_t *sl = smartlist_new(); + int retval = -1; + + tor_assert(err_msg); + + /* guard_line should contain something like this: + <hex digest> <guardfraction> <appearances> */ + smartlist_split_string(sl, guard_line, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3); + if (smartlist_len(sl) < 3) { + tor_asprintf(err_msg, "bad line '%s'", guard_line); + goto done; + } + + inputs_tmp = smartlist_get(sl, 0); + if (strlen(inputs_tmp) != HEX_DIGEST_LEN || + base16_decode(guard_id, DIGEST_LEN, inputs_tmp, HEX_DIGEST_LEN)) { + tor_asprintf(err_msg, "bad digest '%s'", inputs_tmp); + goto done; + } + + inputs_tmp = smartlist_get(sl, 1); + /* Guardfraction is an integer in [0, 100]. */ + guardfraction = + (uint32_t) tor_parse_long(inputs_tmp, 10, 0, 100, &num_ok, NULL); + if (!num_ok) { + tor_asprintf(err_msg, "wrong percentage '%s'", inputs_tmp); + goto done; + } + + /* If routerstatuses were provided, apply this info to actual routers. */ + if (vote_routerstatuses) { + retval = guardfraction_line_apply(guard_id, guardfraction, + vote_routerstatuses); + } else { + retval = 0; /* If we got this far, line was correctly formatted. */ + } + + done: + + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + + return retval; +} + +/** Given an inputs line from a guardfraction file, parse it and + * register its information to <b>total_consensuses</b> and + * <b>total_days</b>. + * + * Return 0 if it parsed well. Return -1 if there was an error, and + * set <b>err_msg</b> to a newly allocated string containing the + * error message. + */ +static int +guardfraction_file_parse_inputs_line(const char *inputs_line, + int *total_consensuses, + int *total_days, + char **err_msg) +{ + int retval = -1; + char *inputs_tmp = NULL; + int num_ok = 1; + smartlist_t *sl = smartlist_new(); + + tor_assert(err_msg); + + /* Second line is inputs information: + * n-inputs <total_consensuses> <total_days>. */ + smartlist_split_string(sl, inputs_line, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3); + if (smartlist_len(sl) < 2) { + tor_asprintf(err_msg, "incomplete line '%s'", inputs_line); + goto done; + } + + inputs_tmp = smartlist_get(sl, 0); + *total_consensuses = + (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL); + if (!num_ok) { + tor_asprintf(err_msg, "unparseable consensus '%s'", inputs_tmp); + goto done; + } + + inputs_tmp = smartlist_get(sl, 1); + *total_days = + (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL); + if (!num_ok) { + tor_asprintf(err_msg, "unparseable days '%s'", inputs_tmp); + goto done; + } + + retval = 0; + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + + return retval; +} + +/* Maximum age of a guardfraction file that we are willing to accept. */ +#define MAX_GUARDFRACTION_FILE_AGE (7*24*60*60) /* approx a week */ + +/** Static strings of guardfraction files. */ +#define GUARDFRACTION_DATE_STR "written-at" +#define GUARDFRACTION_INPUTS "n-inputs" +#define GUARDFRACTION_GUARD "guard-seen" +#define GUARDFRACTION_VERSION "guardfraction-file-version" + +/** Given a guardfraction file in a string, parse it and register the + * guardfraction information to the provided vote routerstatuses. + * + * This is the rough format of the guardfraction file: + * + * guardfraction-file-version 1 + * written-at <date and time> + * n-inputs <number of consesuses parsed> <number of days considered> + * + * guard-seen <fpr 1> <guardfraction percentage> <consensus appearances> + * guard-seen <fpr 2> <guardfraction percentage> <consensus appearances> + * guard-seen <fpr 3> <guardfraction percentage> <consensus appearances> + * guard-seen <fpr 4> <guardfraction percentage> <consensus appearances> + * guard-seen <fpr 5> <guardfraction percentage> <consensus appearances> + * ... + * + * Return -1 if the parsing failed and 0 if it went smoothly. Parsing + * should tolerate errors in all lines but the written-at header. + */ +STATIC int +dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str, + smartlist_t *vote_routerstatuses) +{ + config_line_t *front=NULL, *line; + int ret_tmp; + int retval = -1; + int current_line_n = 0; /* line counter for better log messages */ + + /* Guardfraction info to be parsed */ + int total_consensuses = 0; + int total_days = 0; + + /* Stats */ + int guards_read_n = 0; + int guards_applied_n = 0; + + /* Parse file and split it in lines */ + ret_tmp = config_get_lines(guardfraction_file_str, &front, 0); + if (ret_tmp < 0) { + log_warn(LD_CONFIG, "Error reading from guardfraction file"); + goto done; + } + + /* Sort routerstatuses (needed later when applying guardfraction info) */ + if (vote_routerstatuses) + smartlist_sort(vote_routerstatuses, compare_vote_routerstatus_entries); + + for (line = front; line; line=line->next) { + current_line_n++; + + if (!strcmp(line->key, GUARDFRACTION_VERSION)) { + int num_ok = 1; + unsigned int version; + + version = + (unsigned int) tor_parse_long(line->value, + 10, 0, INT_MAX, &num_ok, NULL); + + if (!num_ok || version != 1) { + log_warn(LD_GENERAL, "Got unknown guardfraction version %d.", version); + goto done; + } + } else if (!strcmp(line->key, GUARDFRACTION_DATE_STR)) { + time_t file_written_at; + time_t now = time(NULL); + + /* First line is 'written-at <date>' */ + if (parse_iso_time(line->value, &file_written_at) < 0) { + log_warn(LD_CONFIG, "Guardfraction:%d: Bad date '%s'. Ignoring", + current_line_n, line->value); + goto done; /* don't tolerate failure here. */ + } + if (file_written_at < now - MAX_GUARDFRACTION_FILE_AGE) { + log_warn(LD_CONFIG, "Guardfraction:%d: was written very long ago '%s'", + current_line_n, line->value); + goto done; /* don't tolerate failure here. */ + } + } else if (!strcmp(line->key, GUARDFRACTION_INPUTS)) { + char *err_msg = NULL; + + if (guardfraction_file_parse_inputs_line(line->value, + &total_consensuses, + &total_days, + &err_msg) < 0) { + log_warn(LD_CONFIG, "Guardfraction:%d: %s", + current_line_n, err_msg); + tor_free(err_msg); + continue; + } + + } else if (!strcmp(line->key, GUARDFRACTION_GUARD)) { + char *err_msg = NULL; + + ret_tmp = guardfraction_file_parse_guard_line(line->value, + vote_routerstatuses, + &err_msg); + if (ret_tmp < 0) { /* failed while parsing the guard line */ + log_warn(LD_CONFIG, "Guardfraction:%d: %s", + current_line_n, err_msg); + tor_free(err_msg); + continue; + } + + /* Successfully parsed guard line. Check if it was applied properly. */ + guards_read_n++; + if (ret_tmp > 0) { + guards_applied_n++; + } + } else { + log_warn(LD_CONFIG, "Unknown guardfraction line %d (%s %s)", + current_line_n, line->key, line->value); + } + } + + retval = 0; + + log_info(LD_CONFIG, + "Successfully parsed guardfraction file with %d consensuses over " + "%d days. Parsed %d nodes and applied %d of them%s.", + total_consensuses, total_days, guards_read_n, guards_applied_n, + vote_routerstatuses ? "" : " (no routerstatus provided)" ); + + done: + config_free_lines(front); + + if (retval < 0) { + return retval; + } else { + return guards_read_n; + } +} + +/** Read a guardfraction file at <b>fname</b> and load all its + * information to <b>vote_routerstatuses</b>. */ +int +dirserv_read_guardfraction_file(const char *fname, + smartlist_t *vote_routerstatuses) +{ + char *guardfraction_file_str; + + /* Read file to a string */ + guardfraction_file_str = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + if (!guardfraction_file_str) { + log_warn(LD_FS, "Cannot open guardfraction file '%s'. Failing.", fname); + return -1; + } + + return dirserv_read_guardfraction_file_from_str(guardfraction_file_str, + vote_routerstatuses); +} + /** * Helper function to parse out a line in the measured bandwidth file * into a measured_bw_line_t output structure. Returns -1 on failure @@ -2563,9 +2658,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, smartlist_t *routers, *routerstatuses; char identity_digest[DIGEST_LEN]; char signing_key_digest[DIGEST_LEN]; - int naming = options->NamingAuthoritativeDir; int listbadexits = options->AuthDirListBadExits; - int listbaddirs = options->AuthDirListBadDirs; int vote_on_hsdirs = options->VoteOnHidServDirectoriesV2; routerlist_t *rl = router_get_routerlist(); time_t now = time(NULL); @@ -2657,7 +2750,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; set_routerstatus_from_routerinfo(rs, node, ri, now, - naming, listbadexits, listbaddirs, + listbadexits, vote_on_hsdirs); if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) @@ -2685,6 +2778,12 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, smartlist_free(routers); digestmap_free(omit_as_sybil, NULL); + /* Apply guardfraction information to routerstatuses. */ + if (options->GuardfractionFile) { + dirserv_read_guardfraction_file(options->GuardfractionFile, + routerstatuses); + } + /* This pass through applies the measured bw lines to the routerstatuses */ if (options->V3BandwidthsFile) { dirserv_read_measured_bandwidths(options->V3BandwidthsFile, @@ -2733,20 +2832,23 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, v3_out->client_versions = client_versions; v3_out->server_versions = server_versions; + v3_out->package_lines = smartlist_new(); + { + config_line_t *cl; + for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) { + if (validate_recommended_package_line(cl->value)) + smartlist_add(v3_out->package_lines, tor_strdup(cl->value)); + } + } + v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, "Authority Exit Fast Guard Stable V2Dir Valid", 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add(v3_out->known_flags, tor_strdup("Running")); - if (listbaddirs) - smartlist_add(v3_out->known_flags, tor_strdup("BadDirectory")); if (listbadexits) smartlist_add(v3_out->known_flags, tor_strdup("BadExit")); - if (naming) { - smartlist_add(v3_out->known_flags, tor_strdup("Named")); - smartlist_add(v3_out->known_flags, tor_strdup("Unnamed")); - } if (vote_on_hsdirs) smartlist_add(v3_out->known_flags, tor_strdup("HSDir")); smartlist_sort_strings(v3_out->known_flags); @@ -3431,7 +3533,7 @@ connection_dirserv_add_networkstatus_bytes_to_outbuf(dir_connection_t *conn) if (uncompressing && ! conn->zlib_state && conn->fingerprint_stack && smartlist_len(conn->fingerprint_stack)) { - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD); + conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); } } if (r) return r; @@ -3484,6 +3586,80 @@ connection_dirserv_flushed_some(dir_connection_t *conn) } } +/** Return true iff <b>line</b> is a valid RecommendedPackages line. + */ +/* + The grammar is: + + "package" SP PACKAGENAME SP VERSION SP URL SP DIGESTS NL + + PACKAGENAME = NONSPACE + VERSION = NONSPACE + URL = NONSPACE + DIGESTS = DIGEST | DIGESTS SP DIGEST + DIGEST = DIGESTTYPE "=" DIGESTVAL + + NONSPACE = one or more non-space printing characters + + DIGESTVAL = DIGESTTYPE = one or more non-=, non-" " characters. + + SP = " " + NL = a newline + + */ +int +validate_recommended_package_line(const char *line) +{ + const char *cp = line; + +#define WORD() \ + do { \ + if (*cp == ' ') \ + return 0; \ + cp = strchr(cp, ' '); \ + if (!cp) \ + return 0; \ + } while (0) + + WORD(); /* skip packagename */ + ++cp; + WORD(); /* skip version */ + ++cp; + WORD(); /* Skip URL */ + ++cp; + + /* Skip digesttype=digestval + */ + int n_entries = 0; + while (1) { + const char *start_of_word = cp; + const char *end_of_word = strchr(cp, ' '); + if (! end_of_word) + end_of_word = cp + strlen(cp); + + if (start_of_word == end_of_word) + return 0; + + const char *eq = memchr(start_of_word, '=', end_of_word - start_of_word); + + if (!eq) + return 0; + if (eq == start_of_word) + return 0; + if (eq == end_of_word - 1) + return 0; + if (memchr(eq+1, '=', end_of_word - (eq+1))) + return 0; + + ++n_entries; + if (0 == *end_of_word) + break; + + cp = end_of_word + 1; + } + + return (n_entries == 0) ? 0 : 1; +} + /** Release all storage used by the directory server. */ void dirserv_free_all(void) diff --git a/src/or/dirserv.h b/src/or/dirserv.h index 858e6e3a07..311a513dbe 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -34,10 +34,9 @@ int connection_dirserv_flushed_some(dir_connection_t *conn); -int dirserv_add_own_fingerprint(const char *nickname, crypto_pk_t *pk); +int dirserv_add_own_fingerprint(crypto_pk_t *pk); int dirserv_load_fingerprint_file(void); void dirserv_free_fingerprint_list(void); -const char *dirserv_get_nickname_by_digest(const char *digest); enum was_router_added_t dirserv_add_multiple_descriptors( const char *desc, uint8_t purpose, const char *source, @@ -105,6 +104,8 @@ void dirserv_free_all(void); void cached_dir_decref(cached_dir_t *d); cached_dir_t *new_cached_dir(char *s, time_t published); +int validate_recommended_package_line(const char *line); + #ifdef DIRSERV_PRIVATE /* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */ @@ -124,10 +125,17 @@ STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_out, time_t *as_of_out); STATIC int dirserv_has_measured_bw(const char *node_id); + +STATIC int +dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str, + smartlist_t *vote_routerstatuses); #endif int dirserv_read_measured_bandwidths(const char *from_file, smartlist_t *routerstatuses); +int dirserv_read_guardfraction_file(const char *fname, + smartlist_t *vote_routerstatuses); + #endif diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 137d6c1a8c..7a5154dae5 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define DIRVOTE_PRIVATE @@ -16,6 +16,7 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "entrynodes.h" /* needed for guardfraction methods */ /** * \file dirvote.c @@ -64,8 +65,9 @@ STATIC char * format_networkstatus_vote(crypto_pk_t *private_signing_key, networkstatus_t *v3_ns) { - smartlist_t *chunks; + smartlist_t *chunks = smartlist_new(); const char *client_versions = NULL, *server_versions = NULL; + char *packages = NULL; char fingerprint[FINGERPRINT_LEN+1]; char digest[DIGEST_LEN]; uint32_t addr; @@ -98,7 +100,18 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, server_versions_line = tor_strdup(""); } - chunks = smartlist_new(); + if (v3_ns->package_lines) { + smartlist_t *tmp = smartlist_new(); + SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p, + if (validate_recommended_package_line(p)) + smartlist_add_asprintf(tmp, "package %s\n", p)); + packages = smartlist_join_strings(tmp, "", 0, NULL); + SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp)); + smartlist_free(tmp); + } else { + packages = tor_strdup(""); + } + { char published[ISO_TIME_LEN+1]; char va[ISO_TIME_LEN+1]; @@ -110,7 +123,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, char *params; authority_cert_t *cert = v3_ns->cert; char *methods = - make_consensus_method_list(1, MAX_SUPPORTED_CONSENSUS_METHOD, " "); + make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD, + MAX_SUPPORTED_CONSENSUS_METHOD, " "); format_iso_time(published, v3_ns->published); format_iso_time(va, v3_ns->valid_after); format_iso_time(fu, v3_ns->fresh_until); @@ -132,6 +146,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, "valid-until %s\n" "voting-delay %d %d\n" "%s%s" /* versions */ + "%s" /* packages */ "known-flags %s\n" "flag-thresholds %s\n" "params %s\n" @@ -143,6 +158,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, v3_ns->vote_seconds, v3_ns->dist_seconds, client_versions_line, server_versions_line, + packages, flags, flag_thresholds, params, @@ -230,10 +246,10 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, done: tor_free(client_versions_line); tor_free(server_versions_line); - if (chunks) { - SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); - smartlist_free(chunks); - } + tor_free(packages); + + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); return status; } @@ -458,8 +474,7 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, smartlist_free(alt_orports); } - if (consensus_method >= MIN_METHOD_FOR_MICRODESC && - microdesc_digest256_out) { + if (microdesc_digest256_out) { smartlist_t *digests = smartlist_new(); const char *best_microdesc_digest; SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { @@ -541,7 +556,8 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - return (method >= 1) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); + return (method >= MIN_SUPPORTED_CONSENSUS_METHOD) && + (method <= MAX_SUPPORTED_CONSENSUS_METHOD); } /** Return a newly allocated string holding the numbers between low and high @@ -605,13 +621,14 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) const int n_votes = smartlist_len(votes); smartlist_t *output; smartlist_t *param_list = smartlist_new(); + (void) method; /* We require that the parameter lists in the votes are well-formed: that is, that their keywords are unique and sorted, and that their values are between INT32_MIN and INT32_MAX inclusive. This should be guaranteed by the parsing code. */ - vals = tor_malloc(sizeof(int)*n_votes); + vals = tor_calloc(n_votes, sizeof(int)); SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { if (!v->net_params) @@ -647,12 +664,13 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) next_param = NULL; else next_param = smartlist_get(param_list, param_sl_idx+1); + /* resolve spurious clang shallow analysis null pointer errors */ + tor_assert(param); if (!next_param || strncmp(next_param, param, cur_param_len)) { /* We've reached the end of a series. */ /* Make sure enough authorities voted on this param, unless the * the consensus method we use is too old for that. */ - if (method < MIN_METHOD_FOR_MAJORITY_PARAMS || - i > total_authorities/2 || + if (i > total_authorities/2 || i >= MIN_VOTES_FOR_PARAM) { int32_t median = median_int32(vals, i); char *out_string = tor_malloc(64+cur_param_len); @@ -1005,299 +1023,87 @@ networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G, I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); return 1; } -/** - * This function computes the bandwidth weights for consensus method 9. - * - * It has been obsoleted in favor of consensus method 10. - */ + +/** Update total bandwidth weights (G/M/E/D/T) with the bandwidth of + * the router in <b>rs</b>. */ static void -networkstatus_compute_bw_weights_v9(smartlist_t *chunks, int64_t G, int64_t M, - int64_t E, int64_t D, int64_t T, - int64_t weight_scale) +update_total_bandwidth_weights(const routerstatus_t *rs, + int is_exit, int is_guard, + int64_t *G, int64_t *M, int64_t *E, int64_t *D, + int64_t *T) { - int64_t Wgg = -1, Wgd = -1; - int64_t Wmg = -1, Wme = -1, Wmd = -1; - int64_t Wed = -1, Wee = -1; - const char *casename; + int default_bandwidth = rs->bandwidth_kb; + int guardfraction_bandwidth = 0; - if (G <= 0 || M <= 0 || E <= 0 || D <= 0) { - log_warn(LD_DIR, "Consensus with empty bandwidth: " - "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT - " D="I64_FORMAT" T="I64_FORMAT, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + if (!rs->has_bandwidth) { + log_info(LD_BUG, "Missing consensus bandwidth for router %s", + rs->nickname); return; } - /* - * Computed from cases in 3.4.3 of dir-spec.txt + /* If this routerstatus represents a guard that we have + * guardfraction information on, use it to calculate its actual + * bandwidth. From proposal236: * - * 1. Neither are scarce - * 2. Both Guard and Exit are scarce - * a. R+D <= S - * b. R+D > S - * 3. One of Guard or Exit is scarce - * a. S+D < T/3 - * b. S+D >= T/3 + * Similarly, when calculating the bandwidth-weights line as in + * section 3.8.3 of dir-spec.txt, directory authorities should treat N + * as if fraction F of its bandwidth has the guard flag and (1-F) does + * not. So when computing the totals G,M,E,D, each relay N with guard + * visibility fraction F and bandwidth B should be added as follows: + * + * G' = G + F*B, if N does not have the exit flag + * M' = M + (1-F)*B, if N does not have the exit flag + * + * or + * + * D' = D + F*B, if N has the exit flag + * E' = E + (1-F)*B, if N has the exit flag + * + * In this block of code, we prepare the bandwidth values by setting + * the default_bandwidth to F*B and guardfraction_bandwidth to (1-F)*B. */ - if (3*E >= T && 3*G >= T) { // E >= T/3 && G >= T/3 - bw_weights_error_t berr = 0; - /* Case 1: Neither are scarce. - * - * Attempt to ensure that we have a large amount of exit bandwidth - * in the middle position. - */ - casename = "Case 1 (Wme*E = Wmd*D)"; - Wgg = (weight_scale*(D+E+G+M))/(3*G); - if (D==0) Wmd = 0; - else Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); - Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); - Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); - Wgd = 0; - Wmg = weight_scale - Wgg; - Wed = weight_scale - Wmd; + if (rs->has_guardfraction) { + guardfraction_bandwidth_t guardfraction_bw; - berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, - weight_scale, G, M, E, D, T, 10, 1); + tor_assert(is_guard); - if (berr) { - log_warn(LD_DIR, "Bw Weights error %d for case %s. " - "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT - " D="I64_FORMAT" T="I64_FORMAT, - berr, casename, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - } - } else if (3*E < T && 3*G < T) { // E < T/3 && G < T/3 - int64_t R = MIN(E, G); - int64_t S = MAX(E, G); - /* - * Case 2: Both Guards and Exits are scarce - * Balance D between E and G, depending upon - * D capacity and scarcity. - */ - if (R+D < S) { // Subcase a - Wgg = weight_scale; - Wee = weight_scale; - Wmg = 0; - Wme = 0; - Wmd = 0; - if (E < G) { - casename = "Case 2a (E scarce)"; - Wed = weight_scale; - Wgd = 0; - } else { /* E >= G */ - casename = "Case 2a (G scarce)"; - Wed = 0; - Wgd = weight_scale; - } - } else { // Subcase b: R+D > S - bw_weights_error_t berr = 0; - casename = "Case 2b (Wme*E == Wmd*D)"; - if (D != 0) { - Wgg = weight_scale; - Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); // T/3 >= G (Ok) - Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); // T/3 >= M - Wme = (weight_scale*(D + E + G - 2*M))/(6*E); - Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); // 2E+M >= T/3 - Wmg = 0; - Wed = weight_scale - Wgd - Wmd; + guard_get_guardfraction_bandwidth(&guardfraction_bw, + rs->bandwidth_kb, + rs->guardfraction_percentage); - berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed, - weight_scale, G, M, E, D, T, 10, 1); - } + default_bandwidth = guardfraction_bw.guard_bw; + guardfraction_bandwidth = guardfraction_bw.non_guard_bw; + } - if (D == 0 || berr) { // Can happen if M > T/3 - casename = "Case 2b (E=G)"; - Wgg = weight_scale; - Wee = weight_scale; - Wmg = 0; - Wme = 0; - Wmd = 0; - if (D == 0) Wgd = 0; - else Wgd = (weight_scale*(D+E-G))/(2*D); - Wed = weight_scale - Wgd; - berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, - Wed, weight_scale, G, M, E, D, T, 10, 1); - } - if (berr != BW_WEIGHTS_NO_ERROR && - berr != BW_WEIGHTS_BALANCE_MID_ERROR) { - log_warn(LD_DIR, "Bw Weights error %d for case %s. " - "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT - " D="I64_FORMAT" T="I64_FORMAT, - berr, casename, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - } - } - } else { // if (E < T/3 || G < T/3) { - int64_t S = MIN(E, G); - // Case 3: Exactly one of Guard or Exit is scarce - if (!(3*E < T || 3*G < T) || !(3*G >= T || 3*E >= T)) { - log_warn(LD_BUG, - "Bw-Weights Case 3 but with G="I64_FORMAT" M=" - I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + /* Now calculate the total bandwidth weights with or without + * guardfraction. Depending on the flags of the relay, add its + * bandwidth to the appropriate weight pool. If it's a guard and + * guardfraction is enabled, add its bandwidth to both pools as + * indicated by the previous comment. + */ + *T += default_bandwidth; + if (is_exit && is_guard) { + + *D += default_bandwidth; + if (rs->has_guardfraction) { + *E += guardfraction_bandwidth; } - if (3*(S+D) < T) { // Subcase a: S+D < T/3 - if (G < E) { - casename = "Case 3a (G scarce)"; - Wgg = Wgd = weight_scale; - Wmd = Wed = Wmg = 0; - // Minor subcase, if E is more scarce than M, - // keep its bandwidth in place. - if (E < M) Wme = 0; - else Wme = (weight_scale*(E-M))/(2*E); - Wee = weight_scale-Wme; - } else { // G >= E - casename = "Case 3a (E scarce)"; - Wee = Wed = weight_scale; - Wmd = Wgd = Wme = 0; - // Minor subcase, if G is more scarce than M, - // keep its bandwidth in place. - if (G < M) Wmg = 0; - else Wmg = (weight_scale*(G-M))/(2*G); - Wgg = weight_scale-Wmg; - } - } else { // Subcase b: S+D >= T/3 - bw_weights_error_t berr = 0; - // D != 0 because S+D >= T/3 - if (G < E) { - casename = "Case 3b (G scarce, Wme*E == Wmd*D)"; - Wgd = (weight_scale*(D + E - 2*G + M))/(3*D); - Wmd = (weight_scale*(D + E + G - 2*M))/(6*D); - Wme = (weight_scale*(D + E + G - 2*M))/(6*E); - Wee = (weight_scale*(-D + 5*E - G + 2*M))/(6*E); - Wgg = weight_scale; - Wmg = 0; - Wed = weight_scale - Wgd - Wmd; + } else if (is_exit) { - berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, - Wed, weight_scale, G, M, E, D, T, 10, 1); - } else { // G >= E - casename = "Case 3b (E scarce, Wme*E == Wmd*D)"; - Wgg = (weight_scale*(D + E + G + M))/(3*G); - Wmd = (weight_scale*(2*D + 2*E - G - M))/(6*D); - Wme = (weight_scale*(2*D + 2*E - G - M))/(6*E); - Wee = (weight_scale*(-2*D + 4*E + G + M))/(6*E); - Wgd = 0; - Wmg = weight_scale - Wgg; - Wed = weight_scale - Wmd; + *E += default_bandwidth; - berr = networkstatus_check_weights(Wgg, Wgd, Wmg, Wme, Wmd, Wee, - Wed, weight_scale, G, M, E, D, T, 10, 1); - } - if (berr) { - log_warn(LD_DIR, "Bw Weights error %d for case %s. " - "G="I64_FORMAT" M="I64_FORMAT - " E="I64_FORMAT" D="I64_FORMAT" T="I64_FORMAT, - berr, casename, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - } - } - } + } else if (is_guard) { - /* We cast down the weights to 32 bit ints on the assumption that - * weight_scale is ~= 10000. We need to ensure a rogue authority - * doesn't break this assumption to rig our weights */ - tor_assert(0 < weight_scale && weight_scale <= INT32_MAX); + *G += default_bandwidth; + if (rs->has_guardfraction) { + *M += guardfraction_bandwidth; + } - if (Wgg < 0 || Wgg > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wgg="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wgg), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); + } else { - Wgg = MAX(MIN(Wgg, weight_scale), 0); - } - if (Wgd < 0 || Wgd > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wgd="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wgd), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wgd = MAX(MIN(Wgd, weight_scale), 0); - } - if (Wmg < 0 || Wmg > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wmg="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wmg), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wmg = MAX(MIN(Wmg, weight_scale), 0); - } - if (Wme < 0 || Wme > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wme="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wme), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wme = MAX(MIN(Wme, weight_scale), 0); + *M += default_bandwidth; } - if (Wmd < 0 || Wmd > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wmd="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wmd), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wmd = MAX(MIN(Wmd, weight_scale), 0); - } - if (Wee < 0 || Wee > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wee="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wee), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wee = MAX(MIN(Wee, weight_scale), 0); - } - if (Wed < 0 || Wed > weight_scale) { - log_warn(LD_DIR, "Bw %s: Wed="I64_FORMAT"! G="I64_FORMAT - " M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, I64_PRINTF_ARG(Wed), - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); - Wed = MAX(MIN(Wed, weight_scale), 0); - } - - // Add consensus weight keywords - smartlist_add(chunks, tor_strdup("bandwidth-weights ")); - /* - * Provide Wgm=Wgg, Wmm=1, Wem=Wee, Weg=Wed. May later determine - * that middle nodes need different bandwidth weights for dirport traffic, - * or that weird exit policies need special weight, or that bridges - * need special weight. - * - * NOTE: This list is sorted. - */ - smartlist_add_asprintf(chunks, - "Wbd=%d Wbe=%d Wbg=%d Wbm=%d " - "Wdb=%d " - "Web=%d Wed=%d Wee=%d Weg=%d Wem=%d " - "Wgb=%d Wgd=%d Wgg=%d Wgm=%d " - "Wmb=%d Wmd=%d Wme=%d Wmg=%d Wmm=%d\n", - (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale, - (int)weight_scale, - (int)weight_scale, (int)Wed, (int)Wee, (int)Wed, (int)Wee, - (int)weight_scale, (int)Wgd, (int)Wgg, (int)Wgg, - (int)weight_scale, (int)Wmd, (int)Wme, (int)Wmg, (int)weight_scale); - - log_notice(LD_CIRC, "Computed bandwidth weights for %s with v9: " - "G="I64_FORMAT" M="I64_FORMAT" E="I64_FORMAT" D="I64_FORMAT - " T="I64_FORMAT, - casename, - I64_PRINTF_ARG(G), I64_PRINTF_ARG(M), I64_PRINTF_ARG(E), - I64_PRINTF_ARG(D), I64_PRINTF_ARG(T)); } /** Given a list of vote networkstatus_t in <b>votes</b>, our public @@ -1330,6 +1136,7 @@ networkstatus_compute_consensus(smartlist_t *votes, const routerstatus_format_type_t rs_format = flavor == FLAV_NS ? NS_V3_CONSENSUS : NS_V3_CONSENSUS_MICRODESC; char *params = NULL; + char *packages = NULL; int added_weights = 0; tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); @@ -1350,18 +1157,18 @@ networkstatus_compute_consensus(smartlist_t *votes, log_warn(LD_DIR, "The other authorities will use consensus method %d, " "which I don't support. Maybe I should upgrade!", consensus_method); - consensus_method = 1; + consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD; } /* Compute medians of time-related things, and figure out how many * routers we might need to talk about. */ { int n_votes = smartlist_len(votes); - time_t *va_times = tor_malloc(n_votes * sizeof(time_t)); - time_t *fu_times = tor_malloc(n_votes * sizeof(time_t)); - time_t *vu_times = tor_malloc(n_votes * sizeof(time_t)); - int *votesec_list = tor_malloc(n_votes * sizeof(int)); - int *distsec_list = tor_malloc(n_votes * sizeof(int)); + time_t *va_times = tor_calloc(n_votes, sizeof(time_t)); + time_t *fu_times = tor_calloc(n_votes, sizeof(time_t)); + time_t *vu_times = tor_calloc(n_votes, sizeof(time_t)); + int *votesec_list = tor_calloc(n_votes, sizeof(int)); + int *distsec_list = tor_calloc(n_votes, sizeof(int)); int n_versioning_clients = 0, n_versioning_servers = 0; smartlist_t *combined_client_versions = smartlist_new(); smartlist_t *combined_server_versions = smartlist_new(); @@ -1400,8 +1207,12 @@ networkstatus_compute_consensus(smartlist_t *votes, vote_seconds = median_int(votesec_list, n_votes); dist_seconds = median_int(distsec_list, n_votes); - tor_assert(valid_after+MIN_VOTE_INTERVAL <= fresh_until); - tor_assert(fresh_until+MIN_VOTE_INTERVAL <= valid_until); + tor_assert(valid_after + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) <= fresh_until); + tor_assert(fresh_until + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) <= valid_until); tor_assert(vote_seconds >= MIN_VOTE_SECONDS); tor_assert(dist_seconds >= MIN_DIST_SECONDS); @@ -1409,6 +1220,11 @@ networkstatus_compute_consensus(smartlist_t *votes, n_versioning_servers); client_versions = compute_consensus_versions_list(combined_client_versions, n_versioning_clients); + if (consensus_method >= MIN_METHOD_FOR_PACKAGE_LINES) { + packages = compute_consensus_package_lines(votes); + } else { + packages = tor_strdup(""); + } SMARTLIST_FOREACH(combined_server_versions, char *, cp, tor_free(cp)); SMARTLIST_FOREACH(combined_client_versions, char *, cp, tor_free(cp)); @@ -1441,10 +1257,8 @@ networkstatus_compute_consensus(smartlist_t *votes, flavor == FLAV_NS ? "" : " ", flavor == FLAV_NS ? "" : flavor_name); - if (consensus_method >= 2) { - smartlist_add_asprintf(chunks, "consensus-method %d\n", - consensus_method); - } + smartlist_add_asprintf(chunks, "consensus-method %d\n", + consensus_method); smartlist_add_asprintf(chunks, "valid-after %s\n" @@ -1453,22 +1267,23 @@ networkstatus_compute_consensus(smartlist_t *votes, "voting-delay %d %d\n" "client-versions %s\n" "server-versions %s\n" + "%s" /* packages */ "known-flags %s\n", va_buf, fu_buf, vu_buf, vote_seconds, dist_seconds, - client_versions, server_versions, flaglist); + client_versions, server_versions, + packages, + flaglist); tor_free(flaglist); } - if (consensus_method >= MIN_METHOD_FOR_PARAMS) { - params = dirvote_compute_params(votes, consensus_method, - total_authorities); - if (params) { - smartlist_add(chunks, tor_strdup("params ")); - smartlist_add(chunks, params); - smartlist_add(chunks, tor_strdup("\n")); - } + params = dirvote_compute_params(votes, consensus_method, + total_authorities); + if (params) { + smartlist_add(chunks, tor_strdup("params ")); + smartlist_add(chunks, params); + smartlist_add(chunks, tor_strdup("\n")); } /* Sort the votes. */ @@ -1482,8 +1297,7 @@ networkstatus_compute_consensus(smartlist_t *votes, e->digest = get_voter(v)->identity_digest; e->is_legacy = 0; smartlist_add(dir_sources, e); - if (consensus_method >= 3 && - !tor_digest_is_zero(get_voter(v)->legacy_id_digest)) { + if (!tor_digest_is_zero(get_voter(v)->legacy_id_digest)) { dir_src_ent_t *e_legacy = tor_malloc_zero(sizeof(dir_src_ent_t)); e_legacy->v = v; e_legacy->digest = get_voter(v)->legacy_id_digest; @@ -1499,9 +1313,6 @@ networkstatus_compute_consensus(smartlist_t *votes, networkstatus_t *v = e->v; networkstatus_voter_info_t *voter = get_voter(v); - if (e->is_legacy) - tor_assert(consensus_method >= 2); - base16_encode(fingerprint, sizeof(fingerprint), e->digest, DIGEST_LEN); base16_encode(votedigest, sizeof(votedigest), voter->vote_digest, DIGEST_LEN); @@ -1559,12 +1370,15 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_t *chosen_flags = smartlist_new(); smartlist_t *versions = smartlist_new(); smartlist_t *exitsummaries = smartlist_new(); - uint32_t *bandwidths_kb = tor_malloc(sizeof(uint32_t) * - smartlist_len(votes)); - uint32_t *measured_bws_kb = tor_malloc(sizeof(uint32_t) * - smartlist_len(votes)); + uint32_t *bandwidths_kb = tor_calloc(smartlist_len(votes), + sizeof(uint32_t)); + uint32_t *measured_bws_kb = tor_calloc(smartlist_len(votes), + sizeof(uint32_t)); + uint32_t *measured_guardfraction = tor_calloc(smartlist_len(votes), + sizeof(uint32_t)); int num_bandwidths; int num_mbws; + int num_guardfraction_inputs; int *n_voter_flags; /* n_voter_flags[j] is the number of flags that * votes[j] knows about. */ @@ -1574,7 +1388,6 @@ networkstatus_compute_consensus(smartlist_t *votes, * is the same flag as votes[j]->known_flags[b]. */ int *named_flag; /* Index of the flag "Named" for votes[j] */ int *unnamed_flag; /* Index of the flag "Unnamed" for votes[j] */ - int chosen_named_idx; int n_authorities_measuring_bandwidth; strmap_t *name_to_id_map = strmap_new(); @@ -1583,16 +1396,15 @@ networkstatus_compute_consensus(smartlist_t *votes, memset(conflict, 0, sizeof(conflict)); memset(unknown, 0xff, sizeof(conflict)); - index = tor_malloc_zero(sizeof(int)*smartlist_len(votes)); - size = tor_malloc_zero(sizeof(int)*smartlist_len(votes)); - n_voter_flags = tor_malloc_zero(sizeof(int) * smartlist_len(votes)); - n_flag_voters = tor_malloc_zero(sizeof(int) * smartlist_len(flags)); - flag_map = tor_malloc_zero(sizeof(int*) * smartlist_len(votes)); - named_flag = tor_malloc_zero(sizeof(int) * smartlist_len(votes)); - unnamed_flag = tor_malloc_zero(sizeof(int) * smartlist_len(votes)); + index = tor_calloc(smartlist_len(votes), sizeof(int)); + size = tor_calloc(smartlist_len(votes), sizeof(int)); + n_voter_flags = tor_calloc(smartlist_len(votes), sizeof(int)); + n_flag_voters = tor_calloc(smartlist_len(flags), sizeof(int)); + flag_map = tor_calloc(smartlist_len(votes), sizeof(int *)); + named_flag = tor_calloc(smartlist_len(votes), sizeof(int)); + unnamed_flag = tor_calloc(smartlist_len(votes), sizeof(int)); for (i = 0; i < smartlist_len(votes); ++i) unnamed_flag[i] = named_flag[i] = -1; - chosen_named_idx = smartlist_string_pos(flags, "Named"); /* Build the flag indexes. Note that no vote can have more than 64 members * for known_flags, so no value will be greater than 63, so it's safe to @@ -1601,8 +1413,8 @@ networkstatus_compute_consensus(smartlist_t *votes, * that they're actually set before doing U64_LITERAL(1) << index with * them.*/ SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { - flag_map[v_sl_idx] = tor_malloc_zero( - sizeof(int)*smartlist_len(v->known_flags)); + flag_map[v_sl_idx] = tor_calloc(smartlist_len(v->known_flags), + sizeof(int)); if (smartlist_len(v->known_flags) > MAX_KNOWN_FLAGS_IN_VOTE) { log_warn(LD_BUG, "Somehow, a vote has %d entries in known_flags", smartlist_len(v->known_flags)); @@ -1622,7 +1434,7 @@ networkstatus_compute_consensus(smartlist_t *votes, } SMARTLIST_FOREACH_END(v); /* Named and Unnamed get treated specially */ - if (consensus_method >= 2) { + { SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { uint64_t nf; if (named_flag[v_sl_idx]<0) @@ -1675,14 +1487,14 @@ networkstatus_compute_consensus(smartlist_t *votes, /* We need to know how many votes measure bandwidth. */ n_authorities_measuring_bandwidth = 0; - SMARTLIST_FOREACH(votes, networkstatus_t *, v, + SMARTLIST_FOREACH(votes, const networkstatus_t *, v, if (v->has_measured_bws) { ++n_authorities_measuring_bandwidth; } ); /* Now go through all the votes */ - flag_counts = tor_malloc(sizeof(int) * smartlist_len(flags)); + flag_counts = tor_calloc(smartlist_len(flags), sizeof(int)); while (1) { vote_routerstatus_t *rs; routerstatus_t rs_out; @@ -1717,6 +1529,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_clear(versions); num_bandwidths = 0; num_mbws = 0; + num_guardfraction_inputs = 0; /* Okay, go through all the entries for this digest. */ SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { @@ -1750,6 +1563,12 @@ networkstatus_compute_consensus(smartlist_t *votes, chosen_name = rs->status.nickname; } + /* Count guardfraction votes and note down the values. */ + if (rs->status.has_guardfraction) { + measured_guardfraction[num_guardfraction_inputs++] = + rs->status.guardfraction_percentage; + } + /* count bandwidths */ if (rs->has_measured_bw) measured_bws_kb[num_mbws++] = rs->measured_bw_kb; @@ -1791,10 +1610,7 @@ networkstatus_compute_consensus(smartlist_t *votes, strlcpy(rs_out.nickname, rs->status.nickname, sizeof(rs_out.nickname)); } - if (consensus_method == 1) { - is_named = chosen_named_idx >= 0 && - (!naming_conflict && flag_counts[chosen_named_idx]); - } else { + { const char *d = strmap_get_lc(name_to_id_map, rs_out.nickname); if (!d) { is_named = is_unnamed = 0; @@ -1811,7 +1627,7 @@ networkstatus_compute_consensus(smartlist_t *votes, if (!strcmp(fl, "Named")) { if (is_named) smartlist_add(chosen_flags, (char*)fl); - } else if (!strcmp(fl, "Unnamed") && consensus_method >= 2) { + } else if (!strcmp(fl, "Unnamed")) { if (is_unnamed) smartlist_add(chosen_flags, (char*)fl); } else { @@ -1831,7 +1647,7 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Starting with consensus method 4 we do not list servers * that are not running in a consensus. See Proposal 138 */ - if (consensus_method >= 4 && !is_running) + if (!is_running) continue; /* Pick the version. */ @@ -1842,12 +1658,23 @@ networkstatus_compute_consensus(smartlist_t *votes, chosen_version = NULL; } + /* If it's a guard and we have enough guardfraction votes, + calculate its consensus guardfraction value. */ + if (is_guard && num_guardfraction_inputs > 2 && + consensus_method >= MIN_METHOD_FOR_GUARDFRACTION) { + rs_out.has_guardfraction = 1; + rs_out.guardfraction_percentage = median_uint32(measured_guardfraction, + num_guardfraction_inputs); + /* final value should be an integer percentage! */ + tor_assert(rs_out.guardfraction_percentage <= 100); + } + /* Pick a bandwidth */ - if (consensus_method >= 6 && num_mbws > 2) { + if (num_mbws > 2) { rs_out.has_bandwidth = 1; rs_out.bw_is_unmeasured = 0; rs_out.bandwidth_kb = median_uint32(measured_bws_kb, num_mbws); - } else if (consensus_method >= 5 && num_bandwidths > 0) { + } else if (num_bandwidths > 0) { rs_out.has_bandwidth = 1; rs_out.bw_is_unmeasured = 1; rs_out.bandwidth_kb = median_uint32(bandwidths_kb, num_bandwidths); @@ -1861,25 +1688,13 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Fix bug 2203: Do not count BadExit nodes as Exits for bw weights */ - if (consensus_method >= MIN_METHOD_TO_CUT_BADEXIT_WEIGHT) { - is_exit = is_exit && !is_bad_exit; - } + is_exit = is_exit && !is_bad_exit; - if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { - if (rs_out.has_bandwidth) { - T += rs_out.bandwidth_kb; - if (is_exit && is_guard) - D += rs_out.bandwidth_kb; - else if (is_exit) - E += rs_out.bandwidth_kb; - else if (is_guard) - G += rs_out.bandwidth_kb; - else - M += rs_out.bandwidth_kb; - } else { - log_warn(LD_BUG, "Missing consensus bandwidth for router %s", - rs_out.nickname); - } + /* Update total bandwidth weights with the bandwidths of this router. */ + { + update_total_bandwidth_weights(&rs_out, + is_exit, is_guard, + &G, &M, &E, &D, &T); } /* Ok, we already picked a descriptor digest we want to list @@ -1896,7 +1711,7 @@ networkstatus_compute_consensus(smartlist_t *votes, * the policy that was most often listed in votes, again breaking * ties like in the previous case. */ - if (consensus_method >= 5) { + { /* Okay, go through all the votes for this router. We prepared * that list previously */ const char *chosen_exitsummary = NULL; @@ -1967,7 +1782,6 @@ networkstatus_compute_consensus(smartlist_t *votes, } if (flavor == FLAV_MICRODESC && - consensus_method >= MIN_METHOD_FOR_MANDATORY_MICRODESC && tor_digest256_is_zero(microdesc_digest)) { /* With no microdescriptor digest, we omit the entry entirely. */ continue; @@ -1999,11 +1813,21 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add(chunks, tor_strdup("\n")); /* Now the weight line. */ if (rs_out.has_bandwidth) { + char *guardfraction_str = NULL; int unmeasured = rs_out.bw_is_unmeasured && consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW; - smartlist_add_asprintf(chunks, "w Bandwidth=%d%s\n", + + /* If we have guardfraction info, include it in the 'w' line. */ + if (rs_out.has_guardfraction) { + tor_asprintf(&guardfraction_str, + " GuardFraction=%u", rs_out.guardfraction_percentage); + } + smartlist_add_asprintf(chunks, "w Bandwidth=%d%s%s\n", rs_out.bandwidth_kb, - unmeasured?" Unmeasured=1":""); + unmeasured?" Unmeasured=1":"", + guardfraction_str ? guardfraction_str : ""); + + tor_free(guardfraction_str); } /* Now the exitpolicy summary line. */ @@ -2031,15 +1855,13 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(exitsummaries); tor_free(bandwidths_kb); tor_free(measured_bws_kb); + tor_free(measured_guardfraction); } - if (consensus_method >= MIN_METHOD_FOR_FOOTER) { - /* Starting with consensus method 9, we clearly mark the directory - * footer region */ - smartlist_add(chunks, tor_strdup("directory-footer\n")); - } + /* Mark the directory footer region */ + smartlist_add(chunks, tor_strdup("directory-footer\n")); - if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS) { + { int64_t weight_scale = BW_WEIGHT_SCALE; char *bw_weight_param = NULL; @@ -2072,13 +1894,8 @@ networkstatus_compute_consensus(smartlist_t *votes, } } - if (consensus_method < 10) { - networkstatus_compute_bw_weights_v9(chunks, G, M, E, D, T, weight_scale); - added_weights = 1; - } else { - added_weights = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, - T, weight_scale); - } + added_weights = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, + T, weight_scale); } /* Add a signature. */ @@ -2119,7 +1936,7 @@ networkstatus_compute_consensus(smartlist_t *votes, } smartlist_add(chunks, signature); - if (legacy_id_key_digest && legacy_signing_key && consensus_method >= 3) { + if (legacy_id_key_digest && legacy_signing_key) { smartlist_add(chunks, tor_strdup("directory-signature ")); base16_encode(fingerprint, sizeof(fingerprint), legacy_id_key_digest, DIGEST_LEN); @@ -2155,7 +1972,7 @@ networkstatus_compute_consensus(smartlist_t *votes, goto done; } // Verify balancing parameters - if (consensus_method >= MIN_METHOD_FOR_BW_WEIGHTS && added_weights) { + if (added_weights) { networkstatus_verify_bw_weights(c, consensus_method); } networkstatus_vote_free(c); @@ -2165,6 +1982,7 @@ networkstatus_compute_consensus(smartlist_t *votes, tor_free(client_versions); tor_free(server_versions); + tor_free(packages); SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp)); smartlist_free(flags); SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); @@ -2173,6 +1991,78 @@ networkstatus_compute_consensus(smartlist_t *votes, return result; } +/** Given a list of networkstatus_t for each vote, return a newly allocated + * string containing the "package" lines for the vote. */ +STATIC char * +compute_consensus_package_lines(smartlist_t *votes) +{ + const int n_votes = smartlist_len(votes); + + /* This will be a map from "packagename version" strings to arrays + * of const char *, with the i'th member of the array corresponding to the + * package line from the i'th vote. + */ + strmap_t *package_status = strmap_new(); + + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { + if (! v->package_lines) + continue; + SMARTLIST_FOREACH_BEGIN(v->package_lines, const char *, line) { + if (! validate_recommended_package_line(line)) + continue; + + /* Skip 'cp' to the second space in the line. */ + const char *cp = strchr(line, ' '); + if (!cp) continue; + ++cp; + cp = strchr(cp, ' '); + if (!cp) continue; + + char *key = tor_strndup(line, cp - line); + + const char **status = strmap_get(package_status, key); + if (!status) { + status = tor_calloc(n_votes, sizeof(const char *)); + strmap_set(package_status, key, status); + } + status[v_sl_idx] = line; /* overwrite old value */ + tor_free(key); + } SMARTLIST_FOREACH_END(line); + } SMARTLIST_FOREACH_END(v); + + smartlist_t *entries = smartlist_new(); /* temporary */ + smartlist_t *result_list = smartlist_new(); /* output */ + STRMAP_FOREACH(package_status, key, const char **, values) { + int i, count=-1; + for (i = 0; i < n_votes; ++i) { + if (values[i]) + smartlist_add(entries, (void*) values[i]); + } + smartlist_sort_strings(entries); + int n_voting_for_entry = smartlist_len(entries); + const char *most_frequent = + smartlist_get_most_frequent_string_(entries, &count); + + if (n_voting_for_entry >= 3 && count > n_voting_for_entry / 2) { + smartlist_add_asprintf(result_list, "package %s\n", most_frequent); + } + + smartlist_clear(entries); + + } STRMAP_FOREACH_END; + + smartlist_sort_strings(result_list); + + char *result = smartlist_join_strings(result_list, "", 0, NULL); + + SMARTLIST_FOREACH(result_list, char *, cp, tor_free(cp)); + smartlist_free(result_list); + smartlist_free(entries); + strmap_free(package_status, tor_free_); + + return result; +} + /** Given a consensus vote <b>target</b> and a set of detached signatures in * <b>sigs</b> that correspond to the same consensus, check whether there are * any new signatures in <b>src_voter_list</b> that should be added to @@ -2279,8 +2169,11 @@ networkstatus_add_detached_signatures(networkstatus_t *target, if (!sig->good_signature && !sig->bad_signature) { cert = authority_cert_get_by_digests(sig->identity_digest, sig->signing_key_digest); - if (cert) - networkstatus_check_document_signature(target, sig, cert); + if (cert) { + /* Not checking the return value here, since we are going to look + * at the status of sig->good_signature in a moment. */ + (void) networkstatus_check_document_signature(target, sig, cert); + } } /* If this signature is good, or we don't have any signature yet, @@ -3020,7 +2913,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) goto discard; } else if (v->vote->published < vote->published) { log_notice(LD_DIR, "Replacing an older pending vote from this " - "directory."); + "directory (%s)", vi->address); cached_dir_decref(v->vote_body); networkstatus_vote_free(v->vote); v->vote_body = new_cached_dir(tor_strndup(vote_body, @@ -3602,8 +3495,8 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) { smartlist_t *lst = microdescs_parse_from_string(output, - output+strlen(output), 0, - SAVED_NOWHERE); + output+strlen(output), 0, + SAVED_NOWHERE, NULL); if (smartlist_len(lst) != 1) { log_warn(LD_DIR, "We generated a microdescriptor we couldn't parse."); SMARTLIST_FOREACH(lst, microdesc_t *, md, microdesc_free(md)); @@ -3664,7 +3557,7 @@ static const struct consensus_method_range_t { int low; int high; } microdesc_consensus_methods[] = { - {MIN_METHOD_FOR_MICRODESC, MIN_METHOD_FOR_A_LINES - 1}, + {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_A_LINES - 1}, {MIN_METHOD_FOR_A_LINES, MIN_METHOD_FOR_P6_LINES - 1}, {MIN_METHOD_FOR_P6_LINES, MIN_METHOD_FOR_NTOR_KEY - 1}, {MIN_METHOD_FOR_NTOR_KEY, MIN_METHOD_FOR_ID_HASH_IN_MD - 1}, diff --git a/src/or/dirvote.h b/src/or/dirvote.h index 4c57e43661..542563b708 100644 --- a/src/or/dirvote.h +++ b/src/or/dirvote.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,34 +14,48 @@ #include "testsupport.h" +/* + * Ideally, assuming synced clocks, we should only need 1 second for each of: + * - Vote + * - Distribute + * - Consensus Publication + * As we can gather descriptors continuously. + * (Could we even go as far as publishing the previous consensus, + * in the same second that we vote for the next one?) + * But we're not there yet: these are the lowest working values at this time. + */ + /** Lowest allowable value for VoteSeconds. */ #define MIN_VOTE_SECONDS 2 +/** Lowest allowable value for VoteSeconds when TestingTorNetwork is 1 */ +#define MIN_VOTE_SECONDS_TESTING 2 + /** Lowest allowable value for DistSeconds. */ #define MIN_DIST_SECONDS 2 -/** Smallest allowable voting interval. */ +/** Lowest allowable value for DistSeconds when TestingTorNetwork is 1 */ +#define MIN_DIST_SECONDS_TESTING 2 + +/** Lowest allowable voting interval. */ #define MIN_VOTE_INTERVAL 300 +/** Lowest allowable voting interval when TestingTorNetwork is 1: + * Voting Interval can be: + * 10, 12, 15, 18, 20, 24, 25, 30, 36, 40, 45, 50, 60, ... + * Testing Initial Voting Interval can be: + * 5, 6, 8, 9, or any of the possible values for Voting Interval, + * as they both need to evenly divide 30 minutes. + * If clock desynchronisation is an issue, use an interval of at least: + * 18 * drift in seconds, to allow for a clock slop factor */ +#define MIN_VOTE_INTERVAL_TESTING \ + (((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)*2) + +#define MIN_VOTE_INTERVAL_TESTING_INITIAL \ + ((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1) + +/** The lowest consensus method that we currently support. */ +#define MIN_SUPPORTED_CONSENSUS_METHOD 13 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 18 - -/** Lowest consensus method that contains a 'directory-footer' marker */ -#define MIN_METHOD_FOR_FOOTER 9 - -/** Lowest consensus method that contains bandwidth weights */ -#define MIN_METHOD_FOR_BW_WEIGHTS 9 - -/** Lowest consensus method that contains consensus params */ -#define MIN_METHOD_FOR_PARAMS 7 - -/** Lowest consensus method that generates microdescriptors */ -#define MIN_METHOD_FOR_MICRODESC 8 - -/** Lowest consensus method that doesn't count bad exits as exits for weight */ -#define MIN_METHOD_TO_CUT_BADEXIT_WEIGHT 11 - -/** Lowest consensus method that ensures a majority of authorities voted - * for a param. */ -#define MIN_METHOD_FOR_MAJORITY_PARAMS 12 +#define MAX_SUPPORTED_CONSENSUS_METHOD 20 /** Lowest consensus method where microdesc consensuses omit any entry * with no microdesc. */ @@ -65,8 +79,16 @@ * microdescriptors. */ #define MIN_METHOD_FOR_ID_HASH_IN_MD 18 +/** Lowest consensus method where we include "package" lines*/ +#define MIN_METHOD_FOR_PACKAGE_LINES 19 + +/** Lowest consensus method where authorities may include + * GuardFraction information in microdescriptors. */ +#define MIN_METHOD_FOR_GUARDFRACTION 20 + /** Default bandwidth to clip unmeasured bandwidths to using method >= - * MIN_METHOD_TO_CLIP_UNMEASURED_BW */ + * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not + * get confused with the above macros.) */ #define DEFAULT_MAX_UNMEASURED_BW_KB 20 void dirvote_free_all(void); @@ -116,8 +138,8 @@ const cached_dir_t *dirvote_get_vote(const char *fp, int flags); void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, routerinfo_t *ri, time_t now, - int naming, int listbadexits, - int listbaddirs, int vote_on_hsdirs); + int listbadexits, + int vote_on_hsdirs); networkstatus_t * dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, authority_cert_t *cert); @@ -146,6 +168,7 @@ STATIC char *format_networkstatus_vote(crypto_pk_t *private_key, networkstatus_t *v3_ns); STATIC char *dirvote_compute_params(smartlist_t *votes, int method, int total_authorities); +STATIC char *compute_consensus_package_lines(smartlist_t *votes); #endif #endif diff --git a/src/or/dns.c b/src/or/dns.c index b55bf7384e..cc4a169422 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -244,8 +244,8 @@ cached_resolve_hash(cached_resolve_t *a) HT_PROTOTYPE(cache_map, cached_resolve_t, node, cached_resolve_hash, cached_resolves_eq) -HT_GENERATE(cache_map, cached_resolve_t, node, cached_resolve_hash, - cached_resolves_eq, 0.6, malloc, realloc, free) +HT_GENERATE2(cache_map, cached_resolve_t, node, cached_resolve_hash, + cached_resolves_eq, 0.6, tor_reallocarray_, tor_free_) /** Initialize the DNS cache. */ static void diff --git a/src/or/dns.h b/src/or/dns.h index 022cd4ac63..b13ab0f890 100644 --- a/src/or/dns.h +++ b/src/or/dns.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index 9b0368dd09..f3618cc2c5 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -139,13 +139,13 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) } if (q->type == EVDNS_TYPE_A || q->type == EVDNS_QTYPE_ALL) { - entry_conn->ipv4_traffic_ok = 1; - entry_conn->ipv6_traffic_ok = 0; - entry_conn->prefer_ipv6_traffic = 0; + entry_conn->entry_cfg.ipv4_traffic = 1; + entry_conn->entry_cfg.ipv6_traffic = 0; + entry_conn->entry_cfg.prefer_ipv6 = 0; } else if (q->type == EVDNS_TYPE_AAAA) { - entry_conn->ipv4_traffic_ok = 0; - entry_conn->ipv6_traffic_ok = 1; - entry_conn->prefer_ipv6_traffic = 1; + entry_conn->entry_cfg.ipv4_traffic = 0; + entry_conn->entry_cfg.ipv6_traffic = 1; + entry_conn->entry_cfg.prefer_ipv6 = 1; } strlcpy(entry_conn->socks_request->address, q->name, @@ -153,8 +153,8 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) entry_conn->socks_request->listener_type = listener->base_.type; entry_conn->dns_server_request = req; - entry_conn->isolation_flags = listener->isolation_flags; - entry_conn->session_group = listener->session_group; + entry_conn->entry_cfg.isolation_flags = listener->entry_cfg.isolation_flags; + entry_conn->entry_cfg.session_group = listener->entry_cfg.session_group; entry_conn->nym_epoch = get_signewnym_epoch(); if (connection_add(ENTRY_TO_CONN(entry_conn)) < 0) { @@ -230,9 +230,9 @@ dnsserv_launch_request(const char *name, int reverse, entry_conn->socks_request->listener_type = CONN_TYPE_CONTROL_LISTENER; entry_conn->original_dest_address = tor_strdup(name); - entry_conn->session_group = SESSION_GROUP_CONTROL_RESOLVE; + entry_conn->entry_cfg.session_group = SESSION_GROUP_CONTROL_RESOLVE; entry_conn->nym_epoch = get_signewnym_epoch(); - entry_conn->isolation_flags = ISO_DEFAULT; + entry_conn->entry_cfg.isolation_flags = ISO_DEFAULT; if (connection_add(TO_CONN(conn))<0) { log_warn(LD_APP, "Couldn't register dummy connection for RESOLVE request"); diff --git a/src/or/dnsserv.h b/src/or/dnsserv.h index 687a77e59e..09ad5d7759 100644 --- a/src/or/dnsserv.h +++ b/src/or/dnsserv.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 66b7201187..30108b6041 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,6 +12,8 @@ * circumvention). **/ +#define ENTRYNODES_PRIVATE + #include "or.h" #include "circpathbias.h" #include "circuitbuild.h" @@ -154,21 +156,41 @@ entry_guard_set_status(entry_guard_t *e, const node_t *node, /** Return true iff enough time has passed since we last tried to connect * to the unreachable guard <b>e</b> that we're willing to try again. */ -static int -entry_is_time_to_retry(entry_guard_t *e, time_t now) +STATIC int +entry_is_time_to_retry(const entry_guard_t *e, time_t now) { - long diff; + struct guard_retry_period_s { + time_t period_duration; + time_t interval_during_period; + }; + + struct guard_retry_period_s periods[] = { + { 6*60*60, 60*60 }, /* For first 6 hrs., retry hourly; */ + { 3*24*60*60, 4*60*60 }, /* Then retry every 4 hrs. until the + 3-day mark; */ + { 7*24*60*60, 18*60*60 }, /* After 3 days, retry every 18 hours until + 1 week mark. */ + { TIME_MAX, 36*60*60 } /* After 1 week, retry every 36 hours. */ + }; + + time_t ith_deadline_for_retry; + time_t unreachable_for; + unsigned i; + if (e->last_attempted < e->unreachable_since) return 1; - diff = now - e->unreachable_since; - if (diff < 6*60*60) - return now > (e->last_attempted + 60*60); - else if (diff < 3*24*60*60) - return now > (e->last_attempted + 4*60*60); - else if (diff < 7*24*60*60) - return now > (e->last_attempted + 18*60*60); - else - return now > (e->last_attempted + 36*60*60); + + unreachable_for = now - e->unreachable_since; + + for (i = 0; i < ARRAY_LENGTH(periods); i++) { + if (unreachable_for <= periods[i].period_duration) { + ith_deadline_for_retry = e->last_attempted + + periods[i].interval_during_period; + + return (now > ith_deadline_for_retry); + } + } + return 0; } /** Return the node corresponding to <b>e</b>, if <b>e</b> is @@ -188,12 +210,17 @@ entry_is_time_to_retry(entry_guard_t *e, time_t now) * If need_descriptor is true, only return the node if we currently have * a descriptor (routerinfo or microdesc) for it. */ -static INLINE const node_t * -entry_is_live(entry_guard_t *e, int need_uptime, int need_capacity, - int assume_reachable, int need_descriptor, const char **msg) +STATIC const node_t * +entry_is_live(const entry_guard_t *e, entry_is_live_flags_t flags, + const char **msg) { const node_t *node; const or_options_t *options = get_options(); + int need_uptime = (flags & ENTRY_NEED_UPTIME) != 0; + int need_capacity = (flags & ENTRY_NEED_CAPACITY) != 0; + const int assume_reachable = (flags & ENTRY_ASSUME_REACHABLE) != 0; + const int need_descriptor = (flags & ENTRY_NEED_DESCRIPTOR) != 0; + tor_assert(msg); if (e->path_bias_disabled) { @@ -255,12 +282,18 @@ num_live_entry_guards(int for_directory) { int n = 0; const char *msg; + /* Set the entry node attributes we are interested in. */ + entry_is_live_flags_t entry_flags = ENTRY_NEED_CAPACITY; + if (!for_directory) { + entry_flags |= ENTRY_NEED_DESCRIPTOR; + } + if (! entry_guards) return 0; SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { if (for_directory && !entry->is_dir_cache) continue; - if (entry_is_live(entry, 0, 1, 0, !for_directory, &msg)) + if (entry_is_live(entry, entry_flags, &msg)) ++n; } SMARTLIST_FOREACH_END(entry); return n; @@ -289,7 +322,7 @@ log_entry_guards(int severity) SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { const char *msg = NULL; - if (entry_is_live(e, 0, 1, 0, 0, &msg)) + if (entry_is_live(e, ENTRY_NEED_CAPACITY, &msg)) smartlist_add_asprintf(elements, "%s [%s] (up %s)", e->nickname, hex_str(e->identity, DIGEST_LEN), @@ -350,7 +383,7 @@ control_event_guard_deferred(void) * If <b>chosen</b> is defined, use that one, and if it's not * already in our entry_guards list, put it at the *beginning*. * Else, put the one we pick at the end of the list. */ -static const node_t * +STATIC const node_t * add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, int for_discovery, int for_directory) { @@ -437,7 +470,7 @@ add_an_entry_guard(const node_t *chosen, int reset_status, int prepend, /** Choose how many entry guards or directory guards we'll use. If * <b>for_directory</b> is true, we return how many directory guards to * use; else we return how many entry guards to use. */ -static int +STATIC int decide_num_guards(const or_options_t *options, int for_directory) { if (for_directory) { @@ -676,7 +709,7 @@ entry_guards_compute_status(const or_options_t *options, time_t now) SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { const char *reason = digestmap_get(reasons, entry->identity); const char *live_msg = ""; - const node_t *r = entry_is_live(entry, 0, 1, 0, 0, &live_msg); + const node_t *r = entry_is_live(entry, ENTRY_NEED_CAPACITY, &live_msg); log_info(LD_CIRC, "Summary: Entry %s [%s] is %s, %s%s%s, and %s%s.", entry->nickname, hex_str(entry->identity, DIGEST_LEN), @@ -794,7 +827,9 @@ entry_guard_register_connect_status(const char *digest, int succeeded, break; if (e->made_contact) { const char *msg; - const node_t *r = entry_is_live(e, 0, 1, 1, 0, &msg); + const node_t *r = entry_is_live(e, + ENTRY_NEED_CAPACITY | ENTRY_ASSUME_REACHABLE, + &msg); if (r && e->unreachable_since) { refuse_conn = 1; e->can_retry = 1; @@ -847,7 +882,7 @@ update_node_guard_status(void) /** Adjust the entry guards list so that it only contains entries from * EntryNodes, adding new entries from EntryNodes to the list as needed. */ -static void +STATIC void entry_guards_set_from_config(const or_options_t *options) { smartlist_t *entry_nodes, *worse_entry_nodes, *entry_fps; @@ -968,7 +1003,8 @@ node_understands_microdescriptors(const node_t *node) } /** Return true iff <b>node</b> is able to answer directory questions - * of type <b>dirinfo</b>. */ + * of type <b>dirinfo</b>. Always returns true if <b>dirinfo</b> is + * NO_DIRINFO (zero). */ static int node_can_handle_dirinfo(const node_t *node, dirinfo_type_t dirinfo) { @@ -990,13 +1026,13 @@ node_can_handle_dirinfo(const node_t *node, dirinfo_type_t dirinfo) * <b>state</b> is non-NULL, this is for a specific circuit -- * make sure not to pick this circuit's exit or any node in the * exit's family. If <b>state</b> is NULL, we're looking for a random - * guard (likely a bridge). If <b>dirinfo</b> is not NO_DIRINFO, then - * only select from nodes that know how to answer directory questions + * guard (likely a bridge). If <b>dirinfo</b> is not NO_DIRINFO (zero), + * then only select from nodes that know how to answer directory questions * of that type. */ const node_t * choose_random_entry(cpath_build_state_t *state) { - return choose_random_entry_impl(state, 0, 0, NULL); + return choose_random_entry_impl(state, 0, NO_DIRINFO, NULL); } /** Pick a live (up and listed) directory guard from entry_guards for @@ -1007,47 +1043,61 @@ choose_random_dirguard(dirinfo_type_t type) return choose_random_entry_impl(NULL, 1, type, NULL); } -/** Helper for choose_random{entry,dirguard}. */ -static const node_t * -choose_random_entry_impl(cpath_build_state_t *state, int for_directory, - dirinfo_type_t dirinfo_type, int *n_options_out) +/** Filter <b>all_entry_guards</b> for usable entry guards and put them + * in <b>live_entry_guards</b>. We filter based on whether the node is + * currently alive, and on whether it satisfies the restrictions + * imposed by the other arguments of this function. + * + * We don't place more guards than NumEntryGuards in <b>live_entry_guards</b>. + * + * If <b>chosen_exit</b> is set, it contains the exit node of this + * circuit. Make sure to not use it or its family as an entry guard. + * + * If <b>need_uptime</b> is set, we are looking for a stable entry guard. + * if <b>need_capacity</b> is set, we are looking for a fast entry guard. + * + * The rest of the arguments are the same as in choose_random_entry_impl(). + * + * Return 1 if we should choose a guard right away. Return 0 if we + * should try to add more nodes to our list before deciding on a + * guard. + */ +STATIC int +populate_live_entry_guards(smartlist_t *live_entry_guards, + const smartlist_t *all_entry_guards, + const node_t *chosen_exit, + dirinfo_type_t dirinfo_type, + int for_directory, + int need_uptime, int need_capacity) { const or_options_t *options = get_options(); - smartlist_t *live_entry_guards = smartlist_new(); - smartlist_t *exit_family = smartlist_new(); - const node_t *chosen_exit = - state?build_state_get_exit_node(state) : NULL; const node_t *node = NULL; - int need_uptime = state ? state->need_uptime : 0; - int need_capacity = state ? state->need_capacity : 0; - int preferred_min, consider_exit_family = 0; - int need_descriptor = !for_directory; const int num_needed = decide_num_guards(options, for_directory); + smartlist_t *exit_family = smartlist_new(); + int retval = 0; + entry_is_live_flags_t entry_flags = 0; - if (n_options_out) - *n_options_out = 0; + { /* Set the flags we want our entry node to have */ + if (need_uptime) { + entry_flags |= ENTRY_NEED_UPTIME; + } + if (need_capacity) { + entry_flags |= ENTRY_NEED_CAPACITY; + } + if (!for_directory) { + entry_flags |= ENTRY_NEED_DESCRIPTOR; + } + } + + tor_assert(all_entry_guards); if (chosen_exit) { nodelist_add_node_and_family(exit_family, chosen_exit); - consider_exit_family = 1; } - if (!entry_guards) - entry_guards = smartlist_new(); - - if (should_add_entry_nodes) - entry_guards_set_from_config(options); - - if (!entry_list_is_constrained(options) && - smartlist_len(entry_guards) < num_needed) - pick_entry_guards(options, for_directory); - - retry: - smartlist_clear(live_entry_guards); - SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, entry) { + SMARTLIST_FOREACH_BEGIN(all_entry_guards, const entry_guard_t *, entry) { const char *msg; - node = entry_is_live(entry, need_uptime, need_capacity, 0, - need_descriptor, &msg); + node = entry_is_live(entry, entry_flags, &msg); if (!node) continue; /* down, no point */ if (for_directory) { @@ -1056,39 +1106,96 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, } if (node == chosen_exit) continue; /* don't pick the same node for entry and exit */ - if (consider_exit_family && smartlist_contains(exit_family, node)) + if (smartlist_contains(exit_family, node)) continue; /* avoid relays that are family members of our exit */ if (dirinfo_type != NO_DIRINFO && !node_can_handle_dirinfo(node, dirinfo_type)) continue; /* this node won't be able to answer our dir questions */ -#if 0 /* since EntryNodes is always strict now, this clause is moot */ - if (options->EntryNodes && - !routerset_contains_node(options->EntryNodes, node)) { - /* We've come to the end of our preferred entry nodes. */ - if (smartlist_len(live_entry_guards)) - goto choose_and_finish; /* only choose from the ones we like */ - if (options->StrictNodes) { - /* in theory this case should never happen, since - * entry_guards_set_from_config() drops unwanted relays */ - tor_fragile_assert(); - } else { - log_info(LD_CIRC, - "No relays from EntryNodes available. Using others."); - } - } -#endif smartlist_add(live_entry_guards, (void*)node); if (!entry->made_contact) { /* Always start with the first not-yet-contacted entry * guard. Otherwise we might add several new ones, pick * the second new one, and now we've expanded our entry * guard list without needing to. */ - goto choose_and_finish; + retval = 1; + goto done; + } + if (smartlist_len(live_entry_guards) >= num_needed) { + retval = 1; + goto done; /* We picked enough entry guards. Done! */ } - if (smartlist_len(live_entry_guards) >= num_needed) - goto choose_and_finish; /* we have enough */ } SMARTLIST_FOREACH_END(entry); + done: + smartlist_free(exit_family); + + return retval; +} + +/** Pick a node to be used as the entry guard of a circuit. + * + * If <b>state</b> is set, it contains the information we know about + * the upcoming circuit. + * + * If <b>for_directory</b> is set, we are looking for a directory guard. + * + * <b>dirinfo_type</b> contains the kind of directory information we + * are looking for in our node, or NO_DIRINFO (zero) if we are not + * looking for any particular directory information (when set to + * NO_DIRINFO, the <b>dirinfo_type</b> filter is ignored). + * + * If <b>n_options_out</b> is set, we set it to the number of + * candidate guard nodes we had before picking a specific guard node. + * + * On success, return the node that should be used as the entry guard + * of the circuit. Return NULL if no such node could be found. + * + * Helper for choose_random{entry,dirguard}. +*/ +static const node_t * +choose_random_entry_impl(cpath_build_state_t *state, int for_directory, + dirinfo_type_t dirinfo_type, int *n_options_out) +{ + const or_options_t *options = get_options(); + smartlist_t *live_entry_guards = smartlist_new(); + const node_t *chosen_exit = + state?build_state_get_exit_node(state) : NULL; + const node_t *node = NULL; + int need_uptime = state ? state->need_uptime : 0; + int need_capacity = state ? state->need_capacity : 0; + int preferred_min = 0; + const int num_needed = decide_num_guards(options, for_directory); + int retval = 0; + + if (n_options_out) + *n_options_out = 0; + + if (!entry_guards) + entry_guards = smartlist_new(); + + if (should_add_entry_nodes) + entry_guards_set_from_config(options); + + if (!entry_list_is_constrained(options) && + smartlist_len(entry_guards) < num_needed) + pick_entry_guards(options, for_directory); + + retry: + smartlist_clear(live_entry_guards); + + /* Populate the list of live entry guards so that we pick one of + them. */ + retval = populate_live_entry_guards(live_entry_guards, + entry_guards, + chosen_exit, + dirinfo_type, + for_directory, + need_uptime, need_capacity); + + if (retval == 1) { /* We should choose a guard right now. */ + goto choose_and_finish; + } + if (entry_list_is_constrained(options)) { /* If we prefer the entry nodes we've got, and we have at least * one choice, that's great. Use it. */ @@ -1127,18 +1234,7 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, need_capacity = 0; goto retry; } -#if 0 - /* Removing this retry logic: if we only allow one exit, and it is in the - same family as all our entries, then we are just plain not going to win - here. */ - if (!node && entry_list_is_constrained(options) && consider_exit_family) { - /* still no? if we're using bridges or have strictentrynodes - * set, and our chosen exit is in the same family as all our - * bridges/entry guards, then be flexible about families. */ - consider_exit_family = 0; - goto retry; - } -#endif + /* live_entry_guards may be empty below. Oh well, we tried. */ } @@ -1156,7 +1252,6 @@ choose_random_entry_impl(cpath_build_state_t *state, int for_directory, if (n_options_out) *n_options_out = smartlist_len(live_entry_guards); smartlist_free(live_entry_guards); - smartlist_free(exit_family); return node; } @@ -1224,7 +1319,7 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) "EntryGuardDownSince/UnlistedSince without EntryGuard"); break; } - if (parse_iso_time(line->value, &when)<0) { + if (parse_iso_time_(line->value, &when, 0)<0) { *msg = tor_strdup("Unable to parse entry nodes: " "Bad time in EntryGuardDownSince/UnlistedSince"); break; @@ -1428,6 +1523,13 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) return *msg ? -1 : 0; } +/** How long will we let a change in our guard nodes stay un-saved + * when we are trying to avoid disk writes? */ +#define SLOW_GUARD_STATE_FLUSH_TIME 600 +/** How long will we let a change in our guard nodes stay un-saved + * when we are not trying to avoid disk writes? */ +#define FAST_GUARD_STATE_FLUSH_TIME 30 + /** Our list of entry guards has changed, or some element of one * of our entry guards has changed. Write the changes to disk within * the next few minutes. @@ -1438,8 +1540,12 @@ entry_guards_changed(void) time_t when; entry_guards_dirty = 1; + if (get_options()->AvoidDiskWrites) + when = time(NULL) + SLOW_GUARD_STATE_FLUSH_TIME; + else + when = time(NULL) + FAST_GUARD_STATE_FLUSH_TIME; + /* or_state_save() will call entry_guards_update_state(). */ - when = get_options()->AvoidDiskWrites ? time(NULL) + 3600 : time(NULL)+600; or_state_mark_dirty(get_or_state(), when); } @@ -1560,6 +1666,9 @@ getinfo_helper_entry_guards(control_connection_t *conn, } else if (e->bad_since) { when = e->bad_since; status = "unusable"; + } else if (e->unreachable_since) { + when = e->unreachable_since; + status = "down"; } else { status = "up"; } @@ -1588,6 +1697,63 @@ getinfo_helper_entry_guards(control_connection_t *conn, return 0; } +/** Return 0 if we should apply guardfraction information found in the + * consensus. A specific consensus can be specified with the + * <b>ns</b> argument, if NULL the most recent one will be picked.*/ +int +should_apply_guardfraction(const networkstatus_t *ns) +{ + /* We need to check the corresponding torrc option and the consensus + * parameter if we need to. */ + const or_options_t *options = get_options(); + + /* If UseGuardFraction is 'auto' then check the same-named consensus + * parameter. If the consensus parameter is not present, default to + * "off". */ + if (options->UseGuardFraction == -1) { + return networkstatus_get_param(ns, "UseGuardFraction", + 0, /* default to "off" */ + 0, 1); + } + + return options->UseGuardFraction; +} + +/* Given the original bandwidth of a guard and its guardfraction, + * calculate how much bandwidth the guard should have as a guard and + * as a non-guard. + * + * Quoting from proposal236: + * + * Let Wpf denote the weight from the 'bandwidth-weights' line a + * client would apply to N for position p if it had the guard + * flag, Wpn the weight if it did not have the guard flag, and B the + * measured bandwidth of N in the consensus. Then instead of choosing + * N for position p proportionally to Wpf*B or Wpn*B, clients should + * choose N proportionally to F*Wpf*B + (1-F)*Wpn*B. + * + * This function fills the <b>guardfraction_bw</b> structure. It sets + * <b>guard_bw</b> to F*B and <b>non_guard_bw</b> to (1-F)*B. + */ +void +guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw, + int orig_bandwidth, + uint32_t guardfraction_percentage) +{ + double guardfraction_fraction; + + /* Turn the percentage into a fraction. */ + tor_assert(guardfraction_percentage <= 100); + guardfraction_fraction = guardfraction_percentage / 100.0; + + long guard_bw = tor_lround(guardfraction_fraction * orig_bandwidth); + tor_assert(guard_bw <= INT_MAX); + + guardfraction_bw->guard_bw = (int) guard_bw; + + guardfraction_bw->non_guard_bw = orig_bandwidth - (int) guard_bw; +} + /** A list of configured bridges. Whenever we actually get a descriptor * for one, we add it as an entry guard. Note that the order of bridges * in this list does not necessarily correspond to the order of bridges @@ -1824,8 +1990,8 @@ bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port, /** Return True if we have a bridge that uses a transport with name * <b>transport_name</b>. */ -int -transport_is_needed(const char *transport_name) +MOCK_IMPL(int, +transport_is_needed, (const char *transport_name)) { if (!bridge_list) return 0; @@ -2199,6 +2365,13 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) node = node_get_mutable_by_id(ri->cache_info.identity_digest); tor_assert(node); rewrite_node_address_for_bridge(bridge, node); + if (tor_digest_is_zero(bridge->identity)) { + memcpy(bridge->identity,ri->cache_info.identity_digest, DIGEST_LEN); + log_notice(LD_DIR, "Learned identity %s for bridge at %s:%d", + hex_str(bridge->identity, DIGEST_LEN), + fmt_and_decorate_addr(&bridge->addr), + (int) bridge->port); + } add_an_entry_guard(node, 1, 1, 0, 0); log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname, @@ -2255,7 +2428,9 @@ entries_retry_helper(const or_options_t *options, int act) SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { node = node_get_by_id(e->identity); if (node && node_has_descriptor(node) && - node_is_bridge(node) == need_bridges) { + node_is_bridge(node) == need_bridges && + (!need_bridges || (!e->bad_since && + node_is_a_configured_bridge(node)))) { any_known = 1; if (node->is_running) any_running = 1; /* some entry is both known and running */ diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h index e229f3b79a..107a562506 100644 --- a/src/or/entrynodes.h +++ b/src/or/entrynodes.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -77,6 +77,38 @@ int num_live_entry_guards(int for_directory); #endif +#ifdef ENTRYNODES_PRIVATE +STATIC const node_t *add_an_entry_guard(const node_t *chosen, + int reset_status, int prepend, + int for_discovery, int for_directory); + +STATIC int populate_live_entry_guards(smartlist_t *live_entry_guards, + const smartlist_t *all_entry_guards, + const node_t *chosen_exit, + dirinfo_type_t dirinfo_type, + int for_directory, + int need_uptime, int need_capacity); +STATIC int decide_num_guards(const or_options_t *options, int for_directory); + +STATIC void entry_guards_set_from_config(const or_options_t *options); + +/** Flags to be passed to entry_is_live() to indicate what kind of + * entry nodes we are looking for. */ +typedef enum { + ENTRY_NEED_UPTIME = 1<<0, + ENTRY_NEED_CAPACITY = 1<<1, + ENTRY_ASSUME_REACHABLE = 1<<2, + ENTRY_NEED_DESCRIPTOR = 1<<3, +} entry_is_live_flags_t; + +STATIC const node_t *entry_is_live(const entry_guard_t *e, + entry_is_live_flags_t flags, + const char **msg); + +STATIC int entry_is_time_to_retry(const entry_guard_t *e, time_t now); + +#endif + void remove_all_entry_guards(void); void entry_guards_compute_status(const or_options_t *options, time_t now); @@ -122,11 +154,27 @@ struct transport_t; int get_transport_by_bridge_addrport(const tor_addr_t *addr, uint16_t port, const struct transport_t **transport); -int transport_is_needed(const char *transport_name); +MOCK_DECL(int, transport_is_needed, (const char *transport_name)); int validate_pluggable_transports_config(void); double pathbias_get_close_success_count(entry_guard_t *guard); double pathbias_get_use_success_count(entry_guard_t *guard); +/** Contains the bandwidth of a relay as a guard and as a non-guard + * after the guardfraction has been considered. */ +typedef struct guardfraction_bandwidth_t { + /** Bandwidth as a guard after guardfraction has been considered. */ + int guard_bw; + /** Bandwidth as a non-guard after guardfraction has been considered. */ + int non_guard_bw; +} guardfraction_bandwidth_t; + +int should_apply_guardfraction(const networkstatus_t *ns); + +void +guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw, + int orig_bandwidth, + uint32_t guardfraction_percentage); + #endif diff --git a/src/or/eventdns_tor.h b/src/or/eventdns_tor.h index 69662281bc..9d51f0960e 100644 --- a/src/or/eventdns_tor.h +++ b/src/or/eventdns_tor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_EVENTDNS_TOR_H diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c index 9b550ee90e..e8c8aa60a4 100644 --- a/src/or/ext_orport.c +++ b/src/or/ext_orport.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012, The Tor Project, Inc. */ +/* Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/ext_orport.h b/src/or/ext_orport.h index ce45e5f418..8b2542f937 100644 --- a/src/or/ext_orport.h +++ b/src/or/ext_orport.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef EXT_ORPORT_H diff --git a/src/or/fp_pair.c b/src/or/fp_pair.c index 55e4c89a42..42bebcd847 100644 --- a/src/or/fp_pair.c +++ b/src/or/fp_pair.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013, The Tor Project, Inc. */ +/* Copyright (c) 2013-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" @@ -42,9 +42,9 @@ fp_pair_map_entry_hash(const fp_pair_map_entry_t *a) HT_PROTOTYPE(fp_pair_map_impl, fp_pair_map_entry_s, node, fp_pair_map_entry_hash, fp_pair_map_entries_eq) -HT_GENERATE(fp_pair_map_impl, fp_pair_map_entry_s, node, - fp_pair_map_entry_hash, fp_pair_map_entries_eq, - 0.6, tor_malloc, tor_realloc, tor_free) +HT_GENERATE2(fp_pair_map_impl, fp_pair_map_entry_s, node, + fp_pair_map_entry_hash, fp_pair_map_entries_eq, + 0.6, tor_reallocarray_, tor_free_) /** Constructor to create a new empty map from fp_pair_t to void * */ diff --git a/src/or/fp_pair.h b/src/or/fp_pair.h index 89f664a813..0830ab1f36 100644 --- a/src/or/fp_pair.h +++ b/src/or/fp_pair.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013, The Tor Project, Inc. */ +/* Copyright (c) 2013-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/geoip.c b/src/or/geoip.c index f722bac468..120ce479cc 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2013, The Tor Project, Inc. */ +/* Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -58,8 +58,8 @@ static char geoip6_digest[DIGEST_LEN]; /** Return the index of the <b>country</b>'s entry in the GeoIP * country list if it is a valid 2-letter country code, otherwise * return -1. */ -country_t -geoip_get_country(const char *country) +MOCK_IMPL(country_t, +geoip_get_country,(const char *country)) { void *idxplus1_; intptr_t idx; @@ -396,8 +396,8 @@ geoip_get_country_by_ipv6(const struct in6_addr *addr) * the 'unknown country'. The return value will always be less than * geoip_get_n_countries(). To decode it, call geoip_get_country_name(). */ -int -geoip_get_country_by_addr(const tor_addr_t *addr) +MOCK_IMPL(int, +geoip_get_country_by_addr,(const tor_addr_t *addr)) { if (tor_addr_family(addr) == AF_INET) { return geoip_get_country_by_ipv4(tor_addr_to_ipv4h(addr)); @@ -409,8 +409,8 @@ geoip_get_country_by_addr(const tor_addr_t *addr) } /** Return the number of countries recognized by the GeoIP country list. */ -int -geoip_get_n_countries(void) +MOCK_IMPL(int, +geoip_get_n_countries,(void)) { if (!geoip_countries) init_geoip_countries(); @@ -430,8 +430,8 @@ geoip_get_country_name(country_t num) } /** Return true iff we have loaded a GeoIP database.*/ -int -geoip_is_loaded(sa_family_t family) +MOCK_IMPL(int, +geoip_is_loaded,(sa_family_t family)) { tor_assert(family == AF_INET || family == AF_INET6); if (geoip_countries == NULL) @@ -506,8 +506,8 @@ clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq); -HT_GENERATE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, - clientmap_entries_eq, 0.6, malloc, realloc, free); +HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash, + clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_) /** Free all storage held by <b>ent</b>. */ static void @@ -720,8 +720,8 @@ dirreq_map_ent_hash(const dirreq_map_entry_t *entry) HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, dirreq_map_ent_eq); -HT_GENERATE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, - dirreq_map_ent_eq, 0.6, malloc, realloc, free); +HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, + dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) /** Helper: Put <b>entry</b> into map of directory requests using * <b>type</b> and <b>dirreq_id</b> as key parts. If there is @@ -963,7 +963,7 @@ geoip_get_dirreq_history(dirreq_type_t type) /* We may have rounded 'completed' up. Here we want to use the * real value. */ complete = smartlist_len(dirreq_completed); - dltimes = tor_malloc_zero(sizeof(uint32_t) * complete); + dltimes = tor_calloc(complete, sizeof(uint32_t)); SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) { uint32_t bytes_per_second; uint32_t time_diff = (uint32_t) tv_mdiff(&ent->request_time, @@ -1033,7 +1033,7 @@ geoip_get_client_history(geoip_client_action_t action, if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6)) return -1; - counts = tor_malloc_zero(sizeof(unsigned)*n_countries); + counts = tor_calloc(n_countries, sizeof(unsigned)); HT_FOREACH(ent, clientmap, &client_history) { int country; if ((*ent)->action != (int)action) @@ -1436,6 +1436,39 @@ format_bridge_stats_controller(time_t now) return out; } +/** Return a newly allocated string holding our bridge usage stats by + * country in a format suitable for inclusion in our heartbeat + * message. Return NULL on failure. */ +char * +format_client_stats_heartbeat(time_t now) +{ + const int n_hours = 6; + char *out = NULL; + int n_clients = 0; + clientmap_entry_t **ent; + unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 ); + + if (!start_of_bridge_stats_interval) + return NULL; /* Not initialized. */ + + /* count unique IPs */ + HT_FOREACH(ent, clientmap, &client_history) { + /* only count directly connecting clients */ + if ((*ent)->action != GEOIP_CLIENT_CONNECT) + continue; + if ((*ent)->last_seen_in_minutes < cutoff) + continue; + n_clients++; + } + + tor_asprintf(&out, "Heartbeat: " + "In the last %d hours, I have seen %d unique clients.", + n_hours, + n_clients); + + return out; +} + /** Write bridge statistics to $DATADIR/stats/bridge-stats and return * when we should next try to write statistics. */ time_t diff --git a/src/or/geoip.h b/src/or/geoip.h index b9b53c3006..8a3486c7ac 100644 --- a/src/or/geoip.h +++ b/src/or/geoip.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -21,12 +21,12 @@ STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr); #endif int should_record_bridge_info(const or_options_t *options); int geoip_load_file(sa_family_t family, const char *filename); -int geoip_get_country_by_addr(const tor_addr_t *addr); -int geoip_get_n_countries(void); +MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr)); +MOCK_DECL(int, geoip_get_n_countries, (void)); const char *geoip_get_country_name(country_t num); -int geoip_is_loaded(sa_family_t family); +MOCK_DECL(int, geoip_is_loaded, (sa_family_t family)); const char *geoip_db_digest(sa_family_t family); -country_t geoip_get_country(const char *countrycode); +MOCK_DECL(country_t, geoip_get_country, (const char *countrycode)); void geoip_note_client_seen(geoip_client_action_t action, const tor_addr_t *addr, const char *transport_name, @@ -64,6 +64,7 @@ time_t geoip_bridge_stats_write(time_t now); void geoip_bridge_stats_term(void); const char *geoip_get_bridge_stats_extrainfo(time_t); char *geoip_get_bridge_stats_controller(time_t); +char *format_client_stats_heartbeat(time_t now); #endif diff --git a/src/or/hibernate.c b/src/or/hibernate.c index c433ac1be9..356e11f6ec 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -410,6 +410,17 @@ configure_accounting(time_t now) accounting_set_wakeup_time(); } +/** Return the relevant number of bytes sent/received this interval + * based on the set AccountingRule */ +static uint64_t +get_accounting_bytes(void) +{ + if (get_options()->AccountingRule == ACCT_SUM) + return n_bytes_read_in_interval+n_bytes_written_in_interval; + else + return MAX(n_bytes_read_in_interval, n_bytes_written_in_interval); +} + /** Set expected_bandwidth_usage based on how much we sent/received * per minute last interval (if we were up for at least 30 minutes), * or based on our declared bandwidth otherwise. */ @@ -421,6 +432,11 @@ update_expected_bandwidth(void) uint64_t max_configured = (options->RelayBandwidthRate > 0 ? options->RelayBandwidthRate : options->BandwidthRate) * 60; + /* max_configured is the larger of bytes read and bytes written + * If we are accounting based on sum, worst case is both are + * at max, doubling the expected sum of bandwidth */ + if (get_options()->AccountingRule == ACCT_SUM) + max_configured *= 2; #define MIN_TIME_FOR_MEASUREMENT (1800) @@ -439,8 +455,7 @@ update_expected_bandwidth(void) * doesn't know to store soft-limit info. Just take rate at which * we were reading/writing in the last interval as our expected rate. */ - uint64_t used = MAX(n_bytes_written_in_interval, - n_bytes_read_in_interval); + uint64_t used = get_accounting_bytes(); expected = used / (n_seconds_active_in_interval / 60); } else { /* If we haven't gotten enough data last interval, set 'expected' @@ -715,8 +730,7 @@ hibernate_hard_limit_reached(void) uint64_t hard_limit = get_options()->AccountingMax; if (!hard_limit) return 0; - return n_bytes_read_in_interval >= hard_limit - || n_bytes_written_in_interval >= hard_limit; + return get_accounting_bytes() >= hard_limit; } /** Return true iff we have sent/received almost all the bytes we are willing @@ -747,8 +761,7 @@ hibernate_soft_limit_reached(void) if (!soft_limit) return 0; - return n_bytes_read_in_interval >= soft_limit - || n_bytes_written_in_interval >= soft_limit; + return get_accounting_bytes() >= soft_limit; } /** Called when we get a SIGINT, or when bandwidth soft limit is @@ -772,8 +785,7 @@ hibernate_begin(hibernate_state_t new_state, time_t now) hibernate_state == HIBERNATE_STATE_LIVE) { soft_limit_hit_at = now; n_seconds_to_hit_soft_limit = n_seconds_active_in_interval; - n_bytes_at_soft_limit = MAX(n_bytes_read_in_interval, - n_bytes_written_in_interval); + n_bytes_at_soft_limit = get_accounting_bytes(); } /* close listeners. leave control listener(s). */ @@ -1003,13 +1015,22 @@ getinfo_helper_accounting(control_connection_t *conn, U64_PRINTF_ARG(n_bytes_written_in_interval)); } else if (!strcmp(question, "accounting/bytes-left")) { uint64_t limit = get_options()->AccountingMax; - uint64_t read_left = 0, write_left = 0; - if (n_bytes_read_in_interval < limit) - read_left = limit - n_bytes_read_in_interval; - if (n_bytes_written_in_interval < limit) - write_left = limit - n_bytes_written_in_interval; - tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, - U64_PRINTF_ARG(read_left), U64_PRINTF_ARG(write_left)); + if (get_options()->AccountingRule == ACCT_SUM) { + uint64_t total_left = 0; + uint64_t total_bytes = get_accounting_bytes(); + if (total_bytes < limit) + total_left = limit - total_bytes; + tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, + U64_PRINTF_ARG(total_left), U64_PRINTF_ARG(total_left)); + } else { + uint64_t read_left = 0, write_left = 0; + if (n_bytes_read_in_interval < limit) + read_left = limit - n_bytes_read_in_interval; + if (n_bytes_written_in_interval < limit) + write_left = limit - n_bytes_written_in_interval; + tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, + U64_PRINTF_ARG(read_left), U64_PRINTF_ARG(write_left)); + } } else if (!strcmp(question, "accounting/interval-start")) { *answer = tor_malloc(ISO_TIME_LEN+1); format_iso_time(*answer, interval_start_time); diff --git a/src/or/hibernate.h b/src/or/hibernate.h index 38ecb75129..b9e619c5ad 100644 --- a/src/or/hibernate.h +++ b/src/or/hibernate.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -28,6 +28,7 @@ void consider_hibernation(time_t now); int getinfo_helper_accounting(control_connection_t *conn, const char *question, char **answer, const char **errmsg); +uint64_t get_accounting_max_total(void); #ifdef HIBERNATE_PRIVATE /** Possible values of hibernate_state */ diff --git a/src/or/include.am b/src/or/include.am index 47bdd09901..b44e1099dc 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -23,12 +23,6 @@ else evdns_source=src/ext/eventdns.c endif -if CURVE25519_ENABLED -onion_ntor_source=src/or/onion_ntor.c -else -onion_ntor_source= -endif - LIBTOR_A_SOURCES = \ src/or/addressmap.c \ src/or/buffers.c \ @@ -80,11 +74,12 @@ LIBTOR_A_SOURCES = \ src/or/routerlist.c \ src/or/routerparse.c \ src/or/routerset.c \ + src/or/scheduler.c \ src/or/statefile.c \ src/or/status.c \ + src/or/onion_ntor.c \ $(evdns_source) \ $(tor_platform_source) \ - $(onion_ntor_source) \ src/or/config_codedigest.c src_or_libtor_a_SOURCES = $(LIBTOR_A_SOURCES) @@ -116,7 +111,7 @@ src_or_tor_LDADD = src/or/libtor.a src/common/libor.a \ src/common/libor-crypto.a $(LIBDONNA) \ src/common/libor-event.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ if COVERAGE_ENABLED src_or_tor_cov_SOURCES = src/or/tor_main.c @@ -127,7 +122,10 @@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ src/common/libor-crypto-testing.a $(LIBDONNA) \ src/common/libor-event-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ +TESTING_TOR_BINARY = ./src/or/tor-cov +else +TESTING_TOR_BINARY = ./src/or/tor endif ORHEADERS = \ @@ -185,6 +183,7 @@ ORHEADERS = \ src/or/routerlist.h \ src/or/routerset.h \ src/or/routerparse.h \ + src/or/scheduler.h \ src/or/statefile.h \ src/or/status.h diff --git a/src/or/main.c b/src/or/main.c index 31fbdcd433..8badd7d382 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -28,6 +28,7 @@ #include "connection_or.h" #include "control.h" #include "cpuworker.h" +#include "crypto_s2k.h" #include "directory.h" #include "dirserv.h" #include "dirvote.h" @@ -52,6 +53,7 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "scheduler.h" #include "statefile.h" #include "status.h" #include "util_process.h" @@ -73,6 +75,16 @@ #include <event2/bufferevent.h> #endif +#ifdef HAVE_SYSTEMD +# if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) +/* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse + * Coverity. Here's a kludge to unconfuse it. + */ +# define __INCLUDE_LEVEL__ 2 +# endif +#include <systemd/sd-daemon.h> +#endif + void evdns_shutdown(int); /********* PROTOTYPES **********/ @@ -85,6 +97,7 @@ static void second_elapsed_callback(periodic_timer_t *timer, void *args); static int conn_close_if_marked(int i); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); +static int run_main_loop_until_done(void); /********* START VARIABLES **********/ @@ -149,7 +162,7 @@ static int called_loop_once = 0; * any longer (a big time jump happened, when we notice our directory is * heinously out-of-date, etc. */ -int can_complete_circuit=0; +static int can_complete_circuits = 0; /** How often do we check for router descriptors that we should download * when we have too little directory info? */ @@ -170,11 +183,11 @@ int quiet_level = 0; /********* END VARIABLES ************/ /**************************************************************************** -* -* This section contains accessors and other methods on the connection_array -* variables (which are global within this file and unavailable outside it). -* -****************************************************************************/ + * + * This section contains accessors and other methods on the connection_array + * variables (which are global within this file and unavailable outside it). + * + ****************************************************************************/ #if 0 && defined(USE_BUFFEREVENTS) static void @@ -222,6 +235,31 @@ set_buffer_lengths_to_zero(tor_socket_t s) } #endif +/** Return 1 if we have successfully built a circuit, and nothing has changed + * to make us think that maybe we can't. + */ +int +have_completed_a_circuit(void) +{ + return can_complete_circuits; +} + +/** Note that we have successfully built a circuit, so that reachability + * testing and introduction points and so on may be attempted. */ +void +note_that_we_completed_a_circuit(void) +{ + can_complete_circuits = 1; +} + +/** Note that something has happened (like a clock jump, or DisableNetwork) to + * make us think that maybe we can't complete circuits. */ +void +note_that_we_maybe_cant_complete_circuits(void) +{ + can_complete_circuits = 0; +} + /** Add <b>conn</b> to the array of connections that we can poll on. The * connection's socket must be set; the connection starts out * non-reading and non-writing. @@ -354,6 +392,10 @@ connection_remove(connection_t *conn) (int)conn->s, conn_type_to_string(conn->type), smartlist_len(connection_array)); + if (conn->type == CONN_TYPE_AP && conn->socket_family == AF_UNIX) { + log_info(LD_NET, "Closing SOCKS SocksSocket connection"); + } + control_event_conn_bandwidth(conn); tor_assert(conn->conn_array_index >= 0); @@ -1045,7 +1087,7 @@ directory_info_has_arrived(time_t now, int from_cache) } if (server_mode(options) && !net_is_disabled() && !from_cache && - (can_complete_circuit || !any_predicted_circuits(now))) + (have_completed_a_circuit() || !any_predicted_circuits(now))) consider_testing_reachability(1, 1); } @@ -1229,7 +1271,6 @@ run_scheduled_events(time_t now) static time_t time_to_check_v3_certificate = 0; static time_t time_to_check_listeners = 0; static time_t time_to_download_networkstatus = 0; - static time_t time_to_shrink_memory = 0; static time_t time_to_try_getting_descriptors = 0; static time_t time_to_reset_descriptor_failures = 0; static time_t time_to_add_entropy = 0; @@ -1277,7 +1318,7 @@ run_scheduled_events(time_t now) get_onion_key_set_at()+MIN_ONION_KEY_LIFETIME < now) { log_info(LD_GENERAL,"Rotating onion key."); rotate_onion_key(); - cpuworkers_rotate(); + cpuworkers_rotate_keyinfo(); if (router_rebuild_descriptor(1)<0) { log_info(LD_CONFIG, "Couldn't rebuild router descriptor"); } @@ -1404,6 +1445,11 @@ run_scheduled_events(time_t now) if (next_write && next_write < next_time_to_write_stats_files) next_time_to_write_stats_files = next_write; } + if (options->HiddenServiceStatistics) { + time_t next_write = rep_hist_hs_stats_write(time_to_write_stats_files); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + } if (options->ExitPortStatistics) { time_t next_write = rep_hist_exit_stats_write(time_to_write_stats_files); if (next_write && next_write < next_time_to_write_stats_files) @@ -1448,7 +1494,7 @@ run_scheduled_events(time_t now) if (time_to_clean_caches < now) { rep_history_clean(now - options->RephistTrackTime); rend_cache_clean(now); - rend_cache_clean_v2_descs_as_dir(now); + rend_cache_clean_v2_descs_as_dir(now, 0); microdesc_cache_rebuild(NULL, 0); #define CLEAN_CACHES_INTERVAL (30*60) time_to_clean_caches = now + CLEAN_CACHES_INTERVAL; @@ -1482,7 +1528,7 @@ run_scheduled_events(time_t now) /* also, check religiously for reachability, if it's within the first * 20 minutes of our uptime. */ if (is_server && - (can_complete_circuit || !any_predicted_circuits(now)) && + (have_completed_a_circuit() || !any_predicted_circuits(now)) && !we_are_hibernating()) { if (stats_n_seconds_working < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { consider_testing_reachability(1, dirport_reachability_count==0); @@ -1574,28 +1620,12 @@ run_scheduled_events(time_t now) for (i=0;i<smartlist_len(connection_array);i++) { run_connection_housekeeping(i, now); } - if (time_to_shrink_memory < now) { - SMARTLIST_FOREACH(connection_array, connection_t *, conn, { - if (conn->outbuf) - buf_shrink(conn->outbuf); - if (conn->inbuf) - buf_shrink(conn->inbuf); - }); -#ifdef ENABLE_MEMPOOL - clean_cell_pool(); -#endif /* ENABLE_MEMPOOL */ - buf_shrink_freelists(0); -/** How often do we check buffers and pools for empty space that can be - * deallocated? */ -#define MEM_SHRINK_INTERVAL (60) - time_to_shrink_memory = now + MEM_SHRINK_INTERVAL; - } /* 6. And remove any marked circuits... */ circuit_close_all_marked(); /* 7. And upload service descriptors if necessary. */ - if (can_complete_circuit && !net_is_disabled()) { + if (have_completed_a_circuit() && !net_is_disabled()) { rend_consider_services_upload(now); rend_consider_descriptor_republication(); } @@ -1726,7 +1756,7 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) if (server_mode(options) && !net_is_disabled() && seconds_elapsed > 0 && - can_complete_circuit && + have_completed_a_circuit() && stats_n_seconds_working / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT != (stats_n_seconds_working+seconds_elapsed) / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { @@ -1774,6 +1804,19 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) current_second = now; /* remember which second it is, for next time */ } +#ifdef HAVE_SYSTEMD_209 +static periodic_timer_t *systemd_watchdog_timer = NULL; + +/** Libevent callback: invoked to reset systemd watchdog. */ +static void +systemd_watchdog_callback(periodic_timer_t *timer, void *arg) +{ + (void)timer; + (void)arg; + sd_notify(0, "WATCHDOG=1"); +} +#endif + #ifndef USE_BUFFEREVENTS /** Timer: used to invoke refill_callback(). */ static periodic_timer_t *refill_timer = NULL; @@ -1908,6 +1951,10 @@ do_hup(void) return -1; } options = get_options(); /* they have changed now */ + /* Logs are only truncated the first time they are opened, but were + probably intended to be cleaned up on signal. */ + if (options->TruncateLogFile) + truncate_logs(); } else { char *msg = NULL; log_notice(LD_GENERAL, "Not reloading config file: the controller told " @@ -1944,9 +1991,9 @@ do_hup(void) * force a retry there. */ if (server_mode(options)) { - /* Restart cpuworker and dnsworker processes, so they get up-to-date + /* Update cpuworker and dnsworker processes, so they get up-to-date * configuration options. */ - cpuworkers_rotate(); + cpuworkers_rotate_keyinfo(); dns_reset(); } return 0; @@ -1956,7 +2003,6 @@ do_hup(void) int do_main_loop(void) { - int loop_result; time_t now; /* initialize dns resolve map, spawn workers if needed */ @@ -1988,11 +2034,6 @@ do_main_loop(void) } } -#ifdef ENABLE_MEMPOOLS - /* Set up the packed_cell_t memory pool. */ - init_cell_pool(); -#endif /* ENABLE_MEMPOOLS */ - /* Set up our buckets */ connection_bucket_init(); #ifndef USE_BUFFEREVENTS @@ -2038,6 +2079,28 @@ do_main_loop(void) tor_assert(second_timer); } +#ifdef HAVE_SYSTEMD_209 + uint64_t watchdog_delay; + /* set up systemd watchdog notification. */ + if (sd_watchdog_enabled(1, &watchdog_delay) > 0) { + if (! systemd_watchdog_timer) { + struct timeval watchdog; + /* The manager will "act on" us if we don't send them a notification + * every 'watchdog_delay' microseconds. So, send notifications twice + * that often. */ + watchdog_delay /= 2; + watchdog.tv_sec = watchdog_delay / 1000000; + watchdog.tv_usec = watchdog_delay % 1000000; + + systemd_watchdog_timer = periodic_timer_new(tor_libevent_get_base(), + &watchdog, + systemd_watchdog_callback, + NULL); + tor_assert(systemd_watchdog_timer); + } + } +#endif + #ifndef USE_BUFFEREVENTS if (!refill_timer) { struct timeval refill_interval; @@ -2054,51 +2117,92 @@ do_main_loop(void) } #endif - for (;;) { - if (nt_service_is_stopping()) - return 0; +#ifdef HAVE_SYSTEMD + { + const int r = sd_notify(0, "READY=1"); + if (r < 0) { + log_warn(LD_GENERAL, "Unable to send readiness to systemd: %s", + strerror(r)); + } else if (r > 0) { + log_notice(LD_GENERAL, "Signaled readiness to systemd"); + } else { + log_info(LD_GENERAL, "Systemd NOTIFY_SOCKET not present."); + } + } +#endif + + return run_main_loop_until_done(); +} + +/** + * Run the main loop a single time. Return 0 for "exit"; -1 for "exit with + * error", and 1 for "run this again." + */ +static int +run_main_loop_once(void) +{ + int loop_result; + + if (nt_service_is_stopping()) + return 0; #ifndef _WIN32 - /* Make it easier to tell whether libevent failure is our fault or not. */ - errno = 0; + /* Make it easier to tell whether libevent failure is our fault or not. */ + errno = 0; #endif - /* All active linked conns should get their read events activated. */ - SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn, - event_active(conn->read_event, EV_READ, 1)); - called_loop_once = smartlist_len(active_linked_connection_lst) ? 1 : 0; - - update_approx_time(time(NULL)); - - /* poll until we have an event, or the second ends, or until we have - * some active linked connections to trigger events for. */ - loop_result = event_base_loop(tor_libevent_get_base(), - called_loop_once ? EVLOOP_ONCE : 0); - - /* let catch() handle things like ^c, and otherwise don't worry about it */ - if (loop_result < 0) { - int e = tor_socket_errno(-1); - /* let the program survive things like ^z */ - if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { - log_err(LD_NET,"libevent call with %s failed: %s [%d]", - tor_libevent_get_method(), tor_socket_strerror(e), e); - return -1; + /* All active linked conns should get their read events activated. */ + SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn, + event_active(conn->read_event, EV_READ, 1)); + called_loop_once = smartlist_len(active_linked_connection_lst) ? 1 : 0; + + update_approx_time(time(NULL)); + + /* poll until we have an event, or the second ends, or until we have + * some active linked connections to trigger events for. */ + loop_result = event_base_loop(tor_libevent_get_base(), + called_loop_once ? EVLOOP_ONCE : 0); + + /* let catch() handle things like ^c, and otherwise don't worry about it */ + if (loop_result < 0) { + int e = tor_socket_errno(-1); + /* let the program survive things like ^z */ + if (e != EINTR && !ERRNO_IS_EINPROGRESS(e)) { + log_err(LD_NET,"libevent call with %s failed: %s [%d]", + tor_libevent_get_method(), tor_socket_strerror(e), e); + return -1; #ifndef _WIN32 - } else if (e == EINVAL) { - log_warn(LD_NET, "EINVAL from libevent: should you upgrade libevent?"); - if (got_libevent_error()) - return -1; + } else if (e == EINVAL) { + log_warn(LD_NET, "EINVAL from libevent: should you upgrade libevent?"); + if (got_libevent_error()) + return -1; #endif - } else { - if (ERRNO_IS_EINPROGRESS(e)) - log_warn(LD_BUG, - "libevent call returned EINPROGRESS? Please report."); - log_debug(LD_NET,"libevent call interrupted."); - /* You can't trust the results of this poll(). Go back to the - * top of the big for loop. */ - continue; - } + } else { + if (ERRNO_IS_EINPROGRESS(e)) + log_warn(LD_BUG, + "libevent call returned EINPROGRESS? Please report."); + log_debug(LD_NET,"libevent call interrupted."); + /* You can't trust the results of this poll(). Go back to the + * top of the big for loop. */ + return 1; } } + + return 1; +} + +/** Run the run_main_loop_once() function until it declares itself done, + * and return its final return value. + * + * Shadow won't invoke this function, so don't fill it up with things. + */ +static int +run_main_loop_until_done(void) +{ + int loop_result = 1; + do { + loop_result = run_main_loop_once(); + } while (loop_result == 1); + return loop_result; } #ifndef _WIN32 /* Only called when we're willing to use signals */ @@ -2132,6 +2236,9 @@ process_signal(uintptr_t sig) tor_cleanup(); exit(0); } +#ifdef HAVE_SYSTEMD + sd_notify(0, "STOPPING=1"); +#endif hibernate_begin_shutdown(); break; #ifdef SIGPIPE @@ -2151,11 +2258,17 @@ process_signal(uintptr_t sig) control_event_signal(sig); break; case SIGHUP: +#ifdef HAVE_SYSTEMD + sd_notify(0, "RELOADING=1"); +#endif if (do_hup() < 0) { log_warn(LD_CONFIG,"Restart failed (config error?). Exiting."); tor_cleanup(); exit(1); } +#ifdef HAVE_SYSTEMD + sd_notify(0, "READY=1"); +#endif control_event_signal(sig); break; #ifdef SIGCHLD @@ -2179,6 +2292,10 @@ process_signal(uintptr_t sig) addressmap_clear_transient(); control_event_signal(sig); break; + case SIGHEARTBEAT: + log_heartbeat(time(NULL)); + control_event_signal(sig); + break; } } @@ -2204,7 +2321,6 @@ dumpmemusage(int severity) dump_routerlist_mem_usage(severity); dump_cell_pool_usage(severity); dump_dns_mem_usage(severity); - buf_dump_freelist_sizes(severity); tor_log_mallinfo(severity); } @@ -2595,7 +2711,7 @@ tor_free_all(int postfork) channel_tls_free_all(); channel_free_all(); connection_free_all(); - buf_shrink_freelists(1); + scheduler_free_all(); memarea_clear_freelist(); nodelist_free_all(); microdesc_free_all(); @@ -2608,9 +2724,6 @@ tor_free_all(int postfork) router_free_all(); policies_free_all(); } -#ifdef ENABLE_MEMPOOLS - free_cell_pool(); -#endif /* ENABLE_MEMPOOLS */ if (!postfork) { tor_tls_free_all(); #ifndef _WIN32 @@ -2717,11 +2830,11 @@ do_hash_password(void) { char output[256]; - char key[S2K_SPECIFIER_LEN+DIGEST_LEN]; + char key[S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN]; - crypto_rand(key, S2K_SPECIFIER_LEN-1); - key[S2K_SPECIFIER_LEN-1] = (uint8_t)96; /* Hash 64 K of data. */ - secret_to_key(key+S2K_SPECIFIER_LEN, DIGEST_LEN, + crypto_rand(key, S2K_RFC2440_SPECIFIER_LEN-1); + key[S2K_RFC2440_SPECIFIER_LEN-1] = (uint8_t)96; /* Hash 64 K of data. */ + secret_to_key_rfc2440(key+S2K_RFC2440_SPECIFIER_LEN, DIGEST_LEN, get_options()->command_arg, strlen(get_options()->command_arg), key); base16_encode(output, sizeof(output), key, sizeof(key)); @@ -2756,31 +2869,6 @@ do_dump_config(void) return 0; } -#if defined (WINCE) -int -find_flashcard_path(PWCHAR path, size_t size) -{ - WIN32_FIND_DATA d = {0}; - HANDLE h = NULL; - - if (!path) - return -1; - - h = FindFirstFlashCard(&d); - if (h == INVALID_HANDLE_VALUE) - return -1; - - if (wcslen(d.cFileName) == 0) { - FindClose(h); - return -1; - } - - wcsncpy(path,d.cFileName,size); - FindClose(h); - return 0; -} -#endif - static void init_addrinfo(void) { @@ -2801,43 +2889,47 @@ sandbox_init_filter(void) sandbox_cfg_allow_openat_filename(&cfg, get_datadir_fname("cached-status")); - sandbox_cfg_allow_open_filename_array(&cfg, - get_datadir_fname("cached-certs"), - get_datadir_fname("cached-certs.tmp"), - get_datadir_fname("cached-consensus"), - get_datadir_fname("cached-consensus.tmp"), - get_datadir_fname("unverified-consensus"), - get_datadir_fname("unverified-consensus.tmp"), - get_datadir_fname("unverified-microdesc-consensus"), - get_datadir_fname("unverified-microdesc-consensus.tmp"), - get_datadir_fname("cached-microdesc-consensus"), - get_datadir_fname("cached-microdesc-consensus.tmp"), - get_datadir_fname("cached-microdescs"), - get_datadir_fname("cached-microdescs.tmp"), - get_datadir_fname("cached-microdescs.new"), - get_datadir_fname("cached-microdescs.new.tmp"), - get_datadir_fname("cached-descriptors"), - get_datadir_fname("cached-descriptors.new"), - get_datadir_fname("cached-descriptors.tmp"), - get_datadir_fname("cached-descriptors.new.tmp"), - get_datadir_fname("cached-descriptors.tmp.tmp"), - get_datadir_fname("cached-extrainfo"), - get_datadir_fname("cached-extrainfo.new"), - get_datadir_fname("cached-extrainfo.tmp"), - get_datadir_fname("cached-extrainfo.new.tmp"), - get_datadir_fname("cached-extrainfo.tmp.tmp"), - get_datadir_fname("state.tmp"), - get_datadir_fname("unparseable-desc.tmp"), - get_datadir_fname("unparseable-desc"), - get_datadir_fname("v3-status-votes"), - get_datadir_fname("v3-status-votes.tmp"), - tor_strdup("/dev/srandom"), - tor_strdup("/dev/urandom"), - tor_strdup("/dev/random"), - tor_strdup("/etc/hosts"), - tor_strdup("/proc/meminfo"), - NULL, 0 - ); +#define OPEN(name) \ + sandbox_cfg_allow_open_filename(&cfg, tor_strdup(name)) + +#define OPEN_DATADIR(name) \ + sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname(name)) + +#define OPEN_DATADIR2(name, name2) \ + sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname2((name), (name2))) + +#define OPEN_DATADIR_SUFFIX(name, suffix) do { \ + OPEN_DATADIR(name); \ + OPEN_DATADIR(name suffix); \ + } while (0) + +#define OPEN_DATADIR2_SUFFIX(name, name2, suffix) do { \ + OPEN_DATADIR2(name, name2); \ + OPEN_DATADIR2(name, name2 suffix); \ + } while (0) + + OPEN_DATADIR_SUFFIX("cached-certs", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("unverified-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("unverified-microdesc-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-microdesc-consensus", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-microdescs", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-microdescs.new", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-descriptors", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-descriptors.new", ".tmp"); + OPEN_DATADIR("cached-descriptors.tmp.tmp"); + OPEN_DATADIR_SUFFIX("cached-extrainfo", ".tmp"); + OPEN_DATADIR_SUFFIX("cached-extrainfo.new", ".tmp"); + OPEN_DATADIR("cached-extrainfo.tmp.tmp"); + OPEN_DATADIR_SUFFIX("state", ".tmp"); + OPEN_DATADIR_SUFFIX("unparseable-desc", ".tmp"); + OPEN_DATADIR_SUFFIX("v3-status-votes", ".tmp"); + OPEN("/dev/srandom"); + OPEN("/dev/urandom"); + OPEN("/dev/random"); + OPEN("/etc/hosts"); + OPEN("/proc/meminfo"); + if (options->ServerDNSResolvConfFile) sandbox_cfg_allow_open_filename(&cfg, tor_strdup(options->ServerDNSResolvConfFile)); @@ -2878,14 +2970,17 @@ sandbox_init_filter(void) RENAME_SUFFIX("unparseable-desc", ".tmp"); RENAME_SUFFIX("v3-status-votes", ".tmp"); - sandbox_cfg_allow_stat_filename_array(&cfg, - get_datadir_fname(NULL), - get_datadir_fname("lock"), - get_datadir_fname("state"), - get_datadir_fname("router-stability"), - get_datadir_fname("cached-extrainfo.new"), - NULL, 0 - ); +#define STAT_DATADIR(name) \ + sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname(name)) + +#define STAT_DATADIR2(name, name2) \ + sandbox_cfg_allow_stat_filename(&cfg, get_datadir_fname2((name), (name2))) + + STAT_DATADIR(NULL); + STAT_DATADIR("lock"); + STAT_DATADIR("state"); + STAT_DATADIR("router-stability"); + STAT_DATADIR("cached-extrainfo.new"); { smartlist_t *files = smartlist_new(); @@ -2907,7 +3002,8 @@ sandbox_init_filter(void) sandbox_cfg_allow_rename(&cfg, tor_strdup(tmp_name), tor_strdup(file_name)); /* steals references */ - sandbox_cfg_allow_open_filename_array(&cfg, file_name, tmp_name, NULL); + sandbox_cfg_allow_open_filename(&cfg, file_name); + sandbox_cfg_allow_open_filename(&cfg, tmp_name); }); SMARTLIST_FOREACH(dirs, char *, dir, { /* steals reference */ @@ -2934,38 +3030,28 @@ sandbox_init_filter(void) // orport if (server_mode(get_options())) { - sandbox_cfg_allow_open_filename_array(&cfg, - get_datadir_fname2("keys", "secret_id_key"), - get_datadir_fname2("keys", "secret_onion_key"), - get_datadir_fname2("keys", "secret_onion_key_ntor"), - get_datadir_fname2("keys", "secret_onion_key_ntor.tmp"), - get_datadir_fname2("keys", "secret_id_key.old"), - get_datadir_fname2("keys", "secret_onion_key.old"), - get_datadir_fname2("keys", "secret_onion_key_ntor.old"), - get_datadir_fname2("keys", "secret_onion_key.tmp"), - get_datadir_fname2("keys", "secret_id_key.tmp"), - get_datadir_fname2("stats", "bridge-stats"), - get_datadir_fname2("stats", "bridge-stats.tmp"), - get_datadir_fname2("stats", "dirreq-stats"), - get_datadir_fname2("stats", "dirreq-stats.tmp"), - get_datadir_fname2("stats", "entry-stats"), - get_datadir_fname2("stats", "entry-stats.tmp"), - get_datadir_fname2("stats", "exit-stats"), - get_datadir_fname2("stats", "exit-stats.tmp"), - get_datadir_fname2("stats", "buffer-stats"), - get_datadir_fname2("stats", "buffer-stats.tmp"), - get_datadir_fname2("stats", "conn-stats"), - get_datadir_fname2("stats", "conn-stats.tmp"), - get_datadir_fname("approved-routers"), - get_datadir_fname("fingerprint"), - get_datadir_fname("fingerprint.tmp"), - get_datadir_fname("hashed-fingerprint"), - get_datadir_fname("hashed-fingerprint.tmp"), - get_datadir_fname("router-stability"), - get_datadir_fname("router-stability.tmp"), - tor_strdup("/etc/resolv.conf"), - NULL, 0 - ); + + OPEN_DATADIR2_SUFFIX("keys", "secret_id_key", ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "secret_onion_key", ".tmp"); + OPEN_DATADIR2_SUFFIX("keys", "secret_onion_key_ntor", ".tmp"); + OPEN_DATADIR2("keys", "secret_id_key.old"); + OPEN_DATADIR2("keys", "secret_onion_key.old"); + OPEN_DATADIR2("keys", "secret_onion_key_ntor.old"); + + OPEN_DATADIR2_SUFFIX("stats", "bridge-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "dirreq-stats", ".tmp"); + + OPEN_DATADIR2_SUFFIX("stats", "entry-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "exit-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "buffer-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "conn-stats", ".tmp"); + + OPEN_DATADIR("approved-routers"); + OPEN_DATADIR_SUFFIX("fingerprint", ".tmp"); + OPEN_DATADIR_SUFFIX("hashed-fingerprint", ".tmp"); + OPEN_DATADIR_SUFFIX("router-stability", ".tmp"); + + OPEN("/etc/resolv.conf"); RENAME_SUFFIX("fingerprint", ".tmp"); RENAME_SUFFIX2("keys", "secret_onion_key_ntor", ".tmp"); @@ -2989,12 +3075,9 @@ sandbox_init_filter(void) get_datadir_fname2("keys", "secret_onion_key_ntor"), get_datadir_fname2("keys", "secret_onion_key_ntor.old")); - sandbox_cfg_allow_stat_filename_array(&cfg, - get_datadir_fname("keys"), - get_datadir_fname("stats"), - get_datadir_fname2("stats", "dirreq-stats"), - NULL, 0 - ); + STAT_DATADIR("keys"); + STAT_DATADIR("stats"); + STAT_DATADIR2("stats", "dirreq-stats"); } init_addrinfo(); @@ -3009,31 +3092,6 @@ int tor_main(int argc, char *argv[]) { int result = 0; -#if defined (WINCE) - WCHAR path [MAX_PATH] = {0}; - WCHAR fullpath [MAX_PATH] = {0}; - PWCHAR p = NULL; - FILE* redir = NULL; - FILE* redirdbg = NULL; - - // this is to facilitate debugging by opening - // a file on a folder shared by the wm emulator. - // if no flashcard (real or emulated) is present, - // log files will be written in the root folder - if (find_flashcard_path(path,MAX_PATH) == -1) { - redir = _wfreopen( L"\\stdout.log", L"w", stdout ); - redirdbg = _wfreopen( L"\\stderr.log", L"w", stderr ); - } else { - swprintf(fullpath,L"\\%s\\tor",path); - CreateDirectory(fullpath,NULL); - - swprintf(fullpath,L"\\%s\\tor\\stdout.log",path); - redir = _wfreopen( fullpath, L"w", stdout ); - - swprintf(fullpath,L"\\%s\\tor\\stderr.log",path); - redirdbg = _wfreopen( fullpath, L"w", stderr ); - } -#endif #ifdef _WIN32 /* Call SetProcessDEPPolicy to permanently enable DEP. @@ -3052,7 +3110,7 @@ tor_main(int argc, char *argv[]) update_approx_time(time(NULL)); tor_threads_init(); - init_logging(); + init_logging(0); #ifdef USE_DMALLOC { /* Instruct OpenSSL to use our internal wrappers for malloc, diff --git a/src/or/main.h b/src/or/main.h index a3bce3486f..f77b4711c5 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,7 +12,9 @@ #ifndef TOR_MAIN_H #define TOR_MAIN_H -extern int can_complete_circuit; +int have_completed_a_circuit(void); +void note_that_we_completed_a_circuit(void); +void note_that_we_maybe_cant_complete_circuits(void); int connection_add_impl(connection_t *conn, int is_connecting); #define connection_add(conn) connection_add_impl((conn), 0) diff --git a/src/or/microdesc.c b/src/or/microdesc.c index fdb549a9ac..0511e870d1 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2013, The Tor Project, Inc. */ +/* Copyright (c) 2009-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" @@ -57,9 +57,9 @@ microdesc_eq_(microdesc_t *a, microdesc_t *b) HT_PROTOTYPE(microdesc_map, microdesc_t, node, microdesc_hash_, microdesc_eq_); -HT_GENERATE(microdesc_map, microdesc_t, node, +HT_GENERATE2(microdesc_map, microdesc_t, node, microdesc_hash_, microdesc_eq_, 0.6, - malloc, realloc, free); + tor_reallocarray_, tor_free_) /** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations. * On success, return the total number of bytes written, and set @@ -147,39 +147,81 @@ microdescs_add_to_cache(microdesc_cache_t *cache, int no_save, time_t listed_at, smartlist_t *requested_digests256) { + void * const DIGEST_REQUESTED = (void*)1; + void * const DIGEST_RECEIVED = (void*)2; + void * const DIGEST_INVALID = (void*)3; + smartlist_t *descriptors, *added; const int allow_annotations = (where != SAVED_NOWHERE); + smartlist_t *invalid_digests = smartlist_new(); descriptors = microdescs_parse_from_string(s, eos, allow_annotations, - where); + where, invalid_digests); if (listed_at != (time_t)-1) { SMARTLIST_FOREACH(descriptors, microdesc_t *, md, md->last_listed = listed_at); } if (requested_digests256) { - digestmap_t *requested; /* XXXX actually we should just use a - digest256map */ - requested = digestmap_new(); - SMARTLIST_FOREACH(requested_digests256, const char *, cp, - digestmap_set(requested, cp, (void*)1)); + digest256map_t *requested; + requested = digest256map_new(); + /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */ + SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp, + digest256map_set(requested, cp, DIGEST_REQUESTED)); + /* Set requested[d] to DIGEST_INVALID for every md we requested which we + * will never be able to parse. Remove the ones we didn't request from + * invalid_digests. + */ + SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) { + if (digest256map_get(requested, cp)) { + digest256map_set(requested, cp, DIGEST_INVALID); + } else { + tor_free(cp); + SMARTLIST_DEL_CURRENT(invalid_digests, cp); + } + } SMARTLIST_FOREACH_END(cp); + /* Update requested[d] to 2 for the mds we asked for and got. Delete the + * ones we never requested from the 'descriptors' smartlist. + */ SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) { - if (digestmap_get(requested, md->digest)) { - digestmap_set(requested, md->digest, (void*)2); + if (digest256map_get(requested, (const uint8_t*)md->digest)) { + digest256map_set(requested, (const uint8_t*)md->digest, + DIGEST_RECEIVED); } else { log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc"); microdesc_free(md); SMARTLIST_DEL_CURRENT(descriptors, md); } } SMARTLIST_FOREACH_END(md); - SMARTLIST_FOREACH_BEGIN(requested_digests256, char *, cp) { - if (digestmap_get(requested, cp) == (void*)2) { + /* Remove the ones we got or the invalid ones from requested_digests256. + */ + SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) { + void *status = digest256map_get(requested, cp); + if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) { tor_free(cp); SMARTLIST_DEL_CURRENT(requested_digests256, cp); } } SMARTLIST_FOREACH_END(cp); - digestmap_free(requested, NULL); + digest256map_free(requested, NULL); + } + + /* For every requested microdescriptor that was unparseable, mark it + * as not to be retried. */ + if (smartlist_len(invalid_digests)) { + networkstatus_t *ns = + networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC); + if (ns) { + SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) { + routerstatus_t *rs = + router_get_mutable_consensus_status_by_descriptor_digest(ns, d); + if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) { + download_status_mark_impossible(&rs->dl_status); + } + } SMARTLIST_FOREACH_END(d); + } } + SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d)); + smartlist_free(invalid_digests); added = microdescs_add_list_to_cache(cache, descriptors, where, no_save); smartlist_free(descriptors); @@ -576,6 +618,7 @@ microdesc_cache_rebuild(microdesc_cache_t *cache, int force) microdesc_wipe_body(md); } } + smartlist_free(wrote); return -1; } @@ -751,7 +794,7 @@ microdesc_average_size(microdesc_cache_t *cache) * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */ smartlist_t * microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, - int downloadable_only, digestmap_t *skip) + int downloadable_only, digest256map_t *skip) { smartlist_t *result = smartlist_new(); time_t now = time(NULL); @@ -763,7 +806,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, !download_status_is_ready(&rs->dl_status, now, get_options()->TestingMicrodescMaxDownloadTries)) continue; - if (skip && digestmap_get(skip, rs->descriptor_digest)) + if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest)) continue; if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN)) continue; @@ -788,7 +831,7 @@ update_microdesc_downloads(time_t now) const or_options_t *options = get_options(); networkstatus_t *consensus; smartlist_t *missing; - digestmap_t *pending; + digest256map_t *pending; if (should_delay_dir_fetches(options, NULL)) return; @@ -802,14 +845,14 @@ update_microdesc_downloads(time_t now) if (!we_fetch_microdescriptors(options)) return; - pending = digestmap_new(); + pending = digest256map_new(); list_pending_microdesc_downloads(pending); missing = microdesc_list_missing_digest256(consensus, get_microdesc_cache(), 1, pending); - digestmap_free(pending, NULL); + digest256map_free(pending, NULL); launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC, missing, NULL, now); diff --git a/src/or/microdesc.h b/src/or/microdesc.h index 7adb8c68af..08571e4bd5 100644 --- a/src/or/microdesc.h +++ b/src/or/microdesc.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -37,7 +37,7 @@ size_t microdesc_average_size(microdesc_cache_t *cache); smartlist_t *microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, int downloadable_only, - digestmap_t *skip); + digest256map_t *skip); void microdesc_free_(microdesc_t *md, const char *fname, int line); #define microdesc_free(md) \ diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 890da0ad17..da110fdff6 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -83,7 +83,11 @@ static consensus_waiting_for_certs_t * before the current consensus becomes invalid. */ static time_t time_to_download_next_consensus[N_CONSENSUS_FLAVORS]; /** Download status for the current consensus networkstatus. */ -static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS]; +static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS] = + { + { 0, 0, DL_SCHED_CONSENSUS }, + { 0, 0, DL_SCHED_CONSENSUS }, + }; /** True iff we have logged a warning about this OR's version being older than * listed by the authorities. */ @@ -253,6 +257,10 @@ networkstatus_vote_free(networkstatus_t *ns) SMARTLIST_FOREACH(ns->supported_methods, char *, c, tor_free(c)); smartlist_free(ns->supported_methods); } + if (ns->package_lines) { + SMARTLIST_FOREACH(ns->package_lines, char *, c, tor_free(c)); + smartlist_free(ns->package_lines); + } if (ns->voters) { SMARTLIST_FOREACH_BEGIN(ns->voters, networkstatus_voter_info_t *, voter) { tor_free(voter->nickname); @@ -591,10 +599,10 @@ networkstatus_vote_find_entry_idx(networkstatus_t *ns, /** As router_get_consensus_status_by_descriptor_digest, but does not return * a const pointer. */ -routerstatus_t * -router_get_mutable_consensus_status_by_descriptor_digest( +MOCK_IMPL(routerstatus_t *, +router_get_mutable_consensus_status_by_descriptor_digest,( networkstatus_t *consensus, - const char *digest) + const char *digest)) { if (!consensus) consensus = current_consensus; @@ -624,8 +632,8 @@ router_get_consensus_status_by_descriptor_digest(networkstatus_t *consensus, /** Given the digest of a router descriptor, return its current download * status, or NULL if the digest is unrecognized. */ -download_status_t * -router_get_dl_status_by_descriptor_digest(const char *d) +MOCK_IMPL(download_status_t *, +router_get_dl_status_by_descriptor_digest,(const char *d)) { routerstatus_t *rs; if (!current_ns_consensus) @@ -754,6 +762,9 @@ update_consensus_networkstatus_downloads(time_t now) resource = networkstatus_get_flavor_name(i); + /* Let's make sure we remembered to update consensus_dl_status */ + tor_assert(consensus_dl_status[i].schedule == DL_SCHED_CONSENSUS); + if (!download_status_is_ready(&consensus_dl_status[i], now, options->TestingConsensusMaxDownloadTries)) continue; /* We failed downloading a consensus too recently. */ @@ -825,6 +836,10 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) a crazy-fast voting interval, though, 2 minutes may be too much. */ min_sec_before_caching = interval/16; + /* make sure we always delay by at least a second before caching */ + if (min_sec_before_caching == 0) { + min_sec_before_caching = 1; + } } if (directory_fetches_dir_info_early(options)) { @@ -856,8 +871,17 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav) dl_interval = (c->valid_until - start) - min_sec_before_caching; } } + /* catch low dl_interval in crazy-fast networks */ if (dl_interval < 1) dl_interval = 1; + /* catch late start in crazy-fast networks */ + if (start+dl_interval >= c->valid_until) + start = c->valid_until - dl_interval - 1; + log_debug(LD_DIR, + "fresh_until: %ld start: %ld " + "dl_interval: %ld valid_until: %ld ", + (long)c->fresh_until, (long)start, dl_interval, + (long)c->valid_until); /* We must not try to replace c while it's still fresh: */ tor_assert(c->fresh_until < start); /* We must download the next one before c is invalid: */ @@ -988,8 +1012,8 @@ networkstatus_get_latest_consensus(void) /** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL * if we don't have one. */ -networkstatus_t * -networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f) +MOCK_IMPL(networkstatus_t *, +networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f)) { if (f == FLAV_NS) return current_ns_consensus; @@ -1055,7 +1079,6 @@ routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b) a->is_valid != b->is_valid || a->is_possible_guard != b->is_possible_guard || a->is_bad_exit != b->is_bad_exit || - a->is_bad_directory != b->is_bad_directory || a->is_hs_dir != b->is_hs_dir || a->version_known != b->version_known; } @@ -1117,7 +1140,7 @@ networkstatus_copy_old_consensus_info(networkstatus_t *new_c, rs_new->last_dir_503_at = rs_old->last_dir_503_at; if (tor_memeq(rs_old->descriptor_digest, rs_new->descriptor_digest, - DIGEST_LEN)) { + DIGEST256_LEN)) { /* And the same descriptor too! */ memcpy(&rs_new->dl_status, &rs_old->dl_status,sizeof(download_status_t)); } @@ -1655,7 +1678,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now) if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE) dirserv_set_router_is_running(ri, now); /* then generate and write out status lines for each of them */ - set_routerstatus_from_routerinfo(&rs, node, ri, now, 0, 0, 0, 0); + set_routerstatus_from_routerinfo(&rs, node, ri, now, 0, 0); smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs)); } SMARTLIST_FOREACH_END(ri); @@ -1672,17 +1695,22 @@ networkstatus_dump_bridge_status_to_file(time_t now) char *status = networkstatus_getinfo_by_purpose("bridge", now); const or_options_t *options = get_options(); char *fname = NULL; - char *thresholds = NULL, *thresholds_and_status = NULL; + char *thresholds = NULL; + char *published_thresholds_and_status = NULL; routerlist_t *rl = router_get_routerlist(); + char published[ISO_TIME_LEN+1]; + + format_iso_time(published, now); dirserv_compute_bridge_flag_thresholds(rl); thresholds = dirserv_get_flag_thresholds_line(); - tor_asprintf(&thresholds_and_status, "flag-thresholds %s\n%s", - thresholds, status); + tor_asprintf(&published_thresholds_and_status, + "published %s\nflag-thresholds %s\n%s", + published, thresholds, status); tor_asprintf(&fname, "%s"PATH_SEPARATOR"networkstatus-bridges", options->DataDirectory); - write_str_to_file(fname,thresholds_and_status,0); + write_str_to_file(fname,published_thresholds_and_status,0); tor_free(thresholds); - tor_free(thresholds_and_status); + tor_free(published_thresholds_and_status); tor_free(fname); tor_free(status); } @@ -1885,6 +1913,33 @@ getinfo_helper_networkstatus(control_connection_t *conn, } else if (!strcmpstart(question, "ns/purpose/")) { *answer = networkstatus_getinfo_by_purpose(question+11, time(NULL)); return *answer ? 0 : -1; + } else if (!strcmp(question, "consensus/packages")) { + const networkstatus_t *ns = networkstatus_get_latest_consensus(); + if (ns && ns->package_lines) + *answer = smartlist_join_strings(ns->package_lines, "\n", 0, NULL); + else + *errmsg = "No consensus available"; + return *answer ? 0 : -1; + } else if (!strcmp(question, "consensus/valid-after") || + !strcmp(question, "consensus/fresh-until") || + !strcmp(question, "consensus/valid-until")) { + const networkstatus_t *ns = networkstatus_get_latest_consensus(); + if (ns) { + time_t t; + if (!strcmp(question, "consensus/valid-after")) + t = ns->valid_after; + else if (!strcmp(question, "consensus/fresh-until")) + t = ns->fresh_until; + else + t = ns->valid_until; + + char tbuf[ISO_TIME_LEN+1]; + format_iso_time(tbuf, t); + *answer = tor_strdup(tbuf); + } else { + *errmsg = "No consensus available"; + } + return *answer ? 0 : -1; } else { return 0; } diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index be0a86cdd8..d6e9e37013 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,6 +12,8 @@ #ifndef TOR_NETWORKSTATUS_H #define TOR_NETWORKSTATUS_H +#include "testsupport.h" + void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); int router_reload_consensus_networkstatus(void); @@ -35,16 +37,19 @@ routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest); int networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out); -download_status_t *router_get_dl_status_by_descriptor_digest(const char *d); + +MOCK_DECL(download_status_t *,router_get_dl_status_by_descriptor_digest, + (const char *d)); + const routerstatus_t *router_get_consensus_status_by_id(const char *digest); routerstatus_t *router_get_mutable_consensus_status_by_id( const char *digest); const routerstatus_t *router_get_consensus_status_by_descriptor_digest( networkstatus_t *consensus, const char *digest); -routerstatus_t *router_get_mutable_consensus_status_by_descriptor_digest( - networkstatus_t *consensus, - const char *digest); +MOCK_DECL(routerstatus_t *, + router_get_mutable_consensus_status_by_descriptor_digest, + (networkstatus_t *consensus, const char *digest)); const routerstatus_t *router_get_consensus_status_by_nickname( const char *nickname, int warn_if_unnamed); @@ -60,8 +65,8 @@ int consensus_is_waiting_for_certs(void); int client_would_use_router(const routerstatus_t *rs, time_t now, const or_options_t *options); networkstatus_t *networkstatus_get_latest_consensus(void); -networkstatus_t *networkstatus_get_latest_consensus_by_flavor( - consensus_flavor_t f); +MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor, + (consensus_flavor_t f)); networkstatus_t *networkstatus_get_live_consensus(time_t now); networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now, int flavor); diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 7b1f338bd4..8f8adb42b5 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" @@ -24,6 +24,23 @@ static void nodelist_drop_node(node_t *node, int remove_from_ht); static void node_free(node_t *node); + +/** count_usable_descriptors counts descriptors with these flag(s) + */ +typedef enum { + /* All descriptors regardless of flags */ + USABLE_DESCRIPTOR_ALL = 0, + /* Only descriptors with the Exit flag */ + USABLE_DESCRIPTOR_EXIT_ONLY = 1 +} usable_descriptor_t; +static void count_usable_descriptors(int *num_present, + int *num_usable, + smartlist_t *descs_out, + const networkstatus_t *consensus, + const or_options_t *options, + time_t now, + routerset_t *in_set, + usable_descriptor_t exit_only); static void update_router_have_minimum_dir_info(void); static double get_frac_paths_needed_for_circs(const or_options_t *options, const networkstatus_t *ns); @@ -53,8 +70,8 @@ node_id_eq(const node_t *node1, const node_t *node2) } HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq); -HT_GENERATE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq, - 0.6, malloc, realloc, free); +HT_GENERATE2(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq, + 0.6, tor_reallocarray_, tor_free_) /** The global nodelist. */ static nodelist_t *the_nodelist=NULL; @@ -241,7 +258,6 @@ nodelist_set_consensus(networkstatus_t *ns) node->is_stable = rs->is_stable; node->is_possible_guard = rs->is_possible_guard; node->is_exit = rs->is_exit; - node->is_bad_directory = rs->is_bad_directory; node->is_bad_exit = rs->is_bad_exit; node->is_hs_dir = rs->is_hs_dir; node->ipv6_preferred = 0; @@ -267,8 +283,7 @@ nodelist_set_consensus(networkstatus_t *ns) node->is_valid = node->is_running = node->is_hs_dir = node->is_fast = node->is_stable = node->is_possible_guard = node->is_exit = - node->is_bad_exit = node->is_bad_directory = - node->ipv6_preferred = 0; + node->is_bad_exit = node->ipv6_preferred = 0; } } } SMARTLIST_FOREACH_END(node); @@ -474,8 +489,8 @@ nodelist_assert_ok(void) /** Return a list of a node_t * for every node we know about. The caller * MUST NOT modify the list. (You can set and clear flags in the nodes if * you must, but you must not add or remove nodes.) */ -smartlist_t * -nodelist_get_list(void) +MOCK_IMPL(smartlist_t *, +nodelist_get_list,(void)) { init_nodelist(); return the_nodelist->nodes; @@ -517,8 +532,8 @@ node_get_by_hex_id(const char *hex_id) * the corresponding node_t, or NULL if none exists. Warn the user if * <b>warn_if_unnamed</b> is set, and they have specified a router by * nickname, but the Named flag isn't set for that router. */ -const node_t * -node_get_by_nickname(const char *nickname, int warn_if_unnamed) +MOCK_IMPL(const node_t *, +node_get_by_nickname,(const char *nickname, int warn_if_unnamed)) { const node_t *node; if (!the_nodelist) @@ -1258,20 +1273,28 @@ router_set_status(const char *digest, int up) } /** True iff, the last time we checked whether we had enough directory info - * to build circuits, the answer was "yes". */ + * to build circuits, the answer was "yes". If there are no exits in the + * consensus, we act as if we have 100% of the exit directory info. */ static int have_min_dir_info = 0; + +/** Does the consensus contain nodes that can exit? */ +static consensus_path_type_t have_consensus_path = CONSENSUS_PATH_UNKNOWN; + /** True iff enough has changed since the last time we checked whether we had * enough directory info to build circuits that our old answer can no longer * be trusted. */ static int need_to_update_have_min_dir_info = 1; /** String describing what we're missing before we have enough directory * info. */ -static char dir_info_status[256] = ""; - -/** Return true iff we have enough networkstatus and router information to - * start building circuits. Right now, this means "more than half the - * networkstatus documents, and at least 1/4 of expected routers." */ -//XXX should consider whether we have enough exiting nodes here. +static char dir_info_status[512] = ""; + +/** Return true iff we have enough consensus information to + * start building circuits. Right now, this means "a consensus that's + * less than a day old, and at least 60% of router descriptors (configurable), + * weighted by bandwidth. Treat the exit fraction as 100% if there are + * no exits in the consensus." + * To obtain the final weighted bandwidth, we multiply the + * weighted bandwidth fraction for each position (guard, middle, exit). */ int router_have_minimum_dir_info(void) { @@ -1293,6 +1316,24 @@ router_have_minimum_dir_info(void) return have_min_dir_info; } +/** Set to CONSENSUS_PATH_EXIT if there is at least one exit node + * in the consensus. We update this flag in compute_frac_paths_available if + * there is at least one relay that has an Exit flag in the consensus. + * Used to avoid building exit circuits when they will almost certainly fail. + * Set to CONSENSUS_PATH_INTERNAL if there are no exits in the consensus. + * (This situation typically occurs during bootstrap of a test network.) + * Set to CONSENSUS_PATH_UNKNOWN if we have never checked, or have + * reason to believe our last known value was invalid or has expired. + * If we're in a network with TestingDirAuthVoteExit set, + * this can cause router_have_consensus_path() to be set to + * CONSENSUS_PATH_EXIT, even if there are no nodes with accept exit policies. + */ +consensus_path_type_t +router_have_consensus_path(void) +{ + return have_consensus_path; +} + /** Called when our internal view of the directory has changed. This can be * when the authorities change, networkstatuses change, the list of routerdescs * changes, or number of running routers changes. @@ -1313,22 +1354,26 @@ get_dir_info_status_string(void) } /** Iterate over the servers listed in <b>consensus</b>, and count how many of - * them seem like ones we'd use, and how many of <em>those</em> we have - * descriptors for. Store the former in *<b>num_usable</b> and the latter in - * *<b>num_present</b>. If <b>in_set</b> is non-NULL, only consider those - * routers in <b>in_set</b>. If <b>exit_only</b> is true, only consider nodes - * with the Exit flag. If *descs_out is present, add a node_t for each - * usable descriptor to it. + * them seem like ones we'd use (store this in *<b>num_usable</b>), and how + * many of <em>those</em> we have descriptors for (store this in + * *<b>num_present</b>). + * + * If <b>in_set</b> is non-NULL, only consider those routers in <b>in_set</b>. + * If <b>exit_only</b> is USABLE_DESCRIPTOR_EXIT_ONLY, only consider nodes + * with the Exit flag. + * If *<b>descs_out</b> is present, add a node_t for each usable descriptor + * to it. */ static void count_usable_descriptors(int *num_present, int *num_usable, smartlist_t *descs_out, const networkstatus_t *consensus, const or_options_t *options, time_t now, - routerset_t *in_set, int exit_only) + routerset_t *in_set, + usable_descriptor_t exit_only) { const int md = (consensus->flavor == FLAV_MICRODESC); - *num_present = 0, *num_usable=0; + *num_present = 0, *num_usable = 0; SMARTLIST_FOREACH_BEGIN(consensus->routerstatus_list, routerstatus_t *, rs) { @@ -1336,7 +1381,7 @@ count_usable_descriptors(int *num_present, int *num_usable, if (!node) continue; /* This would be a bug: every entry in the consensus is * supposed to have a node. */ - if (exit_only && ! rs->is_exit) + if (exit_only == USABLE_DESCRIPTOR_EXIT_ONLY && ! rs->is_exit) continue; if (in_set && ! routerset_contains_routerstatus(in_set, rs, -1)) continue; @@ -1360,11 +1405,22 @@ count_usable_descriptors(int *num_present, int *num_usable, log_debug(LD_DIR, "%d usable, %d present (%s%s).", *num_usable, *num_present, - md ? "microdesc" : "desc", exit_only ? " exits" : "s"); + md ? "microdesc" : "desc", + exit_only == USABLE_DESCRIPTOR_EXIT_ONLY ? " exits" : "s"); } /** Return an estimate of which fraction of usable paths through the Tor - * network we have available for use. */ + * network we have available for use. Count how many routers seem like ones + * we'd use (store this in *<b>num_usable_out</b>), and how many of + * <em>those</em> we have descriptors for (store this in + * *<b>num_present_out</b>.) + * + * If **<b>status_out</b> is present, allocate a new string and print the + * available percentages of guard, middle, and exit nodes to it, noting + * whether there are exits in the consensus. + * If there are no guards in the consensus, + * we treat the exit fraction as 100%. + */ static double compute_frac_paths_available(const networkstatus_t *consensus, const or_options_t *options, time_t now, @@ -1374,17 +1430,20 @@ compute_frac_paths_available(const networkstatus_t *consensus, smartlist_t *guards = smartlist_new(); smartlist_t *mid = smartlist_new(); smartlist_t *exits = smartlist_new(); - smartlist_t *myexits= smartlist_new(); - smartlist_t *myexits_unflagged = smartlist_new(); - double f_guard, f_mid, f_exit, f_myexit, f_myexit_unflagged; - int np, nu; /* Ignored */ + double f_guard, f_mid, f_exit; + double f_path = 0.0; + /* Used to determine whether there are any exits in the consensus */ + int np = 0; + /* Used to determine whether there are any exits with descriptors */ + int nu = 0; const int authdir = authdir_mode_v3(options); count_usable_descriptors(num_present_out, num_usable_out, - mid, consensus, options, now, NULL, 0); + mid, consensus, options, now, NULL, + USABLE_DESCRIPTOR_ALL); if (options->EntryNodes) { count_usable_descriptors(&np, &nu, guards, consensus, options, now, - options->EntryNodes, 0); + options->EntryNodes, USABLE_DESCRIPTOR_ALL); } else { SMARTLIST_FOREACH(mid, const node_t *, node, { if (authdir) { @@ -1397,60 +1456,148 @@ compute_frac_paths_available(const networkstatus_t *consensus, }); } - /* All nodes with exit flag */ + /* All nodes with exit flag + * If we're in a network with TestingDirAuthVoteExit set, + * this can cause false positives on have_consensus_path, + * incorrectly setting it to CONSENSUS_PATH_EXIT. This is + * an unavoidable feature of forcing authorities to declare + * certain nodes as exits. + */ count_usable_descriptors(&np, &nu, exits, consensus, options, now, - NULL, 1); - /* All nodes with exit flag in ExitNodes option */ - count_usable_descriptors(&np, &nu, myexits, consensus, options, now, - options->ExitNodes, 1); - /* Now compute the nodes in the ExitNodes option where which we don't know - * what their exit policy is, or we know it permits something. */ - count_usable_descriptors(&np, &nu, myexits_unflagged, - consensus, options, now, - options->ExitNodes, 0); - SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) { - if (node_has_descriptor(node) && node_exit_policy_rejects_all(node)) - SMARTLIST_DEL_CURRENT(myexits_unflagged, node); - } SMARTLIST_FOREACH_END(node); + NULL, USABLE_DESCRIPTOR_EXIT_ONLY); + log_debug(LD_NET, + "%s: %d present, %d usable", + "exits", + np, + nu); + + /* We need at least 1 exit present in the consensus to consider + * building exit paths */ + /* Update our understanding of whether the consensus has exits */ + consensus_path_type_t old_have_consensus_path = have_consensus_path; + have_consensus_path = ((nu > 0) ? + CONSENSUS_PATH_EXIT : + CONSENSUS_PATH_INTERNAL); + + if (have_consensus_path == CONSENSUS_PATH_INTERNAL + && old_have_consensus_path != have_consensus_path) { + log_notice(LD_NET, + "The current consensus has no exit nodes. " + "Tor can only build internal paths, " + "such as paths to hidden services."); + + /* However, exit nodes can reachability self-test using this consensus, + * join the network, and appear in a later consensus. This will allow + * the network to build exit paths, such as paths for world wide web + * browsing (as distinct from hidden service web browsing). */ + } f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD); f_mid = frac_nodes_with_descriptors(mid, WEIGHT_FOR_MID); f_exit = frac_nodes_with_descriptors(exits, WEIGHT_FOR_EXIT); - f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT); - f_myexit_unflagged= - frac_nodes_with_descriptors(myexits_unflagged,WEIGHT_FOR_EXIT); - - /* If our ExitNodes list has eliminated every possible Exit node, and there - * were some possible Exit nodes, then instead consider nodes that permit - * exiting to some ports. */ - if (smartlist_len(myexits) == 0 && - smartlist_len(myexits_unflagged)) { - f_myexit = f_myexit_unflagged; - } + + log_debug(LD_NET, + "f_guard: %.2f, f_mid: %.2f, f_exit: %.2f", + f_guard, + f_mid, + f_exit); smartlist_free(guards); smartlist_free(mid); smartlist_free(exits); - smartlist_free(myexits); - smartlist_free(myexits_unflagged); - /* This is a tricky point here: we don't want to make it easy for a - * directory to trickle exits to us until it learns which exits we have - * configured, so require that we have a threshold both of total exits - * and usable exits. */ - if (f_myexit < f_exit) - f_exit = f_myexit; + if (options->ExitNodes) { + double f_myexit, f_myexit_unflagged; + smartlist_t *myexits= smartlist_new(); + smartlist_t *myexits_unflagged = smartlist_new(); + + /* All nodes with exit flag in ExitNodes option */ + count_usable_descriptors(&np, &nu, myexits, consensus, options, now, + options->ExitNodes, USABLE_DESCRIPTOR_EXIT_ONLY); + log_debug(LD_NET, + "%s: %d present, %d usable", + "myexits", + np, + nu); + + /* Now compute the nodes in the ExitNodes option where which we don't know + * what their exit policy is, or we know it permits something. */ + count_usable_descriptors(&np, &nu, myexits_unflagged, + consensus, options, now, + options->ExitNodes, USABLE_DESCRIPTOR_ALL); + log_debug(LD_NET, + "%s: %d present, %d usable", + "myexits_unflagged (initial)", + np, + nu); + + SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) { + if (node_has_descriptor(node) && node_exit_policy_rejects_all(node)) { + SMARTLIST_DEL_CURRENT(myexits_unflagged, node); + /* this node is not actually an exit */ + np--; + /* this node is unusable as an exit */ + nu--; + } + } SMARTLIST_FOREACH_END(node); + + log_debug(LD_NET, + "%s: %d present, %d usable", + "myexits_unflagged (final)", + np, + nu); + + f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT); + f_myexit_unflagged= + frac_nodes_with_descriptors(myexits_unflagged,WEIGHT_FOR_EXIT); + + log_debug(LD_NET, + "f_exit: %.2f, f_myexit: %.2f, f_myexit_unflagged: %.2f", + f_exit, + f_myexit, + f_myexit_unflagged); + + /* If our ExitNodes list has eliminated every possible Exit node, and there + * were some possible Exit nodes, then instead consider nodes that permit + * exiting to some ports. */ + if (smartlist_len(myexits) == 0 && + smartlist_len(myexits_unflagged)) { + f_myexit = f_myexit_unflagged; + } + + smartlist_free(myexits); + smartlist_free(myexits_unflagged); + + /* This is a tricky point here: we don't want to make it easy for a + * directory to trickle exits to us until it learns which exits we have + * configured, so require that we have a threshold both of total exits + * and usable exits. */ + if (f_myexit < f_exit) + f_exit = f_myexit; + } + + /* if the consensus has no exits, treat the exit fraction as 100% */ + if (router_have_consensus_path() != CONSENSUS_PATH_EXIT) { + f_exit = 1.0; + } + + f_path = f_guard * f_mid * f_exit; if (status_out) tor_asprintf(status_out, "%d%% of guards bw, " "%d%% of midpoint bw, and " - "%d%% of exit bw", + "%d%% of exit bw%s = " + "%d%% of path bw", (int)(f_guard*100), (int)(f_mid*100), - (int)(f_exit*100)); + (int)(f_exit*100), + (router_have_consensus_path() == CONSENSUS_PATH_EXIT ? + "" : + " (no exits in consensus)"), + (int)(f_path*100)); - return f_guard * f_mid * f_exit; + return f_path; } /** We just fetched a new set of descriptors. Compute how far through @@ -1523,6 +1670,7 @@ update_router_have_minimum_dir_info(void) using_md = consensus->flavor == FLAV_MICRODESC; + /* Check fraction of available paths */ { char *status = NULL; int num_present=0, num_usable=0; @@ -1536,7 +1684,6 @@ update_router_have_minimum_dir_info(void) "can only build %d%% of likely paths. (We have %s.)", using_md?"micro":"", num_present, num_usable, (int)(paths*100), status); - /* log_notice(LD_NET, "%s", dir_info_status); */ tor_free(status); res = 0; control_event_bootstrap(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0); @@ -1548,12 +1695,17 @@ update_router_have_minimum_dir_info(void) } done: + + /* If paths have just become available in this update. */ if (res && !have_min_dir_info) { - log_notice(LD_DIR, - "We now have enough directory information to build circuits."); control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO"); - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_OR, 0); + if (control_event_bootstrap(BOOTSTRAP_STATUS_CONN_OR, 0) == 0) { + log_notice(LD_DIR, + "We now have enough directory information to build circuits."); + } } + + /* If paths have just become unavailable in this update. */ if (!res && have_min_dir_info) { int quiet = directory_too_idle_to_fetch_descriptors(options, now); tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR, @@ -1564,8 +1716,8 @@ update_router_have_minimum_dir_info(void) * is back up and usable, and b) disable some activities that Tor * should only do while circuits are working, like reachability tests * and fetching bridge descriptors only over circuits. */ - can_complete_circuit = 0; - + note_that_we_maybe_cant_complete_circuits(); + have_consensus_path = CONSENSUS_PATH_UNKNOWN; control_event_client_status(LOG_NOTICE, "NOT_ENOUGH_DIR_INFO"); } have_min_dir_info = res; diff --git a/src/or/nodelist.h b/src/or/nodelist.h index 8e719e012d..a131e0dd4e 100644 --- a/src/or/nodelist.h +++ b/src/or/nodelist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -31,7 +31,8 @@ smartlist_t *nodelist_find_nodes_with_microdesc(const microdesc_t *md); void nodelist_free_all(void); void nodelist_assert_ok(void); -const node_t *node_get_by_nickname(const char *nickname, int warn_if_unnamed); +MOCK_DECL(const node_t *, node_get_by_nickname, + (const char *nickname, int warn_if_unnamed)); void node_get_verbose_nickname(const node_t *node, char *verbose_name_out); void node_get_verbose_nickname_by_id(const char *id_digest, @@ -60,7 +61,7 @@ void node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out); int node_has_curve25519_onion_key(const node_t *node); -smartlist_t *nodelist_get_list(void); +MOCK_DECL(smartlist_t *, nodelist_get_list, (void)); /* Temporary during transition to multiple addresses. */ void node_get_addr(const node_t *node, tor_addr_t *addr_out); @@ -78,7 +79,37 @@ int node_is_unreliable(const node_t *router, int need_uptime, int router_exit_policy_all_nodes_reject(const tor_addr_t *addr, uint16_t port, int need_uptime); void router_set_status(const char *digest, int up); + +/** router_have_minimum_dir_info tests to see if we have enough + * descriptor information to create circuits. + * If there are exits in the consensus, we wait until we have enough + * info to create exit paths before creating any circuits. If there are + * no exits in the consensus, we wait for enough info to create internal + * paths, and should avoid creating exit paths, as they will simply fail. + * We make sure we create all available circuit types at the same time. */ int router_have_minimum_dir_info(void); + +/** Set to CONSENSUS_PATH_EXIT if there is at least one exit node + * in the consensus. We update this flag in compute_frac_paths_available if + * there is at least one relay that has an Exit flag in the consensus. + * Used to avoid building exit circuits when they will almost certainly fail. + * Set to CONSENSUS_PATH_INTERNAL if there are no exits in the consensus. + * (This situation typically occurs during bootstrap of a test network.) + * Set to CONSENSUS_PATH_UNKNOWN if we have never checked, or have + * reason to believe our last known value was invalid or has expired. + */ +typedef enum { + /* we haven't checked yet, or we have invalidated our previous check */ + CONSENSUS_PATH_UNKNOWN = -1, + /* The consensus only has internal relays, and we should only + * create internal paths, circuits, streams, ... */ + CONSENSUS_PATH_INTERNAL = 0, + /* The consensus has at least one exit, and can therefore (potentially) + * create exit and internal paths, circuits, streams, ... */ + CONSENSUS_PATH_EXIT = 1 +} consensus_path_type_t; +consensus_path_type_t router_have_consensus_path(void); + void router_dir_info_changed(void); const char *get_dir_info_status_string(void); int count_loading_descriptors_progress(void); diff --git a/src/or/ntmain.c b/src/or/ntmain.c index e848314043..833d870041 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "or.h" diff --git a/src/or/ntmain.h b/src/or/ntmain.h index d3027936cd..eb55a296f6 100644 --- a/src/or/ntmain.h +++ b/src/or/ntmain.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,10 +13,8 @@ #define TOR_NTMAIN_H #ifdef _WIN32 -#if !defined (WINCE) #define NT_SERVICE #endif -#endif #ifdef NT_SERVICE int nt_service_parse_options(int argc, char **argv, int *should_exit); diff --git a/src/or/onion.c b/src/or/onion.c index ae39f451f4..4864792511 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -111,15 +111,11 @@ have_room_for_onionskin(uint16_t type) (uint64_t)options->MaxOnionQueueDelay) return 0; -#ifdef CURVE25519_ENABLED /* If we support the ntor handshake, then don't let TAP handshakes use * more than 2/3 of the space on the queue. */ if (type == ONION_HANDSHAKE_TYPE_TAP && tap_usec / 1000 > (uint64_t)options->MaxOnionQueueDelay * 2 / 3) return 0; -#else - (void) type; -#endif return 1; } @@ -299,6 +295,8 @@ onion_pending_remove(or_circuit_t *circ) victim = circ->onionqueue_entry; if (victim) onion_queue_entry_remove(victim); + + cpuworker_cancel_circ_handshake(circ); } /** Remove a queue entry <b>victim</b> from the queue, unlinking it from @@ -343,38 +341,35 @@ clear_pending_onions(void) /* ============================================================ */ -/** Fill in a server_onion_keys_t object at <b>keys</b> with all of the keys +/** Return a new server_onion_keys_t object with all of the keys * and other info we might need to do onion handshakes. (We make a copy of * our keys for each cpuworker to avoid race conditions with the main thread, * and to avoid locking) */ -void -setup_server_onion_keys(server_onion_keys_t *keys) +server_onion_keys_t * +server_onion_keys_new(void) { - memset(keys, 0, sizeof(server_onion_keys_t)); + server_onion_keys_t *keys = tor_malloc_zero(sizeof(server_onion_keys_t)); memcpy(keys->my_identity, router_get_my_id_digest(), DIGEST_LEN); dup_onion_keys(&keys->onion_key, &keys->last_onion_key); -#ifdef CURVE25519_ENABLED keys->curve25519_key_map = construct_ntor_key_map(); keys->junk_keypair = tor_malloc_zero(sizeof(curve25519_keypair_t)); curve25519_keypair_generate(keys->junk_keypair, 0); -#endif + return keys; } -/** Release all storage held in <b>keys</b>, but do not free <b>keys</b> - * itself (as it's likely to be stack-allocated.) */ +/** Release all storage held in <b>keys</b>. */ void -release_server_onion_keys(server_onion_keys_t *keys) +server_onion_keys_free(server_onion_keys_t *keys) { if (! keys) return; crypto_pk_free(keys->onion_key); crypto_pk_free(keys->last_onion_key); -#ifdef CURVE25519_ENABLED ntor_key_map_free(keys->curve25519_key_map); tor_free(keys->junk_keypair); -#endif - memset(keys, 0, sizeof(server_onion_keys_t)); + memwipe(keys, 0, sizeof(server_onion_keys_t)); + tor_free(keys); } /** Release whatever storage is held in <b>state</b>, depending on its @@ -391,12 +386,10 @@ onion_handshake_state_release(onion_handshake_state_t *state) fast_handshake_state_free(state->u.fast); state->u.fast = NULL; break; -#ifdef CURVE25519_ENABLED case ONION_HANDSHAKE_TYPE_NTOR: ntor_handshake_state_free(state->u.ntor); state->u.ntor = NULL; break; -#endif default: log_warn(LD_BUG, "called with unknown handshake state type %d", (int)state->tag); @@ -436,7 +429,6 @@ onion_skin_create(int type, r = CREATE_FAST_LEN; break; case ONION_HANDSHAKE_TYPE_NTOR: -#ifdef CURVE25519_ENABLED if (tor_mem_is_zero((const char*)node->curve25519_onion_key.public_key, CURVE25519_PUBKEY_LEN)) return -1; @@ -447,9 +439,6 @@ onion_skin_create(int type, return -1; r = NTOR_ONIONSKIN_LEN; -#else - return -1; -#endif break; default: log_warn(LD_BUG, "called with unknown handshake state type %d", type); @@ -501,7 +490,6 @@ onion_skin_server_handshake(int type, memcpy(rend_nonce_out, reply_out+DIGEST_LEN, DIGEST_LEN); break; case ONION_HANDSHAKE_TYPE_NTOR: -#ifdef CURVE25519_ENABLED if (onionskin_len < NTOR_ONIONSKIN_LEN) return -1; { @@ -522,9 +510,6 @@ onion_skin_server_handshake(int type, tor_free(keys_tmp); r = NTOR_REPLY_LEN; } -#else - return -1; -#endif break; default: log_warn(LD_BUG, "called with unknown handshake state type %d", type); @@ -541,13 +526,15 @@ onion_skin_server_handshake(int type, * bytes worth of key material in <b>keys_out_len</b>, set * <b>rend_authenticator_out</b> to the "KH" field that can be used to * establish introduction points at this hop, and return 0. On failure, - * return -1. */ + * return -1, and set *msg_out to an error message if this is worth + * complaining to the usre about. */ int onion_skin_client_handshake(int type, const onion_handshake_state_t *handshake_state, const uint8_t *reply, size_t reply_len, uint8_t *keys_out, size_t keys_out_len, - uint8_t *rend_authenticator_out) + uint8_t *rend_authenticator_out, + const char **msg_out) { if (handshake_state->tag != type) return -1; @@ -555,12 +542,14 @@ onion_skin_client_handshake(int type, switch (type) { case ONION_HANDSHAKE_TYPE_TAP: if (reply_len != TAP_ONIONSKIN_REPLY_LEN) { - log_warn(LD_CIRC, "TAP reply was not of the correct length."); + if (msg_out) + *msg_out = "TAP reply was not of the correct length."; return -1; } if (onion_skin_TAP_client_handshake(handshake_state->u.tap, (const char*)reply, - (char *)keys_out, keys_out_len) < 0) + (char *)keys_out, keys_out_len, + msg_out) < 0) return -1; memcpy(rend_authenticator_out, reply+DH_KEY_LEN, DIGEST_LEN); @@ -568,27 +557,28 @@ onion_skin_client_handshake(int type, return 0; case ONION_HANDSHAKE_TYPE_FAST: if (reply_len != CREATED_FAST_LEN) { - log_warn(LD_CIRC, "CREATED_FAST reply was not of the correct length."); + if (msg_out) + *msg_out = "TAP reply was not of the correct length."; return -1; } if (fast_client_handshake(handshake_state->u.fast, reply, - keys_out, keys_out_len) < 0) + keys_out, keys_out_len, msg_out) < 0) return -1; memcpy(rend_authenticator_out, reply+DIGEST_LEN, DIGEST_LEN); return 0; -#ifdef CURVE25519_ENABLED case ONION_HANDSHAKE_TYPE_NTOR: if (reply_len < NTOR_REPLY_LEN) { - log_warn(LD_CIRC, "ntor reply was not of the correct length."); + if (msg_out) + *msg_out = "ntor reply was not of the correct length."; return -1; } { size_t keys_tmp_len = keys_out_len + DIGEST_LEN; uint8_t *keys_tmp = tor_malloc(keys_tmp_len); if (onion_skin_ntor_client_handshake(handshake_state->u.ntor, - reply, - keys_tmp, keys_tmp_len) < 0) { + reply, + keys_tmp, keys_tmp_len, msg_out) < 0) { tor_free(keys_tmp); return -1; } @@ -598,7 +588,6 @@ onion_skin_client_handshake(int type, tor_free(keys_tmp); } return 0; -#endif default: log_warn(LD_BUG, "called with unknown handshake state type %d", type); tor_fragile_assert(); @@ -637,12 +626,10 @@ check_create_cell(const create_cell_t *cell, int unknown_ok) if (cell->handshake_len != CREATE_FAST_LEN) return -1; break; -#ifdef CURVE25519_ENABLED case ONION_HANDSHAKE_TYPE_NTOR: if (cell->handshake_len != NTOR_ONIONSKIN_LEN) return -1; break; -#endif default: if (! unknown_ok) return -1; diff --git a/src/or/onion.h b/src/or/onion.h index d62f032b87..45454f480d 100644 --- a/src/or/onion.h +++ b/src/or/onion.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -23,17 +23,15 @@ typedef struct server_onion_keys_t { uint8_t my_identity[DIGEST_LEN]; crypto_pk_t *onion_key; crypto_pk_t *last_onion_key; -#ifdef CURVE25519_ENABLED di_digest256_map_t *curve25519_key_map; curve25519_keypair_t *junk_keypair; -#endif } server_onion_keys_t; #define MAX_ONIONSKIN_CHALLENGE_LEN 255 #define MAX_ONIONSKIN_REPLY_LEN 255 -void setup_server_onion_keys(server_onion_keys_t *keys); -void release_server_onion_keys(server_onion_keys_t *keys); +server_onion_keys_t *server_onion_keys_new(void); +void server_onion_keys_free(server_onion_keys_t *keys); void onion_handshake_state_release(onion_handshake_state_t *state); @@ -51,7 +49,8 @@ int onion_skin_client_handshake(int type, const onion_handshake_state_t *handshake_state, const uint8_t *reply, size_t reply_len, uint8_t *keys_out, size_t key_out_len, - uint8_t *rend_authenticator_out); + uint8_t *rend_authenticator_out, + const char **msg_out); /** A parsed CREATE, CREATE_FAST, or CREATE2 cell. */ typedef struct create_cell_t { diff --git a/src/or/onion_fast.c b/src/or/onion_fast.c index 38b62decc3..7584112570 100644 --- a/src/or/onion_fast.c +++ b/src/or/onion_fast.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -92,7 +92,8 @@ int fast_client_handshake(const fast_handshake_state_t *handshake_state, const uint8_t *handshake_reply_out,/*DIGEST_LEN*2 bytes*/ uint8_t *key_out, - size_t key_out_len) + size_t key_out_len, + const char **msg_out) { uint8_t tmp[DIGEST_LEN+DIGEST_LEN]; uint8_t *out; @@ -104,13 +105,14 @@ fast_client_handshake(const fast_handshake_state_t *handshake_state, out_len = key_out_len+DIGEST_LEN; out = tor_malloc(out_len); if (crypto_expand_key_material_TAP(tmp, sizeof(tmp), out, out_len)) { - log_warn(LD_CIRC, "Failed to expand key material"); + if (msg_out) + *msg_out = "Failed to expand key material"; goto done; } if (tor_memneq(out, handshake_reply_out+DIGEST_LEN, DIGEST_LEN)) { /* H(K) does *not* match. Something fishy. */ - log_warn(LD_PROTOCOL,"Digest DOES NOT MATCH on fast handshake. " - "Bug or attack."); + if (msg_out) + *msg_out = "Digest DOES NOT MATCH on fast handshake. Bug or attack."; goto done; } memcpy(key_out, out+DIGEST_LEN, key_out_len); diff --git a/src/or/onion_fast.h b/src/or/onion_fast.h index 8c078378d2..d50d503a73 100644 --- a/src/or/onion_fast.h +++ b/src/or/onion_fast.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -32,7 +32,8 @@ int fast_server_handshake(const uint8_t *message_in, int fast_client_handshake(const fast_handshake_state_t *handshake_state, const uint8_t *handshake_reply_out, uint8_t *key_out, - size_t key_out_len); + size_t key_out_len, + const char **msg_out); #endif diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c index ef501f69da..539f06f61f 100644 --- a/src/or/onion_ntor.c +++ b/src/or/onion_ntor.c @@ -1,10 +1,10 @@ -/* Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" -#include "crypto.h" #define ONION_NTOR_PRIVATE +#include "crypto.h" #include "onion_ntor.h" #include "torlog.h" #include "util.h" @@ -226,7 +226,8 @@ onion_skin_ntor_client_handshake( const ntor_handshake_state_t *handshake_state, const uint8_t *handshake_reply, uint8_t *key_out, - size_t key_out_len) + size_t key_out_len, + const char **msg_out) { const tweakset_t *T = &proto1_tweaks; /* Sensitive stack-allocated material. Kept in an anonymous struct to make @@ -292,7 +293,19 @@ onion_skin_ntor_client_handshake( memwipe(&s, 0, sizeof(s)); if (bad) { - log_warn(LD_PROTOCOL, "Invalid result from curve25519 handshake: %d", bad); + if (bad & 4) { + if (msg_out) + *msg_out = NULL; /* Don't report this one; we probably just had the + * wrong onion key.*/ + log_fn(LOG_INFO, LD_PROTOCOL, + "Invalid result from curve25519 handshake: %d", bad); + } + if (bad & 3) { + if (msg_out) + *msg_out = "Zero output from curve25519 handshake"; + log_fn(LOG_WARN, LD_PROTOCOL, + "Invalid result from curve25519 handshake: %d", bad); + } } return bad ? -1 : 0; diff --git a/src/or/onion_ntor.h b/src/or/onion_ntor.h index c942e6e0f0..0a39c1de16 100644 --- a/src/or/onion_ntor.h +++ b/src/or/onion_ntor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ONION_NTOR_H @@ -17,7 +17,6 @@ typedef struct ntor_handshake_state_t ntor_handshake_state_t; /** Length of an ntor reply, as sent from server to client. */ #define NTOR_REPLY_LEN 64 -#ifdef CURVE25519_ENABLED void ntor_handshake_state_free(ntor_handshake_state_t *state); int onion_skin_ntor_create(const uint8_t *router_id, @@ -37,7 +36,8 @@ int onion_skin_ntor_client_handshake( const ntor_handshake_state_t *handshake_state, const uint8_t *handshake_reply, uint8_t *key_out, - size_t key_out_len); + size_t key_out_len, + const char **msg_out); #ifdef ONION_NTOR_PRIVATE @@ -59,5 +59,3 @@ struct ntor_handshake_state_t { #endif -#endif - diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index 65f8275f75..487cbeec04 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -183,7 +183,8 @@ int onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state, const char *handshake_reply, /* TAP_ONIONSKIN_REPLY_LEN bytes */ char *key_out, - size_t key_out_len) + size_t key_out_len, + const char **msg_out) { ssize_t len; char *key_material=NULL; @@ -196,14 +197,15 @@ onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state, handshake_reply, DH_KEY_LEN, key_material, key_material_len); if (len < 0) { - log_warn(LD_PROTOCOL,"DH computation failed."); + if (msg_out) + *msg_out = "DH computation failed."; goto err; } if (tor_memneq(key_material, handshake_reply+DH_KEY_LEN, DIGEST_LEN)) { /* H(K) does *not* match. Something fishy. */ - log_warn(LD_PROTOCOL,"Digest DOES NOT MATCH on onion handshake. " - "Bug or attack."); + if (msg_out) + *msg_out = "Digest DOES NOT MATCH on onion handshake. Bug or attack."; goto err; } diff --git a/src/or/onion_tap.h b/src/or/onion_tap.h index b978b66737..c548b3d99f 100644 --- a/src/or/onion_tap.h +++ b/src/or/onion_tap.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -31,7 +31,8 @@ int onion_skin_TAP_server_handshake(const char *onion_skin, int onion_skin_TAP_client_handshake(crypto_dh_t *handshake_state, const char *handshake_reply, char *key_out, - size_t key_out_len); + size_t key_out_len, + const char **msg_out); #endif diff --git a/src/or/or.h b/src/or/or.h index adf3cfa866..0d81b54d94 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,7 +14,7 @@ #include "orconfig.h" -#ifdef __COVERITY__ +#if defined(__clang_analyzer__) || defined(__COVERITY__) /* If we're building for a static analysis, turn on all the off-by-default * features. */ #ifndef INSTRUMENT_DOWNLOADS @@ -119,6 +119,7 @@ * conflict with system-defined signals. */ #define SIGNEWNYM 129 #define SIGCLEARDNSCACHE 130 +#define SIGHEARTBEAT 131 #if (SIZEOF_CELL_T != 0) /* On Irix, stdlib.h defines a cell_t type, so we need to make sure @@ -212,8 +213,7 @@ typedef enum { #define CONN_TYPE_DIR_LISTENER 8 /** Type for HTTP connections to the directory server. */ #define CONN_TYPE_DIR 9 -/** Connection from the main process to a CPU worker process. */ -#define CONN_TYPE_CPUWORKER 10 +/* Type 10 is unused. */ /** Type for listening for connections from user interface process. */ #define CONN_TYPE_CONTROL_LISTENER 11 /** Type for connections from user interface process. */ @@ -241,7 +241,7 @@ typedef enum { #define PROXY_CONNECT 1 #define PROXY_SOCKS4 2 #define PROXY_SOCKS5 3 -/* !!!! If there is ever a PROXY_* type over 2, we must grow the proxy_type +/* !!!! If there is ever a PROXY_* type over 3, we must grow the proxy_type * field in or_connection_t */ /* Pluggable transport proxy type. Don't use this in or_connection_t, @@ -275,17 +275,6 @@ typedef enum { /** State for any listener connection. */ #define LISTENER_STATE_READY 0 -#define CPUWORKER_STATE_MIN_ 1 -/** State for a connection to a cpuworker process that's idle. */ -#define CPUWORKER_STATE_IDLE 1 -/** State for a connection to a cpuworker process that's processing a - * handshake. */ -#define CPUWORKER_STATE_BUSY_ONION 2 -#define CPUWORKER_STATE_MAX_ 2 - -#define CPUWORKER_TASK_ONION CPUWORKER_STATE_BUSY_ONION -#define CPUWORKER_TASK_SHUTDOWN 255 - #define OR_CONN_STATE_MIN_ 1 /** State for a connection to an OR: waiting for connect() to finish. */ #define OR_CONN_STATE_CONNECTING 1 @@ -676,6 +665,10 @@ typedef enum { /* Negative reasons are internal: we never send them in a DESTROY or TRUNCATE * call; they only go to the controller for tracking */ + +/* Closing introduction point that were opened in parallel. */ +#define END_CIRC_REASON_IP_NOW_REDUNDANT -4 + /** Our post-timeout circuit time measurement period expired. * We must give up now */ #define END_CIRC_REASON_MEASUREMENT_EXPIRED -3 @@ -1138,6 +1131,51 @@ typedef struct socks_request_t socks_request_t; #define generic_buffer_t buf_t #endif +typedef struct entry_port_cfg_t { + /* Client port types (socks, dns, trans, natd) only: */ + uint8_t isolation_flags; /**< Zero or more isolation flags */ + int session_group; /**< A session group, or -1 if this port is not in a + * session group. */ + + /* Socks only: */ + /** When both no-auth and user/pass are advertised by a SOCKS client, select + * no-auth. */ + unsigned int socks_prefer_no_auth : 1; + + /* Client port types only: */ + unsigned int ipv4_traffic : 1; + unsigned int ipv6_traffic : 1; + unsigned int prefer_ipv6 : 1; + + /** For a socks listener: should we cache IPv4/IPv6 DNS information that + * exit nodes tell us? + * + * @{ */ + unsigned int cache_ipv4_answers : 1; + unsigned int cache_ipv6_answers : 1; + /** @} */ + /** For a socks listeners: if we find an answer in our client-side DNS cache, + * should we use it? + * + * @{ */ + unsigned int use_cached_ipv4_answers : 1; + unsigned int use_cached_ipv6_answers : 1; + /** @} */ + /** For socks listeners: When we can automap an address to IPv4 or IPv6, + * do we prefer IPv6? */ + unsigned int prefer_ipv6_virtaddr : 1; + +} entry_port_cfg_t; + +typedef struct server_port_cfg_t { + /* Server port types (or, dir) only: */ + unsigned int no_advertise : 1; + unsigned int no_listen : 1; + unsigned int all_addrs : 1; + unsigned int bind_ipv4_only : 1; + unsigned int bind_ipv6_only : 1; +} server_port_cfg_t; + /* Values for connection_t.magic: used to make sure that downcasts (casts from * connection_t to foo_connection_t) are safe. */ #define BASE_CONNECTION_MAGIC 0x7C3C304Eu @@ -1273,52 +1311,7 @@ typedef struct listener_connection_t { * to the evdns_server_port it uses to listen to and answer connections. */ struct evdns_server_port *dns_server_port; - /** @name Isolation parameters - * - * For an AP listener, these fields describe how to isolate streams that - * arrive on the listener. - * - * @{ - */ - /** The session group for this listener. */ - int session_group; - /** One or more ISO_ flags to describe how to isolate streams. */ - uint8_t isolation_flags; - /**@}*/ - /** For SOCKS connections only: If this is set, we will choose "no - * authentication" instead of "username/password" authentication if both - * are offered. Used as input to parse_socks. */ - unsigned int socks_prefer_no_auth : 1; - - /** For a SOCKS listeners, these fields describe whether we should - * allow IPv4 and IPv6 addresses from our exit nodes, respectively. - * - * @{ - */ - unsigned int socks_ipv4_traffic : 1; - unsigned int socks_ipv6_traffic : 1; - /** @} */ - /** For a socks listener: should we tell the exit that we prefer IPv6 - * addresses? */ - unsigned int socks_prefer_ipv6 : 1; - - /** For a socks listener: should we cache IPv4/IPv6 DNS information that - * exit nodes tell us? - * - * @{ */ - unsigned int cache_ipv4_answers : 1; - unsigned int cache_ipv6_answers : 1; - /** @} */ - /** For a socks listeners: if we find an answer in our client-side DNS cache, - * should we use it? - * - * @{ */ - unsigned int use_cached_ipv4_answers : 1; - unsigned int use_cached_ipv6_answers : 1; - /** @} */ - /** For socks listeners: When we can automap an address to IPv4 or IPv6, - * do we prefer IPv6? */ - unsigned int prefer_ipv6_virtaddr : 1; + entry_port_cfg_t entry_cfg; } listener_connection_t; @@ -1426,6 +1419,18 @@ typedef struct or_handshake_state_t { /** Length of Extended ORPort connection identifier. */ #define EXT_OR_CONN_ID_LEN DIGEST_LEN /* 20 */ +/* + * OR_CONN_HIGHWATER and OR_CONN_LOWWATER moved from connection_or.c so + * channeltls.c can see them too. + */ + +/** When adding cells to an OR connection's outbuf, keep adding until the + * outbuf is at least this long, or we run out of cells. */ +#define OR_CONN_HIGHWATER (32*1024) + +/** Add cells to an OR connection's outbuf whenever the outbuf's data length + * drops below this size. */ +#define OR_CONN_LOWWATER (16*1024) /** Subtype of connection_t for an "OR connection" -- that is, one that speaks * cells over TLS. */ @@ -1517,6 +1522,12 @@ typedef struct or_connection_t { /** Last emptied write token bucket in msec since midnight; only used if * TB_EMPTY events are enabled. */ uint32_t write_emptied_time; + + /* + * Count the number of bytes flushed out on this orconn, and the number of + * bytes TLS actually sent - used for overhead estimation for scheduling. + */ + uint64_t bytes_xmitted, bytes_xmitted_by_tls; } or_connection_t; /** Subtype of connection_t for an "edge connection" -- that is, an entry (ap) @@ -1588,12 +1599,10 @@ typedef struct entry_connection_t { * only.) */ /* === Isolation related, AP only. === */ - /** AP only: based on which factors do we isolate this stream? */ - uint8_t isolation_flags; - /** AP only: what session group is this stream in? */ - int session_group; + entry_port_cfg_t entry_cfg; /** AP only: The newnym epoch in which we created this connection. */ unsigned nym_epoch; + /** AP only: The original requested address before we rewrote it. */ char *original_dest_address; /* Other fields to isolate on already exist. The ClientAddr is addr. The @@ -1652,33 +1661,8 @@ typedef struct entry_connection_t { */ unsigned int may_use_optimistic_data : 1; - /** Should we permit IPv4 and IPv6 traffic to use this connection? - * - * @{ */ - unsigned int ipv4_traffic_ok : 1; - unsigned int ipv6_traffic_ok : 1; - /** @} */ - /** Should we say we prefer IPv6 traffic? */ - unsigned int prefer_ipv6_traffic : 1; - - /** For a socks listener: should we cache IPv4/IPv6 DNS information that - * exit nodes tell us? - * - * @{ */ - unsigned int cache_ipv4_answers : 1; - unsigned int cache_ipv6_answers : 1; - /** @} */ - /** For a socks listeners: if we find an answer in our client-side DNS cache, - * should we use it? - * - * @{ */ - unsigned int use_cached_ipv4_answers : 1; - unsigned int use_cached_ipv6_answers : 1; - /** @} */ - /** For socks listeners: When we can automap an address to IPv4 or IPv6, - * do we prefer IPv6? */ - unsigned int prefer_ipv6_virtaddr : 1; - + /** Are we a socks SocksSocket listener? */ + unsigned int is_socks_socket:1; } entry_connection_t; typedef enum { @@ -1958,6 +1942,7 @@ typedef struct download_status_t { uint8_t n_download_failures; /**< Number of failures trying to download the * most recent descriptor. */ download_schedule_bitfield_t schedule : 8; + } download_status_t; /** If n_download_failures is this high, the download can never happen. */ @@ -2139,8 +2124,6 @@ typedef struct routerstatus_t { * choice as an entry guard. */ unsigned int is_bad_exit:1; /**< True iff this node is a bad choice for * an exit node. */ - unsigned int is_bad_directory:1; /**< Do we think this directory is junky, - * underpowered, or otherwise useless? */ unsigned int is_hs_dir:1; /**< True iff this router is a v2-or-later hidden * service directory. */ /** True iff we know version info for this router. (i.e., a "v" entry was @@ -2151,9 +2134,6 @@ typedef struct routerstatus_t { /** True iff this router is a version that, if it caches directory info, * we can get microdescriptors from. */ unsigned int version_supports_microdesc_cache:1; - /** True iff this router is a version that allows DATA cells to arrive on - * a stream before it has sent a CONNECTED cell. */ - unsigned int version_supports_optimistic_data:1; /** True iff this router has a version that allows it to accept EXTEND2 * cells */ unsigned int version_supports_extend2_cells:1; @@ -2165,6 +2145,12 @@ typedef struct routerstatus_t { uint32_t bandwidth_kb; /**< Bandwidth (capacity) of the router as reported in * the vote/consensus, in kilobytes/sec. */ + + /** The consensus has guardfraction information for this router. */ + unsigned int has_guardfraction:1; + /** The guardfraction value of this router. */ + uint32_t guardfraction_percentage; + char *exitsummary; /**< exit policy summary - * XXX weasel: this probably should not stay a string. */ @@ -2300,8 +2286,6 @@ typedef struct node_t { unsigned int is_exit:1; /**< Do we think this is an OK exit? */ unsigned int is_bad_exit:1; /**< Do we think this exit is censored, borked, * or otherwise nasty? */ - unsigned int is_bad_directory:1; /**< Do we think this directory is junky, - * underpowered, or otherwise useless? */ unsigned int is_hs_dir:1; /**< True iff this router is a hidden service * directory according to the authorities. */ @@ -2439,6 +2423,9 @@ typedef struct networkstatus_t { /** Vote only: what methods is this voter willing to use? */ smartlist_t *supported_methods; + /** List of 'package' lines describing hashes of downloadable packages */ + smartlist_t *package_lines; + /** How long does this vote/consensus claim that authorities take to * distribute their votes to one another? */ int vote_seconds; @@ -2560,9 +2547,7 @@ typedef struct extend_info_t { uint16_t port; /**< OR port. */ tor_addr_t addr; /**< IP address. */ crypto_pk_t *onion_key; /**< Current onionskin key. */ -#ifdef CURVE25519_ENABLED curve25519_public_key_t curve25519_onion_key; -#endif } extend_info_t; /** Certificate for v3 directory protocol: binds long-term authority identity @@ -2719,8 +2704,14 @@ typedef struct { time_t expiry_time; } cpath_build_state_t; +/** "magic" value for an origin_circuit_t */ #define ORIGIN_CIRCUIT_MAGIC 0x35315243u +/** "magic" value for an or_circuit_t */ #define OR_CIRCUIT_MAGIC 0x98ABC04Fu +/** "magic" value for a circuit that would have been freed by circuit_free, + * but which we're keeping around until a cpuworker reply arrives. See + * circuit_free() for more documentation. */ +#define DEAD_CIRCUIT_MAGIC 0xdeadc14c struct create_cell_t; @@ -2865,8 +2856,8 @@ typedef struct circuit_t { /** Unique ID for measuring tunneled network status requests. */ uint64_t dirreq_id; - /** Next circuit in linked list of all circuits (global_circuitlist). */ - TOR_LIST_ENTRY(circuit_t) head; + /** Index in smartlist of all circuits (global_circuitlist). */ + int global_circuitlist_idx; /** Next circuit in the doubly-linked ring of circuits waiting to add * cells to n_conn. NULL if we have no cells pending, or if we're not @@ -3140,6 +3131,10 @@ typedef struct or_circuit_t { /** Pointer to an entry on the onion queue, if this circuit is waiting for a * chance to give an onionskin to a cpuworker. Used only in onion.c */ struct onion_queue_t *onionqueue_entry; + /** Pointer to a workqueue entry, if this circuit has given an onionskin to + * a cpuworker and is waiting for a response. Used to decide whether it is + * safe to free a circuit or if it is still in use by a cpuworker. */ + struct workqueue_entry_s *workqueue_entry; /** The circuit_id used in the previous (backward) hop of this circuit. */ circid_t p_circ_id; @@ -3192,6 +3187,10 @@ typedef struct or_circuit_t { /** True iff this circuit was made with a CREATE_FAST cell. */ unsigned int is_first_hop : 1; + /** If set, this circuit carries HS traffic. Consider it in any HS + * statistics. */ + unsigned int circuit_carries_hs_traffic_stats : 1; + /** Number of cells that were removed from circuit queue; reset every * time when writing buffer stats to disk. */ uint32_t processed_cells; @@ -3239,6 +3238,14 @@ static const or_circuit_t *CONST_TO_OR_CIRCUIT(const circuit_t *); static origin_circuit_t *TO_ORIGIN_CIRCUIT(circuit_t *); static const origin_circuit_t *CONST_TO_ORIGIN_CIRCUIT(const circuit_t *); +/** Return 1 iff <b>node</b> has Exit flag and no BadExit flag. + * Otherwise, return 0. + */ +static INLINE int node_is_good_exit(const node_t *node) +{ + return node->is_exit && ! node->is_bad_exit; +} + static INLINE or_circuit_t *TO_OR_CIRCUIT(circuit_t *x) { tor_assert(x->magic == OR_CIRCUIT_MAGIC); @@ -3318,44 +3325,9 @@ typedef struct port_cfg_t { uint8_t type; /**< One of CONN_TYPE_*_LISTENER */ unsigned is_unix_addr : 1; /**< True iff this is an AF_UNIX address. */ - /* Client port types (socks, dns, trans, natd) only: */ - uint8_t isolation_flags; /**< Zero or more isolation flags */ - int session_group; /**< A session group, or -1 if this port is not in a - * session group. */ - /* Socks only: */ - /** When both no-auth and user/pass are advertised by a SOCKS client, select - * no-auth. */ - unsigned int socks_prefer_no_auth : 1; - - /* Server port types (or, dir) only: */ - unsigned int no_advertise : 1; - unsigned int no_listen : 1; - unsigned int all_addrs : 1; - unsigned int bind_ipv4_only : 1; - unsigned int bind_ipv6_only : 1; - - /* Client port types only: */ - unsigned int ipv4_traffic : 1; - unsigned int ipv6_traffic : 1; - unsigned int prefer_ipv6 : 1; + entry_port_cfg_t entry_cfg; - /** For a socks listener: should we cache IPv4/IPv6 DNS information that - * exit nodes tell us? - * - * @{ */ - unsigned int cache_ipv4_answers : 1; - unsigned int cache_ipv6_answers : 1; - /** @} */ - /** For a socks listeners: if we find an answer in our client-side DNS cache, - * should we use it? - * - * @{ */ - unsigned int use_cached_ipv4_answers : 1; - unsigned int use_cached_ipv6_answers : 1; - /** @} */ - /** For socks listeners: When we can automap an address to IPv4 or IPv6, - * do we prefer IPv6? */ - unsigned int prefer_ipv6_virtaddr : 1; + server_port_cfg_t server_cfg; /* Unix sockets only: */ /** Path for an AF_UNIX address */ @@ -3406,6 +3378,8 @@ typedef struct { int LogMessageDomains; /**< Boolean: Should we log the domain(s) in which * each log message occurs? */ + int TruncateLogFile; /**< Boolean: Should we truncate the log file + before we start writing? */ char *DebugLogFile; /**< Where to send verbose log messages. */ char *DataDirectory; /**< OR only: where to store long-term data. */ @@ -3472,6 +3446,7 @@ typedef struct { config_line_t *RecommendedVersions; config_line_t *RecommendedClientVersions; config_line_t *RecommendedServerVersions; + config_line_t *RecommendedPackages; /** Whether dirservers allow router descriptors with private IPs. */ int DirAllowPrivateAddresses; /** Whether routers accept EXTEND cells to routers with private IPs. */ @@ -3502,6 +3477,7 @@ typedef struct { * for control connections. */ int ControlSocketsGroupWritable; /**< Boolean: Are control sockets g+rw? */ + int SocksSocketsGroupWritable; /**< Boolean: Are SOCKS sockets g+rw? */ /** Ports to listen on for directory connections. */ config_line_t *DirPort_lines; config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */ @@ -3511,6 +3487,8 @@ typedef struct { uint64_t MaxMemInQueues_raw; uint64_t MaxMemInQueues;/**< If we have more memory than this allocated * for queues and buffers, run the OOM handler */ + /** Above this value, consider ourselves low on RAM. */ + uint64_t MaxMemInQueues_low_threshold; /** @name port booleans * @@ -3534,8 +3512,6 @@ typedef struct { int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */ int V3AuthoritativeDir; /**< Boolean: is this an authoritative directory * for version 3 directories? */ - int NamingAuthoritativeDir; /**< Boolean: is this an authoritative directory - * that's willing to bind names? */ int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative * directory that's willing to recommend * versions? */ @@ -3600,6 +3576,9 @@ typedef struct { * circuits.) */ int Tor2webMode; + /** A routerset that should be used when picking RPs for HS circuits. */ + routerset_t *Tor2webRendezvousPoints; + /** Close hidden service client circuits immediately when they reach * the normal circuit-build timeout, even if they have already sent * an INTRODUCE1 cell on its way to the service. */ @@ -3649,8 +3628,9 @@ typedef struct { * hostname ending with one of the suffixes in * <b>AutomapHostsSuffixes</b>, map it to a * virtual address. */ - smartlist_t *AutomapHostsSuffixes; /**< List of suffixes for - * <b>AutomapHostsOnResolve</b>. */ + /** List of suffixes for <b>AutomapHostsOnResolve</b>. The special value + * "." means "match everything." */ + smartlist_t *AutomapHostsSuffixes; int RendPostPeriod; /**< How often do we post each rendezvous service * descriptor? Remember to publish them independently. */ int KeepalivePeriod; /**< How often do we send padding cells to keep @@ -3745,8 +3725,6 @@ typedef struct { config_line_t *NodeFamilies; /**< List of config lines for * node families */ smartlist_t *NodeFamilySets; /**< List of parsed NodeFamilies values. */ - config_line_t *AuthDirBadDir; /**< Address policy for descriptors to - * mark as bad dir mirrors. */ config_line_t *AuthDirBadExit; /**< Address policy for descriptors to * mark as bad exits. */ config_line_t *AuthDirReject; /**< Address policy for descriptors to @@ -3755,23 +3733,18 @@ typedef struct { * never mark as valid. */ /** @name AuthDir...CC * - * Lists of country codes to mark as BadDir, BadExit, or Invalid, or to + * Lists of country codes to mark as BadExit, or Invalid, or to * reject entirely. * * @{ */ - smartlist_t *AuthDirBadDirCCs; smartlist_t *AuthDirBadExitCCs; smartlist_t *AuthDirInvalidCCs; smartlist_t *AuthDirRejectCCs; /**@}*/ - int AuthDirListBadDirs; /**< True iff we should list bad dirs, - * and vote for all other dir mirrors as good. */ int AuthDirListBadExits; /**< True iff we should list bad exits, * and vote for all other exits as good. */ - int AuthDirRejectUnlisted; /**< Boolean: do we reject all routers that - * aren't named in our fingerprint file? */ int AuthDirMaxServersPerAddr; /**< Do not permit more than this * number of servers per IP address. */ int AuthDirMaxServersPerAuthAddr; /**< Do not permit more than this @@ -3792,6 +3765,11 @@ typedef struct { uint64_t AccountingMax; /**< How many bytes do we allow per accounting * interval before hibernation? 0 for "never * hibernate." */ + /** How do we determine when our AccountingMax has been reached? + * "max" for when in or out reaches AccountingMax + * "sum for when in plus out reaches AccountingMax */ + char *AccountingRule_option; + enum { ACCT_MAX, ACCT_SUM } AccountingRule; /** Base64-encoded hash of accepted passwords for the control system. */ config_line_t *HashedControlPassword; @@ -3847,6 +3825,12 @@ typedef struct { int NumEntryGuards; /**< How many entry guards do we try to establish? */ int UseEntryGuardsAsDirGuards; /** Boolean: Do we try to get directory info * from a smallish number of fixed nodes? */ + + /** If 1, we use any guardfraction information we see in the + * consensus. If 0, we don't. If -1, let the consensus parameter + * decide. */ + int UseGuardFraction; + int NumDirectoryGuards; /**< How many dir guards do we try to establish? * If 0, use value from NumEntryGuards. */ int RephistTrackTime; /**< How many seconds do we keep rephist info? */ @@ -3924,8 +3908,11 @@ typedef struct { * instead of a hostname. */ int WarnUnsafeSocks; - /** If true, the user wants us to collect statistics on clients + /** If true, we're configured to collect statistics on clients * requesting network statuses from us as directory. */ + int DirReqStatistics_option; + /** Internal variable to remember whether we're actually acting on + * DirReqStatistics_option -- yes if it's set and we're a server, else no. */ int DirReqStatistics; /** If true, the user wants us to collect statistics on port usage. */ @@ -3940,6 +3927,10 @@ typedef struct { /** If true, the user wants us to collect statistics as entry node. */ int EntryStatistics; + /** If true, the user wants us to collect statistics as hidden service + * directory, introduction point, or rendezvous point. */ + int HiddenServiceStatistics; + /** If true, include statistics file contents in extra-info documents. */ int ExtraInfoStatistics; @@ -3975,6 +3966,9 @@ typedef struct { /** Location of bandwidth measurement file */ char *V3BandwidthsFile; + /** Location of guardfraction file */ + char *GuardfractionFile; + /** Authority only: key=value pairs that we add to our networkstatus * consensus vote on the 'params' line. */ char *ConsensusParams; @@ -4065,10 +4059,19 @@ typedef struct { /** Minimum value for the Fast flag threshold on testing networks. */ uint64_t TestingMinFastFlagThreshold; + /** Relays in a testing network which should be voted Exit + * regardless of exit policy. */ + routerset_t *TestingDirAuthVoteExit; + /** Relays in a testing network which should be voted Guard * regardless of uptime and bandwidth. */ routerset_t *TestingDirAuthVoteGuard; + /** Relays in a testing network which should be voted HSDir + * regardless of uptime and ORPort connectivity. + * Respects VoteOnHidServDirectoriesV2. */ + routerset_t *TestingDirAuthVoteHSDir; + /** Enable CONN_BW events. Only altered on testing networks. */ int TestingEnableConnBwEvent; @@ -4223,8 +4226,26 @@ typedef struct { /** How long (seconds) do we keep a guard before picking a new one? */ int GuardLifetime; - /** Should we send the timestamps that pre-023 hidden services want? */ - int Support022HiddenServices; + /** Low-water mark for global scheduler - start sending when estimated + * queued size falls below this threshold. + */ + uint64_t SchedulerLowWaterMark__; + /** High-water mark for global scheduler - stop sending when estimated + * queued size exceeds this threshold. + */ + uint64_t SchedulerHighWaterMark__; + /** Flush size for global scheduler - flush this many cells at a time + * when sending. + */ + int SchedulerMaxFlushCells__; + + /** Is this an exit node? This is a tristate, where "1" means "yes, and use + * the default exit policy if none is given" and "0" means "no; exit policy + * is 'reject *'" and "auto" (-1) means "same as 1, but warn the user." + * + * XXXX Eventually, the default will be 0. */ + int ExitRelay; + } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -4315,7 +4336,8 @@ static INLINE void or_state_mark_dirty(or_state_t *state, time_t when) /** Please turn this IP address into an FQDN, privately. */ #define SOCKS_COMMAND_RESOLVE_PTR 0xF1 -#define SOCKS_COMMAND_IS_CONNECT(c) ((c)==SOCKS_COMMAND_CONNECT) +/* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */ +#define SOCKS_COMMAND_IS_CONNECT(c) (((c)==SOCKS_COMMAND_CONNECT) || 0) #define SOCKS_COMMAND_IS_RESOLVE(c) ((c)==SOCKS_COMMAND_RESOLVE || \ (c)==SOCKS_COMMAND_RESOLVE_PTR) @@ -4894,7 +4916,8 @@ typedef struct rend_service_descriptor_t { /** A cached rendezvous descriptor. */ typedef struct rend_cache_entry_t { size_t len; /**< Length of <b>desc</b> */ - time_t received; /**< When was the descriptor received? */ + time_t last_served; /**< When did we last write this one to somebody? + * (HSDir only) */ char *desc; /**< Service descriptor */ rend_service_descriptor_t *parsed; /**< Parsed value of 'desc' */ } rend_cache_entry_t; @@ -4936,7 +4959,8 @@ typedef struct dir_server_t { **/ } dir_server_t; -#define ROUTER_REQUIRED_MIN_BANDWIDTH (20*1024) +#define RELAY_REQUIRED_MIN_BANDWIDTH (75*1024) +#define BRIDGE_REQUIRED_MIN_BANDWIDTH (50*1024) #define ROUTER_MAX_DECLARED_BANDWIDTH INT32_MAX @@ -4960,14 +4984,13 @@ typedef struct dir_server_t { * or extrainfo documents. * * Passed to router_pick_directory_server (et al) - * - * [XXXX NOTE: This option is only implemented for pick_trusteddirserver, - * not pick_directory_server. If we make it work on pick_directory_server - * too, we could conservatively make it only prevent multiple fetches to - * the same authority, or we could aggressively make it prevent multiple - * fetches to _any_ single directory server.] */ #define PDS_NO_EXISTING_SERVERDESC_FETCH (1<<3) +/** Flag to indicate that we should not use any directory authority to which + * we have an existing directory connection for downloading microdescs. + * + * Passed to router_pick_directory_server (et al) + */ #define PDS_NO_EXISTING_MICRODESC_FETCH (1<<4) /** This node is to be chosen as a directory guard, so don't choose any @@ -4995,14 +5018,31 @@ typedef enum { /** Return value for router_add_to_routerlist() and dirserv_add_descriptor() */ typedef enum was_router_added_t { + /* Router was added successfully. */ ROUTER_ADDED_SUCCESSFULLY = 1, + /* Router descriptor was added with warnings to submitter. */ ROUTER_ADDED_NOTIFY_GENERATOR = 0, + /* Extrainfo document was rejected because no corresponding router + * descriptor was found OR router descriptor was rejected because + * it was incompatible with its extrainfo document. */ ROUTER_BAD_EI = -1, - ROUTER_WAS_NOT_NEW = -2, + /* Router descriptor was rejected because it is already known. */ + ROUTER_IS_ALREADY_KNOWN = -2, + /* General purpose router was rejected, because it was not listed + * in consensus. */ ROUTER_NOT_IN_CONSENSUS = -3, + /* Router was neither in directory consensus nor in any of + * networkstatus documents. Caching it to access later. + * (Applies to fetched descriptors only.) */ ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS = -4, + /* Router was rejected by directory authority. */ ROUTER_AUTHDIR_REJECTS = -5, - ROUTER_WAS_NOT_WANTED = -6 + /* Bridge descriptor was rejected because such bridge was not one + * of the bridges we have listed in our configuration. */ + ROUTER_WAS_NOT_WANTED = -6, + /* Router descriptor was rejected because it was older than + * OLD_ROUTER_DESC_MAX_AGE. */ + ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */ } was_router_added_t; /********************************* routerparse.c ************************/ diff --git a/src/or/policies.c b/src/or/policies.c index 8a91509a77..560b8cb4c3 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,9 +29,6 @@ static smartlist_t *authdir_reject_policy = NULL; * to be marked as valid in our networkstatus. */ static smartlist_t *authdir_invalid_policy = NULL; /** Policy that addresses for incoming router descriptors must <b>not</b> - * match in order to not be marked as BadDirectory. */ -static smartlist_t *authdir_baddir_policy = NULL; -/** Policy that addresses for incoming router descriptors must <b>not</b> * match in order to not be marked as BadExit. */ static smartlist_t *authdir_badexit_policy = NULL; @@ -65,6 +62,13 @@ static const char *private_nets[] = { NULL }; +static int policies_parse_exit_policy_internal(config_line_t *cfg, + smartlist_t **dest, + int ipv6_exit, + int rejectprivate, + uint32_t local_address, + int add_default_policy); + /** Replace all "private" entries in *<b>policy</b> with their expanded * equivalents. */ void @@ -400,17 +404,6 @@ authdir_policy_valid_address(uint32_t addr, uint16_t port) return !addr_is_in_cc_list(addr, get_options()->AuthDirInvalidCCs); } -/** Return 1 if <b>addr</b>:<b>port</b> should be marked as a bad dir, - * based on <b>authdir_baddir_policy</b>. Else return 0. - */ -int -authdir_policy_baddir_address(uint32_t addr, uint16_t port) -{ - if (! addr_policy_permits_address(addr, port, authdir_baddir_policy)) - return 1; - return addr_is_in_cc_list(addr, get_options()->AuthDirBadDirCCs); -} - /** Return 1 if <b>addr</b>:<b>port</b> should be marked as a bad exit, * based on <b>authdir_badexit_policy</b>. Else return 0. */ @@ -437,11 +430,36 @@ validate_addr_policies(const or_options_t *options, char **msg) smartlist_t *addr_policy=NULL; *msg = NULL; - if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, - options->IPv6Exit, - options->ExitPolicyRejectPrivate, 0, - !options->BridgeRelay)) + if (policies_parse_exit_policy_from_options(options,0,&addr_policy)) { REJECT("Error in ExitPolicy entry."); + } + + static int warned_about_exitrelay = 0; + + const int exitrelay_setting_is_auto = options->ExitRelay == -1; + const int policy_accepts_something = + ! (policy_is_reject_star(addr_policy, AF_INET) && + policy_is_reject_star(addr_policy, AF_INET6)); + + if (server_mode(options) && + ! warned_about_exitrelay && + exitrelay_setting_is_auto && + policy_accepts_something) { + /* Policy accepts something */ + warned_about_exitrelay = 1; + log_warn(LD_CONFIG, + "Tor is running as an exit relay%s. If you did not want this " + "behavior, please set the ExitRelay option to 0. If you do " + "want to run an exit Relay, please set the ExitRelay option " + "to 1 to disable this warning, and for forward compatibility.", + options->ExitPolicy == NULL ? + " with the default exit policy" : ""); + if (options->ExitPolicy == NULL) { + log_warn(LD_CONFIG, + "In a future version of Tor, ExitRelay 0 may become the " + "default when no ExitPolicy is given."); + } + } /* The rest of these calls *append* to addr_policy. So don't actually * use the results for anything other than checking if they parse! */ @@ -455,9 +473,6 @@ validate_addr_policies(const or_options_t *options, char **msg) if (parse_addr_policy(options->AuthDirInvalid, &addr_policy, ADDR_POLICY_REJECT)) REJECT("Error in AuthDirInvalid entry."); - if (parse_addr_policy(options->AuthDirBadDir, &addr_policy, - ADDR_POLICY_REJECT)) - REJECT("Error in AuthDirBadDir entry."); if (parse_addr_policy(options->AuthDirBadExit, &addr_policy, ADDR_POLICY_REJECT)) REJECT("Error in AuthDirBadExit entry."); @@ -535,9 +550,6 @@ policies_parse_from_options(const or_options_t *options) if (load_policy_from_option(options->AuthDirInvalid, "AuthDirInvalid", &authdir_invalid_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirBadDir, "AuthDirBadDir", - &authdir_baddir_policy, ADDR_POLICY_REJECT) < 0) - ret = -1; if (load_policy_from_option(options->AuthDirBadExit, "AuthDirBadExit", &authdir_badexit_policy, ADDR_POLICY_REJECT) < 0) ret = -1; @@ -629,8 +641,8 @@ policy_hash(const policy_map_ent_t *ent) HT_PROTOTYPE(policy_map, policy_map_ent_t, node, policy_hash, policy_eq) -HT_GENERATE(policy_map, policy_map_ent_t, node, policy_hash, - policy_eq, 0.6, malloc, realloc, free) +HT_GENERATE2(policy_map, policy_map_ent_t, node, policy_hash, + policy_eq, 0.6, tor_reallocarray_, tor_free_) /** Given a pointer to an addr_policy_t, return a copy of the pointer to the * "canonical" copy of that addr_policy_t; the canonical copy is a single @@ -769,9 +781,9 @@ compare_unknown_tor_addr_to_addr_policy(uint16_t port, * We could do better by assuming that some ranges never match typical * addresses (127.0.0.1, and so on). But we'll try this for now. */ -addr_policy_result_t -compare_tor_addr_to_addr_policy(const tor_addr_t *addr, uint16_t port, - const smartlist_t *policy) +MOCK_IMPL(addr_policy_result_t, +compare_tor_addr_to_addr_policy,(const tor_addr_t *addr, uint16_t port, + const smartlist_t *policy)) { if (!policy) { /* no policy? accept all. */ @@ -968,11 +980,12 @@ exit_policy_remove_redundancies(smartlist_t *dest) * the functions used to parse the exit policy from a router descriptor, * see router_add_exit_policy. */ -int -policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, - int ipv6_exit, - int rejectprivate, uint32_t local_address, - int add_default_policy) +static int +policies_parse_exit_policy_internal(config_line_t *cfg, smartlist_t **dest, + int ipv6_exit, + int rejectprivate, + uint32_t local_address, + int add_default_policy) { if (!ipv6_exit) { append_exit_policy_string(dest, "reject *6:*"); @@ -998,6 +1011,77 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, return 0; } +/** Parse exit policy in <b>cfg</b> into <b>dest</b> smartlist. + * + * Add entry that rejects all IPv6 destinations unless + * <b>EXIT_POLICY_IPV6_ENABLED</b> bit is set in <b>options</b> bitmask. + * + * If <b>EXIT_POLICY_REJECT_PRIVATE</b> bit is set in <b>options</b>, + * do add entry that rejects all destinations in private subnetwork + * Tor is running in. + * + * Respectively, if <b>EXIT_POLICY_ADD_DEFAULT</b> bit is set, add + * default exit policy entries to <b>result</b> smartlist. + */ +int +policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, + exit_policy_parser_cfg_t options, + uint32_t local_address) +{ + int ipv6_enabled = (options & EXIT_POLICY_IPV6_ENABLED) ? 1 : 0; + int reject_private = (options & EXIT_POLICY_REJECT_PRIVATE) ? 1 : 0; + int add_default = (options & EXIT_POLICY_ADD_DEFAULT) ? 1 : 0; + + return policies_parse_exit_policy_internal(cfg,dest,ipv6_enabled, + reject_private, + local_address, + add_default); +} + +/** Parse <b>ExitPolicy</b> member of <b>or_options</b> into <b>result</b> + * smartlist. + * If <b>or_options->IPv6Exit</b> is false, add an entry that + * rejects all IPv6 destinations. + * + * If <b>or_options->ExitPolicyRejectPrivate</b> is true, add entry that + * rejects all destinations in the private subnetwork of machine Tor + * instance is running in. + * + * If <b>or_options->BridgeRelay</b> is false, add entries of default + * Tor exit policy into <b>result</b> smartlist. + * + * If or_options->ExitRelay is false, then make our exit policy into + * "reject *:*" regardless. + */ +int +policies_parse_exit_policy_from_options(const or_options_t *or_options, + uint32_t local_address, + smartlist_t **result) +{ + exit_policy_parser_cfg_t parser_cfg = 0; + + if (or_options->ExitRelay == 0) { + append_exit_policy_string(result, "reject *4:*"); + append_exit_policy_string(result, "reject *6:*"); + return 0; + } + + if (or_options->IPv6Exit) { + parser_cfg |= EXIT_POLICY_IPV6_ENABLED; + } + + if (or_options->ExitPolicyRejectPrivate) { + parser_cfg |= EXIT_POLICY_REJECT_PRIVATE; + } + + if (!or_options->BridgeRelay) { + parser_cfg |= EXIT_POLICY_ADD_DEFAULT; + } + + return policies_parse_exit_policy(or_options->ExitPolicy,result, + parser_cfg,local_address); +} + /** Add "reject *:*" to the end of the policy in *<b>dest</b>, allocating * *<b>dest</b> as needed. */ void @@ -1334,9 +1418,9 @@ policy_summary_add_item(smartlist_t *summary, addr_policy_t *p) * The summary will either be an "accept" plus a comma-separated list of port * ranges or a "reject" plus port-ranges, depending on which is shorter. * - * If no exits are allowed at all then NULL is returned, if no ports - * are blocked instead of "reject " we return "accept 1-65535" (this - * is an exception to the shorter-representation-wins rule). + * If no exits are allowed at all then "reject 1-65535" is returned. If no + * ports are blocked instead of "reject " we return "accept 1-65535". (These + * are an exception to the shorter-representation-wins rule). */ char * policy_summarize(smartlist_t *policy, sa_family_t family) @@ -1766,8 +1850,6 @@ policies_free_all(void) authdir_reject_policy = NULL; addr_policy_list_free(authdir_invalid_policy); authdir_invalid_policy = NULL; - addr_policy_list_free(authdir_baddir_policy); - authdir_baddir_policy = NULL; addr_policy_list_free(authdir_badexit_policy); authdir_badexit_policy = NULL; diff --git a/src/or/policies.h b/src/or/policies.h index 91ac427492..0225b57a2c 100644 --- a/src/or/policies.h +++ b/src/or/policies.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -18,6 +18,12 @@ */ #define POLICY_BUF_LEN 72 +#define EXIT_POLICY_IPV6_ENABLED (1 << 0) +#define EXIT_POLICY_REJECT_PRIVATE (1 << 1) +#define EXIT_POLICY_ADD_DEFAULT (1 << 2) + +typedef int exit_policy_parser_cfg_t; + int firewall_is_fascist_or(void); int fascist_firewall_allows_address_or(const tor_addr_t *addr, uint16_t port); int fascist_firewall_allows_or(const routerinfo_t *ri); @@ -27,7 +33,6 @@ int dir_policy_permits_address(const tor_addr_t *addr); int socks_policy_permits_address(const tor_addr_t *addr); int authdir_policy_permits_address(uint32_t addr, uint16_t port); int authdir_policy_valid_address(uint32_t addr, uint16_t port); -int authdir_policy_baddir_address(uint32_t addr, uint16_t port); int authdir_policy_badexit_address(uint32_t addr, uint16_t port); int validate_addr_policies(const or_options_t *options, char **msg); @@ -37,16 +42,24 @@ int policies_parse_from_options(const or_options_t *options); addr_policy_t *addr_policy_get_canonical_entry(addr_policy_t *ent); int cmp_addr_policies(smartlist_t *a, smartlist_t *b); -addr_policy_result_t compare_tor_addr_to_addr_policy(const tor_addr_t *addr, - uint16_t port, const smartlist_t *policy); +MOCK_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy, + (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy)); addr_policy_result_t compare_tor_addr_to_node_policy(const tor_addr_t *addr, uint16_t port, const node_t *node); +/* int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, int ipv6exit, int rejectprivate, uint32_t local_address, int add_default_policy); +*/ +int policies_parse_exit_policy_from_options(const or_options_t *or_options, + uint32_t local_address, + smartlist_t **result); +int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, + exit_policy_parser_cfg_t options, + uint32_t local_address); void policies_exit_policy_append_reject_star(smartlist_t **dest); void addr_policy_append_reject_addr(smartlist_t **dest, const tor_addr_t *addr); diff --git a/src/or/reasons.c b/src/or/reasons.c index 750e89bbe7..23ab6041a6 100644 --- a/src/or/reasons.c +++ b/src/or/reasons.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -350,6 +350,8 @@ circuit_end_reason_to_control_string(int reason) return "NOSUCHSERVICE"; case END_CIRC_REASON_MEASUREMENT_EXPIRED: return "MEASUREMENT_EXPIRED"; + case END_CIRC_REASON_IP_NOW_REDUNDANT: + return "IP_NOW_REDUNDANT"; default: if (is_remote) { /* @@ -367,7 +369,7 @@ circuit_end_reason_to_control_string(int reason) } } -/** Return a string corresponding to a SOCKS4 reponse code. */ +/** Return a string corresponding to a SOCKS4 response code. */ const char * socks4_response_code_to_string(uint8_t code) { @@ -385,7 +387,7 @@ socks4_response_code_to_string(uint8_t code) } } -/** Return a string corresponding to a SOCKS5 reponse code. */ +/** Return a string corresponding to a SOCKS5 response code. */ const char * socks5_response_code_to_string(uint8_t code) { diff --git a/src/or/reasons.h b/src/or/reasons.h index fe7e67722a..00a099061b 100644 --- a/src/or/reasons.h +++ b/src/or/reasons.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/relay.c b/src/or/relay.c index 9407df0559..50eaab83c8 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -26,9 +26,6 @@ #include "control.h" #include "geoip.h" #include "main.h" -#ifdef ENABLE_MEMPOOLS -#include "mempool.h" -#endif #include "networkstatus.h" #include "nodelist.h" #include "onion.h" @@ -39,6 +36,7 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "scheduler.h" static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -803,8 +801,10 @@ connection_ap_process_end_not_open( return 0; } - if ((tor_addr_family(&addr) == AF_INET && !conn->ipv4_traffic_ok) || - (tor_addr_family(&addr) == AF_INET6 && !conn->ipv6_traffic_ok)) { + if ((tor_addr_family(&addr) == AF_INET && + !conn->entry_cfg.ipv4_traffic) || + (tor_addr_family(&addr) == AF_INET6 && + !conn->entry_cfg.ipv6_traffic)) { log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got an EXITPOLICY failure on a connection with a " "mismatched family. Closing."); @@ -1155,11 +1155,11 @@ connection_ap_handshake_socks_got_resolved_cell(entry_connection_t *conn, addr_hostname = addr; } } else if (tor_addr_family(&addr->addr) == AF_INET) { - if (!addr_ipv4 && conn->ipv4_traffic_ok) { + if (!addr_ipv4 && conn->entry_cfg.ipv4_traffic) { addr_ipv4 = addr; } } else if (tor_addr_family(&addr->addr) == AF_INET6) { - if (!addr_ipv6 && conn->ipv6_traffic_ok) { + if (!addr_ipv6 && conn->entry_cfg.ipv6_traffic) { addr_ipv6 = addr; } } @@ -1180,7 +1180,7 @@ connection_ap_handshake_socks_got_resolved_cell(entry_connection_t *conn, return; } - if (conn->prefer_ipv6_traffic) { + if (conn->entry_cfg.prefer_ipv6) { addr_best = addr_ipv6 ? addr_ipv6 : addr_ipv4; } else { addr_best = addr_ipv4 ? addr_ipv4 : addr_ipv6; @@ -1326,8 +1326,8 @@ connection_edge_process_relay_cell_not_open( return 0; } - if ((family == AF_INET && ! entry_conn->ipv4_traffic_ok) || - (family == AF_INET6 && ! entry_conn->ipv6_traffic_ok)) { + if ((family == AF_INET && ! entry_conn->entry_cfg.ipv4_traffic) || + (family == AF_INET6 && ! entry_conn->entry_cfg.ipv6_traffic)) { log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got a connected cell to %s with unsupported address family." " Closing.", fmt_addr(&addr)); @@ -1643,8 +1643,9 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } if ((reason = circuit_finish_handshake(TO_ORIGIN_CIRCUIT(circ), &extended_cell.created_cell)) < 0) { - log_warn(domain,"circuit_finish_handshake failed."); - return reason; + circuit_mark_for_close(circ, -reason); + return 0; /* We don't want to cause a warning, so we mark the circuit + * here. */ } } if ((reason=circuit_send_next_onion_skin(TO_ORIGIN_CIRCUIT(circ)))<0) { @@ -2248,62 +2249,12 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) /** The total number of cells we have allocated. */ static size_t total_cells_allocated = 0; -#ifdef ENABLE_MEMPOOLS -/** A memory pool to allocate packed_cell_t objects. */ -static mp_pool_t *cell_pool = NULL; - -/** Allocate structures to hold cells. */ -void -init_cell_pool(void) -{ - tor_assert(!cell_pool); - cell_pool = mp_pool_new(sizeof(packed_cell_t), 128*1024); -} - -/** Free all storage used to hold cells (and insertion times/commands if we - * measure cell statistics and/or if CELL_STATS events are enabled). */ -void -free_cell_pool(void) -{ - /* Maybe we haven't called init_cell_pool yet; need to check for it. */ - if (cell_pool) { - mp_pool_destroy(cell_pool); - cell_pool = NULL; - } -} - -/** Free excess storage in cell pool. */ -void -clean_cell_pool(void) -{ - tor_assert(cell_pool); - mp_pool_clean(cell_pool, 0, 1); -} - -#define relay_alloc_cell() \ - mp_pool_get(cell_pool) -#define relay_free_cell(cell) \ - mp_pool_release(cell) - -#define RELAY_CELL_MEM_COST (sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD) - -#else /* !ENABLE_MEMPOOLS case */ - -#define relay_alloc_cell() \ - tor_malloc_zero(sizeof(packed_cell_t)) -#define relay_free_cell(cell) \ - tor_free(cell) - -#define RELAY_CELL_MEM_COST (sizeof(packed_cell_t)) - -#endif /* ENABLE_MEMPOOLS */ - /** Release storage held by <b>cell</b>. */ static INLINE void packed_cell_free_unchecked(packed_cell_t *cell) { --total_cells_allocated; - relay_free_cell(cell); + tor_free(cell); } /** Allocate and return a new packed_cell_t. */ @@ -2311,7 +2262,7 @@ STATIC packed_cell_t * packed_cell_new(void) { ++total_cells_allocated; - return relay_alloc_cell(); + return tor_malloc_zero(sizeof(packed_cell_t)); } /** Return a packed cell used outside by channel_t lower layer */ @@ -2328,21 +2279,18 @@ packed_cell_free(packed_cell_t *cell) void dump_cell_pool_usage(int severity) { - circuit_t *c; int n_circs = 0; int n_cells = 0; - TOR_LIST_FOREACH(c, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, c) { n_cells += c->n_chan_cells.n; if (!CIRCUIT_IS_ORIGIN(c)) n_cells += TO_OR_CIRCUIT(c)->p_chan_cells.n; ++n_circs; } + SMARTLIST_FOREACH_END(c); tor_log(severity, LD_MM, "%d cells allocated on %d circuits. %d cells leaked.", n_cells, n_circs, (int)total_cells_allocated - n_cells); -#ifdef ENABLE_MEMPOOLS - mp_pool_log_status(cell_pool, severity); -#endif } /** Allocate a new copy of packed <b>cell</b>. */ @@ -2422,7 +2370,7 @@ cell_queue_pop(cell_queue_t *queue) size_t packed_cell_mem_cost(void) { - return RELAY_CELL_MEM_COST; + return sizeof(packed_cell_t); } /** DOCDOC */ @@ -2432,6 +2380,12 @@ cell_queues_get_total_allocation(void) return total_cells_allocated * packed_cell_mem_cost(); } +/** How long after we've been low on memory should we try to conserve it? */ +#define MEMORY_PRESSURE_INTERVAL (30*60) + +/** The time at which we were last low on memory. */ +static time_t last_time_under_memory_pressure = 0; + /** Check whether we've got too much space used for cells. If so, * call the OOM handler and return 1. Otherwise, return 0. */ STATIC int @@ -2439,13 +2393,38 @@ cell_queues_check_size(void) { size_t alloc = cell_queues_get_total_allocation(); alloc += buf_get_total_allocation(); - if (alloc >= get_options()->MaxMemInQueues) { - circuits_handle_oom(alloc); - return 1; + alloc += tor_zlib_get_total_allocation(); + const size_t rend_cache_total = rend_cache_get_total_allocation(); + alloc += rend_cache_total; + if (alloc >= get_options()->MaxMemInQueues_low_threshold) { + last_time_under_memory_pressure = approx_time(); + if (alloc >= get_options()->MaxMemInQueues) { + /* If we're spending over 20% of the memory limit on hidden service + * descriptors, free them until we're down to 10%. + */ + if (rend_cache_total > get_options()->MaxMemInQueues / 5) { + const size_t bytes_to_remove = + rend_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); + rend_cache_clean_v2_descs_as_dir(time(NULL), bytes_to_remove); + alloc -= rend_cache_total; + alloc += rend_cache_get_total_allocation(); + } + circuits_handle_oom(alloc); + return 1; + } } return 0; } +/** Return true if we've been under memory pressure in the last + * MEMORY_PRESSURE_INTERVAL seconds. */ +int +have_been_under_memory_pressure(void) +{ + return last_time_under_memory_pressure + MEMORY_PRESSURE_INTERVAL + < approx_time(); +} + /** * Update the number of cells available on the circuit's n_chan or p_chan's * circuit mux. @@ -2590,8 +2569,8 @@ packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids) * queue of the first active circuit on <b>chan</b>, and write them to * <b>chan</b>->outbuf. Return the number of cells written. Advance * the active circuit pointer to the next active circuit in the ring. */ -int -channel_flush_from_first_active_circuit(channel_t *chan, int max) +MOCK_IMPL(int, +channel_flush_from_first_active_circuit, (channel_t *chan, int max)) { circuitmux_t *cmux = NULL; int n_flushed = 0; @@ -2867,14 +2846,8 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, log_debug(LD_GENERAL, "Made a circuit active."); } - if (!channel_has_queued_writes(chan)) { - /* There is no data at all waiting to be sent on the outbuf. Add a - * cell, so that we can notice when it gets flushed, flushed_some can - * get called, and we can start putting more data onto the buffer then. - */ - log_debug(LD_GENERAL, "Primed a buffer."); - channel_flush_from_first_active_circuit(chan, 1); - } + /* New way: mark this as having waiting cells for the scheduler */ + scheduler_channel_has_waiting_cells(chan); } /** Append an encoded value of <b>addr</b> to <b>payload_out</b>, which must diff --git a/src/or/relay.h b/src/or/relay.h index 969c6fb61d..a4f583d11e 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -42,14 +42,11 @@ extern uint64_t stats_n_data_bytes_packaged; extern uint64_t stats_n_data_cells_received; extern uint64_t stats_n_data_bytes_received; -#ifdef ENABLE_MEMPOOLS -void init_cell_pool(void); -void free_cell_pool(void); -void clean_cell_pool(void); -#endif /* ENABLE_MEMPOOLS */ void dump_cell_pool_usage(int severity); size_t packed_cell_mem_cost(void); +int have_been_under_memory_pressure(void); + /* For channeltls.c */ void packed_cell_free(packed_cell_t *cell); @@ -64,7 +61,8 @@ void append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, cell_t *cell, cell_direction_t direction, streamid_t fromstream); void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); -int channel_flush_from_first_active_circuit(channel_t *chan, int max); +MOCK_DECL(int, channel_flush_from_first_active_circuit, + (channel_t *chan, int max)); void assert_circuit_mux_okay(channel_t *chan); void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, const char *file, int lineno); diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 19a8cef1bf..162e0ac53e 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -130,16 +130,6 @@ rend_client_reextend_intro_circuit(origin_circuit_t *circ) return result; } -/** Return true iff we should send timestamps in our INTRODUCE1 cells */ -static int -rend_client_should_send_timestamp(void) -{ - if (get_options()->Support022HiddenServices >= 0) - return get_options()->Support022HiddenServices; - - return networkstatus_get_param(NULL, "Support022HiddenServices", 1, 0, 1); -} - /** Called when we're trying to connect an ap conn; sends an INTRODUCE1 cell * down introcirc if possible. */ @@ -251,14 +241,8 @@ rend_client_send_introduction(origin_circuit_t *introcirc, REND_DESC_COOKIE_LEN); v3_shift += 2+REND_DESC_COOKIE_LEN; } - if (rend_client_should_send_timestamp()) { - uint32_t now = (uint32_t)time(NULL); - now += 300; - now -= now % 600; - set_uint32(tmp+v3_shift+1, htonl(now)); - } else { - set_uint32(tmp+v3_shift+1, 0); - } + /* Once this held a timestamp. */ + set_uint32(tmp+v3_shift+1, 0); v3_shift += 4; } /* if version 2 only write version number */ else if (entry->parsed->protocols & (1<<2)) { @@ -370,15 +354,13 @@ rend_client_rendcirc_has_opened(origin_circuit_t *circ) } /** - * Called to close other intro circuits we launched in parallel - * due to timeout. + * Called to close other intro circuits we launched in parallel. */ static void rend_client_close_other_intros(const char *onion_address) { - circuit_t *c; /* abort parallel intro circs, if any */ - TOR_LIST_FOREACH(c, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, c) { if ((c->purpose == CIRCUIT_PURPOSE_C_INTRODUCING || c->purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) && !c->marked_for_close && CIRCUIT_IS_ORIGIN(c)) { @@ -389,10 +371,11 @@ rend_client_close_other_intros(const char *onion_address) log_info(LD_REND|LD_CIRC, "Closing introduction circuit %d that we " "built in parallel (Purpose %d).", oc->global_identifier, c->purpose); - circuit_mark_for_close(c, END_CIRC_REASON_TIMEOUT); + circuit_mark_for_close(c, END_CIRC_REASON_IP_NOW_REDUNDANT); } } } + SMARTLIST_FOREACH_END(c); } /** Called when get an ACK or a NAK for a REND_INTRODUCE1 cell. @@ -468,6 +451,13 @@ rend_client_introduction_acked(origin_circuit_t *circ, /* XXXX If that call failed, should we close the rend circuit, * too? */ return result; + } else { + /* Close circuit because no more intro points are usable thus not + * useful anymore. Change it's purpose before so we don't report an + * intro point failure again triggering an extra descriptor fetch. */ + circuit_change_purpose(TO_CIRCUIT(circ), + CIRCUIT_PURPOSE_C_INTRODUCE_ACKED); + circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED); } } return 0; @@ -564,7 +554,12 @@ directory_clean_last_hid_serv_requests(time_t now) /** Remove all requests related to the hidden service named * <b>onion_address</b> from the history of times of requests to - * hidden service directories. */ + * hidden service directories. + * + * This is called from rend_client_note_connection_attempt_ended(), which + * must be idempotent, so any future changes to this function must leave + * it idempotent too. + */ static void purge_hid_serv_from_last_hid_serv_requests(const char *onion_address) { @@ -625,7 +620,12 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; time_t now = time(NULL); char descriptor_cookie_base64[3*REND_DESC_COOKIE_LEN_BASE64]; +#ifdef ENABLE_TOR2WEB_MODE const int tor2web_mode = options->Tor2webMode; + const int how_to_fetch = tor2web_mode ? DIRIND_ONEHOP : DIRIND_ANONYMOUS; +#else + const int how_to_fetch = DIRIND_ANONYMOUS; +#endif int excluded_some; tor_assert(desc_id); tor_assert(rend_query); @@ -702,7 +702,7 @@ directory_get_from_hs_dir(const char *desc_id, const rend_data_t *rend_query) directory_initiate_command_routerstatus_rend(hs_dir, DIR_PURPOSE_FETCH_RENDDESC_V2, ROUTER_PURPOSE_GENERAL, - tor2web_mode?DIRIND_ONEHOP:DIRIND_ANONYMOUS, + how_to_fetch, desc_id_base32, NULL, 0, 0, rend_query); @@ -1093,8 +1093,11 @@ rend_client_desc_trynow(const char *query) /** Clear temporary state used only during an attempt to connect to * the hidden service named <b>onion_address</b>. Called when a - * connection attempt has ended; may be called occasionally at other - * times, and should be reasonably harmless. */ + * connection attempt has ended; it is possible for this to be called + * multiple times while handling an ended connection attempt, and + * any future changes to this function must ensure it remains + * idempotent. + */ void rend_client_note_connection_attempt_ended(const char *onion_address) { diff --git a/src/or/rendclient.h b/src/or/rendclient.h index 1f731d0ae5..098c61d0a1 100644 --- a/src/or/rendclient.h +++ b/src/or/rendclient.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 9637d4d838..0451ae62ee 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -155,7 +155,7 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id, } /* Calculate current time-period. */ time_period = get_time_period(now, 0, service_id_binary); - /* Calculate secret-id-part = h(time-period + replica). */ + /* Calculate secret-id-part = h(time-period | replica). */ get_secret_id_part_bytes(secret_id_part, time_period, descriptor_cookie, replica); /* Calculate descriptor ID. */ @@ -411,7 +411,7 @@ rend_desc_v2_is_parsable(rend_encoded_v2_service_descriptor_t *desc) &test_intro_content, &test_intro_size, &test_encoded_size, - &test_next, desc->desc_str); + &test_next, desc->desc_str, 1); rend_service_descriptor_free(test_parsed); tor_free(test_intro_content); return (res >= 0); @@ -528,7 +528,7 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, return -1; } /* Base64-encode introduction points. */ - ipos_base64 = tor_malloc_zero(ipos_len * 2); + ipos_base64 = tor_calloc(ipos_len, 2); if (base64_encode(ipos_base64, ipos_len * 2, ipos, ipos_len)<0) { log_warn(LD_REND, "Could not encode introduction point string to " "base64. length=%d", (int)ipos_len); @@ -556,7 +556,7 @@ rend_encode_v2_descriptors(smartlist_t *descs_out, char desc_digest[DIGEST_LEN]; rend_encoded_v2_service_descriptor_t *enc = tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t)); - /* Calculate secret-id-part = h(time-period + cookie + replica). */ + /* Calculate secret-id-part = h(time-period | cookie | replica). */ get_secret_id_part_bytes(secret_id_part, time_period, descriptor_cookie, k); base32_encode(secret_id_part_base32, sizeof(secret_id_part_base32), @@ -704,6 +704,9 @@ static strmap_t *rend_cache = NULL; * directories. */ static digestmap_t *rend_cache_v2_dir = NULL; +/** DOCDOC */ +static size_t rend_cache_total_allocation = 0; + /** Initializes the service descriptor cache. */ void @@ -713,12 +716,64 @@ rend_cache_init(void) rend_cache_v2_dir = digestmap_new(); } +/** Return the approximate number of bytes needed to hold <b>e</b>. */ +static size_t +rend_cache_entry_allocation(const rend_cache_entry_t *e) +{ + if (!e) + return 0; + + /* This doesn't count intro_nodes or key size */ + return sizeof(*e) + e->len + sizeof(*e->parsed); +} + +/** DOCDOC */ +size_t +rend_cache_get_total_allocation(void) +{ + return rend_cache_total_allocation; +} + +/** Decrement the total bytes attributed to the rendezvous cache by n. */ +static void +rend_cache_decrement_allocation(size_t n) +{ + static int have_underflowed = 0; + + if (rend_cache_total_allocation >= n) { + rend_cache_total_allocation -= n; + } else { + rend_cache_total_allocation = 0; + if (! have_underflowed) { + have_underflowed = 1; + log_warn(LD_BUG, "Underflow in rend_cache_decrement_allocation"); + } + } +} + +/** Increase the total bytes attributed to the rendezvous cache by n. */ +static void +rend_cache_increment_allocation(size_t n) +{ + static int have_overflowed = 0; + if (rend_cache_total_allocation <= SIZE_MAX - n) { + rend_cache_total_allocation += n; + } else { + rend_cache_total_allocation = SIZE_MAX; + if (! have_overflowed) { + have_overflowed = 1; + log_warn(LD_BUG, "Overflow in rend_cache_increment_allocation"); + } + } +} + /** Helper: free storage held by a single service descriptor cache entry. */ static void rend_cache_entry_free(rend_cache_entry_t *e) { if (!e) return; + rend_cache_decrement_allocation(rend_cache_entry_allocation(e)); rend_service_descriptor_free(e->parsed); tor_free(e->desc); tor_free(e); @@ -740,6 +795,7 @@ rend_cache_free_all(void) digestmap_free(rend_cache_v2_dir, rend_cache_entry_free_); rend_cache = NULL; rend_cache_v2_dir = NULL; + rend_cache_total_allocation = 0; } /** Removes all old entries from the service descriptor cache. @@ -777,31 +833,46 @@ rend_cache_purge(void) } /** Remove all old v2 descriptors and those for which this hidden service - * directory is not responsible for any more. */ + * directory is not responsible for any more. + * + * If at all possible, remove at least <b>force_remove</b> bytes of data. + */ void -rend_cache_clean_v2_descs_as_dir(time_t now) +rend_cache_clean_v2_descs_as_dir(time_t now, size_t force_remove) { digestmap_iter_t *iter; time_t cutoff = now - REND_CACHE_MAX_AGE - REND_CACHE_MAX_SKEW; - for (iter = digestmap_iter_init(rend_cache_v2_dir); - !digestmap_iter_done(iter); ) { - const char *key; - void *val; - rend_cache_entry_t *ent; - digestmap_iter_get(iter, &key, &val); - ent = val; - if (ent->parsed->timestamp < cutoff || - !hid_serv_responsible_for_desc_id(key)) { - char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; - base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); - log_info(LD_REND, "Removing descriptor with ID '%s' from cache", - safe_str_client(key_base32)); - iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); - rend_cache_entry_free(ent); - } else { - iter = digestmap_iter_next(rend_cache_v2_dir, iter); + const int LAST_SERVED_CUTOFF_STEP = 1800; + time_t last_served_cutoff = cutoff; + size_t bytes_removed = 0; + do { + for (iter = digestmap_iter_init(rend_cache_v2_dir); + !digestmap_iter_done(iter); ) { + const char *key; + void *val; + rend_cache_entry_t *ent; + digestmap_iter_get(iter, &key, &val); + ent = val; + if (ent->parsed->timestamp < cutoff || + ent->last_served < last_served_cutoff || + !hid_serv_responsible_for_desc_id(key)) { + char key_base32[REND_DESC_ID_V2_LEN_BASE32 + 1]; + base32_encode(key_base32, sizeof(key_base32), key, DIGEST_LEN); + log_info(LD_REND, "Removing descriptor with ID '%s' from cache", + safe_str_client(key_base32)); + bytes_removed += rend_cache_entry_allocation(ent); + iter = digestmap_iter_next_rmv(rend_cache_v2_dir, iter); + rend_cache_entry_free(ent); + } else { + iter = digestmap_iter_next(rend_cache_v2_dir, iter); + } } - } + + /* In case we didn't remove enough bytes, advance the cutoff a little. */ + last_served_cutoff += LAST_SERVED_CUTOFF_STEP; + if (last_served_cutoff > now) + break; + } while (bytes_removed < force_remove); } /** Determines whether <b>a</b> is in the interval of <b>b</b> (excluded) and @@ -903,6 +974,7 @@ rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc) e = digestmap_get(rend_cache_v2_dir, desc_id_digest); if (e) { *desc = e->desc; + e->last_served = approx_time(); return 1; } return 0; @@ -924,6 +996,7 @@ rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc) rend_cache_store_status_t rend_cache_store_v2_desc_as_dir(const char *desc) { + const or_options_t *options = get_options(); rend_service_descriptor_t *parsed; char desc_id[DIGEST_LEN]; char *intro_content; @@ -945,7 +1018,7 @@ rend_cache_store_v2_desc_as_dir(const char *desc) } while (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content, &intro_size, &encoded_size, - &next_desc, current_desc) >= 0) { + &next_desc, current_desc, 1) >= 0) { number_parsed++; /* We don't care about the introduction points. */ tor_free(intro_content); @@ -985,24 +1058,35 @@ rend_cache_store_v2_desc_as_dir(const char *desc) if (e && !strcmp(desc, e->desc)) { log_info(LD_REND, "We already have this service descriptor with desc " "ID %s.", safe_str(desc_id_base32)); - e->received = time(NULL); goto skip; } /* Store received descriptor. */ if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); digestmap_set(rend_cache_v2_dir, desc_id, e); + /* Treat something just uploaded as having been served a little + * while ago, so that flooding with new descriptors doesn't help + * too much. + */ + e->last_served = approx_time() - 3600; } else { + rend_cache_decrement_allocation(rend_cache_entry_allocation(e)); rend_service_descriptor_free(e->parsed); tor_free(e->desc); } - e->received = time(NULL); e->parsed = parsed; e->desc = tor_strndup(current_desc, encoded_size); e->len = encoded_size; + rend_cache_increment_allocation(rend_cache_entry_allocation(e)); log_info(LD_REND, "Successfully stored service descriptor with desc ID " "'%s' and len %d.", safe_str(desc_id_base32), (int)encoded_size); + + /* Statistics: Note down this potentially new HS. */ + if (options->HiddenServiceStatistics) { + rep_hist_stored_maybe_new_hs(e->parsed->pk); + } + number_stored++; goto advance; skip: @@ -1034,10 +1118,14 @@ rend_cache_store_v2_desc_as_dir(const char *desc) * If the descriptor's service ID does not match * <b>rend_query</b>-\>onion_address, reject it. * + * If the descriptor's descriptor ID doesn't match <b>desc_id_base32</b>, + * reject it. + * * Return an appropriate rend_cache_store_status_t. */ rend_cache_store_status_t rend_cache_store_v2_desc_as_client(const char *desc, + const char *desc_id_base32, const rend_data_t *rend_query) { /*XXXX this seems to have a bit of duplicate code with @@ -1064,14 +1152,23 @@ rend_cache_store_v2_desc_as_client(const char *desc, time_t now = time(NULL); char key[REND_SERVICE_ID_LEN_BASE32+2]; char service_id[REND_SERVICE_ID_LEN_BASE32+1]; + char want_desc_id[DIGEST_LEN]; rend_cache_entry_t *e; rend_cache_store_status_t retval = RCS_BADDESC; tor_assert(rend_cache); tor_assert(desc); + tor_assert(desc_id_base32); + memset(want_desc_id, 0, sizeof(want_desc_id)); + if (base32_decode(want_desc_id, sizeof(want_desc_id), + desc_id_base32, strlen(desc_id_base32)) != 0) { + log_warn(LD_BUG, "Couldn't decode base32 %s for descriptor id.", + escaped_safe_str_client(desc_id_base32)); + goto err; + } /* Parse the descriptor. */ if (rend_parse_v2_service_descriptor(&parsed, desc_id, &intro_content, &intro_size, &encoded_size, - &next_desc, desc) < 0) { + &next_desc, desc, 0) < 0) { log_warn(LD_REND, "Could not parse descriptor."); goto err; } @@ -1086,6 +1183,12 @@ rend_cache_store_v2_desc_as_client(const char *desc, service_id, safe_str(rend_query->onion_address)); goto err; } + if (tor_memneq(desc_id, want_desc_id, DIGEST_LEN)) { + log_warn(LD_REND, "Received service descriptor for %s with incorrect " + "descriptor ID.", service_id); + goto err; + } + /* Decode/decrypt introduction points. */ if (intro_content && intro_size > 0) { int n_intro_points; @@ -1158,21 +1261,21 @@ rend_cache_store_v2_desc_as_client(const char *desc, if (e && !strcmp(desc, e->desc)) { log_info(LD_REND,"We already have this service descriptor %s.", safe_str_client(service_id)); - e->received = time(NULL); goto okay; } if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); strmap_set_lc(rend_cache, key, e); } else { + rend_cache_decrement_allocation(rend_cache_entry_allocation(e)); rend_service_descriptor_free(e->parsed); tor_free(e->desc); } - e->received = time(NULL); e->parsed = parsed; e->desc = tor_malloc_zero(encoded_size + 1); strlcpy(e->desc, desc, encoded_size + 1); e->len = encoded_size; + rend_cache_increment_allocation(rend_cache_entry_allocation(e)); log_debug(LD_REND,"Successfully stored rend desc '%s', len %d.", safe_str_client(service_id), (int)encoded_size); return RCS_OKAY; diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h index 07a47accfe..8396cc3551 100644 --- a/src/or/rendcommon.h +++ b/src/or/rendcommon.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -33,7 +33,7 @@ void rend_intro_point_free(rend_intro_point_t *intro); void rend_cache_init(void); void rend_cache_clean(time_t now); -void rend_cache_clean_v2_descs_as_dir(time_t now); +void rend_cache_clean_v2_descs_as_dir(time_t now, size_t min_to_remove); void rend_cache_purge(void); void rend_cache_free_all(void); int rend_valid_service_id(const char *query); @@ -49,8 +49,8 @@ typedef enum { rend_cache_store_status_t rend_cache_store_v2_desc_as_dir(const char *desc); rend_cache_store_status_t rend_cache_store_v2_desc_as_client(const char *desc, + const char *desc_id_base32, const rend_data_t *rend_query); - int rend_encode_v2_descriptors(smartlist_t *descs_out, rend_service_descriptor_t *desc, time_t now, uint8_t period, rend_auth_type_t auth_type, @@ -63,6 +63,7 @@ int rend_id_is_in_interval(const char *a, const char *b, const char *c); void rend_get_descriptor_id_bytes(char *descriptor_id_out, const char *service_id, const char *secret_id_part); +size_t rend_cache_get_total_allocation(void); #endif diff --git a/src/or/rendmid.c b/src/or/rendmid.c index 0e1f91c302..2451acb514 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -202,7 +202,7 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, "Unable to send INTRODUCE2 cell to Tor client."); goto err; } - /* And sent an ack down Alice's circuit. Empty body means succeeded. */ + /* And send an ack down Alice's circuit. Empty body means succeeded. */ if (relay_send_command_from_edge(0,TO_CIRCUIT(circ), RELAY_COMMAND_INTRODUCE_ACK, NULL,0,NULL)) { @@ -213,7 +213,7 @@ rend_mid_introduce(or_circuit_t *circ, const uint8_t *request, return 0; err: - /* Send the client an NACK */ + /* Send the client a NACK */ nak_body[0] = 1; if (relay_send_command_from_edge(0,TO_CIRCUIT(circ), RELAY_COMMAND_INTRODUCE_ACK, @@ -295,6 +295,7 @@ int rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, size_t request_len) { + const or_options_t *options = get_options(); or_circuit_t *rend_circ; char hexid[9]; int reason = END_CIRC_REASON_INTERNAL; @@ -330,6 +331,12 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, goto err; } + /* Statistics: Mark this circuit as an RP circuit so that we collect + stats from it. */ + if (options->HiddenServiceStatistics) { + circ->circuit_carries_hs_traffic_stats = 1; + } + /* Send the RENDEZVOUS2 cell to Alice. */ if (relay_send_command_from_edge(0, TO_CIRCUIT(rend_circ), RELAY_COMMAND_RENDEZVOUS2, diff --git a/src/or/rendmid.h b/src/or/rendmid.h index 310276ac96..6bd691a740 100644 --- a/src/or/rendmid.h +++ b/src/or/rendmid.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/rendservice.c b/src/or/rendservice.c index d958de9df9..111b369b1c 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,6 +16,7 @@ #include "circuituse.h" #include "config.h" #include "directory.h" +#include "main.h" #include "networkstatus.h" #include "nodelist.h" #include "rendclient.h" @@ -65,9 +66,16 @@ static ssize_t rend_service_parse_intro_for_v3( * a real port on some IP. */ typedef struct rend_service_port_config_t { + /* The incoming HS virtual port we're mapping */ uint16_t virtual_port; + /* Is this an AF_UNIX port? */ + unsigned int is_unix_addr:1; + /* The outgoing TCP port to use, if !is_unix_addr */ uint16_t real_port; + /* The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */ tor_addr_t real_addr; + /* The socket path to connect to, if is_unix_addr */ + char unix_addr[FLEXIBLE_ARRAY_MEMBER]; } rend_service_port_config_t; /** Try to maintain this many intro points per service by default. */ @@ -82,7 +90,7 @@ typedef struct rend_service_port_config_t { #define MAX_INTRO_CIRCS_PER_PERIOD 10 /** How many times will a hidden service operator attempt to connect to * a requested rendezvous point before giving up? */ -#define MAX_REND_FAILURES 8 +#define MAX_REND_FAILURES 1 /** How many seconds should we spend trying to connect to a requested * rendezvous point before giving up? */ #define MAX_REND_TIMEOUT 30 @@ -95,6 +103,8 @@ typedef struct rend_service_port_config_t { typedef struct rend_service_t { /* Fields specified in config file */ char *directory; /**< where in the filesystem it stores it */ + int dir_group_readable; /**< if 1, allow group read + permissions on directory */ smartlist_t *ports; /**< List of rend_service_port_config_t */ rend_auth_type_t auth_type; /**< Client authorization type or 0 if no client * authorization is performed. */ @@ -126,6 +136,9 @@ typedef struct rend_service_t { * when they do, this keeps us from launching multiple simultaneous attempts * to connect to the same rend point. */ replaycache_t *accepted_intro_dh_parts; + /** If true, we don't close circuits for making requests to unsupported + * ports. */ + int allow_unknown_ports; } rend_service_t; /** A list of rend_service_t's for services run on this OP. @@ -273,16 +286,48 @@ rend_add_service(rend_service_t *service) service->directory); for (i = 0; i < smartlist_len(service->ports); ++i) { p = smartlist_get(service->ports, i); - log_debug(LD_REND,"Service maps port %d to %s", - p->virtual_port, fmt_addrport(&p->real_addr, p->real_port)); + if (!(p->is_unix_addr)) { + log_debug(LD_REND, + "Service maps port %d to %s", + p->virtual_port, + fmt_addrport(&p->real_addr, p->real_port)); + } else { +#ifdef HAVE_SYS_UN_H + log_debug(LD_REND, + "Service maps port %d to socket at \"%s\"", + p->virtual_port, p->unix_addr); +#else + log_debug(LD_REND, + "Service maps port %d to an AF_UNIX socket, but we " + "have no AF_UNIX support on this platform. This is " + "probably a bug.", + p->virtual_port); +#endif /* defined(HAVE_SYS_UN_H) */ + } } } } +/** Return a new rend_service_port_config_t with its path set to + * <b>socket_path</b> or empty if <b>socket_path</b> is NULL */ +static rend_service_port_config_t * +rend_service_port_config_new(const char *socket_path) +{ + if (!socket_path) + return tor_malloc_zero(sizeof(rend_service_port_config_t) + 1); + + const size_t pathlen = strlen(socket_path) + 1; + rend_service_port_config_t *conf = + tor_malloc_zero(sizeof(rend_service_port_config_t) + pathlen); + memcpy(conf->unix_addr, socket_path, pathlen); + conf->is_unix_addr = 1; + return conf; +} + /** Parses a real-port to virtual-port mapping and returns a new * rend_service_port_config_t. * - * The format is: VirtualPort (IP|RealPort|IP:RealPort)? + * The format is: VirtualPort (IP|RealPort|IP:RealPort|'socket':path)? * * IP defaults to 127.0.0.1; RealPort defaults to VirtualPort. */ @@ -291,11 +336,13 @@ parse_port_config(const char *string) { smartlist_t *sl; int virtport; - int realport; + int realport = 0; uint16_t p; tor_addr_t addr; const char *addrport; rend_service_port_config_t *result = NULL; + unsigned int is_unix_addr = 0; + char *socket_path = NULL; sl = smartlist_new(); smartlist_split_string(sl, string, " ", @@ -317,8 +364,21 @@ parse_port_config(const char *string) realport = virtport; tor_addr_from_ipv4h(&addr, 0x7F000001u); /* 127.0.0.1 */ } else { + int ret; + addrport = smartlist_get(sl,1); - if (strchr(addrport, ':') || strchr(addrport, '.')) { + ret = config_parse_unix_port(addrport, &socket_path); + if (ret < 0 && ret != -ENOENT) { + if (ret == -EINVAL) { + log_warn(LD_CONFIG, + "Empty socket path in hidden service port configuration."); + } + goto err; + } + if (socket_path) { + is_unix_addr = 1; + } else if (strchr(addrport, ':') || strchr(addrport, '.')) { + /* else try it as an IP:port pair if it has a : or . in it */ if (tor_addr_port_lookup(addrport, &addr, &p)<0) { log_warn(LD_CONFIG,"Unparseable address in hidden service port " "configuration."); @@ -337,13 +397,21 @@ parse_port_config(const char *string) } } - result = tor_malloc(sizeof(rend_service_port_config_t)); + /* Allow room for unix_addr */ + result = rend_service_port_config_new(socket_path); result->virtual_port = virtport; - result->real_port = realport; - tor_addr_copy(&result->real_addr, &addr); + result->is_unix_addr = is_unix_addr; + if (!is_unix_addr) { + result->real_port = realport; + tor_addr_copy(&result->real_addr, &addr); + result->unix_addr[0] = '\0'; + } + err: SMARTLIST_FOREACH(sl, char *, c, tor_free(c)); smartlist_free(sl); + if (socket_path) tor_free(socket_path); + return result; } @@ -359,6 +427,7 @@ rend_config_services(const or_options_t *options, int validate_only) rend_service_t *service = NULL; rend_service_port_config_t *portcfg; smartlist_t *old_service_list = NULL; + int ok = 0; if (!validate_only) { old_service_list = rend_service_list; @@ -369,87 +438,114 @@ rend_config_services(const or_options_t *options, int validate_only) if (!strcasecmp(line->key, "HiddenServiceDir")) { if (service) { /* register the one we just finished parsing */ if (validate_only) - rend_service_free(service); - else - rend_add_service(service); - } - service = tor_malloc_zero(sizeof(rend_service_t)); - service->directory = tor_strdup(line->value); - service->ports = smartlist_new(); - service->intro_period_started = time(NULL); - service->n_intro_points_wanted = NUM_INTRO_POINTS_DEFAULT; - continue; - } - if (!service) { - log_warn(LD_CONFIG, "%s with no preceding HiddenServiceDir directive", - line->key); - rend_service_free(service); - return -1; - } - if (!strcasecmp(line->key, "HiddenServicePort")) { - portcfg = parse_port_config(line->value); - if (!portcfg) { - rend_service_free(service); - return -1; - } - smartlist_add(service->ports, portcfg); - } else if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) { - /* Parse auth type and comma-separated list of client names and add a - * rend_authorized_client_t for each client to the service's list - * of authorized clients. */ - smartlist_t *type_names_split, *clients; - const char *authname; - int num_clients; - if (service->auth_type != REND_NO_AUTH) { - log_warn(LD_CONFIG, "Got multiple HiddenServiceAuthorizeClient " - "lines for a single service."); - rend_service_free(service); - return -1; - } - type_names_split = smartlist_new(); - smartlist_split_string(type_names_split, line->value, " ", 0, 2); - if (smartlist_len(type_names_split) < 1) { - log_warn(LD_BUG, "HiddenServiceAuthorizeClient has no value. This " - "should have been prevented when parsing the " - "configuration."); - smartlist_free(type_names_split); - rend_service_free(service); - return -1; - } - authname = smartlist_get(type_names_split, 0); - if (!strcasecmp(authname, "basic")) { - service->auth_type = REND_BASIC_AUTH; - } else if (!strcasecmp(authname, "stealth")) { - service->auth_type = REND_STEALTH_AUTH; - } else { - log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains " - "unrecognized auth-type '%s'. Only 'basic' or 'stealth' " - "are recognized.", - (char *) smartlist_get(type_names_split, 0)); - SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp)); - smartlist_free(type_names_split); - rend_service_free(service); - return -1; - } - service->clients = smartlist_new(); - if (smartlist_len(type_names_split) < 2) { - log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains " - "auth-type '%s', but no client names.", - service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth"); - SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp)); - smartlist_free(type_names_split); - continue; - } - clients = smartlist_new(); - smartlist_split_string(clients, smartlist_get(type_names_split, 1), - ",", SPLIT_SKIP_SPACE, 0); - SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp)); - smartlist_free(type_names_split); - /* Remove duplicate client names. */ - num_clients = smartlist_len(clients); - smartlist_sort_strings(clients); - smartlist_uniq_strings(clients); - if (smartlist_len(clients) < num_clients) { + rend_service_free(service); + else + rend_add_service(service); + } + service = tor_malloc_zero(sizeof(rend_service_t)); + service->directory = tor_strdup(line->value); + service->ports = smartlist_new(); + service->intro_period_started = time(NULL); + service->n_intro_points_wanted = NUM_INTRO_POINTS_DEFAULT; + continue; + } + if (!service) { + log_warn(LD_CONFIG, "%s with no preceding HiddenServiceDir directive", + line->key); + rend_service_free(service); + return -1; + } + if (!strcasecmp(line->key, "HiddenServicePort")) { + portcfg = parse_port_config(line->value); + if (!portcfg) { + rend_service_free(service); + return -1; + } + smartlist_add(service->ports, portcfg); + } else if (!strcasecmp(line->key, "HiddenServiceAllowUnknownPorts")) { + service->allow_unknown_ports = (int)tor_parse_long(line->value, + 10, 0, 1, &ok, NULL); + if (!ok) { + log_warn(LD_CONFIG, + "HiddenServiceAllowUnknownPorts should be 0 or 1, not %s", + line->value); + rend_service_free(service); + return -1; + } + log_info(LD_CONFIG, + "HiddenServiceAllowUnknownPorts=%d for %s", + (int)service->allow_unknown_ports, service->directory); + } else if (!strcasecmp(line->key, + "HiddenServiceDirGroupReadable")) { + service->dir_group_readable = (int)tor_parse_long(line->value, + 10, 0, 1, &ok, NULL); + if (!ok) { + log_warn(LD_CONFIG, + "HiddenServiceDirGroupReadable should be 0 or 1, not %s", + line->value); + rend_service_free(service); + return -1; + } + log_info(LD_CONFIG, + "HiddenServiceDirGroupReadable=%d for %s", + service->dir_group_readable, service->directory); + } else if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) { + /* Parse auth type and comma-separated list of client names and add a + * rend_authorized_client_t for each client to the service's list + * of authorized clients. */ + smartlist_t *type_names_split, *clients; + const char *authname; + int num_clients; + if (service->auth_type != REND_NO_AUTH) { + log_warn(LD_CONFIG, "Got multiple HiddenServiceAuthorizeClient " + "lines for a single service."); + rend_service_free(service); + return -1; + } + type_names_split = smartlist_new(); + smartlist_split_string(type_names_split, line->value, " ", 0, 2); + if (smartlist_len(type_names_split) < 1) { + log_warn(LD_BUG, "HiddenServiceAuthorizeClient has no value. This " + "should have been prevented when parsing the " + "configuration."); + smartlist_free(type_names_split); + rend_service_free(service); + return -1; + } + authname = smartlist_get(type_names_split, 0); + if (!strcasecmp(authname, "basic")) { + service->auth_type = REND_BASIC_AUTH; + } else if (!strcasecmp(authname, "stealth")) { + service->auth_type = REND_STEALTH_AUTH; + } else { + log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains " + "unrecognized auth-type '%s'. Only 'basic' or 'stealth' " + "are recognized.", + (char *) smartlist_get(type_names_split, 0)); + SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp)); + smartlist_free(type_names_split); + rend_service_free(service); + return -1; + } + service->clients = smartlist_new(); + if (smartlist_len(type_names_split) < 2) { + log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains " + "auth-type '%s', but no client names.", + service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth"); + SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp)); + smartlist_free(type_names_split); + continue; + } + clients = smartlist_new(); + smartlist_split_string(clients, smartlist_get(type_names_split, 1), + ",", SPLIT_SKIP_SPACE, 0); + SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp)); + smartlist_free(type_names_split); + /* Remove duplicate client names. */ + num_clients = smartlist_len(clients); + smartlist_sort_strings(clients); + smartlist_uniq_strings(clients); + if (smartlist_len(clients) < num_clients) { log_info(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d " "duplicate client name(s); removing.", num_clients - smartlist_len(clients)); @@ -513,10 +609,21 @@ rend_config_services(const or_options_t *options, int validate_only) } } if (service) { - if (validate_only) + cpd_check_t check_opts = CPD_CHECK_MODE_ONLY|CPD_CHECK; + if (service->dir_group_readable) { + check_opts |= CPD_GROUP_READ; + } + + if (check_private_dir(service->directory, check_opts, options->User) < 0) { + rend_service_free(service); + return -1; + } + + if (validate_only) { rend_service_free(service); - else + } else { rend_add_service(service); + } } /* If this is a reload and there were hidden services configured before, @@ -524,7 +631,6 @@ rend_config_services(const or_options_t *options, int validate_only) * other ones. */ if (old_service_list && !validate_only) { smartlist_t *surviving_services = smartlist_new(); - circuit_t *circ; /* Copy introduction points to new services. */ /* XXXX This is O(n^2), but it's only called on reconfigure, so it's @@ -544,7 +650,7 @@ rend_config_services(const or_options_t *options, int validate_only) /* XXXX it would be nicer if we had a nicer abstraction to use here, * so we could just iterate over the list of services to close, but * once again, this isn't critical-path code. */ - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN && (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || @@ -569,6 +675,7 @@ rend_config_services(const or_options_t *options, int validate_only) /* XXXX Is there another reason we should use here? */ } } + SMARTLIST_FOREACH_END(circ); smartlist_free(surviving_services); SMARTLIST_FOREACH(old_service_list, rend_service_t *, ptr, rend_service_free(ptr)); @@ -693,10 +800,23 @@ rend_service_load_keys(rend_service_t *s) { char fname[512]; char buf[128]; + cpd_check_t check_opts = CPD_CREATE; + if (s->dir_group_readable) { + check_opts |= CPD_GROUP_READ; + } /* Check/create directory */ - if (check_private_dir(s->directory, CPD_CREATE, get_options()->User) < 0) + if (check_private_dir(s->directory, check_opts, get_options()->User) < 0) { return -1; + } +#ifndef _WIN32 + if (s->dir_group_readable) { + /* Only new dirs created get new opts, also enforce group read. */ + if (chmod(s->directory, 0750)) { + log_warn(LD_FS,"Unable to make %s group-readable.", s->directory); + } + } +#endif /* Load key */ if (strlcpy(fname,s->directory,sizeof(fname)) >= sizeof(fname) || @@ -706,7 +826,7 @@ rend_service_load_keys(rend_service_t *s) s->directory); return -1; } - s->private_key = init_key_from_file(fname, 1, LOG_ERR); + s->private_key = init_key_from_file(fname, 1, LOG_ERR, 0); if (!s->private_key) return -1; @@ -733,6 +853,15 @@ rend_service_load_keys(rend_service_t *s) memwipe(buf, 0, sizeof(buf)); return -1; } +#ifndef _WIN32 + if (s->dir_group_readable) { + /* Also verify hostname file created with group read. */ + if (chmod(fname, 0640)) + log_warn(LD_FS,"Unable to make hidden hostname file %s group-readable.", + fname); + } +#endif + memwipe(buf, 0, sizeof(buf)); /* If client authorization is configured, load or generate keys. */ @@ -1012,8 +1141,8 @@ rend_check_authorization(rend_service_t *service, } /* Allow the request. */ - log_debug(LD_REND, "Client %s authorized for service %s.", - auth_client->client_name, service->service_id); + log_info(LD_REND, "Client %s authorized for service %s.", + auth_client->client_name, service->service_id); return 1; } @@ -1456,10 +1585,7 @@ rend_service_introduce(origin_circuit_t *circuit, const uint8_t *request, memwipe(hexcookie, 0, sizeof(hexcookie)); /* Free the parsed cell */ - if (parsed_req) { - rend_service_free_intro(parsed_req); - parsed_req = NULL; - } + rend_service_free_intro(parsed_req); /* Free rp if we must */ if (need_rp_free) extend_info_free(rp); @@ -1489,8 +1615,7 @@ find_rp_for_intro(const rend_intro_cell_t *intro, } if (intro->version == 0 || intro->version == 1) { - if (intro->version == 1) rp_nickname = (const char *)(intro->u.v1.rp); - else rp_nickname = (const char *)(intro->u.v0.rp); + rp_nickname = (const char *)(intro->u.v0_v1.rp); node = node_get_by_nickname(rp_nickname, 0); if (!node) { @@ -1549,7 +1674,6 @@ void rend_service_free_intro(rend_intro_cell_t *request) { if (!request) { - log_info(LD_BUG, "rend_service_free_intro() called with NULL request!"); return; } @@ -1658,8 +1782,9 @@ rend_service_begin_parse_intro(const uint8_t *request, goto done; err: - if (rv) rend_service_free_intro(rv); + rend_service_free_intro(rv); rv = NULL; + if (err_msg_out && !err_msg) { tor_asprintf(&err_msg, "unknown INTRODUCE%d error", @@ -1739,11 +1864,7 @@ rend_service_parse_intro_for_v0_or_v1( goto err; } - if (intro->version == 1) { - memcpy(intro->u.v1.rp, rp_nickname, endptr - rp_nickname + 1); - } else { - memcpy(intro->u.v0.rp, rp_nickname, endptr - rp_nickname + 1); - } + memcpy(intro->u.v0_v1.rp, rp_nickname, endptr - rp_nickname + 1); return ver_specific_len; @@ -1767,7 +1888,7 @@ rend_service_parse_intro_for_v2( /* * We accept version 3 too so that the v3 parser can call this with - * and adjusted buffer for the latter part of a v3 cell, which is + * an adjusted buffer for the latter part of a v3 cell, which is * identical to a v2 cell. */ if (!(intro->version == 2 || @@ -2005,7 +2126,7 @@ rend_service_decrypt_intro( char service_id[REND_SERVICE_ID_LEN_BASE32+1]; ssize_t key_len; uint8_t buf[RELAY_PAYLOAD_SIZE]; - int result, status = 0; + int result, status = -1; if (!intro || !key) { if (err_msg_out) { @@ -2084,6 +2205,8 @@ rend_service_decrypt_intro( intro->plaintext = tor_malloc(intro->plaintext_len); memcpy(intro->plaintext, buf, intro->plaintext_len); + status = 0; + goto done; err: @@ -2092,7 +2215,6 @@ rend_service_decrypt_intro( "unknown INTRODUCE%d error decrypting encrypted part", intro ? (int)(intro->type) : -1); } - if (status >= 0) status = -1; done: if (err_msg_out) *err_msg_out = err_msg; @@ -2119,7 +2241,7 @@ rend_service_parse_intro_plaintext( char *err_msg = NULL; ssize_t ver_specific_len, ver_invariant_len; uint8_t version; - int status = 0; + int status = -1; if (!intro) { if (err_msg_out) { @@ -2178,6 +2300,7 @@ rend_service_parse_intro_plaintext( (int)(intro->type), (long)(intro->plaintext_len)); status = -6; + goto err; } else { memcpy(intro->rc, intro->plaintext + ver_specific_len, @@ -2190,6 +2313,7 @@ rend_service_parse_intro_plaintext( /* Flag it as being fully parsed */ intro->parsed = 1; + status = 0; goto done; err: @@ -2198,7 +2322,6 @@ rend_service_parse_intro_plaintext( "unknown INTRODUCE%d error parsing encrypted part", intro ? (int)(intro->type) : -1); } - if (status >= 0) status = -1; done: if (err_msg_out) *err_msg_out = err_msg; @@ -2404,8 +2527,7 @@ static int count_established_intro_points(const char *query) { int num_ipos = 0; - circuit_t *circ; - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->marked_for_close && circ->state == CIRCUIT_STATE_OPEN && (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || @@ -2416,6 +2538,7 @@ count_established_intro_points(const char *query) num_ipos++; } } + SMARTLIST_FOREACH_END(circ); return num_ipos; } @@ -3049,15 +3172,20 @@ rend_services_introduce(void) int intro_point_set_changed, prev_intro_nodes; unsigned int n_intro_points_unexpired; unsigned int n_intro_points_to_open; - smartlist_t *intro_nodes; time_t now; const or_options_t *options = get_options(); + /* List of nodes we need to _exclude_ when choosing a new node to establish + * an intro point to. */ + smartlist_t *exclude_nodes; + + if (!have_completed_a_circuit()) + return; - intro_nodes = smartlist_new(); + exclude_nodes = smartlist_new(); now = time(NULL); for (i=0; i < smartlist_len(rend_service_list); ++i) { - smartlist_clear(intro_nodes); + smartlist_clear(exclude_nodes); service = smartlist_get(rend_service_list, i); tor_assert(service); @@ -3156,8 +3284,10 @@ rend_services_introduce(void) if (intro != NULL && intro->time_expiring == -1) ++n_intro_points_unexpired; + /* Add the valid node to the exclusion list so we don't try to establish + * an introduction point to it again. */ if (node) - smartlist_add(intro_nodes, (void*)node); + smartlist_add(exclude_nodes, (void*)node); } SMARTLIST_FOREACH_END(intro); if (!intro_point_set_changed && @@ -3193,7 +3323,7 @@ rend_services_introduce(void) router_crn_flags_t flags = CRN_NEED_UPTIME|CRN_NEED_DESC; if (get_options()->AllowInvalid_ & ALLOW_INVALID_INTRODUCTION) flags |= CRN_ALLOW_INVALID; - node = router_choose_random_node(intro_nodes, + node = router_choose_random_node(exclude_nodes, options->ExcludeNodes, flags); if (!node) { log_warn(LD_REND, @@ -3204,7 +3334,9 @@ rend_services_introduce(void) break; } intro_point_set_changed = 1; - smartlist_add(intro_nodes, (void*)node); + /* Add the choosen node to the exclusion list in order to avoid to pick + * it again in the next iteration. */ + smartlist_add(exclude_nodes, (void*)node); intro = tor_malloc_zero(sizeof(rend_intro_point_t)); intro->extend_info = extend_info_from_node(node, 0); intro->intro_key = crypto_pk_new(); @@ -3233,9 +3365,12 @@ rend_services_introduce(void) } } } - smartlist_free(intro_nodes); + smartlist_free(exclude_nodes); } +#define MIN_REND_INITIAL_POST_DELAY (30) +#define MIN_REND_INITIAL_POST_DELAY_TESTING (5) + /** Regenerate and upload rendezvous service descriptors for all * services, if necessary. If the descriptor has been dirty enough * for long enough, definitely upload; else only upload when the @@ -3250,6 +3385,9 @@ rend_consider_services_upload(time_t now) int i; rend_service_t *service; int rendpostperiod = get_options()->RendPostPeriod; + int rendinitialpostdelay = (get_options()->TestingTorNetwork ? + MIN_REND_INITIAL_POST_DELAY_TESTING : + MIN_REND_INITIAL_POST_DELAY); if (!get_options()->PublishHidServDescriptors) return; @@ -3257,17 +3395,17 @@ rend_consider_services_upload(time_t now) for (i=0; i < smartlist_len(rend_service_list); ++i) { service = smartlist_get(rend_service_list, i); if (!service->next_upload_time) { /* never been uploaded yet */ - /* The fixed lower bound of 30 seconds ensures that the descriptor - * is stable before being published. See comment below. */ + /* The fixed lower bound of rendinitialpostdelay seconds ensures that + * the descriptor is stable before being published. See comment below. */ service->next_upload_time = - now + 30 + crypto_rand_int(2*rendpostperiod); + now + rendinitialpostdelay + crypto_rand_int(2*rendpostperiod); } if (service->next_upload_time < now || (service->desc_is_dirty && - service->desc_is_dirty < now-30)) { + service->desc_is_dirty < now-rendinitialpostdelay)) { /* if it's time, or if the directory servers have a wrong service - * descriptor and ours has been stable for 30 seconds, upload a - * new one of each format. */ + * descriptor and ours has been stable for rendinitialpostdelay seconds, + * upload a new one of each format. */ rend_service_update_descriptor(service); upload_service_descriptor(service); } @@ -3346,9 +3484,64 @@ rend_service_dump_stats(int severity) } } +#ifdef HAVE_SYS_UN_H + +/** Given <b>ports</b>, a smarlist containing rend_service_port_config_t, + * add the given <b>p</b>, a AF_UNIX port to the list. Return 0 on success + * else return -ENOSYS if AF_UNIX is not supported (see function in the + * #else statement below). */ +static int +add_unix_port(smartlist_t *ports, rend_service_port_config_t *p) +{ + tor_assert(ports); + tor_assert(p); + tor_assert(p->is_unix_addr); + + smartlist_add(ports, p); + return 0; +} + +/** Given <b>conn</b> set it to use the given port <b>p</b> values. Return 0 + * on success else return -ENOSYS if AF_UNIX is not supported (see function + * in the #else statement below). */ +static int +set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p) +{ + tor_assert(conn); + tor_assert(p); + tor_assert(p->is_unix_addr); + + conn->base_.socket_family = AF_UNIX; + tor_addr_make_unspec(&conn->base_.addr); + conn->base_.port = 1; + conn->base_.address = tor_strdup(p->unix_addr); + return 0; +} + +#else /* defined(HAVE_SYS_UN_H) */ + +static int +set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p) +{ + (void) conn; + (void) p; + return -ENOSYS; +} + +static int +add_unix_port(smartlist_t *ports, rend_service_port_config_t *p) +{ + (void) ports; + (void) p; + return -ENOSYS; +} + +#endif /* HAVE_SYS_UN_H */ + /** Given <b>conn</b>, a rendezvous exit stream, look up the hidden service for * 'circ', and look up the port and address based on conn-\>port. - * Assign the actual conn-\>addr and conn-\>port. Return -1 if failure, + * Assign the actual conn-\>addr and conn-\>port. Return -2 on failure + * for which the circuit should be closed, -1 on other failure, * or 0 for success. */ int @@ -3359,6 +3552,7 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, char serviceid[REND_SERVICE_ID_LEN_BASE32+1]; smartlist_t *matching_ports; rend_service_port_config_t *chosen_port; + unsigned int warn_once = 0; tor_assert(circ->base_.purpose == CIRCUIT_PURPOSE_S_REND_JOINED); tor_assert(circ->rend_data); @@ -3371,24 +3565,53 @@ rend_service_set_connection_addr_port(edge_connection_t *conn, log_warn(LD_REND, "Couldn't find any service associated with pk %s on " "rendezvous circuit %u; closing.", serviceid, (unsigned)circ->base_.n_circ_id); - return -1; + return -2; } matching_ports = smartlist_new(); SMARTLIST_FOREACH(service->ports, rend_service_port_config_t *, p, { - if (conn->base_.port == p->virtual_port) { + if (conn->base_.port != p->virtual_port) { + continue; + } + if (!(p->is_unix_addr)) { smartlist_add(matching_ports, p); + } else { + if (add_unix_port(matching_ports, p)) { + if (!warn_once) { + /* Unix port not supported so warn only once. */ + log_warn(LD_REND, + "Saw AF_UNIX virtual port mapping for port %d on service " + "%s, which is unsupported on this platform. Ignoring it.", + conn->base_.port, serviceid); + } + warn_once++; + } } }); chosen_port = smartlist_choose(matching_ports); smartlist_free(matching_ports); if (chosen_port) { - tor_addr_copy(&conn->base_.addr, &chosen_port->real_addr); - conn->base_.port = chosen_port->real_port; + if (!(chosen_port->is_unix_addr)) { + /* Get a non-AF_UNIX connection ready for connection_exit_connect() */ + tor_addr_copy(&conn->base_.addr, &chosen_port->real_addr); + conn->base_.port = chosen_port->real_port; + } else { + if (set_unix_port(conn, chosen_port)) { + /* Simply impossible to end up here else we were able to add a Unix + * port without AF_UNIX support... ? */ + tor_assert(0); + } + } return 0; } - log_info(LD_REND, "No virtual port mapping exists for port %d on service %s", - conn->base_.port,serviceid); - return -1; + + log_info(LD_REND, + "No virtual port mapping exists for port %d on service %s", + conn->base_.port, serviceid); + + if (service->allow_unknown_ports) + return -1; + else + return -2; } diff --git a/src/or/rendservice.h b/src/or/rendservice.h index 40198b07ec..754f7c358c 100644 --- a/src/or/rendservice.h +++ b/src/or/rendservice.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -38,13 +38,9 @@ struct rend_intro_cell_s { /* Version-specific parts */ union { struct { - /* Rendezvous point nickname */ - uint8_t rp[20]; - } v0; - struct { /* Rendezvous point nickname or hex-encoded key digest */ uint8_t rp[42]; - } v1; + } v0_v1; struct { /* The extend_info_t struct has everything v2 uses */ extend_info_t *extend_info; diff --git a/src/or/rephist.c b/src/or/rephist.c index cedc56af07..fe0997c891 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1,5 +1,5 @@ /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -1996,12 +1996,9 @@ void rep_hist_exit_stats_init(time_t now) { start_of_exit_stats_interval = now; - exit_bytes_read = tor_malloc_zero(EXIT_STATS_NUM_PORTS * - sizeof(uint64_t)); - exit_bytes_written = tor_malloc_zero(EXIT_STATS_NUM_PORTS * - sizeof(uint64_t)); - exit_streams = tor_malloc_zero(EXIT_STATS_NUM_PORTS * - sizeof(uint32_t)); + exit_bytes_read = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t)); + exit_bytes_written = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t)); + exit_streams = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint32_t)); } /** Reset counters for exit port statistics. */ @@ -2472,7 +2469,6 @@ rep_hist_format_buffer_stats(time_t now) time_t rep_hist_buffer_stats_write(time_t now) { - circuit_t *circ; char *str = NULL; if (!start_of_buffer_stats_interval) @@ -2481,9 +2477,10 @@ rep_hist_buffer_stats_write(time_t now) goto done; /* Not ready to write */ /* Add open circuits to the history. */ - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) { + SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { rep_hist_buffer_stats_add_circ(circ, now); } + SMARTLIST_FOREACH_END(circ); /* Generate history string. */ str = rep_hist_format_buffer_stats(now); @@ -2570,7 +2567,7 @@ rep_hist_format_desc_stats(time_t now) size = digestmap_size(served_descs); if (size > 0) { - vals = tor_malloc(size * sizeof(int)); + vals = tor_calloc(size, sizeof(int)); for (iter = digestmap_iter_init(served_descs); !digestmap_iter_done(iter); iter = digestmap_iter_next(served_descs, iter)) { @@ -2725,8 +2722,8 @@ bidi_map_ent_hash(const bidi_map_entry_t *entry) HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, bidi_map_ent_eq); -HT_GENERATE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, - bidi_map_ent_eq, 0.6, malloc, realloc, free); +HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, + bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) /* DOCDOC bidi_map_free */ static void @@ -2909,11 +2906,271 @@ rep_hist_log_circuit_handshake_stats(time_t now) memset(onion_handshakes_requested, 0, sizeof(onion_handshakes_requested)); } +/* Hidden service statistics section */ + +/** Start of the current hidden service stats interval or 0 if we're + * not collecting hidden service statistics. */ +static time_t start_of_hs_stats_interval; + +/** Carries the various hidden service statistics, and any other + * information needed. */ +typedef struct hs_stats_t { + /** How many relay cells have we seen as rendezvous points? */ + int64_t rp_relay_cells_seen; + + /** Set of unique public key digests we've seen this stat period + * (could also be implemented as sorted smartlist). */ + digestmap_t *onions_seen_this_period; +} hs_stats_t; + +/** Our statistics structure singleton. */ +static hs_stats_t *hs_stats = NULL; + +/** Allocate, initialize and return an hs_stats_t structure. */ +static hs_stats_t * +hs_stats_new(void) +{ + hs_stats_t * hs_stats = tor_malloc_zero(sizeof(hs_stats_t)); + hs_stats->onions_seen_this_period = digestmap_new(); + + return hs_stats; +} + +/** Free an hs_stats_t structure. */ +static void +hs_stats_free(hs_stats_t *hs_stats) +{ + if (!hs_stats) { + return; + } + + digestmap_free(hs_stats->onions_seen_this_period, NULL); + tor_free(hs_stats); +} + +/** Initialize hidden service statistics. */ +void +rep_hist_hs_stats_init(time_t now) +{ + if (!hs_stats) { + hs_stats = hs_stats_new(); + } + + start_of_hs_stats_interval = now; +} + +/** Clear history of hidden service statistics and set the measurement + * interval start to <b>now</b>. */ +static void +rep_hist_reset_hs_stats(time_t now) +{ + if (!hs_stats) { + hs_stats = hs_stats_new(); + } + + hs_stats->rp_relay_cells_seen = 0; + + digestmap_free(hs_stats->onions_seen_this_period, NULL); + hs_stats->onions_seen_this_period = digestmap_new(); + + start_of_hs_stats_interval = now; +} + +/** Stop collecting hidden service stats in a way that we can re-start + * doing so in rep_hist_buffer_stats_init(). */ +void +rep_hist_hs_stats_term(void) +{ + rep_hist_reset_hs_stats(0); +} + +/** We saw a new HS relay cell, Count it! */ +void +rep_hist_seen_new_rp_cell(void) +{ + if (!hs_stats) { + return; // We're not collecting stats + } + + hs_stats->rp_relay_cells_seen++; +} + +/** As HSDirs, we saw another hidden service with public key + * <b>pubkey</b>. Check whether we have counted it before, if not + * count it now! */ +void +rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey) +{ + char pubkey_hash[DIGEST_LEN]; + + if (!hs_stats) { + return; // We're not collecting stats + } + + /* Get the digest of the pubkey which will be used to detect whether + we've seen this hidden service before or not. */ + if (crypto_pk_get_digest(pubkey, pubkey_hash) < 0) { + /* This fail should not happen; key has been validated by + descriptor parsing code first. */ + return; + } + + /* Check if this is the first time we've seen this hidden + service. If it is, count it as new. */ + if (!digestmap_get(hs_stats->onions_seen_this_period, + pubkey_hash)) { + digestmap_set(hs_stats->onions_seen_this_period, + pubkey_hash, (void*)(uintptr_t)1); + } +} + +/* The number of cells that are supposed to be hidden from the adversary + * by adding noise from the Laplace distribution. This value, divided by + * EPSILON, is Laplace parameter b. */ +#define REND_CELLS_DELTA_F 2048 +/* Security parameter for obfuscating number of cells with a value between + * 0 and 1. Smaller values obfuscate observations more, but at the same + * time make statistics less usable. */ +#define REND_CELLS_EPSILON 0.3 +/* The number of cells that are supposed to be hidden from the adversary + * by rounding up to the next multiple of this number. */ +#define REND_CELLS_BIN_SIZE 1024 +/* The number of service identities that are supposed to be hidden from + * the adversary by adding noise from the Laplace distribution. This + * value, divided by EPSILON, is Laplace parameter b. */ +#define ONIONS_SEEN_DELTA_F 8 +/* Security parameter for obfuscating number of service identities with a + * value between 0 and 1. Smaller values obfuscate observations more, but + * at the same time make statistics less usable. */ +#define ONIONS_SEEN_EPSILON 0.3 +/* The number of service identities that are supposed to be hidden from + * the adversary by rounding up to the next multiple of this number. */ +#define ONIONS_SEEN_BIN_SIZE 8 + +/** Allocate and return a string containing hidden service stats that + * are meant to be placed in the extra-info descriptor. */ +static char * +rep_hist_format_hs_stats(time_t now) +{ + char t[ISO_TIME_LEN+1]; + char *hs_stats_string; + int64_t obfuscated_cells_seen; + int64_t obfuscated_onions_seen; + + obfuscated_cells_seen = round_int64_to_next_multiple_of( + hs_stats->rp_relay_cells_seen, + REND_CELLS_BIN_SIZE); + obfuscated_cells_seen = add_laplace_noise(obfuscated_cells_seen, + crypto_rand_double(), + REND_CELLS_DELTA_F, REND_CELLS_EPSILON); + obfuscated_onions_seen = round_int64_to_next_multiple_of(digestmap_size( + hs_stats->onions_seen_this_period), + ONIONS_SEEN_BIN_SIZE); + obfuscated_onions_seen = add_laplace_noise(obfuscated_onions_seen, + crypto_rand_double(), ONIONS_SEEN_DELTA_F, + ONIONS_SEEN_EPSILON); + + format_iso_time(t, now); + tor_asprintf(&hs_stats_string, "hidserv-stats-end %s (%d s)\n" + "hidserv-rend-relayed-cells "I64_FORMAT" delta_f=%d " + "epsilon=%.2f bin_size=%d\n" + "hidserv-dir-onions-seen "I64_FORMAT" delta_f=%d " + "epsilon=%.2f bin_size=%d\n", + t, (unsigned) (now - start_of_hs_stats_interval), + I64_PRINTF_ARG(obfuscated_cells_seen), REND_CELLS_DELTA_F, + REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE, + I64_PRINTF_ARG(obfuscated_onions_seen), + ONIONS_SEEN_DELTA_F, + ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE); + + return hs_stats_string; +} + +/** If 24 hours have passed since the beginning of the current HS + * stats period, write buffer stats to $DATADIR/stats/hidserv-stats + * (possibly overwriting an existing file) and reset counters. Return + * when we would next want to write buffer stats or 0 if we never want to + * write. */ +time_t +rep_hist_hs_stats_write(time_t now) +{ + char *str = NULL; + + if (!start_of_hs_stats_interval) { + return 0; /* Not initialized. */ + } + + if (start_of_hs_stats_interval + WRITE_STATS_INTERVAL > now) { + goto done; /* Not ready to write */ + } + + /* Generate history string. */ + str = rep_hist_format_hs_stats(now); + + /* Reset HS history. */ + rep_hist_reset_hs_stats(now); + + /* Try to write to disk. */ + if (!check_or_create_data_subdir("stats")) { + write_to_data_subdir("stats", "hidserv-stats", str, + "hidden service stats"); + } + + done: + tor_free(str); + return start_of_hs_stats_interval + WRITE_STATS_INTERVAL; +} + +#define MAX_LINK_PROTO_TO_LOG 4 +static uint64_t link_proto_count[MAX_LINK_PROTO_TO_LOG+1][2]; + +/** Note that we negotiated link protocol version <b>link_proto</b>, on + * a connection that started here iff <b>started_here</b> is true. + */ +void +rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here) +{ + started_here = !!started_here; /* force to 0 or 1 */ + if (link_proto > MAX_LINK_PROTO_TO_LOG) { + log_warn(LD_BUG, "Can't log link protocol %u", link_proto); + return; + } + + link_proto_count[link_proto][started_here]++; +} + +/** Log a heartbeat message explaining how many connections of each link + * protocol version we have used. + */ +void +rep_hist_log_link_protocol_counts(void) +{ + log_notice(LD_HEARTBEAT, + "Since startup, we have initiated " + U64_FORMAT" v1 connections, " + U64_FORMAT" v2 connections, " + U64_FORMAT" v3 connections, and " + U64_FORMAT" v4 connections; and received " + U64_FORMAT" v1 connections, " + U64_FORMAT" v2 connections, " + U64_FORMAT" v3 connections, and " + U64_FORMAT" v4 connections.", + U64_PRINTF_ARG(link_proto_count[1][1]), + U64_PRINTF_ARG(link_proto_count[2][1]), + U64_PRINTF_ARG(link_proto_count[3][1]), + U64_PRINTF_ARG(link_proto_count[4][1]), + U64_PRINTF_ARG(link_proto_count[1][0]), + U64_PRINTF_ARG(link_proto_count[2][0]), + U64_PRINTF_ARG(link_proto_count[3][0]), + U64_PRINTF_ARG(link_proto_count[4][0])); +} + /** Free all storage held by the OR/link history caches, by the * bandwidth history arrays, by the port history, or by statistics . */ void rep_hist_free_all(void) { + hs_stats_free(hs_stats); digestmap_free(history_map, free_or_history); tor_free(read_array); tor_free(write_array); diff --git a/src/or/rephist.h b/src/or/rephist.h index cd6231e6e4..f94b4e8ff1 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -99,7 +99,18 @@ void rep_hist_note_circuit_handshake_requested(uint16_t type); void rep_hist_note_circuit_handshake_assigned(uint16_t type); void rep_hist_log_circuit_handshake_stats(time_t now); +void rep_hist_hs_stats_init(time_t now); +void rep_hist_hs_stats_term(void); +time_t rep_hist_hs_stats_write(time_t now); +char *rep_hist_get_hs_stats_string(void); +void rep_hist_seen_new_rp_cell(void); +void rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey); + void rep_hist_free_all(void); +void rep_hist_note_negotiated_link_proto(unsigned link_proto, + int started_here); +void rep_hist_log_link_protocol_counts(void); + #endif diff --git a/src/or/replaycache.c b/src/or/replaycache.c index 90f87c12d5..569e0736cb 100644 --- a/src/or/replaycache.c +++ b/src/or/replaycache.c @@ -1,4 +1,4 @@ - /* Copyright (c) 2012-2013, The Tor Project, Inc. */ + /* Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* diff --git a/src/or/replaycache.h b/src/or/replaycache.h index cd713fe891..9b9daf3831 100644 --- a/src/or/replaycache.h +++ b/src/or/replaycache.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2013, The Tor Project, Inc. */ +/* Copyright (c) 2012-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/or/router.c b/src/or/router.c index 2cdbb0c8bb..2ddaa895fc 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define ROUTER_PRIVATE @@ -55,13 +55,11 @@ static crypto_pk_t *onionkey=NULL; /** Previous private onionskin decryption key: used to decode CREATE cells * generated by clients that have an older version of our descriptor. */ static crypto_pk_t *lastonionkey=NULL; -#ifdef CURVE25519_ENABLED /** Current private ntor secret key: used to perform the ntor handshake. */ static curve25519_keypair_t curve25519_onion_key; /** Previous private ntor secret key: used to perform the ntor handshake * with clients that have an older version of our descriptor. */ static curve25519_keypair_t last_curve25519_onion_key; -#endif /** Private server "identity key": used to sign directory info and TLS * certificates. Never changes. */ static crypto_pk_t *server_identitykey=NULL; @@ -134,7 +132,6 @@ dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last) tor_mutex_release(key_lock); } -#ifdef CURVE25519_ENABLED /** Return the current secret onion key for the ntor handshake. Must only * be called from the main thread. */ static const curve25519_keypair_t * @@ -181,7 +178,6 @@ ntor_key_map_free(di_digest256_map_t *map) return; dimap_free(map, ntor_key_map_free_helper); } -#endif /** Return the time when the onion key was last set. This is either the time * when the process launched, or the time of the most recent key rotation since @@ -313,12 +309,11 @@ rotate_onion_key(void) char *fname, *fname_prev; crypto_pk_t *prkey = NULL; or_state_t *state = get_or_state(); -#ifdef CURVE25519_ENABLED curve25519_keypair_t new_curve25519_keypair; -#endif time_t now; fname = get_datadir_fname2("keys", "secret_onion_key"); fname_prev = get_datadir_fname2("keys", "secret_onion_key.old"); + /* There isn't much point replacing an old key with an empty file */ if (file_status(fname) == FN_FILE) { if (replace_file(fname, fname_prev)) goto error; @@ -335,13 +330,13 @@ rotate_onion_key(void) log_err(LD_FS,"Couldn't write generated onion key to \"%s\".", fname); goto error; } -#ifdef CURVE25519_ENABLED tor_free(fname); tor_free(fname_prev); fname = get_datadir_fname2("keys", "secret_onion_key_ntor"); fname_prev = get_datadir_fname2("keys", "secret_onion_key_ntor.old"); if (curve25519_keypair_generate(&new_curve25519_keypair, 1) < 0) goto error; + /* There isn't much point replacing an old key with an empty file */ if (file_status(fname) == FN_FILE) { if (replace_file(fname, fname_prev)) goto error; @@ -351,18 +346,15 @@ rotate_onion_key(void) log_err(LD_FS,"Couldn't write curve25519 onion key to \"%s\".",fname); goto error; } -#endif log_info(LD_GENERAL, "Rotating onion key"); tor_mutex_acquire(key_lock); crypto_pk_free(lastonionkey); lastonionkey = onionkey; onionkey = prkey; -#ifdef CURVE25519_ENABLED memcpy(&last_curve25519_onion_key, &curve25519_onion_key, sizeof(curve25519_keypair_t)); memcpy(&curve25519_onion_key, &new_curve25519_keypair, sizeof(curve25519_keypair_t)); -#endif now = time(NULL); state->LastRotatedOnionKey = onionkey_set_at = now; tor_mutex_release(key_lock); @@ -374,20 +366,40 @@ rotate_onion_key(void) if (prkey) crypto_pk_free(prkey); done: -#ifdef CURVE25519_ENABLED memwipe(&new_curve25519_keypair, 0, sizeof(new_curve25519_keypair)); -#endif tor_free(fname); tor_free(fname_prev); } +/** Log greeting message that points to new relay lifecycle document the + * first time this function has been called. + */ +static void +log_new_relay_greeting(void) +{ + static int already_logged = 0; + + if (already_logged) + return; + + tor_log(LOG_NOTICE, LD_GENERAL, "You are running a new relay. " + "Thanks for helping the Tor network! If you wish to know " + "what will happen in the upcoming weeks regarding its usage, " + "have a look at https://blog.torproject.org/blog/lifecycle-of" + "-a-new-relay"); + + already_logged = 1; +} + /** Try to read an RSA key from <b>fname</b>. If <b>fname</b> doesn't exist * and <b>generate</b> is true, create a new RSA key and save it in * <b>fname</b>. Return the read/created key, or NULL on error. Log all - * errors at level <b>severity</b>. + * errors at level <b>severity</b>. If <b>log_greeting</b> is non-zero and a + * new key was created, log_new_relay_greeting() is called. */ crypto_pk_t * -init_key_from_file(const char *fname, int generate, int severity) +init_key_from_file(const char *fname, int generate, int severity, + int log_greeting) { crypto_pk_t *prkey = NULL; @@ -401,7 +413,11 @@ init_key_from_file(const char *fname, int generate, int severity) case FN_ERROR: tor_log(severity, LD_FS,"Can't read key from \"%s\"", fname); goto error; + /* treat empty key files as if the file doesn't exist, and, + * if generate is set, replace the empty file in + * crypto_pk_write_private_key_to_filename() */ case FN_NOENT: + case FN_EMPTY: if (generate) { if (!have_lockfile()) { if (try_locking(get_options(), 0)<0) { @@ -425,6 +441,9 @@ init_key_from_file(const char *fname, int generate, int severity) goto error; } log_info(LD_GENERAL, "Generated key seems valid"); + if (log_greeting) { + log_new_relay_greeting(); + } if (crypto_pk_write_private_key_to_filename(prkey, fname)) { tor_log(severity, LD_FS, "Couldn't write generated key to \"%s\".", fname); @@ -450,12 +469,11 @@ init_key_from_file(const char *fname, int generate, int severity) return NULL; } -#ifdef CURVE25519_ENABLED /** Load a curve25519 keypair from the file <b>fname</b>, writing it into - * <b>keys_out</b>. If the file isn't found and <b>generate</b> is true, - * create a new keypair and write it into the file. If there are errors, log - * them at level <b>severity</b>. Generate files using <b>tag</b> in their - * ASCII wrapper. */ + * <b>keys_out</b>. If the file isn't found, or is empty, and <b>generate</b> + * is true, create a new keypair and write it into the file. If there are + * errors, log them at level <b>severity</b>. Generate files using <b>tag</b> + * in their ASCII wrapper. */ static int init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out, const char *fname, @@ -468,7 +486,10 @@ init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out, case FN_ERROR: tor_log(severity, LD_FS,"Can't read key from \"%s\"", fname); goto error; + /* treat empty key files as if the file doesn't exist, and, if generate + * is set, replace the empty file in curve25519_keypair_write_to_file() */ case FN_NOENT: + case FN_EMPTY: if (generate) { if (!have_lockfile()) { if (try_locking(get_options(), 0)<0) { @@ -488,7 +509,7 @@ init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out, if (curve25519_keypair_write_to_file(keys_out, fname, tag)<0) { tor_log(severity, LD_FS, "Couldn't write generated key to \"%s\".", fname); - memset(keys_out, 0, sizeof(*keys_out)); + memwipe(keys_out, 0, sizeof(*keys_out)); goto error; } } else { @@ -519,7 +540,6 @@ init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out, error: return -1; } -#endif /** Try to load the vote-signing private key and certificate for being a v3 * directory authority, and make sure they match. If <b>legacy</b>, load a @@ -538,7 +558,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out, fname = get_datadir_fname2("keys", legacy ? "legacy_signing_key" : "authority_signing_key"); - signing_key = init_key_from_file(fname, 0, LOG_INFO); + signing_key = init_key_from_file(fname, 0, LOG_INFO, 0); if (!signing_key) { log_warn(LD_DIR, "No version 3 directory key found in %s", fname); goto done; @@ -821,7 +841,7 @@ init_keys(void) /* 1b. Read identity key. Make it if none is found. */ keydir = get_datadir_fname2("keys", "secret_id_key"); log_info(LD_GENERAL,"Reading/making identity key \"%s\"...",keydir); - prkey = init_key_from_file(keydir, 1, LOG_ERR); + prkey = init_key_from_file(keydir, 1, LOG_ERR, 1); tor_free(keydir); if (!prkey) return -1; set_server_identity_key(prkey); @@ -844,7 +864,7 @@ init_keys(void) /* 2. Read onion key. Make it if none is found. */ keydir = get_datadir_fname2("keys", "secret_onion_key"); log_info(LD_GENERAL,"Reading/making onion key \"%s\"...",keydir); - prkey = init_key_from_file(keydir, 1, LOG_ERR); + prkey = init_key_from_file(keydir, 1, LOG_ERR, 1); tor_free(keydir); if (!prkey) return -1; set_onion_key(prkey); @@ -869,13 +889,14 @@ init_keys(void) keydir = get_datadir_fname2("keys", "secret_onion_key.old"); if (!lastonionkey && file_status(keydir) == FN_FILE) { - prkey = init_key_from_file(keydir, 1, LOG_ERR); /* XXXX Why 1? */ + /* Load keys from non-empty files only. + * Missing old keys won't be replaced with freshly generated keys. */ + prkey = init_key_from_file(keydir, 0, LOG_ERR, 0); if (prkey) lastonionkey = prkey; } tor_free(keydir); -#ifdef CURVE25519_ENABLED { /* 2b. Load curve25519 onion keys. */ int r; @@ -891,12 +912,13 @@ init_keys(void) last_curve25519_onion_key.pubkey.public_key, CURVE25519_PUBKEY_LEN) && file_status(keydir) == FN_FILE) { + /* Load keys from non-empty files only. + * Missing old keys won't be replaced with freshly generated keys. */ init_curve25519_keypair_from_file(&last_curve25519_onion_key, keydir, 0, LOG_ERR, "onion"); } tor_free(keydir); } -#endif /* 3. Initialize link key and TLS context. */ if (router_initialize_tls_context() < 0) { @@ -911,14 +933,13 @@ init_keys(void) const char *m = NULL; routerinfo_t *ri; /* We need to add our own fingerprint so it gets recognized. */ - if (dirserv_add_own_fingerprint(options->Nickname, - get_server_identity_key())) { - log_err(LD_GENERAL,"Error adding own fingerprint to approved set"); + if (dirserv_add_own_fingerprint(get_server_identity_key())) { + log_err(LD_GENERAL,"Error adding own fingerprint to set of relays"); return -1; } if (mydesc) { was_router_added_t added; - ri = router_parse_entry_from_string(mydesc, NULL, 1, 0, NULL); + ri = router_parse_entry_from_string(mydesc, NULL, 1, 0, NULL, NULL); if (!ri) { log_err(LD_GENERAL,"Generated a routerinfo we couldn't parse."); return -1; @@ -1081,6 +1102,7 @@ decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port) * they're confused or to get statistics. */ int interval_length = accounting_get_interval_length(); uint32_t effective_bw = get_effective_bwrate(options); + uint64_t acc_bytes; if (!interval_length) { log_warn(LD_BUG, "An accounting interval is not allowed to be zero " "seconds long. Raising to 1."); @@ -1091,8 +1113,12 @@ decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port) "accounting interval length %d", effective_bw, U64_PRINTF_ARG(options->AccountingMax), interval_length); + + acc_bytes = options->AccountingMax; + if (get_options()->AccountingRule == ACCT_SUM) + acc_bytes /= 2; if (effective_bw >= - options->AccountingMax / interval_length) { + acc_bytes / interval_length) { new_choice = 0; reason = "AccountingMax enabled"; } @@ -1210,6 +1236,11 @@ router_orport_found_reachable(void) " Publishing server descriptor." : ""); can_reach_or_port = 1; mark_my_descriptor_dirty("ORPort found reachable"); + /* This is a significant enough change to upload immediately, + * at least in a test network */ + if (get_options()->TestingTorNetwork == 1) { + reschedule_descriptor_update_check(); + } control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d", address, me->or_port); @@ -1227,8 +1258,14 @@ router_dirport_found_reachable(void) log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable " "from the outside. Excellent."); can_reach_dir_port = 1; - if (decide_to_advertise_dirport(get_options(), me->dir_port)) + if (decide_to_advertise_dirport(get_options(), me->dir_port)) { mark_my_descriptor_dirty("DirPort found reachable"); + /* This is a significant enough change to upload immediately, + * at least in a test network */ + if (get_options()->TestingTorNetwork == 1) { + reschedule_descriptor_update_check(); + } + } control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d", address, me->dir_port); @@ -1802,19 +1839,17 @@ router_rebuild_descriptor(int force) ri->cache_info.published_on = time(NULL); ri->onion_pkey = crypto_pk_dup_key(get_onion_key()); /* must invoke from * main thread */ -#ifdef CURVE25519_ENABLED ri->onion_curve25519_pkey = tor_memdup(&get_current_curve25519_keypair()->pubkey, sizeof(curve25519_public_key_t)); -#endif /* For now, at most one IPv6 or-address is being advertised. */ { const port_cfg_t *ipv6_orport = NULL; SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) { if (p->type == CONN_TYPE_OR_LISTENER && - ! p->no_advertise && - ! p->bind_ipv4_only && + ! p->server_cfg.no_advertise && + ! p->server_cfg.bind_ipv4_only && tor_addr_family(&p->addr) == AF_INET6) { if (! tor_addr_is_internal(&p->addr, 0)) { ipv6_orport = p; @@ -1856,10 +1891,8 @@ router_rebuild_descriptor(int force) /* DNS is screwed up; don't claim to be an exit. */ policies_exit_policy_append_reject_star(&ri->exit_policy); } else { - policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, - options->IPv6Exit, - options->ExitPolicyRejectPrivate, - ri->addr, !options->BridgeRelay); + policies_parse_exit_policy_from_options(options,ri->addr, + &ri->exit_policy); } ri->policy_is_reject_star = policy_is_reject_star(ri->exit_policy, AF_INET) && @@ -1879,7 +1912,7 @@ router_rebuild_descriptor(int force) family = smartlist_new(); ri->declared_family = smartlist_new(); smartlist_split_string(family, options->MyFamily, ",", - SPLIT_SKIP_SPACE|SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0); SMARTLIST_FOREACH_BEGIN(family, char *, name) { const node_t *member; if (!strcasecmp(name, options->Nickname)) @@ -2063,7 +2096,8 @@ mark_my_descriptor_dirty(const char *reason) } /** How frequently will we republish our descriptor because of large (factor - * of 2) shifts in estimated bandwidth? */ + * of 2) shifts in estimated bandwidth? Note: We don't use this constant + * if our previous bandwidth estimate was exactly 0. */ #define MAX_BANDWIDTH_CHANGE_FREQ (20*60) /** Check whether bandwidth has changed a lot since the last time we announced @@ -2081,7 +2115,7 @@ check_descriptor_bandwidth_changed(time_t now) if ((prev != cur && (!prev || !cur)) || cur > prev*2 || cur < prev/2) { - if (last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now) { + if (last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now || !prev) { log_info(LD_GENERAL, "Measured bandwidth has changed; rebuilding descriptor."); mark_my_descriptor_dirty("bandwidth has changed"); @@ -2371,7 +2405,8 @@ router_dump_router_to_string(routerinfo_t *router, has_extra_info_digest ? "extra-info-digest " : "", has_extra_info_digest ? extra_info_digest : "", has_extra_info_digest ? "\n" : "", - options->DownloadExtraInfo ? "caches-extra-info\n" : "", + (options->DownloadExtraInfo || options->V3AuthoritativeDir) ? + "caches-extra-info\n" : "", onion_pkey, identity_pkey, family_line, we_are_hibernating() ? "hibernating 1\n" : "", @@ -2385,7 +2420,6 @@ router_dump_router_to_string(routerinfo_t *router, smartlist_add_asprintf(chunks, "contact %s\n", ci); } -#ifdef CURVE25519_ENABLED if (router->onion_curve25519_pkey) { char kbuf[128]; base64_encode(kbuf, sizeof(kbuf), @@ -2393,7 +2427,6 @@ router_dump_router_to_string(routerinfo_t *router, CURVE25519_PUBKEY_LEN); smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf); } -#endif /* Write the exit policy to the end of 's'. */ if (!router->exit_policy || !smartlist_len(router->exit_policy)) { @@ -2443,7 +2476,7 @@ router_dump_router_to_string(routerinfo_t *router, const char *cp; routerinfo_t *ri_tmp; cp = s_dup = tor_strdup(output); - ri_tmp = router_parse_entry_from_string(cp, NULL, 1, 0, NULL); + ri_tmp = router_parse_entry_from_string(cp, NULL, 1, 0, NULL, NULL); if (!ri_tmp) { log_err(LD_BUG, "We just generated a router descriptor we can't parse."); @@ -2557,8 +2590,9 @@ router_has_orport(const routerinfo_t *router, const tor_addr_port_t *orport) * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in * the past or more than 1 hour in the future with respect to <b>now</b>, * and write the file contents starting with that line to *<b>out</b>. - * Return 1 for success, 0 if the file does not exist, or -1 if the file - * does not contain a line matching these criteria or other failure. */ + * Return 1 for success, 0 if the file does not exist or is empty, or -1 + * if the file does not contain a line matching these criteria or other + * failure. */ static int load_stats_file(const char *filename, const char *end_line, time_t now, char **out) @@ -2592,7 +2626,9 @@ load_stats_file(const char *filename, const char *end_line, time_t now, notfound: tor_free(contents); break; + /* treat empty stats files as if the file doesn't exist */ case FN_NOENT: + case FN_EMPTY: r = 0; break; case FN_ERROR: @@ -2649,6 +2685,11 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, "dirreq-stats-end", now, &contents) > 0) { smartlist_add(chunks, contents); } + if (options->HiddenServiceStatistics && + load_stats_file("stats"PATH_SEPARATOR"hidserv-stats", + "hidserv-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } if (options->EntryStatistics && load_stats_file("stats"PATH_SEPARATOR"entry-stats", "entry-stats-end", now, &contents) > 0) { @@ -2725,7 +2766,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, s = smartlist_join_strings(chunks, "", 0, NULL); cp = s_dup = tor_strdup(s); - ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL); + ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL, NULL); if (!ei_tmp) { if (write_stats_to_extrainfo) { log_warn(LD_GENERAL, "We just generated an extra-info descriptor " @@ -3069,10 +3110,8 @@ router_free_all(void) crypto_pk_free(legacy_signing_key); authority_cert_free(legacy_key_certificate); -#ifdef CURVE25519_ENABLED memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key)); memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); -#endif if (warned_nonexistent_family) { SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); diff --git a/src/or/router.h b/src/or/router.h index d18ff065ea..8108ffb22f 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,13 +29,11 @@ crypto_pk_t *get_my_v3_legacy_signing_key(void); void dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last); void rotate_onion_key(void); crypto_pk_t *init_key_from_file(const char *fname, int generate, - int severity); + int severity, int log_greeting); void v3_authority_check_key_expiry(void); -#ifdef CURVE25519_ENABLED di_digest256_map_t *construct_ntor_key_map(void); void ntor_key_map_free(di_digest256_map_t *map); -#endif int router_initialize_tls_context(void); int init_keys(void); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 07e87724ba..af8e68e880 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -65,7 +65,7 @@ static int compute_weighted_bandwidths(const smartlist_t *sl, bandwidth_weight_rule_t rule, u64_dbl_t **bandwidths_out); static const routerstatus_t *router_pick_directory_server_impl( - dirinfo_type_t auth, int flags); + dirinfo_type_t auth, int flags, int *n_busy_out); static const routerstatus_t *router_pick_trusteddirserver_impl( const smartlist_t *sourcelist, dirinfo_type_t auth, int flags, int *n_busy_out); @@ -79,6 +79,7 @@ static const char *signed_descriptor_get_body_impl( const signed_descriptor_t *desc, int with_annotations); static void list_pending_downloads(digestmap_t *result, + digest256map_t *result256, int purpose, const char *prefix); static void list_pending_fpsk_downloads(fp_pair_map_t *result); static void launch_dummy_descriptor_download_as_needed(time_t now, @@ -448,46 +449,69 @@ trusted_dirs_flush_certs_to_disk(void) trusted_dir_servers_certs_changed = 0; } -/** Remove all v3 authority certificates that have been superseded for more - * than 48 hours. (If the most recent cert was published more than 48 hours - * ago, then we aren't going to get any consensuses signed with older +static int +compare_certs_by_pubdates(const void **_a, const void **_b) +{ + const authority_cert_t *cert1 = *_a, *cert2=*_b; + + if (cert1->cache_info.published_on < cert2->cache_info.published_on) + return -1; + else if (cert1->cache_info.published_on > cert2->cache_info.published_on) + return 1; + else + return 0; +} + +/** Remove all expired v3 authority certificates that have been superseded for + * more than 48 hours or, if not expired, that were published more than 7 days + * before being superseded. (If the most recent cert was published more than 48 + * hours ago, then we aren't going to get any consensuses signed with older * keys.) */ static void trusted_dirs_remove_old_certs(void) { time_t now = time(NULL); #define DEAD_CERT_LIFETIME (2*24*60*60) -#define OLD_CERT_LIFETIME (7*24*60*60) +#define SUPERSEDED_CERT_LIFETIME (2*24*60*60) if (!trusted_dir_certs) return; DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) { - authority_cert_t *newest = NULL; - SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert, - if (!newest || (cert->cache_info.published_on > - newest->cache_info.published_on)) - newest = cert); - if (newest) { - const time_t newest_published = newest->cache_info.published_on; - SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) { - int expired; - time_t cert_published; - if (newest == cert) - continue; - expired = now > cert->expires; - cert_published = cert->cache_info.published_on; - /* Store expired certs for 48 hours after a newer arrives; + /* Sort the list from first-published to last-published */ + smartlist_sort(cl->certs, compare_certs_by_pubdates); + + SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) { + if (cert_sl_idx == smartlist_len(cl->certs) - 1) { + /* This is the most recently published cert. Keep it. */ + continue; + } + authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1); + const time_t next_cert_published = next_cert->cache_info.published_on; + if (next_cert_published > now) { + /* All later certs are published in the future. Keep everything + * we didn't discard. */ + break; + } + int should_remove = 0; + if (cert->expires + DEAD_CERT_LIFETIME < now) { + /* Certificate has been expired for at least DEAD_CERT_LIFETIME. + * Remove it. */ + should_remove = 1; + } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) { + /* Certificate has been superseded for OLD_CERT_LIFETIME. + * Remove it. */ - if (expired ? - (newest_published + DEAD_CERT_LIFETIME < now) : - (cert_published + OLD_CERT_LIFETIME < newest_published)) { - SMARTLIST_DEL_CURRENT(cl->certs, cert); - authority_cert_free(cert); - trusted_dir_servers_certs_changed = 1; - } - } SMARTLIST_FOREACH_END(cert); - } + should_remove = 1; + } + if (should_remove) { + SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert); + authority_cert_free(cert); + trusted_dir_servers_certs_changed = 1; + } + } SMARTLIST_FOREACH_END(cert); + } DIGESTMAP_FOREACH_END; +#undef DEAD_CERT_LIFETIME #undef OLD_CERT_LIFETIME trusted_dirs_flush_certs_to_disk(); @@ -713,7 +737,8 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) * First, we get the lists of already pending downloads so we don't * duplicate effort. */ - list_pending_downloads(pending_id, DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); + list_pending_downloads(pending_id, NULL, + DIR_PURPOSE_FETCH_CERTIFICATE, "fp/"); list_pending_fpsk_downloads(pending_cert); /* @@ -1200,6 +1225,7 @@ router_reload_router_list_impl(desc_store_t *store) tor_free(fname); fname = get_datadir_fname_suffix(store->fname_base, ".new"); + /* don't load empty files - we wouldn't get any data, even if we tried */ if (file_status(fname) == FN_FILE) contents = read_file_to_str(fname, RFTS_BIN|RFTS_IGNORE_MISSING, &st); if (contents) { @@ -1281,22 +1307,32 @@ router_get_fallback_dir_servers(void) const routerstatus_t * router_pick_directory_server(dirinfo_type_t type, int flags) { + int busy = 0; const routerstatus_t *choice; if (!routerlist) return NULL; - choice = router_pick_directory_server_impl(type, flags); + choice = router_pick_directory_server_impl(type, flags, &busy); if (choice || !(flags & PDS_RETRY_IF_NO_SERVERS)) return choice; + if (busy) { + /* If the reason that we got no server is that servers are "busy", + * we must be excluding good servers because we already have serverdesc + * fetches with them. Do not mark down servers up because of this. */ + tor_assert((flags & (PDS_NO_EXISTING_SERVERDESC_FETCH| + PDS_NO_EXISTING_MICRODESC_FETCH))); + return NULL; + } + log_info(LD_DIR, "No reachable router entries for dirservers. " "Trying them all again."); /* mark all authdirservers as up again */ mark_all_dirservers_up(fallback_dir_servers); /* try again */ - choice = router_pick_directory_server_impl(type, flags); + choice = router_pick_directory_server_impl(type, flags, NULL); return choice; } @@ -1406,11 +1442,15 @@ router_pick_dirserver_generic(smartlist_t *sourcelist, #define DIR_503_TIMEOUT (60*60) /** Pick a random running valid directory server/mirror from our - * routerlist. Arguments are as for router_pick_directory_server(), except - * that RETRY_IF_NO_SERVERS is ignored. + * routerlist. Arguments are as for router_pick_directory_server(), except: + * + * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of + * directories that we excluded for no other reason than + * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH. */ static const routerstatus_t * -router_pick_directory_server_impl(dirinfo_type_t type, int flags) +router_pick_directory_server_impl(dirinfo_type_t type, int flags, + int *n_busy_out) { const or_options_t *options = get_options(); const node_t *result; @@ -1419,10 +1459,12 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) smartlist_t *overloaded_direct, *overloaded_tunnel; time_t now = time(NULL); const networkstatus_t *consensus = networkstatus_get_latest_consensus(); - int requireother = ! (flags & PDS_ALLOW_SELF); - int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - int for_guard = (flags & PDS_FOR_GUARD); - int try_excluding = 1, n_excluded = 0; + const int requireother = ! (flags & PDS_ALLOW_SELF); + const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); + const int no_serverdesc_fetching =(flags & PDS_NO_EXISTING_SERVERDESC_FETCH); + const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH); + const int for_guard = (flags & PDS_FOR_GUARD); + int try_excluding = 1, n_excluded = 0, n_busy = 0; if (!consensus) return NULL; @@ -1438,7 +1480,7 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) /* Find all the running dirservers we know about. */ SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) { - int is_trusted; + int is_trusted, is_trusted_extrainfo; int is_overloaded; tor_addr_t addr; const routerstatus_t *status = node->rs; @@ -1448,13 +1490,13 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) if (!node->is_running || !status->dir_port || !node->is_valid) continue; - if (node->is_bad_directory) - continue; if (requireother && router_digest_is_me(node->identity)) continue; is_trusted = router_digest_is_trusted_dir(node->identity); + is_trusted_extrainfo = router_digest_is_trusted_dir_type( + node->identity, EXTRAINFO_DIRINFO); if ((type & EXTRAINFO_DIRINFO) && - !router_supports_extrainfo(node->identity, 0)) + !router_supports_extrainfo(node->identity, is_trusted_extrainfo)) continue; if ((type & MICRODESC_DIRINFO) && !is_trusted && !node->rs->version_supports_microdesc_cache) @@ -1475,7 +1517,24 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) } /* XXXX IP6 proposal 118 */ - tor_addr_from_ipv4h(&addr, node->rs->addr); + tor_addr_from_ipv4h(&addr, status->addr); + + if (no_serverdesc_fetching && ( + connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &addr, status->dir_port, DIR_PURPOSE_FETCH_SERVERDESC) + || connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &addr, status->dir_port, DIR_PURPOSE_FETCH_EXTRAINFO) + )) { + ++n_busy; + continue; + } + + if (no_microdesc_fetching && connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &addr, status->dir_port, DIR_PURPOSE_FETCH_MICRODESC) + ) { + ++n_busy; + continue; + } is_overloaded = status->last_dir_503_at + DIR_503_TIMEOUT > now; @@ -1515,14 +1574,19 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags) smartlist_free(overloaded_direct); smartlist_free(overloaded_tunnel); - if (result == NULL && try_excluding && !options->StrictNodes && n_excluded) { + if (result == NULL && try_excluding && !options->StrictNodes && n_excluded + && !n_busy) { /* If we got no result, and we are excluding nodes, and StrictNodes is * not set, try again without excluding nodes. */ try_excluding = 0; n_excluded = 0; + n_busy = 0; goto retry_without_exclude; } + if (n_busy_out) + *n_busy_out = n_busy; + return result ? result->rs : NULL; } @@ -1536,7 +1600,7 @@ dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight) u64_dbl_t *weights; const dir_server_t *ds; - weights = tor_malloc(sizeof(u64_dbl_t) * n); + weights = tor_calloc(n, sizeof(u64_dbl_t)); for (i = 0; i < n; ++i) { ds = smartlist_get(servers, i); weights[i].dbl = ds->weight; @@ -1736,7 +1800,7 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router) /** Add every suitable node from our nodelist to <b>sl</b>, so that * we can pick a node for a circuit. */ -static void +void router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid, int need_uptime, int need_capacity, int need_guard, int need_desc) @@ -1808,15 +1872,16 @@ scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, uint64_t *total_out) { double total = 0.0; - double scale_factor; + double scale_factor = 0.0; int i; /* big, but far away from overflowing an int64_t */ -#define SCALE_TO_U64_MAX (INT64_MAX / 4) +#define SCALE_TO_U64_MAX ((int64_t) (INT64_MAX / 4)) for (i = 0; i < n_entries; ++i) total += entries[i].dbl; - scale_factor = SCALE_TO_U64_MAX / total; + if (total > 0.0) + scale_factor = SCALE_TO_U64_MAX / total; for (i = 0; i < n_entries; ++i) entries[i].u64 = tor_llround(entries[i].dbl * scale_factor); @@ -1963,6 +2028,7 @@ compute_weighted_bandwidths(const smartlist_t *sl, double Wg = -1, Wm = -1, We = -1, Wd = -1; double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; uint64_t weighted_bw = 0; + guardfraction_bandwidth_t guardfraction_bw; u64_dbl_t *bandwidths; /* Can't choose exit and guard at same time */ @@ -2029,9 +2095,10 @@ compute_weighted_bandwidths(const smartlist_t *sl, if (Wg < 0 || Wm < 0 || We < 0 || Wd < 0 || Wgb < 0 || Wmb < 0 || Wdb < 0 || Web < 0) { log_debug(LD_CIRC, - "Got negative bandwidth weights. Defaulting to old selection" + "Got negative bandwidth weights. Defaulting to naive selection" " algorithm."); - return -1; // Use old algorithm. + Wg = Wm = We = Wd = weight_scale; + Wgb = Wmb = Web = Wdb = weight_scale; } Wg /= weight_scale; @@ -2044,26 +2111,32 @@ compute_weighted_bandwidths(const smartlist_t *sl, Web /= weight_scale; Wdb /= weight_scale; - bandwidths = tor_malloc_zero(sizeof(u64_dbl_t)*smartlist_len(sl)); + bandwidths = tor_calloc(smartlist_len(sl), sizeof(u64_dbl_t)); // Cycle through smartlist and total the bandwidth. + static int warned_missing_bw = 0; SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { int is_exit = 0, is_guard = 0, is_dir = 0, this_bw = 0; double weight = 1; + double weight_without_guard_flag = 0; /* Used for guardfraction */ + double final_weight = 0; is_exit = node->is_exit && ! node->is_bad_exit; is_guard = node->is_possible_guard; is_dir = node_is_dir(node); if (node->rs) { if (!node->rs->has_bandwidth) { - tor_free(bandwidths); /* This should never happen, unless all the authorites downgrade * to 0.2.0 or rogue routerstatuses get inserted into our consensus. */ - log_warn(LD_BUG, - "Consensus is not listing bandwidths. Defaulting back to " - "old router selection algorithm."); - return -1; + if (! warned_missing_bw) { + log_warn(LD_BUG, + "Consensus is missing some bandwidths. Using a naive " + "router selection algorithm"); + warned_missing_bw = 1; + } + this_bw = 30000; /* Chosen arbitrarily */ + } else { + this_bw = kb_to_bytes(node->rs->bandwidth_kb); } - this_bw = kb_to_bytes(node->rs->bandwidth_kb); } else if (node->ri) { /* bridge or other descriptor not in our consensus */ this_bw = bridge_get_advertised_bandwidth_bounded(node->ri); @@ -2074,8 +2147,10 @@ compute_weighted_bandwidths(const smartlist_t *sl, if (is_guard && is_exit) { weight = (is_dir ? Wdb*Wd : Wd); + weight_without_guard_flag = (is_dir ? Web*We : We); } else if (is_guard) { weight = (is_dir ? Wgb*Wg : Wg); + weight_without_guard_flag = (is_dir ? Wmb*Wm : Wm); } else if (is_exit) { weight = (is_dir ? Web*We : We); } else { // middle @@ -2087,8 +2162,43 @@ compute_weighted_bandwidths(const smartlist_t *sl, this_bw = 0; if (weight < 0.0) weight = 0.0; + if (weight_without_guard_flag < 0.0) + weight_without_guard_flag = 0.0; + + /* If guardfraction information is available in the consensus, we + * want to calculate this router's bandwidth according to its + * guardfraction. Quoting from proposal236: + * + * Let Wpf denote the weight from the 'bandwidth-weights' line a + * client would apply to N for position p if it had the guard + * flag, Wpn the weight if it did not have the guard flag, and B the + * measured bandwidth of N in the consensus. Then instead of choosing + * N for position p proportionally to Wpf*B or Wpn*B, clients should + * choose N proportionally to F*Wpf*B + (1-F)*Wpn*B. + */ + if (node->rs && node->rs->has_guardfraction && rule != WEIGHT_FOR_GUARD) { + /* XXX The assert should actually check for is_guard. However, + * that crashes dirauths because of #13297. This should be + * equivalent: */ + tor_assert(node->rs->is_possible_guard); + + guard_get_guardfraction_bandwidth(&guardfraction_bw, + this_bw, + node->rs->guardfraction_percentage); + + /* Calculate final_weight = F*Wpf*B + (1-F)*Wpn*B */ + final_weight = + guardfraction_bw.guard_bw * weight + + guardfraction_bw.non_guard_bw * weight_without_guard_flag; + + log_debug(LD_GENERAL, "%s: Guardfraction weight %f instead of %f (%s)", + node->rs->nickname, final_weight, weight*this_bw, + bandwidth_weight_rule_to_string(rule)); + } else { /* no guardfraction information. calculate the weight normally. */ + final_weight = weight*this_bw; + } - bandwidths[node_sl_idx].dbl = weight*this_bw + 0.5; + bandwidths[node_sl_idx].dbl = final_weight + 0.5; } SMARTLIST_FOREACH_END(node); log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based " @@ -2140,226 +2250,13 @@ frac_nodes_with_descriptors(const smartlist_t *sl, return present / total; } -/** Helper function: - * choose a random node_t element of smartlist <b>sl</b>, weighted by - * the advertised bandwidth of each element. - * - * If <b>rule</b>==WEIGHT_FOR_EXIT. we're picking an exit node: consider all - * nodes' bandwidth equally regardless of their Exit status, since there may - * be some in the list because they exit to obscure ports. If - * <b>rule</b>==NO_WEIGHTING, we're picking a non-exit node: weight - * exit-node's bandwidth less depending on the smallness of the fraction of - * Exit-to-total bandwidth. If <b>rule</b>==WEIGHT_FOR_GUARD, we're picking a - * guard node: consider all guard's bandwidth equally. Otherwise, weight - * guards proportionally less. - */ -static const node_t * -smartlist_choose_node_by_bandwidth(const smartlist_t *sl, - bandwidth_weight_rule_t rule) -{ - unsigned int i; - u64_dbl_t *bandwidths; - int is_exit; - int is_guard; - int is_fast; - double total_nonexit_bw = 0, total_exit_bw = 0; - double total_nonguard_bw = 0, total_guard_bw = 0; - double exit_weight; - double guard_weight; - int n_unknown = 0; - bitarray_t *fast_bits; - bitarray_t *exit_bits; - bitarray_t *guard_bits; - - // This function does not support WEIGHT_FOR_DIR - // or WEIGHT_FOR_MID - if (rule == WEIGHT_FOR_DIR || rule == WEIGHT_FOR_MID) { - rule = NO_WEIGHTING; - } - - /* Can't choose exit and guard at same time */ - tor_assert(rule == NO_WEIGHTING || - rule == WEIGHT_FOR_EXIT || - rule == WEIGHT_FOR_GUARD); - - if (smartlist_len(sl) == 0) { - log_info(LD_CIRC, - "Empty routerlist passed in to old node selection for rule %s", - bandwidth_weight_rule_to_string(rule)); - return NULL; - } - - /* First count the total bandwidth weight, and make a list - * of each value. We use UINT64_MAX to indicate "unknown". */ - bandwidths = tor_malloc_zero(sizeof(u64_dbl_t)*smartlist_len(sl)); - fast_bits = bitarray_init_zero(smartlist_len(sl)); - exit_bits = bitarray_init_zero(smartlist_len(sl)); - guard_bits = bitarray_init_zero(smartlist_len(sl)); - - /* Iterate over all the routerinfo_t or routerstatus_t, and */ - SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { - /* first, learn what bandwidth we think i has */ - int is_known = 1; - uint32_t this_bw = 0; - i = node_sl_idx; - - is_exit = node->is_exit; - is_guard = node->is_possible_guard; - if (node->rs) { - if (node->rs->has_bandwidth) { - this_bw = kb_to_bytes(node->rs->bandwidth_kb); - } else { /* guess */ - is_known = 0; - } - } else if (node->ri) { - /* Must be a bridge if we're willing to use it */ - this_bw = bridge_get_advertised_bandwidth_bounded(node->ri); - } - - if (is_exit) - bitarray_set(exit_bits, i); - if (is_guard) - bitarray_set(guard_bits, i); - if (node->is_fast) - bitarray_set(fast_bits, i); - - if (is_known) { - bandwidths[i].dbl = this_bw; - if (is_guard) - total_guard_bw += this_bw; - else - total_nonguard_bw += this_bw; - if (is_exit) - total_exit_bw += this_bw; - else - total_nonexit_bw += this_bw; - } else { - ++n_unknown; - bandwidths[i].dbl = -1.0; - } - } SMARTLIST_FOREACH_END(node); - -#define EPSILON .1 - - /* Now, fill in the unknown values. */ - if (n_unknown) { - int32_t avg_fast, avg_slow; - if (total_exit_bw+total_nonexit_bw < EPSILON) { - /* if there's some bandwidth, there's at least one known router, - * so no worries about div by 0 here */ - int n_known = smartlist_len(sl)-n_unknown; - avg_fast = avg_slow = (int32_t) - ((total_exit_bw+total_nonexit_bw)/((uint64_t) n_known)); - } else { - avg_fast = 40000; - avg_slow = 20000; - } - for (i=0; i<(unsigned)smartlist_len(sl); ++i) { - if (bandwidths[i].dbl >= 0.0) - continue; - is_fast = bitarray_is_set(fast_bits, i); - is_exit = bitarray_is_set(exit_bits, i); - is_guard = bitarray_is_set(guard_bits, i); - bandwidths[i].dbl = is_fast ? avg_fast : avg_slow; - if (is_exit) - total_exit_bw += bandwidths[i].dbl; - else - total_nonexit_bw += bandwidths[i].dbl; - if (is_guard) - total_guard_bw += bandwidths[i].dbl; - else - total_nonguard_bw += bandwidths[i].dbl; - } - } - - /* If there's no bandwidth at all, pick at random. */ - if (total_exit_bw+total_nonexit_bw < EPSILON) { - tor_free(bandwidths); - tor_free(fast_bits); - tor_free(exit_bits); - tor_free(guard_bits); - return smartlist_choose(sl); - } - - /* Figure out how to weight exits and guards */ - { - double all_bw = U64_TO_DBL(total_exit_bw+total_nonexit_bw); - double exit_bw = U64_TO_DBL(total_exit_bw); - double guard_bw = U64_TO_DBL(total_guard_bw); - /* - * For detailed derivation of this formula, see - * http://archives.seul.org/or/dev/Jul-2007/msg00056.html - */ - if (rule == WEIGHT_FOR_EXIT || total_exit_bw<EPSILON) - exit_weight = 1.0; - else - exit_weight = 1.0 - all_bw/(3.0*exit_bw); - - if (rule == WEIGHT_FOR_GUARD || total_guard_bw<EPSILON) - guard_weight = 1.0; - else - guard_weight = 1.0 - all_bw/(3.0*guard_bw); - - if (exit_weight <= 0.0) - exit_weight = 0.0; - - if (guard_weight <= 0.0) - guard_weight = 0.0; - - for (i=0; i < (unsigned)smartlist_len(sl); i++) { - tor_assert(bandwidths[i].dbl >= 0.0); - - is_exit = bitarray_is_set(exit_bits, i); - is_guard = bitarray_is_set(guard_bits, i); - if (is_exit && is_guard) - bandwidths[i].dbl *= exit_weight * guard_weight; - else if (is_guard) - bandwidths[i].dbl *= guard_weight; - else if (is_exit) - bandwidths[i].dbl *= exit_weight; - } - } - -#if 0 - log_debug(LD_CIRC, "Total weighted bw = "U64_FORMAT - ", exit bw = "U64_FORMAT - ", nonexit bw = "U64_FORMAT", exit weight = %f " - "(for exit == %d)" - ", guard bw = "U64_FORMAT - ", nonguard bw = "U64_FORMAT", guard weight = %f " - "(for guard == %d)", - U64_PRINTF_ARG(total_bw), - U64_PRINTF_ARG(total_exit_bw), U64_PRINTF_ARG(total_nonexit_bw), - exit_weight, (int)(rule == WEIGHT_FOR_EXIT), - U64_PRINTF_ARG(total_guard_bw), U64_PRINTF_ARG(total_nonguard_bw), - guard_weight, (int)(rule == WEIGHT_FOR_GUARD)); -#endif - - scale_array_elements_to_u64(bandwidths, smartlist_len(sl), NULL); - - { - int idx = choose_array_element_by_weight(bandwidths, - smartlist_len(sl)); - tor_free(bandwidths); - tor_free(fast_bits); - tor_free(exit_bits); - tor_free(guard_bits); - return idx < 0 ? NULL : smartlist_get(sl, idx); - } -} - /** Choose a random element of status list <b>sl</b>, weighted by * the advertised bandwidth of each node */ const node_t * node_sl_choose_by_bandwidth(const smartlist_t *sl, bandwidth_weight_rule_t rule) { /*XXXX MOVE */ - const node_t *ret; - if ((ret = smartlist_choose_node_by_bandwidth_weights(sl, rule))) { - return ret; - } else { - return smartlist_choose_node_by_bandwidth(sl, rule); - } + return smartlist_choose_node_by_bandwidth_weights(sl, rule); } /** Return a random running node from the nodelist. Never @@ -2417,11 +2314,29 @@ router_choose_random_node(smartlist_t *excludedsmartlist, router_add_running_nodes_to_smartlist(sl, allow_invalid, need_uptime, need_capacity, need_guard, need_desc); + log_debug(LD_CIRC, + "We found %d running nodes.", + smartlist_len(sl)); + smartlist_subtract(sl,excludednodes); - if (excludedsmartlist) + log_debug(LD_CIRC, + "We removed %d excludednodes, leaving %d nodes.", + smartlist_len(excludednodes), + smartlist_len(sl)); + + if (excludedsmartlist) { smartlist_subtract(sl,excludedsmartlist); - if (excludedset) + log_debug(LD_CIRC, + "We removed %d excludedsmartlist, leaving %d nodes.", + smartlist_len(excludedsmartlist), + smartlist_len(sl)); + } + if (excludedset) { routerset_subtract_nodes(sl,excludedset); + log_debug(LD_CIRC, + "We removed excludedset, leaving %d nodes.", + smartlist_len(sl)); + } // Always weight by bandwidth choice = node_sl_choose_by_bandwidth(sl, rule); @@ -2535,7 +2450,7 @@ router_is_named(const routerinfo_t *router) /** Return true iff <b>digest</b> is the digest of the identity key of a * trusted directory matching at least one bit of <b>type</b>. If <b>type</b> - * is zero, any authority is okay. */ + * is zero (NO_DIRINFO), or ALL_DIRINFO, any authority is okay. */ int router_digest_is_trusted_dir_type(const char *digest, dirinfo_type_t type) { @@ -2616,8 +2531,8 @@ router_get_by_descriptor_digest(const char *digest) /** Return the signed descriptor for the router in our routerlist whose * 20-byte extra-info digest is <b>digest</b>. Return NULL if no such router * is known. */ -signed_descriptor_t * -router_get_by_extrainfo_digest(const char *digest) +MOCK_IMPL(signed_descriptor_t *, +router_get_by_extrainfo_digest,(const char *digest)) { tor_assert(digest); @@ -2938,17 +2853,19 @@ routerlist_insert(routerlist_t *rl, routerinfo_t *ri) } /** Adds the extrainfo_t <b>ei</b> to the routerlist <b>rl</b>, if there is a - * corresponding router in rl-\>routers or rl-\>old_routers. Return true iff - * we actually inserted <b>ei</b>. Free <b>ei</b> if it isn't inserted. */ -static int -extrainfo_insert(routerlist_t *rl, extrainfo_t *ei) + * corresponding router in rl-\>routers or rl-\>old_routers. Return the status + * of inserting <b>ei</b>. Free <b>ei</b> if it isn't inserted. */ +MOCK_IMPL(STATIC was_router_added_t, +extrainfo_insert,(routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible)) { - int r = 0; + was_router_added_t r; + const char *compatibility_error_msg; routerinfo_t *ri = rimap_get(rl->identity_map, ei->cache_info.identity_digest); signed_descriptor_t *sd = sdmap_get(rl->desc_by_eid_map, ei->cache_info.signed_descriptor_digest); extrainfo_t *ei_tmp; + const int severity = warn_if_incompatible ? LOG_WARN : LOG_INFO; { extrainfo_t *ei_generated = router_get_my_extrainfo(); @@ -2957,9 +2874,41 @@ extrainfo_insert(routerlist_t *rl, extrainfo_t *ei) if (!ri) { /* This router is unknown; we can't even verify the signature. Give up.*/ + r = ROUTER_NOT_IN_CONSENSUS; + goto done; + } + if (! sd) { + /* The extrainfo router doesn't have a known routerdesc to attach it to. + * This just won't work. */; + static ratelim_t no_sd_ratelim = RATELIM_INIT(1800); + r = ROUTER_BAD_EI; + log_fn_ratelim(&no_sd_ratelim, severity, LD_BUG, + "No entry found in extrainfo map."); goto done; } - if (routerinfo_incompatible_with_extrainfo(ri, ei, sd, NULL)) { + if (tor_memneq(ei->cache_info.signed_descriptor_digest, + sd->extra_info_digest, DIGEST_LEN)) { + static ratelim_t digest_mismatch_ratelim = RATELIM_INIT(1800); + /* The sd we got from the map doesn't match the digest we used to look + * it up. This makes no sense. */ + r = ROUTER_BAD_EI; + log_fn_ratelim(&digest_mismatch_ratelim, severity, LD_BUG, + "Mismatch in digest in extrainfo map."); + goto done; + } + if (routerinfo_incompatible_with_extrainfo(ri, ei, sd, + &compatibility_error_msg)) { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + r = (ri->cache_info.extrainfo_is_bogus) ? + ROUTER_BAD_EI : ROUTER_NOT_IN_CONSENSUS; + + base16_encode(d1, sizeof(d1), ri->cache_info.identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), ei->cache_info.identity_digest, DIGEST_LEN); + + log_fn(severity,LD_DIR, + "router info incompatible with extra info (ri id: %s, ei id %s, " + "reason: %s)", d1, d2, compatibility_error_msg); + goto done; } @@ -2969,7 +2918,7 @@ extrainfo_insert(routerlist_t *rl, extrainfo_t *ei) ei_tmp = eimap_set(rl->extra_info_map, ei->cache_info.signed_descriptor_digest, ei); - r = 1; + r = ROUTER_ADDED_SUCCESSFULLY; if (ei_tmp) { rl->extrainfo_store.bytes_dropped += ei_tmp->cache_info.signed_descriptor_len; @@ -2977,7 +2926,7 @@ extrainfo_insert(routerlist_t *rl, extrainfo_t *ei) } done: - if (r == 0) + if (r != ROUTER_ADDED_SUCCESSFULLY) extrainfo_free(ei); #ifdef DEBUG_ROUTERLIST @@ -3252,7 +3201,7 @@ routerlist_reparse_old(routerlist_t *rl, signed_descriptor_t *sd) ri = router_parse_entry_from_string(body, body+sd->signed_descriptor_len+sd->annotations_len, - 0, 1, NULL); + 0, 1, NULL, NULL); if (!ri) return NULL; memcpy(&ri->cache_info, sd, sizeof(signed_descriptor_t)); @@ -3298,6 +3247,14 @@ routerlist_reset_warnings(void) networkstatus_reset_warnings(); } +/** Return 1 if the signed descriptor of this router is older than + * <b>seconds</b> seconds. Otherwise return 0. */ +MOCK_IMPL(int, +router_descriptor_is_older_than,(const routerinfo_t *router, int seconds)) +{ + return router->cache_info.published_on < approx_time() - seconds; +} + /** Add <b>router</b> to the routerlist, if we don't already have it. Replace * older entries (if any) with the same key. Note: Callers should not hold * their pointers to <b>router</b> if this function fails; <b>router</b> @@ -3364,7 +3321,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, router_describe(router)); *msg = "Router descriptor was not new."; routerinfo_free(router); - return ROUTER_WAS_NOT_NEW; + return ROUTER_IS_ALREADY_KNOWN; } } @@ -3449,7 +3406,7 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, &routerlist->desc_store); routerlist_insert_old(routerlist, router); *msg = "Router descriptor was not new."; - return ROUTER_WAS_NOT_NEW; + return ROUTER_IS_ALREADY_KNOWN; } else { /* Same key, and either new, or listed in the consensus. */ log_debug(LD_DIR, "Replacing entry for router %s", @@ -3467,10 +3424,10 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, } if (!in_consensus && from_cache && - router->cache_info.published_on < time(NULL) - OLD_ROUTER_DESC_MAX_AGE) { + router_descriptor_is_older_than(router, OLD_ROUTER_DESC_MAX_AGE)) { *msg = "Router descriptor was really old."; routerinfo_free(router); - return ROUTER_WAS_NOT_NEW; + return ROUTER_WAS_TOO_OLD; } /* We haven't seen a router with this identity before. Add it to the end of @@ -3491,21 +3448,18 @@ was_router_added_t router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, int from_cache, int from_fetch) { - int inserted; + was_router_added_t inserted; (void)from_fetch; if (msg) *msg = NULL; /*XXXX023 Do something with msg */ - inserted = extrainfo_insert(router_get_routerlist(), ei); + inserted = extrainfo_insert(router_get_routerlist(), ei, !from_cache); - if (inserted && !from_cache) + if (WRA_WAS_ADDED(inserted) && !from_cache) signed_desc_append_to_journal(&ei->cache_info, &routerlist->extrainfo_store); - if (inserted) - return ROUTER_ADDED_SUCCESSFULLY; - else - return ROUTER_BAD_EI; + return inserted; } /** Sorting helper: return <0, 0, or >0 depending on whether the @@ -3575,9 +3529,9 @@ routerlist_remove_old_cached_routers_with_id(time_t now, n_extra = n - mdpr; } - lifespans = tor_malloc_zero(sizeof(struct duration_idx_t)*n); - rmv = tor_malloc_zero(sizeof(uint8_t)*n); - must_keep = tor_malloc_zero(sizeof(uint8_t)*n); + lifespans = tor_calloc(n, sizeof(struct duration_idx_t)); + rmv = tor_calloc(n, sizeof(uint8_t)); + must_keep = tor_calloc(n, sizeof(uint8_t)); /* Set lifespans to contain the lifespan and index of each server. */ /* Set rmv[i-lo]=1 if we're going to remove a server for being too old. */ for (i = lo; i <= hi; ++i) { @@ -3800,7 +3754,8 @@ router_load_single_router(const char *s, uint8_t purpose, int cache, "@source controller\n" "@purpose %s\n", router_purpose_to_string(purpose)); - if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0, annotation_buf))) { + if (!(ri = router_parse_entry_from_string(s, NULL, 1, 0, + annotation_buf, NULL))) { log_warn(LD_DIR, "Error parsing router descriptor; dropping."); *msg = "Couldn't parse router descriptor."; return -1; @@ -3864,9 +3819,11 @@ router_load_routers_from_string(const char *s, const char *eos, int from_cache = (saved_location != SAVED_NOWHERE); int allow_annotations = (saved_location != SAVED_NOWHERE); int any_changed = 0; + smartlist_t *invalid_digests = smartlist_new(); router_parse_list_from_string(&s, eos, routers, saved_location, 0, - allow_annotations, prepend_annotations); + allow_annotations, prepend_annotations, + invalid_digests); routers_update_status_from_consensus_networkstatus(routers, !from_cache); @@ -3902,7 +3859,7 @@ router_load_routers_from_string(const char *s, const char *eos, smartlist_add(changed, ri); routerlist_descriptors_added(changed, from_cache); smartlist_clear(changed); - } else if (WRA_WAS_REJECTED(r)) { + } else if (WRA_NEVER_DOWNLOADABLE(r)) { download_status_t *dl_status; dl_status = router_get_dl_status_by_descriptor_digest(d); if (dl_status) { @@ -3913,6 +3870,27 @@ router_load_routers_from_string(const char *s, const char *eos, } } SMARTLIST_FOREACH_END(ri); + SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) { + /* This digest is never going to be parseable. */ + base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN); + if (requested_fingerprints && descriptor_digests) { + if (! smartlist_contains_string(requested_fingerprints, fp)) { + /* But we didn't ask for it, so we should assume shennanegans. */ + continue; + } + smartlist_string_remove(requested_fingerprints, fp); + } + download_status_t *dls; + dls = router_get_dl_status_by_descriptor_digest((char*)bad_digest); + if (dls) { + log_info(LD_GENERAL, "Marking router with descriptor %s as unparseable, " + "and therefore undownloadable", fp); + download_status_mark_impossible(dls); + } + } SMARTLIST_FOREACH_END(bad_digest); + SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d)); + smartlist_free(invalid_digests); + routerlist_assert_ok(routerlist); if (any_changed) @@ -3936,13 +3914,16 @@ router_load_extrainfo_from_string(const char *s, const char *eos, smartlist_t *extrainfo_list = smartlist_new(); const char *msg; int from_cache = (saved_location != SAVED_NOWHERE); + smartlist_t *invalid_digests = smartlist_new(); router_parse_list_from_string(&s, eos, extrainfo_list, saved_location, 1, 0, - NULL); + NULL, invalid_digests); log_info(LD_DIR, "%d elements to add", smartlist_len(extrainfo_list)); SMARTLIST_FOREACH_BEGIN(extrainfo_list, extrainfo_t *, ei) { + uint8_t d[DIGEST_LEN]; + memcpy(d, ei->cache_info.signed_descriptor_digest, DIGEST_LEN); was_router_added_t added = router_add_extrainfo_to_routerlist(ei, &msg, from_cache, !from_cache); if (WRA_WAS_ADDED(added) && requested_fingerprints) { @@ -3956,9 +3937,39 @@ router_load_extrainfo_from_string(const char *s, const char *eos, * so long as we would have wanted them anyway. Since we always fetch * all the extrainfos we want, and we never actually act on them * inside Tor, this should be harmless. */ + } else if (WRA_NEVER_DOWNLOADABLE(added)) { + signed_descriptor_t *sd = router_get_by_extrainfo_digest((char*)d); + if (sd) { + log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as " + "unparseable, and therefore undownloadable", + hex_str((char*)d,DIGEST_LEN)); + download_status_mark_impossible(&sd->ei_dl_status); + } } } SMARTLIST_FOREACH_END(ei); + SMARTLIST_FOREACH_BEGIN(invalid_digests, const uint8_t *, bad_digest) { + /* This digest is never going to be parseable. */ + char fp[HEX_DIGEST_LEN+1]; + base16_encode(fp, sizeof(fp), (char*)bad_digest, DIGEST_LEN); + if (requested_fingerprints) { + if (! smartlist_contains_string(requested_fingerprints, fp)) { + /* But we didn't ask for it, so we should assume shennanegans. */ + continue; + } + smartlist_string_remove(requested_fingerprints, fp); + } + signed_descriptor_t *sd = + router_get_by_extrainfo_digest((char*)bad_digest); + if (sd) { + log_info(LD_GENERAL, "Marking extrainfo with descriptor %s as " + "unparseable, and therefore undownloadable", fp); + download_status_mark_impossible(&sd->ei_dl_status); + } + } SMARTLIST_FOREACH_END(bad_digest); + SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d)); + smartlist_free(invalid_digests); + routerlist_assert_ok(routerlist); router_rebuild_store(0, &router_get_routerlist()->extrainfo_store); @@ -4204,7 +4215,7 @@ clear_dir_servers(void) * corresponding elements of <b>result</b> to a nonzero value. */ static void -list_pending_downloads(digestmap_t *result, +list_pending_downloads(digestmap_t *result, digest256map_t *result256, int purpose, const char *prefix) { const size_t p_len = strlen(prefix); @@ -4214,7 +4225,7 @@ list_pending_downloads(digestmap_t *result, if (purpose == DIR_PURPOSE_FETCH_MICRODESC) flags = DSR_DIGEST256|DSR_BASE64; - tor_assert(result); + tor_assert(result || result256); SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { if (conn->type == CONN_TYPE_DIR && @@ -4227,11 +4238,19 @@ list_pending_downloads(digestmap_t *result, } } SMARTLIST_FOREACH_END(conn); - SMARTLIST_FOREACH(tmp, char *, d, + if (result) { + SMARTLIST_FOREACH(tmp, char *, d, { digestmap_set(result, d, (void*)1); tor_free(d); }); + } else if (result256) { + SMARTLIST_FOREACH(tmp, uint8_t *, d, + { + digest256map_set(result256, d, (void*)1); + tor_free(d); + }); + } smartlist_free(tmp); } @@ -4243,20 +4262,16 @@ list_pending_descriptor_downloads(digestmap_t *result, int extrainfo) { int purpose = extrainfo ? DIR_PURPOSE_FETCH_EXTRAINFO : DIR_PURPOSE_FETCH_SERVERDESC; - list_pending_downloads(result, purpose, "d/"); + list_pending_downloads(result, NULL, purpose, "d/"); } /** For every microdescriptor we are currently downloading by descriptor - * digest, set result[d] to (void*)1. (Note that microdescriptor digests - * are 256-bit, and digestmap_t only holds 160-bit digests, so we're only - * getting the first 20 bytes of each digest here.) - * - * XXXX Let there be a digestmap256_t, and use that instead. + * digest, set result[d] to (void*)1. */ void -list_pending_microdesc_downloads(digestmap_t *result) +list_pending_microdesc_downloads(digest256map_t *result) { - list_pending_downloads(result, DIR_PURPOSE_FETCH_MICRODESC, "d/"); + list_pending_downloads(NULL, result, DIR_PURPOSE_FETCH_MICRODESC, "d/"); } /** For every certificate we are currently downloading by (identity digest, @@ -4299,51 +4314,57 @@ list_pending_fpsk_downloads(fp_pair_map_t *result) * range.) If <b>source</b> is given, download from <b>source</b>; * otherwise, download from an appropriate random directory server. */ -static void -initiate_descriptor_downloads(const routerstatus_t *source, - int purpose, - smartlist_t *digests, - int lo, int hi, int pds_flags) +MOCK_IMPL(STATIC void, initiate_descriptor_downloads, + (const routerstatus_t *source, int purpose, smartlist_t *digests, + int lo, int hi, int pds_flags)) { - int i, n = hi-lo; char *resource, *cp; - size_t r_len; - - int digest_len = DIGEST_LEN, enc_digest_len = HEX_DIGEST_LEN; - char sep = '+'; - int b64_256 = 0; + int digest_len, enc_digest_len; + const char *sep; + int b64_256; + smartlist_t *tmp; if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { /* Microdescriptors are downloaded by "-"-separated base64-encoded * 256-bit digests. */ digest_len = DIGEST256_LEN; - enc_digest_len = BASE64_DIGEST256_LEN; - sep = '-'; + enc_digest_len = BASE64_DIGEST256_LEN + 1; + sep = "-"; b64_256 = 1; + } else { + digest_len = DIGEST_LEN; + enc_digest_len = HEX_DIGEST_LEN + 1; + sep = "+"; + b64_256 = 0; } - if (n <= 0) - return; if (lo < 0) lo = 0; if (hi > smartlist_len(digests)) hi = smartlist_len(digests); - r_len = 8 + (enc_digest_len+1)*n; - cp = resource = tor_malloc(r_len); - memcpy(cp, "d/", 2); - cp += 2; - for (i = lo; i < hi; ++i) { + if (hi-lo <= 0) + return; + + tmp = smartlist_new(); + + for (; lo < hi; ++lo) { + cp = tor_malloc(enc_digest_len); if (b64_256) { - digest256_to_base64(cp, smartlist_get(digests, i)); + digest256_to_base64(cp, smartlist_get(digests, lo)); } else { - base16_encode(cp, r_len-(cp-resource), - smartlist_get(digests,i), digest_len); + base16_encode(cp, enc_digest_len, smartlist_get(digests, lo), + digest_len); } - cp += enc_digest_len; - *cp++ = sep; + smartlist_add(tmp, cp); } - memcpy(cp-1, ".z", 3); + + cp = smartlist_join_strings(tmp, sep, 0, NULL); + tor_asprintf(&resource, "d/%s.z", cp); + + SMARTLIST_FOREACH(tmp, char *, cp1, tor_free(cp1)); + smartlist_free(tmp); + tor_free(cp); if (source) { /* We know which authority we want. */ @@ -4358,14 +4379,28 @@ initiate_descriptor_downloads(const routerstatus_t *source, tor_free(resource); } -/** Max amount of hashes to download per request. - * Since squid does not like URLs >= 4096 bytes we limit it to 96. - * 4096 - strlen(http://255.255.255.255/tor/server/d/.z) == 4058 - * 4058/41 (40 for the hash and 1 for the + that separates them) => 98 - * So use 96 because it's a nice number. +/** Return the max number of hashes to put in a URL for a given request. */ -#define MAX_DL_PER_REQUEST 96 -#define MAX_MICRODESC_DL_PER_REQUEST 92 +static int +max_dl_per_request(const or_options_t *options, int purpose) +{ + /* Since squid does not like URLs >= 4096 bytes we limit it to 96. + * 4096 - strlen(http://255.255.255.255/tor/server/d/.z) == 4058 + * 4058/41 (40 for the hash and 1 for the + that separates them) => 98 + * So use 96 because it's a nice number. + */ + int max = 96; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + max = 92; + } + /* If we're going to tunnel our connections, we can ask for a lot more + * in a request. */ + if (!directory_fetches_from_authorities(options)) { + max = 500; + } + return max; +} + /** Don't split our requests so finely that we are requesting fewer than * this number per server. */ #define MIN_DL_PER_REQUEST 4 @@ -4387,92 +4422,89 @@ launch_descriptor_downloads(int purpose, smartlist_t *downloadable, const routerstatus_t *source, time_t now) { - int should_delay = 0, n_downloadable; const or_options_t *options = get_options(); const char *descname; + const int fetch_microdesc = (purpose == DIR_PURPOSE_FETCH_MICRODESC); + int n_downloadable = smartlist_len(downloadable); + + int i, n_per_request, max_dl_per_req; + const char *req_plural = "", *rtr_plural = ""; + int pds_flags = PDS_RETRY_IF_NO_SERVERS; - tor_assert(purpose == DIR_PURPOSE_FETCH_SERVERDESC || - purpose == DIR_PURPOSE_FETCH_MICRODESC); + tor_assert(fetch_microdesc || purpose == DIR_PURPOSE_FETCH_SERVERDESC); + descname = fetch_microdesc ? "microdesc" : "routerdesc"; - descname = (purpose == DIR_PURPOSE_FETCH_SERVERDESC) ? - "routerdesc" : "microdesc"; + if (!n_downloadable) + return; - n_downloadable = smartlist_len(downloadable); if (!directory_fetches_dir_info_early(options)) { if (n_downloadable >= MAX_DL_TO_DELAY) { log_debug(LD_DIR, "There are enough downloadable %ss to launch requests.", descname); - should_delay = 0; } else { - should_delay = (last_descriptor_download_attempted + - options->TestingClientMaxIntervalWithoutRequest) > now; - if (!should_delay && n_downloadable) { - if (last_descriptor_download_attempted) { - log_info(LD_DIR, - "There are not many downloadable %ss, but we've " - "been waiting long enough (%d seconds). Downloading.", - descname, - (int)(now-last_descriptor_download_attempted)); - } else { - log_info(LD_DIR, - "There are not many downloadable %ss, but we haven't " - "tried downloading descriptors recently. Downloading.", - descname); - } + + /* should delay */ + if ((last_descriptor_download_attempted + + options->TestingClientMaxIntervalWithoutRequest) > now) + return; + + if (last_descriptor_download_attempted) { + log_info(LD_DIR, + "There are not many downloadable %ss, but we've " + "been waiting long enough (%d seconds). Downloading.", + descname, + (int)(now-last_descriptor_download_attempted)); + } else { + log_info(LD_DIR, + "There are not many downloadable %ss, but we haven't " + "tried downloading descriptors recently. Downloading.", + descname); } } } - if (! should_delay && n_downloadable) { - int i, n_per_request; - const char *req_plural = "", *rtr_plural = ""; - int pds_flags = PDS_RETRY_IF_NO_SERVERS; - if (! authdir_mode_any_nonhidserv(options)) { - /* If we wind up going to the authorities, we want to only open one - * connection to each authority at a time, so that we don't overload - * them. We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH - * regardless of whether we're a cache or not; it gets ignored if we're - * not calling router_pick_trusteddirserver. - * - * Setting this flag can make initiate_descriptor_downloads() ignore - * requests. We need to make sure that we do in fact call - * update_router_descriptor_downloads() later on, once the connections - * have succeeded or failed. - */ - pds_flags |= (purpose == DIR_PURPOSE_FETCH_MICRODESC) ? - PDS_NO_EXISTING_MICRODESC_FETCH : - PDS_NO_EXISTING_SERVERDESC_FETCH; - } + if (!authdir_mode_any_nonhidserv(options)) { + /* If we wind up going to the authorities, we want to only open one + * connection to each authority at a time, so that we don't overload + * them. We do this by setting PDS_NO_EXISTING_SERVERDESC_FETCH + * regardless of whether we're a cache or not. + * + * Setting this flag can make initiate_descriptor_downloads() ignore + * requests. We need to make sure that we do in fact call + * update_router_descriptor_downloads() later on, once the connections + * have succeeded or failed. + */ + pds_flags |= fetch_microdesc ? + PDS_NO_EXISTING_MICRODESC_FETCH : + PDS_NO_EXISTING_SERVERDESC_FETCH; + } - n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); - if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { - if (n_per_request > MAX_MICRODESC_DL_PER_REQUEST) - n_per_request = MAX_MICRODESC_DL_PER_REQUEST; - } else { - if (n_per_request > MAX_DL_PER_REQUEST) - n_per_request = MAX_DL_PER_REQUEST; - } - if (n_per_request < MIN_DL_PER_REQUEST) - n_per_request = MIN_DL_PER_REQUEST; - - if (n_downloadable > n_per_request) - req_plural = rtr_plural = "s"; - else if (n_downloadable > 1) - rtr_plural = "s"; - - log_info(LD_DIR, - "Launching %d request%s for %d %s%s, %d at a time", - CEIL_DIV(n_downloadable, n_per_request), req_plural, - n_downloadable, descname, rtr_plural, n_per_request); - smartlist_sort_digests(downloadable); - for (i=0; i < n_downloadable; i += n_per_request) { - initiate_descriptor_downloads(source, purpose, - downloadable, i, i+n_per_request, - pds_flags); - } - last_descriptor_download_attempted = now; + n_per_request = CEIL_DIV(n_downloadable, MIN_REQUESTS); + max_dl_per_req = max_dl_per_request(options, purpose); + + if (n_per_request > max_dl_per_req) + n_per_request = max_dl_per_req; + + if (n_per_request < MIN_DL_PER_REQUEST) + n_per_request = MIN_DL_PER_REQUEST; + + if (n_downloadable > n_per_request) + req_plural = rtr_plural = "s"; + else if (n_downloadable > 1) + rtr_plural = "s"; + + log_info(LD_DIR, + "Launching %d request%s for %d %s%s, %d at a time", + CEIL_DIV(n_downloadable, n_per_request), req_plural, + n_downloadable, descname, rtr_plural, n_per_request); + smartlist_sort_digests(downloadable); + for (i=0; i < n_downloadable; i += n_per_request) { + initiate_descriptor_downloads(source, purpose, + downloadable, i, i+n_per_request, + pds_flags); } + last_descriptor_download_attempted = now; } /** For any descriptor that we want that's currently listed in @@ -4652,8 +4684,8 @@ update_extrainfo_downloads(time_t now) routerlist_t *rl; smartlist_t *wanted; digestmap_t *pending; - int old_routers, i; - int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0; + int old_routers, i, max_dl_per_req; + int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0, n_bogus[2] = {0,0}; if (! options->DownloadExtraInfo) return; if (should_delay_dir_fetches(options, NULL)) @@ -4698,19 +4730,54 @@ update_extrainfo_downloads(time_t now) ++n_pending; continue; } + + const signed_descriptor_t *sd2 = router_get_by_extrainfo_digest(d); + if (sd2 != sd) { + if (sd2 != NULL) { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + char d3[HEX_DIGEST_LEN+1], d4[HEX_DIGEST_LEN+1]; + base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), sd2->identity_digest, DIGEST_LEN); + base16_encode(d3, sizeof(d3), d, DIGEST_LEN); + base16_encode(d4, sizeof(d3), sd2->extra_info_digest, DIGEST_LEN); + + log_info(LD_DIR, "Found an entry in %s with mismatched " + "router_get_by_extrainfo_digest() value. This has ID %s " + "but the entry in the map has ID %s. This has EI digest " + "%s and the entry in the map has EI digest %s.", + old_routers?"old_routers":"routers", + d1, d2, d3, d4); + } else { + char d1[HEX_DIGEST_LEN+1], d2[HEX_DIGEST_LEN+1]; + base16_encode(d1, sizeof(d1), sd->identity_digest, DIGEST_LEN); + base16_encode(d2, sizeof(d2), d, DIGEST_LEN); + + log_info(LD_DIR, "Found an entry in %s with NULL " + "router_get_by_extrainfo_digest() value. This has ID %s " + "and EI digest %s.", + old_routers?"old_routers":"routers", + d1, d2); + } + ++n_bogus[old_routers]; + continue; + } smartlist_add(wanted, d); } } digestmap_free(pending, NULL); log_info(LD_DIR, "Extrainfo download status: %d router with no ei, %d " - "with present ei, %d delaying, %d pending, %d downloadable.", - n_no_ei, n_have, n_delay, n_pending, smartlist_len(wanted)); + "with present ei, %d delaying, %d pending, %d downloadable, %d " + "bogus in routers, %d bogus in old_routers", + n_no_ei, n_have, n_delay, n_pending, smartlist_len(wanted), + n_bogus[0], n_bogus[1]); smartlist_shuffle(wanted); - for (i = 0; i < smartlist_len(wanted); i += MAX_DL_PER_REQUEST) { + + max_dl_per_req = max_dl_per_request(options, DIR_PURPOSE_FETCH_EXTRAINFO); + for (i = 0; i < smartlist_len(wanted); i += max_dl_per_req) { initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_EXTRAINFO, - wanted, i, i + MAX_DL_PER_REQUEST, + wanted, i, i+max_dl_per_req, PDS_RETRY_IF_NO_SERVERS|PDS_NO_EXISTING_SERVERDESC_FETCH); } diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 6e2f2eaea0..78c3fbb880 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -58,6 +58,10 @@ const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type, int router_get_my_share_of_directory_requests(double *v3_share_out); void router_reset_status_download_failures(void); int routers_have_same_or_addrs(const routerinfo_t *r1, const routerinfo_t *r2); +void router_add_running_nodes_to_smartlist(smartlist_t *sl, int allow_invalid, + int need_uptime, int need_capacity, + int need_guard, int need_desc); + const routerinfo_t *routerlist_find_my_routerinfo(void); uint32_t router_get_advertised_bandwidth(const routerinfo_t *router); uint32_t router_get_advertised_bandwidth_capped(const routerinfo_t *router); @@ -82,7 +86,8 @@ int hexdigest_to_digest(const char *hexdigest, char *digest); const routerinfo_t *router_get_by_id_digest(const char *digest); routerinfo_t *router_get_mutable_by_digest(const char *digest); signed_descriptor_t *router_get_by_descriptor_digest(const char *digest); -signed_descriptor_t *router_get_by_extrainfo_digest(const char *digest); +MOCK_DECL(signed_descriptor_t *,router_get_by_extrainfo_digest, + (const char *digest)); signed_descriptor_t *extrainfo_get_by_descriptor_digest(const char *digest); const char *signed_descriptor_get_body(const signed_descriptor_t *desc); const char *signed_descriptor_get_annotations(const signed_descriptor_t *desc); @@ -99,6 +104,7 @@ void routerlist_reset_warnings(void); static int WRA_WAS_ADDED(was_router_added_t s); static int WRA_WAS_OUTDATED(was_router_added_t s); static int WRA_WAS_REJECTED(was_router_added_t s); +static int WRA_NEVER_DOWNLOADABLE(was_router_added_t s); /** Return true iff the outcome code in <b>s</b> indicates that the descriptor * was added. It might still be necessary to check whether the descriptor * generator should be notified. @@ -115,7 +121,8 @@ WRA_WAS_ADDED(was_router_added_t s) { */ static INLINE int WRA_WAS_OUTDATED(was_router_added_t s) { - return (s == ROUTER_WAS_NOT_NEW || + return (s == ROUTER_WAS_TOO_OLD || + s == ROUTER_IS_ALREADY_KNOWN || s == ROUTER_NOT_IN_CONSENSUS || s == ROUTER_NOT_IN_CONSENSUS_OR_NETWORKSTATUS); } @@ -125,6 +132,14 @@ static INLINE int WRA_WAS_REJECTED(was_router_added_t s) { return (s == ROUTER_AUTHDIR_REJECTS); } +/** Return true iff the outcome code in <b>s</b> indicates that the descriptor + * was flat-out rejected. */ +static INLINE int WRA_NEVER_DOWNLOADABLE(was_router_added_t s) +{ + return (s == ROUTER_AUTHDIR_REJECTS || + s == ROUTER_BAD_EI || + s == ROUTER_WAS_TOO_OLD); +} was_router_added_t router_add_to_routerlist(routerinfo_t *router, const char **msg, int from_cache, @@ -185,7 +200,7 @@ int hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, int hid_serv_acting_as_directory(void); int hid_serv_responsible_for_desc_id(const char *id); -void list_pending_microdesc_downloads(digestmap_t *result); +void list_pending_microdesc_downloads(digest256map_t *result); void launch_descriptor_downloads(int purpose, smartlist_t *downloadable, const routerstatus_t *source, @@ -212,6 +227,16 @@ STATIC int choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries); STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, uint64_t *total_out); + +MOCK_DECL(int, router_descriptor_is_older_than, (const routerinfo_t *router, + int seconds)); +MOCK_DECL(STATIC was_router_added_t, extrainfo_insert, + (routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible)); + +MOCK_DECL(STATIC void, initiate_descriptor_downloads, + (const routerstatus_t *source, int purpose, smartlist_t *digests, + int lo, int hi, int pds_flags)); + #endif #endif diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 5b70142a43..22d1a27f39 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1,7 +1,7 @@ -/* Copyright (c) 2001 Matej Pfajfar. + /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -9,6 +9,8 @@ * \brief Code to parse and validate router descriptors and directories. **/ +#define ROUTERPARSE_PRIVATE + #include "or.h" #include "config.h" #include "circuitstats.h" @@ -23,6 +25,7 @@ #include "networkstatus.h" #include "rephist.h" #include "routerparse.h" +#include "entrynodes.h" #undef log #include <math.h> @@ -131,6 +134,7 @@ typedef enum { K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, K_DIRECTORY_FOOTER, + K_PACKAGE, A_PURPOSE, A_LAST_LISTED, @@ -420,6 +424,7 @@ static token_rule_t networkstatus_token_table[] = { T1("known-flags", K_KNOWN_FLAGS, ARGS, NO_OBJ ), T01("params", K_PARAMS, ARGS, NO_OBJ ), T( "fingerprint", K_FINGERPRINT, CONCAT_ARGS, NO_OBJ ), + T0N("package", K_PACKAGE, CONCAT_ARGS, NO_OBJ ), CERTIFICATE_MEMBERS @@ -911,7 +916,9 @@ find_start_of_next_router_or_extrainfo(const char **s_ptr, * descriptor in the signed_descriptor_body field of each routerinfo_t. If it * isn't SAVED_NOWHERE, remember the offset of each descriptor. * - * Returns 0 on success and -1 on failure. + * Returns 0 on success and -1 on failure. Adds a digest to + * <b>invalid_digests_out</b> for every entry that was unparseable or + * invalid. (This may cause duplicate entries.) */ int router_parse_list_from_string(const char **s, const char *eos, @@ -919,7 +926,8 @@ router_parse_list_from_string(const char **s, const char *eos, saved_location_t saved_location, int want_extrainfo, int allow_annotations, - const char *prepend_annotations) + const char *prepend_annotations, + smartlist_t *invalid_digests_out) { routerinfo_t *router; extrainfo_t *extrainfo; @@ -939,6 +947,9 @@ router_parse_list_from_string(const char **s, const char *eos, tor_assert(eos >= *s); while (1) { + char raw_digest[DIGEST_LEN]; + int have_raw_digest = 0; + int dl_again = 0; if (find_start_of_next_router_or_extrainfo(s, eos, &have_extrainfo) < 0) break; @@ -955,18 +966,20 @@ router_parse_list_from_string(const char **s, const char *eos, if (have_extrainfo && want_extrainfo) { routerlist_t *rl = router_get_routerlist(); + have_raw_digest = router_get_extrainfo_hash(*s, end-*s, raw_digest) == 0; extrainfo = extrainfo_parse_entry_from_string(*s, end, saved_location != SAVED_IN_CACHE, - rl->identity_map); + rl->identity_map, &dl_again); if (extrainfo) { signed_desc = &extrainfo->cache_info; elt = extrainfo; } } else if (!have_extrainfo && !want_extrainfo) { + have_raw_digest = router_get_router_hash(*s, end-*s, raw_digest) == 0; router = router_parse_entry_from_string(*s, end, saved_location != SAVED_IN_CACHE, allow_annotations, - prepend_annotations); + prepend_annotations, &dl_again); if (router) { log_debug(LD_DIR, "Read router '%s', purpose '%s'", router_describe(router), @@ -975,6 +988,9 @@ router_parse_list_from_string(const char **s, const char *eos, elt = router; } } + if (! elt && ! dl_again && have_raw_digest && invalid_digests_out) { + smartlist_add(invalid_digests_out, tor_memdup(raw_digest, DIGEST_LEN)); + } if (!elt) { *s = end; continue; @@ -1068,11 +1084,17 @@ find_single_ipv6_orport(const smartlist_t *list, * around when caching the router. * * Only one of allow_annotations and prepend_annotations may be set. + * + * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1 + * if it's okay to try to download a descriptor with this same digest again, + * and 0 if it isn't. (It might not be okay to download it again if part of + * the part covered by the digest is invalid.) */ routerinfo_t * router_parse_entry_from_string(const char *s, const char *end, int cache_copy, int allow_annotations, - const char *prepend_annotations) + const char *prepend_annotations, + int *can_dl_again_out) { routerinfo_t *router = NULL; char digest[128]; @@ -1083,6 +1105,9 @@ router_parse_entry_from_string(const char *s, const char *end, size_t prepend_len = prepend_annotations ? strlen(prepend_annotations) : 0; int ok = 1; memarea_t *area = NULL; + /* Do not set this to '1' until we have parsed everything that we intend to + * parse that's covered by the hash. */ + int can_dl_again = 0; tor_assert(!allow_annotations || !prepend_annotations); @@ -1389,19 +1414,21 @@ router_parse_entry_from_string(const char *s, const char *end, verified_digests = digestmap_new(); digestmap_set(verified_digests, signed_digest, (void*)(uintptr_t)1); #endif - if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, - "router descriptor") < 0) - goto err; if (!router->or_port) { log_warn(LD_DIR,"or_port unreadable or 0. Failing."); goto err; } + /* We've checked everything that's covered by the hash. */ + can_dl_again = 1; + if (check_signature_token(digest, DIGEST_LEN, tok, router->identity_pkey, 0, + "router descriptor") < 0) + goto err; + if (!router->platform) { router->platform = tor_strdup("<unknown>"); } - goto done; err: @@ -1418,6 +1445,8 @@ router_parse_entry_from_string(const char *s, const char *end, DUMP_AREA(area, "routerinfo"); memarea_drop_all(area); } + if (can_dl_again_out) + *can_dl_again_out = can_dl_again; return router; } @@ -1426,10 +1455,16 @@ router_parse_entry_from_string(const char *s, const char *end, * <b>cache_copy</b> is true, make a copy of the extra-info document in the * cache_info fields of the result. If <b>routermap</b> is provided, use it * as a map from router identity to routerinfo_t when looking up signing keys. + * + * If <b>can_dl_again_out</b> is provided, set *<b>can_dl_again_out</b> to 1 + * if it's okay to try to download an extrainfo with this same digest again, + * and 0 if it isn't. (It might not be okay to download it again if part of + * the part covered by the digest is invalid.) */ extrainfo_t * extrainfo_parse_entry_from_string(const char *s, const char *end, - int cache_copy, struct digest_ri_map_t *routermap) + int cache_copy, struct digest_ri_map_t *routermap, + int *can_dl_again_out) { extrainfo_t *extrainfo = NULL; char digest[128]; @@ -1439,6 +1474,9 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, routerinfo_t *router = NULL; memarea_t *area = NULL; const char *s_dup = s; + /* Do not set this to '1' until we have parsed everything that we intend to + * parse that's covered by the hash. */ + int can_dl_again = 0; if (!end) { end = s + strlen(s); @@ -1498,6 +1536,9 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, goto err; } + /* We've checked everything that's covered by the hash. */ + can_dl_again = 1; + if (routermap && (router = digestmap_get((digestmap_t*)routermap, extrainfo->cache_info.identity_digest))) { @@ -1540,6 +1581,8 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, DUMP_AREA(area, "extrainfo"); memarea_drop_all(area); } + if (can_dl_again_out) + *can_dl_again_out = can_dl_again; return extrainfo; } @@ -1754,6 +1797,63 @@ find_start_of_next_routerstatus(const char *s) return eos; } +/** Parse the GuardFraction string from a consensus or vote. + * + * If <b>vote</b> or <b>vote_rs</b> are set the document getting + * parsed is a vote routerstatus. Otherwise it's a consensus. This is + * the same semantic as in routerstatus_parse_entry_from_string(). */ +STATIC int +routerstatus_parse_guardfraction(const char *guardfraction_str, + networkstatus_t *vote, + vote_routerstatus_t *vote_rs, + routerstatus_t *rs) +{ + int ok; + const char *end_of_header = NULL; + int is_consensus = !vote_rs; + uint32_t guardfraction; + + tor_assert(bool_eq(vote, vote_rs)); + + /* If this info comes from a consensus, but we should't apply + guardfraction, just exit. */ + if (is_consensus && !should_apply_guardfraction(NULL)) { + return 0; + } + + end_of_header = strchr(guardfraction_str, '='); + if (!end_of_header) { + return -1; + } + + guardfraction = (uint32_t)tor_parse_ulong(end_of_header+1, + 10, 0, 100, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Invalid GuardFraction %s", escaped(guardfraction_str)); + return -1; + } + + log_debug(LD_GENERAL, "[*] Parsed %s guardfraction '%s' for '%s'.", + is_consensus ? "consensus" : "vote", + guardfraction_str, rs->nickname); + + if (!is_consensus) { /* We are parsing a vote */ + vote_rs->status.guardfraction_percentage = guardfraction; + vote_rs->status.has_guardfraction = 1; + } else { + /* We are parsing a consensus. Only apply guardfraction to guards. */ + if (rs->is_possible_guard) { + rs->guardfraction_percentage = guardfraction; + rs->has_guardfraction = 1; + } else { + log_warn(LD_BUG, "Got GuardFraction for non-guard %s. " + "This is not supposed to happen. Not applying. ", rs->nickname); + } + } + + return 0; +} + /** Given a string at *<b>s</b>, containing a routerstatus object, and an * empty smartlist at <b>tokens</b>, parse and return the first router status * object in the string, and advance *<b>s</b> to just after the end of the @@ -1900,8 +2000,6 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_possible_guard = 1; else if (!strcmp(tok->args[i], "BadExit")) rs->is_bad_exit = 1; - else if (!strcmp(tok->args[i], "BadDirectory")) - rs->is_bad_directory = 1; else if (!strcmp(tok->args[i], "Authority")) rs->is_authority = 1; else if (!strcmp(tok->args[i], "Unnamed") && @@ -1918,12 +2016,9 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->version_known = 1; if (strcmpstart(tok->args[0], "Tor ")) { rs->version_supports_microdesc_cache = 1; - rs->version_supports_optimistic_data = 1; } else { rs->version_supports_microdesc_cache = tor_version_supports_microdescriptors(tok->args[0]); - rs->version_supports_optimistic_data = - tor_version_as_new_as(tok->args[0], "0.2.3.1-alpha"); rs->version_supports_extend2_cells = tor_version_as_new_as(tok->args[0], "0.2.4.8-alpha"); } @@ -1961,6 +2056,11 @@ routerstatus_parse_entry_from_string(memarea_t *area, vote->has_measured_bws = 1; } else if (!strcmpstart(tok->args[i], "Unmeasured=1")) { rs->bw_is_unmeasured = 1; + } else if (!strcmpstart(tok->args[i], "GuardFraction=")) { + if (routerstatus_parse_guardfraction(tok->args[i], + vote, vote_rs, rs) < 0) { + goto err; + } } } } @@ -2048,6 +2148,7 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method) double Gtotal=0, Mtotal=0, Etotal=0; const char *casename = NULL; int valid = 1; + (void) consensus_method; weight_scale = networkstatus_get_weight_scale_param(ns); Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1); @@ -2127,12 +2228,8 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method) // Then, gather G, M, E, D, T to determine case SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) { int is_exit = 0; - if (consensus_method >= MIN_METHOD_TO_CUT_BADEXIT_WEIGHT) { - /* Bug #2203: Don't count bad exits as exits for balancing */ - is_exit = rs->is_exit && !rs->is_bad_exit; - } else { - is_exit = rs->is_exit; - } + /* Bug #2203: Don't count bad exits as exits for balancing */ + is_exit = rs->is_exit && !rs->is_bad_exit; if (rs->has_bandwidth) { T += rs->bandwidth_kb; if (is_exit && rs->is_possible_guard) { @@ -2568,11 +2665,15 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, (int) tor_parse_long(tok->args[1], 10, 0, INT_MAX, &ok, NULL); if (!ok) goto err; - if (ns->valid_after + MIN_VOTE_INTERVAL > ns->fresh_until) { + if (ns->valid_after + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) > ns->fresh_until) { log_warn(LD_DIR, "Vote/consensus freshness interval is too short"); goto err; } - if (ns->valid_after + MIN_VOTE_INTERVAL*2 > ns->valid_until) { + if (ns->valid_after + + (get_options()->TestingTorNetwork ? + MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL)*2 > ns->valid_until) { log_warn(LD_DIR, "Vote/consensus liveness interval is too short"); goto err; } @@ -2592,6 +2693,16 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, ns->server_versions = tor_strdup(tok->args[0]); } + { + smartlist_t *package_lst = find_all_by_keyword(tokens, K_PACKAGE); + ns->package_lines = smartlist_new(); + if (package_lst) { + SMARTLIST_FOREACH(package_lst, directory_token_t *, t, + smartlist_add(ns->package_lines, tor_strdup(t->args[0]))); + } + smartlist_free(package_lst); + } + tok = find_by_keyword(tokens, K_KNOWN_FLAGS); ns->known_flags = smartlist_new(); inorder = 1; @@ -3247,8 +3358,8 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) * AF_UNSPEC for '*'. Use policy_expand_unspec() to turn this into a pair * of AF_INET and AF_INET6 items. */ -addr_policy_t * -router_parse_addr_policy_item_from_string(const char *s, int assume_action) +MOCK_IMPL(addr_policy_t *, +router_parse_addr_policy_item_from_string,(const char *s, int assume_action)) { directory_token_t *tok = NULL; const char *cp, *eos; @@ -4014,12 +4125,15 @@ find_start_of_next_microdesc(const char *s, const char *eos) * If <b>saved_location</b> isn't SAVED_IN_CACHE, make a local copy of each * descriptor in the body field of each microdesc_t. * - * Return all newly - * parsed microdescriptors in a newly allocated smartlist_t. */ + * Return all newly parsed microdescriptors in a newly allocated + * smartlist_t. If <b>invalid_disgests_out</b> is provided, add a SHA256 + * microdesc digest to it for every microdesc that we found to be badly + * formed. (This may cause duplicates) */ smartlist_t * microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, - saved_location_t where) + saved_location_t where, + smartlist_t *invalid_digests_out) { smartlist_t *tokens; smartlist_t *result; @@ -4041,21 +4155,20 @@ microdescs_parse_from_string(const char *s, const char *eos, tokens = smartlist_new(); while (s < eos) { + int okay = 0; + start_of_next_microdesc = find_start_of_next_microdesc(s, eos); if (!start_of_next_microdesc) start_of_next_microdesc = eos; - if (tokenize_string(area, s, start_of_next_microdesc, tokens, - microdesc_token_table, flags)) { - log_warn(LD_DIR, "Unparseable microdescriptor"); - goto next; - } - md = tor_malloc_zero(sizeof(microdesc_t)); { const char *cp = tor_memstr(s, start_of_next_microdesc-s, "onion-key"); - tor_assert(cp); + const int no_onion_key = (cp == NULL); + if (no_onion_key) { + cp = s; /* So that we have *some* junk to put in the body */ + } md->bodylen = start_of_next_microdesc - cp; md->saved_location = where; @@ -4064,6 +4177,18 @@ microdescs_parse_from_string(const char *s, const char *eos, else md->body = (char*)cp; md->off = cp - start; + crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); + if (no_onion_key) { + log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor"); + goto next; + } + } + + + if (tokenize_string(area, s, start_of_next_microdesc, tokens, + microdesc_token_table, flags)) { + log_warn(LD_DIR, "Unparseable microdescriptor"); + goto next; } if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) { @@ -4121,12 +4246,15 @@ microdescs_parse_from_string(const char *s, const char *eos, md->ipv6_exit_policy = parse_short_policy(tok->args[0]); } - crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256); - smartlist_add(result, md); + okay = 1; md = NULL; next: + if (! okay && invalid_digests_out) { + smartlist_add(invalid_digests_out, + tor_memdup(md->digest, DIGEST256_LEN)); + } microdesc_free(md); md = NULL; @@ -4207,40 +4335,50 @@ tor_version_parse(const char *s, tor_version_t *out) char *eos=NULL; const char *cp=NULL; /* Format is: - * "Tor " ? NUM dot NUM dot NUM [ ( pre | rc | dot ) NUM [ - tag ] ] + * "Tor " ? NUM dot NUM [ dot NUM [ ( pre | rc | dot ) NUM ] ] [ - tag ] */ tor_assert(s); tor_assert(out); memset(out, 0, sizeof(tor_version_t)); - + out->status = VER_RELEASE; if (!strcasecmpstart(s, "Tor ")) s += 4; - /* Get major. */ - out->major = (int)strtol(s,&eos,10); - if (!eos || eos==s || *eos != '.') return -1; - cp = eos+1; - - /* Get minor */ - out->minor = (int) strtol(cp,&eos,10); - if (!eos || eos==cp || *eos != '.') return -1; - cp = eos+1; - - /* Get micro */ - out->micro = (int) strtol(cp,&eos,10); - if (!eos || eos==cp) return -1; - if (!*eos) { - out->status = VER_RELEASE; - out->patchlevel = 0; + cp = s; + +#define NUMBER(m) \ + do { \ + out->m = (int)strtol(cp, &eos, 10); \ + if (!eos || eos == cp) \ + return -1; \ + cp = eos; \ + } while (0) + +#define DOT() \ + do { \ + if (*cp != '.') \ + return -1; \ + ++cp; \ + } while (0) + + NUMBER(major); + DOT(); + NUMBER(minor); + if (*cp == 0) return 0; - } - cp = eos; + else if (*cp == '-') + goto status_tag; + DOT(); + NUMBER(micro); /* Get status */ - if (*cp == '.') { - out->status = VER_RELEASE; + if (*cp == 0) { + return 0; + } else if (*cp == '.') { ++cp; + } else if (*cp == '-') { + goto status_tag; } else if (0==strncmp(cp, "pre", 3)) { out->status = VER_PRE; cp += 3; @@ -4251,11 +4389,9 @@ tor_version_parse(const char *s, tor_version_t *out) return -1; } - /* Get patchlevel */ - out->patchlevel = (int) strtol(cp,&eos,10); - if (!eos || eos==cp) return -1; - cp = eos; + NUMBER(patchlevel); + status_tag: /* Get status tag. */ if (*cp == '-' || *cp == '.') ++cp; @@ -4291,6 +4427,8 @@ tor_version_parse(const char *s, tor_version_t *out) } return 0; +#undef NUMBER +#undef DOT } /** Compare two tor versions; Return <0 if a < b; 0 if a ==b, >0 if a > @@ -4378,6 +4516,9 @@ sort_version_list(smartlist_t *versions, int remove_duplicates) * to *<b>encoded_size_out</b>, and a pointer to the possibly next * descriptor to *<b>next_out</b>; return 0 for success (including validation) * and -1 for failure. + * + * If <b>as_hsdir</b> is 1, we're parsing this as an HSDir, and we should + * be strict about time formats. */ int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, @@ -4385,7 +4526,8 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, char **intro_points_encrypted_out, size_t *intro_points_encrypted_size_out, size_t *encoded_size_out, - const char **next_out, const char *desc) + const char **next_out, const char *desc, + int as_hsdir) { rend_service_descriptor_t *result = tor_malloc_zero(sizeof(rend_service_descriptor_t)); @@ -4399,6 +4541,8 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, char public_key_hash[DIGEST_LEN]; char test_desc_id[DIGEST_LEN]; memarea_t *area = NULL; + const int strict_time_fmt = as_hsdir; + tor_assert(desc); /* Check if desc starts correctly. */ if (strncmp(desc, "rendezvous-service-descriptor ", @@ -4493,7 +4637,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, * descriptor. */ tok = find_by_keyword(tokens, R_PUBLICATION_TIME); tor_assert(tok->n_args == 1); - if (parse_iso_time(tok->args[0], &result->timestamp) < 0) { + if (parse_iso_time_(tok->args[0], &result->timestamp, strict_time_fmt) < 0) { log_warn(LD_REND, "Invalid publication time: '%s'", tok->args[0]); goto err; } diff --git a/src/or/routerparse.h b/src/or/routerparse.h index 5d5d9e59ef..fc21cb1041 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,16 +29,19 @@ int router_parse_list_from_string(const char **s, const char *eos, saved_location_t saved_location, int is_extrainfo, int allow_annotations, - const char *prepend_annotations); + const char *prepend_annotations, + smartlist_t *invalid_digests_out); routerinfo_t *router_parse_entry_from_string(const char *s, const char *end, int cache_copy, int allow_annotations, - const char *prepend_annotations); + const char *prepend_annotations, + int *can_dl_again_out); extrainfo_t *extrainfo_parse_entry_from_string(const char *s, const char *end, - int cache_copy, struct digest_ri_map_t *routermap); -addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, - int assume_action); + int cache_copy, struct digest_ri_map_t *routermap, + int *can_dl_again_out); +MOCK_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string, + (const char *s, int assume_action)); version_status_t tor_version_is_obsolete(const char *myversion, const char *versionlist); int tor_version_supports_microdescriptors(const char *platform); @@ -60,7 +63,8 @@ ns_detached_signatures_t *networkstatus_parse_detached_signatures( smartlist_t *microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, - saved_location_t where); + saved_location_t where, + smartlist_t *invalid_digests_out); authority_cert_t *authority_cert_parse_from_string(const char *s, const char **end_of_string); @@ -69,7 +73,8 @@ int rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, char **intro_points_encrypted_out, size_t *intro_points_encrypted_size_out, size_t *encoded_size_out, - const char **next_out, const char *desc); + const char **next_out, const char *desc, + int as_hsdir); int rend_decrypt_introduction_points(char **ipos_decrypted, size_t *ipos_decrypted_size, const char *descriptor_cookie, @@ -80,5 +85,12 @@ int rend_parse_introduction_points(rend_service_descriptor_t *parsed, size_t intro_points_encoded_size); int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); +#ifdef ROUTERPARSE_PRIVATE +STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str, + networkstatus_t *vote, + vote_routerstatus_t *vote_rs, + routerstatus_t *rs); +#endif + #endif diff --git a/src/or/routerset.c b/src/or/routerset.c index 7aee90d6db..99de11ed5e 100644 --- a/src/or/routerset.c +++ b/src/or/routerset.c @@ -1,9 +1,11 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#define ROUTERSET_PRIVATE + #include "or.h" #include "geoip.h" #include "nodelist.h" @@ -12,39 +14,6 @@ #include "routerparse.h" #include "routerset.h" -/** A routerset specifies constraints on a set of possible routerinfos, based - * on their names, identities, or addresses. It is optimized for determining - * whether a router is a member or not, in O(1+P) time, where P is the number - * of address policy constraints. */ -struct routerset_t { - /** A list of strings for the elements of the policy. Each string is either - * a nickname, a hexadecimal identity fingerprint, or an address policy. A - * router belongs to the set if its nickname OR its identity OR its address - * matches an entry here. */ - smartlist_t *list; - /** A map from lowercase nicknames of routers in the set to (void*)1 */ - strmap_t *names; - /** A map from identity digests routers in the set to (void*)1 */ - digestmap_t *digests; - /** An address policy for routers in the set. For implementation reasons, - * a router belongs to the set if it is _rejected_ by this policy. */ - smartlist_t *policies; - - /** A human-readable description of what this routerset is for. Used in - * log messages. */ - char *description; - - /** A list of the country codes in this set. */ - smartlist_t *country_names; - /** Total number of countries we knew about when we built <b>countries</b>.*/ - int n_countries; - /** Bit array mapping the return value of geoip_get_country() to 1 iff the - * country is a member of this routerset. Note that we MUST call - * routerset_refresh_countries() whenever the geoip country list is - * reloaded. */ - bitarray_t *countries; -}; - /** Return a new empty routerset. */ routerset_t * routerset_new(void) @@ -60,7 +29,7 @@ routerset_new(void) /** If <b>c</b> is a country code in the form {cc}, return a newly allocated * string holding the "cc" part. Else, return NULL. */ -static char * +STATIC char * routerset_get_countryname(const char *c) { char *country; @@ -200,7 +169,7 @@ routerset_is_empty(const routerset_t *set) * * (If country is -1, then we take the country * from addr.) */ -static int +STATIC int routerset_contains(const routerset_t *set, const tor_addr_t *addr, uint16_t orport, const char *nickname, const char *id_digest, diff --git a/src/or/routerset.h b/src/or/routerset.h index 8261c7fb09..8d41de8b6b 100644 --- a/src/or/routerset.h +++ b/src/or/routerset.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -39,5 +39,45 @@ char *routerset_to_string(const routerset_t *routerset); int routerset_equal(const routerset_t *old, const routerset_t *new); void routerset_free(routerset_t *routerset); +#ifdef ROUTERSET_PRIVATE +STATIC char * routerset_get_countryname(const char *c); +STATIC int routerset_contains(const routerset_t *set, const tor_addr_t *addr, + uint16_t orport, + const char *nickname, const char *id_digest, + country_t country); + +/** A routerset specifies constraints on a set of possible routerinfos, based + * on their names, identities, or addresses. It is optimized for determining + * whether a router is a member or not, in O(1+P) time, where P is the number + * of address policy constraints. */ +struct routerset_t { + /** A list of strings for the elements of the policy. Each string is either + * a nickname, a hexadecimal identity fingerprint, or an address policy. A + * router belongs to the set if its nickname OR its identity OR its address + * matches an entry here. */ + smartlist_t *list; + /** A map from lowercase nicknames of routers in the set to (void*)1 */ + strmap_t *names; + /** A map from identity digests routers in the set to (void*)1 */ + digestmap_t *digests; + /** An address policy for routers in the set. For implementation reasons, + * a router belongs to the set if it is _rejected_ by this policy. */ + smartlist_t *policies; + + /** A human-readable description of what this routerset is for. Used in + * log messages. */ + char *description; + + /** A list of the country codes in this set. */ + smartlist_t *country_names; + /** Total number of countries we knew about when we built <b>countries</b>.*/ + int n_countries; + /** Bit array mapping the return value of geoip_get_country() to 1 iff the + * country is a member of this routerset. Note that we MUST call + * routerset_refresh_countries() whenever the geoip country list is + * reloaded. */ + bitarray_t *countries; +}; +#endif #endif diff --git a/src/or/scheduler.c b/src/or/scheduler.c new file mode 100644 index 0000000000..931bb6b744 --- /dev/null +++ b/src/or/scheduler.c @@ -0,0 +1,711 @@ +/* * Copyright (c) 2013-2015, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file scheduler.c + * \brief Relay scheduling system + **/ + +#include "or.h" + +#define TOR_CHANNEL_INTERNAL_ /* For channel_flush_some_cells() */ +#include "channel.h" + +#include "compat_libevent.h" +#define SCHEDULER_PRIVATE_ +#include "scheduler.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +/* + * Scheduler high/low watermarks + */ + +static uint32_t sched_q_low_water = 16384; +static uint32_t sched_q_high_water = 32768; + +/* + * Maximum cells to flush in a single call to channel_flush_some_cells(); + * setting this low means more calls, but too high and we could overshoot + * sched_q_high_water. + */ + +static uint32_t sched_max_flush_cells = 16; + +/* + * Write scheduling works by keeping track of which channels can + * accept cells, and have cells to write. From the scheduler's perspective, + * a channel can be in four possible states: + * + * 1.) Not open for writes, no cells to send + * - Not much to do here, and the channel will have scheduler_state == + * SCHED_CHAN_IDLE + * - Transitions from: + * - Open for writes/has cells by simultaneously draining all circuit + * queues and filling the output buffer. + * - Transitions to: + * - Not open for writes/has cells by arrival of cells on an attached + * circuit (this would be driven from append_cell_to_circuit_queue()) + * - Open for writes/no cells by a channel type specific path; + * driven from connection_or_flushed_some() for channel_tls_t. + * + * 2.) Open for writes, no cells to send + * - Not much here either; this will be the state an idle but open channel + * can be expected to settle in. It will have scheduler_state == + * SCHED_CHAN_WAITING_FOR_CELLS + * - Transitions from: + * - Not open for writes/no cells by flushing some of the output + * buffer. + * - Open for writes/has cells by the scheduler moving cells from + * circuit queues to channel output queue, but not having enough + * to fill the output queue. + * - Transitions to: + * - Open for writes/has cells by arrival of new cells on an attached + * circuit, in append_cell_to_circuit_queue() + * + * 3.) Not open for writes, cells to send + * - This is the state of a busy circuit limited by output bandwidth; + * cells have piled up in the circuit queues waiting to be relayed. + * The channel will have scheduler_state == SCHED_CHAN_WAITING_TO_WRITE. + * - Transitions from: + * - Not open for writes/no cells by arrival of cells on an attached + * circuit + * - Open for writes/has cells by filling an output buffer without + * draining all cells from attached circuits + * - Transitions to: + * - Opens for writes/has cells by draining some of the output buffer + * via the connection_or_flushed_some() path (for channel_tls_t). + * + * 4.) Open for writes, cells to send + * - This connection is ready to relay some cells and waiting for + * the scheduler to choose it. The channel will have scheduler_state == + * SCHED_CHAN_PENDING. + * - Transitions from: + * - Not open for writes/has cells by the connection_or_flushed_some() + * path + * - Open for writes/no cells by the append_cell_to_circuit_queue() + * path + * - Transitions to: + * - Not open for writes/no cells by draining all circuit queues and + * simultaneously filling the output buffer. + * - Not open for writes/has cells by writing enough cells to fill the + * output buffer + * - Open for writes/no cells by draining all attached circuit queues + * without also filling the output buffer + * + * Other event-driven parts of the code move channels between these scheduling + * states by calling scheduler functions; the scheduler only runs on open-for- + * writes/has-cells channels and is the only path for those to transition to + * other states. The scheduler_run() function gives us the opportunity to do + * scheduling work, and is called from other scheduler functions whenever a + * state transition occurs, and periodically from the main event loop. + */ + +/* Scheduler global data structures */ + +/* + * We keep a list of channels that are pending - i.e, have cells to write + * and can accept them to send. The enum scheduler_state in channel_t + * is reserved for our use. + */ + +/* Pqueue of channels that can write and have cells (pending work) */ +STATIC smartlist_t *channels_pending = NULL; + +/* + * This event runs the scheduler from its callback, and is manually + * activated whenever a channel enters open for writes/cells to send. + */ + +STATIC struct event *run_sched_ev = NULL; + +/* + * Queue heuristic; this is not the queue size, but an 'effective queuesize' + * that ages out contributions from stalled channels. + */ + +STATIC uint64_t queue_heuristic = 0; + +/* + * Timestamp for last queue heuristic update + */ + +STATIC time_t queue_heuristic_timestamp = 0; + +/* Scheduler static function declarations */ + +static void scheduler_evt_callback(evutil_socket_t fd, + short events, void *arg); +static int scheduler_more_work(void); +static void scheduler_retrigger(void); +#if 0 +static void scheduler_trigger(void); +#endif + +/* Scheduler function implementations */ + +/** Free everything and shut down the scheduling system */ + +void +scheduler_free_all(void) +{ + log_debug(LD_SCHED, "Shutting down scheduler"); + + if (run_sched_ev) { + if (event_del(run_sched_ev) < 0) { + log_warn(LD_BUG, "Problem deleting run_sched_ev"); + } + tor_event_free(run_sched_ev); + run_sched_ev = NULL; + } + + if (channels_pending) { + smartlist_free(channels_pending); + channels_pending = NULL; + } +} + +/** + * Comparison function to use when sorting pending channels + */ + +MOCK_IMPL(STATIC int, +scheduler_compare_channels, (const void *c1_v, const void *c2_v)) +{ + channel_t *c1 = NULL, *c2 = NULL; + /* These are a workaround for -Wbad-function-cast throwing a fit */ + const circuitmux_policy_t *p1, *p2; + uintptr_t p1_i, p2_i; + + tor_assert(c1_v); + tor_assert(c2_v); + + c1 = (channel_t *)(c1_v); + c2 = (channel_t *)(c2_v); + + tor_assert(c1); + tor_assert(c2); + + if (c1 != c2) { + if (circuitmux_get_policy(c1->cmux) == + circuitmux_get_policy(c2->cmux)) { + /* Same cmux policy, so use the mux comparison */ + return circuitmux_compare_muxes(c1->cmux, c2->cmux); + } else { + /* + * Different policies; not important to get this edge case perfect + * because the current code never actually gives different channels + * different cmux policies anyway. Just use this arbitrary but + * definite choice. + */ + p1 = circuitmux_get_policy(c1->cmux); + p2 = circuitmux_get_policy(c2->cmux); + p1_i = (uintptr_t)p1; + p2_i = (uintptr_t)p2; + + return (p1_i < p2_i) ? -1 : 1; + } + } else { + /* c1 == c2, so always equal */ + return 0; + } +} + +/* + * Scheduler event callback; this should get triggered once per event loop + * if any scheduling work was created during the event loop. + */ + +static void +scheduler_evt_callback(evutil_socket_t fd, short events, void *arg) +{ + (void)fd; + (void)events; + (void)arg; + log_debug(LD_SCHED, "Scheduler event callback called"); + + tor_assert(run_sched_ev); + + /* Run the scheduler */ + scheduler_run(); + + /* Do we have more work to do? */ + if (scheduler_more_work()) scheduler_retrigger(); +} + +/** Mark a channel as no longer ready to accept writes */ + +MOCK_IMPL(void, +scheduler_channel_doesnt_want_writes,(channel_t *chan)) +{ + tor_assert(chan); + + tor_assert(channels_pending); + + /* If it's already in pending, we can put it in waiting_to_write */ + if (chan->scheduler_state == SCHED_CHAN_PENDING) { + /* + * It's in channels_pending, so it shouldn't be in any of + * the other lists. It can't write any more, so it goes to + * channels_waiting_to_write. + */ + smartlist_pqueue_remove(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p went from pending " + "to waiting_to_write", + U64_PRINTF_ARG(chan->global_identifier), chan); + } else { + /* + * It's not in pending, so it can't become waiting_to_write; it's + * either not in any of the lists (nothing to do) or it's already in + * waiting_for_cells (remove it, can't write any more). + */ + if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) { + chan->scheduler_state = SCHED_CHAN_IDLE; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p left waiting_for_cells", + U64_PRINTF_ARG(chan->global_identifier), chan); + } + } +} + +/** Mark a channel as having waiting cells */ + +MOCK_IMPL(void, +scheduler_channel_has_waiting_cells,(channel_t *chan)) +{ + int became_pending = 0; + + tor_assert(chan); + tor_assert(channels_pending); + + /* First, check if this one also writeable */ + if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) { + /* + * It's in channels_waiting_for_cells, so it shouldn't be in any of + * the other lists. It has waiting cells now, so it goes to + * channels_pending. + */ + chan->scheduler_state = SCHED_CHAN_PENDING; + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p went from waiting_for_cells " + "to pending", + U64_PRINTF_ARG(chan->global_identifier), chan); + became_pending = 1; + } else { + /* + * It's not in waiting_for_cells, so it can't become pending; it's + * either not in any of the lists (we add it to waiting_to_write) + * or it's already in waiting_to_write or pending (we do nothing) + */ + if (!(chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE || + chan->scheduler_state == SCHED_CHAN_PENDING)) { + chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p entered waiting_to_write", + U64_PRINTF_ARG(chan->global_identifier), chan); + } + } + + /* + * If we made a channel pending, we potentially have scheduling work + * to do. + */ + if (became_pending) scheduler_retrigger(); +} + +/** Set up the scheduling system */ + +void +scheduler_init(void) +{ + log_debug(LD_SCHED, "Initting scheduler"); + + tor_assert(!run_sched_ev); + run_sched_ev = tor_event_new(tor_libevent_get_base(), -1, + 0, scheduler_evt_callback, NULL); + + channels_pending = smartlist_new(); + queue_heuristic = 0; + queue_heuristic_timestamp = approx_time(); +} + +/** Check if there's more scheduling work */ + +static int +scheduler_more_work(void) +{ + tor_assert(channels_pending); + + return ((scheduler_get_queue_heuristic() < sched_q_low_water) && + ((smartlist_len(channels_pending) > 0))) ? 1 : 0; +} + +/** Retrigger the scheduler in a way safe to use from the callback */ + +static void +scheduler_retrigger(void) +{ + tor_assert(run_sched_ev); + event_active(run_sched_ev, EV_TIMEOUT, 1); +} + +/** Notify the scheduler of a channel being closed */ + +MOCK_IMPL(void, +scheduler_release_channel,(channel_t *chan)) +{ + tor_assert(chan); + tor_assert(channels_pending); + + if (chan->scheduler_state == SCHED_CHAN_PENDING) { + smartlist_pqueue_remove(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + } + + chan->scheduler_state = SCHED_CHAN_IDLE; +} + +/** Run the scheduling algorithm if necessary */ + +MOCK_IMPL(void, +scheduler_run, (void)) +{ + int n_cells, n_chans_before, n_chans_after; + uint64_t q_len_before, q_heur_before, q_len_after, q_heur_after; + ssize_t flushed, flushed_this_time; + smartlist_t *to_readd = NULL; + channel_t *chan = NULL; + + log_debug(LD_SCHED, "We have a chance to run the scheduler"); + + if (scheduler_get_queue_heuristic() < sched_q_low_water) { + n_chans_before = smartlist_len(channels_pending); + q_len_before = channel_get_global_queue_estimate(); + q_heur_before = scheduler_get_queue_heuristic(); + + while (scheduler_get_queue_heuristic() <= sched_q_high_water && + smartlist_len(channels_pending) > 0) { + /* Pop off a channel */ + chan = smartlist_pqueue_pop(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx)); + tor_assert(chan); + + /* Figure out how many cells we can write */ + n_cells = channel_num_cells_writeable(chan); + if (n_cells > 0) { + log_debug(LD_SCHED, + "Scheduler saw pending channel " U64_FORMAT " at %p with " + "%d cells writeable", + U64_PRINTF_ARG(chan->global_identifier), chan, n_cells); + + flushed = 0; + while (flushed < n_cells && + scheduler_get_queue_heuristic() <= sched_q_high_water) { + flushed_this_time = + channel_flush_some_cells(chan, + MIN(sched_max_flush_cells, + (size_t) n_cells - flushed)); + if (flushed_this_time <= 0) break; + flushed += flushed_this_time; + } + + if (flushed < n_cells) { + /* We ran out of cells to flush */ + chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p " + "entered waiting_for_cells from pending", + U64_PRINTF_ARG(chan->global_identifier), + chan); + } else { + /* The channel may still have some cells */ + if (channel_more_to_flush(chan)) { + /* The channel goes to either pending or waiting_to_write */ + if (channel_num_cells_writeable(chan) > 0) { + /* Add it back to pending later */ + if (!to_readd) to_readd = smartlist_new(); + smartlist_add(to_readd, chan); + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p " + "is still pending", + U64_PRINTF_ARG(chan->global_identifier), + chan); + } else { + /* It's waiting to be able to write more */ + chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p " + "entered waiting_to_write from pending", + U64_PRINTF_ARG(chan->global_identifier), + chan); + } + } else { + /* No cells left; it can go to idle or waiting_for_cells */ + if (channel_num_cells_writeable(chan) > 0) { + /* + * It can still accept writes, so it goes to + * waiting_for_cells + */ + chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p " + "entered waiting_for_cells from pending", + U64_PRINTF_ARG(chan->global_identifier), + chan); + } else { + /* + * We exactly filled up the output queue with all available + * cells; go to idle. + */ + chan->scheduler_state = SCHED_CHAN_IDLE; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p " + "become idle from pending", + U64_PRINTF_ARG(chan->global_identifier), + chan); + } + } + } + + log_debug(LD_SCHED, + "Scheduler flushed %d cells onto pending channel " + U64_FORMAT " at %p", + (int)flushed, U64_PRINTF_ARG(chan->global_identifier), + chan); + } else { + log_info(LD_SCHED, + "Scheduler saw pending channel " U64_FORMAT " at %p with " + "no cells writeable", + U64_PRINTF_ARG(chan->global_identifier), chan); + /* Put it back to WAITING_TO_WRITE */ + chan->scheduler_state = SCHED_CHAN_WAITING_TO_WRITE; + } + } + + /* Readd any channels we need to */ + if (to_readd) { + SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, chan) { + chan->scheduler_state = SCHED_CHAN_PENDING; + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + } SMARTLIST_FOREACH_END(chan); + smartlist_free(to_readd); + } + + n_chans_after = smartlist_len(channels_pending); + q_len_after = channel_get_global_queue_estimate(); + q_heur_after = scheduler_get_queue_heuristic(); + log_debug(LD_SCHED, + "Scheduler handled %d of %d pending channels, queue size from " + U64_FORMAT " to " U64_FORMAT ", queue heuristic from " + U64_FORMAT " to " U64_FORMAT, + n_chans_before - n_chans_after, n_chans_before, + U64_PRINTF_ARG(q_len_before), U64_PRINTF_ARG(q_len_after), + U64_PRINTF_ARG(q_heur_before), U64_PRINTF_ARG(q_heur_after)); + } +} + +/** Trigger the scheduling event so we run the scheduler later */ + +#if 0 +static void +scheduler_trigger(void) +{ + log_debug(LD_SCHED, "Triggering scheduler event"); + + tor_assert(run_sched_ev); + + event_add(run_sched_ev, EV_TIMEOUT, 1); +} +#endif + +/** Mark a channel as ready to accept writes */ + +void +scheduler_channel_wants_writes(channel_t *chan) +{ + int became_pending = 0; + + tor_assert(chan); + tor_assert(channels_pending); + + /* If it's already in waiting_to_write, we can put it in pending */ + if (chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE) { + /* + * It can write now, so it goes to channels_pending. + */ + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + chan->scheduler_state = SCHED_CHAN_PENDING; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p went from waiting_to_write " + "to pending", + U64_PRINTF_ARG(chan->global_identifier), chan); + became_pending = 1; + } else { + /* + * It's not in SCHED_CHAN_WAITING_TO_WRITE, so it can't become pending; + * it's either idle and goes to WAITING_FOR_CELLS, or it's a no-op. + */ + if (!(chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS || + chan->scheduler_state == SCHED_CHAN_PENDING)) { + chan->scheduler_state = SCHED_CHAN_WAITING_FOR_CELLS; + log_debug(LD_SCHED, + "Channel " U64_FORMAT " at %p entered waiting_for_cells", + U64_PRINTF_ARG(chan->global_identifier), chan); + } + } + + /* + * If we made a channel pending, we potentially have scheduling work + * to do. + */ + if (became_pending) scheduler_retrigger(); +} + +/** + * Notify the scheduler that a channel's position in the pqueue may have + * changed + */ + +void +scheduler_touch_channel(channel_t *chan) +{ + tor_assert(chan); + + if (chan->scheduler_state == SCHED_CHAN_PENDING) { + /* Remove and re-add it */ + smartlist_pqueue_remove(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + smartlist_pqueue_add(channels_pending, + scheduler_compare_channels, + STRUCT_OFFSET(channel_t, sched_heap_idx), + chan); + } + /* else no-op, since it isn't in the queue */ +} + +/** + * Notify the scheduler of a queue size adjustment, to recalculate the + * queue heuristic. + */ + +void +scheduler_adjust_queue_size(channel_t *chan, int dir, uint64_t adj) +{ + time_t now = approx_time(); + + log_debug(LD_SCHED, + "Queue size adjustment by %s" U64_FORMAT " for channel " + U64_FORMAT, + (dir >= 0) ? "+" : "-", + U64_PRINTF_ARG(adj), + U64_PRINTF_ARG(chan->global_identifier)); + + /* Get the queue heuristic up to date */ + scheduler_update_queue_heuristic(now); + + /* Adjust as appropriate */ + if (dir >= 0) { + /* Increasing it */ + queue_heuristic += adj; + } else { + /* Decreasing it */ + if (queue_heuristic > adj) queue_heuristic -= adj; + else queue_heuristic = 0; + } + + log_debug(LD_SCHED, + "Queue heuristic is now " U64_FORMAT, + U64_PRINTF_ARG(queue_heuristic)); +} + +/** + * Query the current value of the queue heuristic + */ + +STATIC uint64_t +scheduler_get_queue_heuristic(void) +{ + time_t now = approx_time(); + + scheduler_update_queue_heuristic(now); + + return queue_heuristic; +} + +/** + * Adjust the queue heuristic value to the present time + */ + +STATIC void +scheduler_update_queue_heuristic(time_t now) +{ + time_t diff; + + if (queue_heuristic_timestamp == 0) { + /* + * Nothing we can sensibly do; must not have been initted properly. + * Oh well. + */ + queue_heuristic_timestamp = now; + } else if (queue_heuristic_timestamp < now) { + diff = now - queue_heuristic_timestamp; + /* + * This is a simple exponential age-out; the other proposed alternative + * was a linear age-out using the bandwidth history in rephist.c; I'm + * going with this out of concern that if an adversary can jam the + * scheduler long enough, it would cause the bandwidth to drop to + * zero and render the aging mechanism ineffective thereafter. + */ + if (0 <= diff && diff < 64) queue_heuristic >>= diff; + else queue_heuristic = 0; + + queue_heuristic_timestamp = now; + + log_debug(LD_SCHED, + "Queue heuristic is now " U64_FORMAT, + U64_PRINTF_ARG(queue_heuristic)); + } + /* else no update needed, or time went backward */ +} + +/** + * Set scheduler watermarks and flush size + */ + +void +scheduler_set_watermarks(uint32_t lo, uint32_t hi, uint32_t max_flush) +{ + /* Sanity assertions - caller should ensure these are true */ + tor_assert(lo > 0); + tor_assert(hi > lo); + tor_assert(max_flush > 0); + + sched_q_low_water = lo; + sched_q_high_water = hi; + sched_max_flush_cells = max_flush; +} + diff --git a/src/or/scheduler.h b/src/or/scheduler.h new file mode 100644 index 0000000000..27dd2d8388 --- /dev/null +++ b/src/or/scheduler.h @@ -0,0 +1,50 @@ +/* * Copyright (c) 2013-2015, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file scheduler.h + * \brief Header file for scheduler.c + **/ + +#ifndef TOR_SCHEDULER_H +#define TOR_SCHEDULER_H + +#include "or.h" +#include "channel.h" +#include "testsupport.h" + +/* Global-visibility scheduler functions */ + +/* Set up and shut down the scheduler from main.c */ +void scheduler_free_all(void); +void scheduler_init(void); +MOCK_DECL(void, scheduler_run, (void)); + +/* Mark channels as having cells or wanting/not wanting writes */ +MOCK_DECL(void,scheduler_channel_doesnt_want_writes,(channel_t *chan)); +MOCK_DECL(void,scheduler_channel_has_waiting_cells,(channel_t *chan)); +void scheduler_channel_wants_writes(channel_t *chan); + +/* Notify the scheduler of a channel being closed */ +MOCK_DECL(void,scheduler_release_channel,(channel_t *chan)); + +/* Notify scheduler of queue size adjustments */ +void scheduler_adjust_queue_size(channel_t *chan, int dir, uint64_t adj); + +/* Notify scheduler that a channel's queue position may have changed */ +void scheduler_touch_channel(channel_t *chan); + +/* Adjust the watermarks from config file*/ +void scheduler_set_watermarks(uint32_t lo, uint32_t hi, uint32_t max_flush); + +/* Things only scheduler.c and its test suite should see */ + +#ifdef SCHEDULER_PRIVATE_ +MOCK_DECL(STATIC int, scheduler_compare_channels, + (const void *c1_v, const void *c2_v)); +STATIC uint64_t scheduler_get_queue_heuristic(void); +STATIC void scheduler_update_queue_heuristic(time_t now); +#endif + +#endif /* !defined(TOR_SCHEDULER_H) */ + diff --git a/src/or/statefile.c b/src/or/statefile.c index 7b9998fc1a..dd1894beb7 100644 --- a/src/or/statefile.c +++ b/src/or/statefile.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #define STATEFILE_PRIVATE @@ -323,7 +323,10 @@ or_state_load(void) goto done; } break; + /* treat empty state files as if the file doesn't exist, and generate + * a new state file, overwriting the empty file in or_state_save() */ case FN_NOENT: + case FN_EMPTY: break; case FN_ERROR: case FN_DIR: diff --git a/src/or/statefile.h b/src/or/statefile.h index 15bb0b4aae..8c790ea206 100644 --- a/src/or/statefile.h +++ b/src/or/statefile.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_STATEFILE_H diff --git a/src/or/status.c b/src/or/status.c index afaa9de840..8f7be0aa3c 100644 --- a/src/or/status.c +++ b/src/or/status.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2013, The Tor Project, Inc. */ +/* Copyright (c) 2010-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -23,18 +23,13 @@ #include "statefile.h" static void log_accounting(const time_t now, const or_options_t *options); +#include "geoip.h" /** Return the total number of circuits. */ STATIC int count_circuits(void) { - circuit_t *circ; - int nr=0; - - TOR_LIST_FOREACH(circ, circuit_get_global_list(), head) - nr++; - - return nr; + return smartlist_len(circuit_get_global_list()); } /** Take seconds <b>secs</b> and return a newly allocated human-readable @@ -98,7 +93,6 @@ log_heartbeat(time_t now) const int hibernating = we_are_hibernating(); const or_options_t *options = get_options(); - (void)now; if (public_server_mode(options) && !hibernating) { /* Let's check if we are in the current cached consensus. */ @@ -116,28 +110,47 @@ log_heartbeat(time_t now) log_fn(LOG_NOTICE, LD_HEARTBEAT, "Heartbeat: Tor's uptime is %s, with %d " "circuits open. I've sent %s and received %s.%s", - uptime, count_circuits(),bw_sent,bw_rcvd, + uptime, count_circuits(), bw_sent, bw_rcvd, hibernating?" We are currently hibernating.":""); if (server_mode(options) && accounting_is_enabled(options) && !hibernating) { log_accounting(now, options); } - if (stats_n_data_cells_packaged && !hibernating) - log_notice(LD_HEARTBEAT, "Average packaged cell fullness: %2.3f%%", - 100*(U64_TO_DBL(stats_n_data_bytes_packaged) / - U64_TO_DBL(stats_n_data_cells_packaged*RELAY_PAYLOAD_SIZE)) ); - - if (r > 1.0) { - double overhead = ( r - 1.0 ) * 100.0; - log_notice(LD_HEARTBEAT, "TLS write overhead: %.f%%", overhead); + double fullness_pct = 100; + if (stats_n_data_cells_packaged && !hibernating) { + fullness_pct = + 100*(U64_TO_DBL(stats_n_data_bytes_packaged) / + U64_TO_DBL(stats_n_data_cells_packaged*RELAY_PAYLOAD_SIZE)); } + const double overhead_pct = ( r - 1.0 ) * 100.0; + +#define FULLNESS_PCT_THRESHOLD 80 +#define TLS_OVERHEAD_THRESHOLD 15 + + const int severity = (fullness_pct < FULLNESS_PCT_THRESHOLD || + overhead_pct > TLS_OVERHEAD_THRESHOLD) + ? LOG_NOTICE : LOG_INFO; - if (public_server_mode(options)) + log_fn(severity, LD_HEARTBEAT, + "Average packaged cell fullness: %2.3f%%. " + "TLS write overhead: %.f%%", fullness_pct, overhead_pct); + + if (public_server_mode(options)) { rep_hist_log_circuit_handshake_stats(now); + rep_hist_log_link_protocol_counts(); + } circuit_log_ancient_one_hop_circuits(1800); + if (options->BridgeRelay) { + char *msg = NULL; + msg = format_client_stats_heartbeat(now); + if (msg) + log_notice(LD_HEARTBEAT, "%s", msg); + tor_free(msg); + } + tor_free(uptime); tor_free(bw_sent); tor_free(bw_rcvd); @@ -151,10 +164,14 @@ log_accounting(const time_t now, const or_options_t *options) or_state_t *state = get_or_state(); char *acc_rcvd = bytes_to_usage(state->AccountingBytesReadInInterval); char *acc_sent = bytes_to_usage(state->AccountingBytesWrittenInInterval); - char *acc_max = bytes_to_usage(options->AccountingMax); + uint64_t acc_bytes = options->AccountingMax; + char *acc_max; time_t interval_end = accounting_get_end_time(); char end_buf[ISO_TIME_LEN + 1]; char *remaining = NULL; + if (options->AccountingRule == ACCT_SUM) + acc_bytes *= 2; + acc_max = bytes_to_usage(acc_bytes); format_local_iso_time(end_buf, interval_end); remaining = secs_to_uptime(interval_end - now); diff --git a/src/or/status.h b/src/or/status.h index 13458ea476..3dd8206e0f 100644 --- a/src/or/status.h +++ b/src/or/status.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2010-2013, The Tor Project, Inc. */ +/* Copyright (c) 2010-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_STATUS_H diff --git a/src/or/tor_main.c b/src/or/tor_main.c index 05dc0bf0bf..af03b8c06a 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -1,6 +1,6 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** String describing which Tor Git repository version the source was diff --git a/src/or/transports.c b/src/or/transports.c index dc30754162..6f07054ea8 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2013, The Tor Project, Inc. */ +/* Copyright (c) 2011-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -112,8 +112,6 @@ static void parse_method_error(const char *line, int is_server_method); #define parse_server_method_error(l) parse_method_error(l, 1) #define parse_client_method_error(l) parse_method_error(l, 0) -static INLINE void free_execve_args(char **arg); - /** Managed proxy protocol strings */ #define PROTO_ENV_ERROR "ENV-ERROR" #define PROTO_NEG_SUCCESS "VERSION" @@ -124,6 +122,8 @@ static INLINE void free_execve_args(char **arg); #define PROTO_SMETHOD_ERROR "SMETHOD-ERROR" #define PROTO_CMETHODS_DONE "CMETHODS DONE" #define PROTO_SMETHODS_DONE "SMETHODS DONE" +#define PROTO_PROXY_DONE "PROXY DONE" +#define PROTO_PROXY_ERROR "PROXY-ERROR" /** The first and only supported - at the moment - configuration protocol version. */ @@ -324,9 +324,9 @@ transport_add(transport_t *t) /** Remember a new pluggable transport proxy at <b>addr</b>:<b>port</b>. * <b>name</b> is set to the name of the protocol this proxy uses. * <b>socks_ver</b> is set to the SOCKS version of the proxy. */ -int -transport_add_from_config(const tor_addr_t *addr, uint16_t port, - const char *name, int socks_ver) +MOCK_IMPL(int, +transport_add_from_config, (const tor_addr_t *addr, uint16_t port, + const char *name, int socks_ver)) { transport_t *t = transport_new(addr, port, name, socks_ver, NULL); @@ -439,6 +439,17 @@ add_transport_to_proxy(const char *transport, managed_proxy_t *mp) static int proxy_needs_restart(const managed_proxy_t *mp) { + int ret = 1; + char* proxy_uri; + + /* If the PT proxy config has changed, then all existing pluggable transports + * should be restarted. + */ + + proxy_uri = get_pt_proxy_uri(); + if (strcmp_opt(proxy_uri, mp->proxy_uri) != 0) + goto needs_restart; + /* mp->transport_to_launch is populated with the names of the transports that must be launched *after* the SIGHUP. mp->transports is populated with the transports that were @@ -459,10 +470,10 @@ proxy_needs_restart(const managed_proxy_t *mp) } SMARTLIST_FOREACH_END(t); - return 0; - + ret = 0; needs_restart: - return 1; + tor_free(proxy_uri); + return ret; } /** Managed proxy <b>mp</b> must be restarted. Do all the necessary @@ -493,6 +504,11 @@ proxy_prepare_for_restart(managed_proxy_t *mp) SMARTLIST_FOREACH(mp->transports, transport_t *, t, transport_free(t)); smartlist_clear(mp->transports); + /* Reset the proxy's HTTPS/SOCKS proxy */ + tor_free(mp->proxy_uri); + mp->proxy_uri = get_pt_proxy_uri(); + mp->proxy_supported = 0; + /* flag it as an infant proxy so that it gets launched on next tick */ mp->conf_state = PT_PROTO_INFANT; unconfigured_proxies_n++; @@ -727,12 +743,54 @@ managed_proxy_destroy(managed_proxy_t *mp, /* free the argv */ free_execve_args(mp->argv); + /* free the outgoing proxy URI */ + tor_free(mp->proxy_uri); + tor_process_handle_destroy(mp->process_handle, also_terminate_process); mp->process_handle = NULL; tor_free(mp); } +/** Convert the tor proxy options to a URI suitable for TOR_PT_PROXY. + * Return a newly allocated string containing the URI, or NULL if no + * proxy is set. */ +STATIC char * +get_pt_proxy_uri(void) +{ + const or_options_t *options = get_options(); + char *uri = NULL; + + if (options->Socks4Proxy || options->Socks5Proxy || options->HTTPSProxy) { + char addr[TOR_ADDR_BUF_LEN+1]; + + if (options->Socks4Proxy) { + tor_addr_to_str(addr, &options->Socks4ProxyAddr, sizeof(addr), 1); + tor_asprintf(&uri, "socks4a://%s:%d", addr, options->Socks4ProxyPort); + } else if (options->Socks5Proxy) { + tor_addr_to_str(addr, &options->Socks5ProxyAddr, sizeof(addr), 1); + if (!options->Socks5ProxyUsername && !options->Socks5ProxyPassword) { + tor_asprintf(&uri, "socks5://%s:%d", addr, options->Socks5ProxyPort); + } else { + tor_asprintf(&uri, "socks5://%s:%s@%s:%d", + options->Socks5ProxyUsername, + options->Socks5ProxyPassword, + addr, options->Socks5ProxyPort); + } + } else if (options->HTTPSProxy) { + tor_addr_to_str(addr, &options->HTTPSProxyAddr, sizeof(addr), 1); + if (!options->HTTPSProxyAuthenticator) { + tor_asprintf(&uri, "http://%s:%d", addr, options->HTTPSProxyPort); + } else { + tor_asprintf(&uri, "http://%s@%s:%d", options->HTTPSProxyAuthenticator, + addr, options->HTTPSProxyPort); + } + } + } + + return uri; +} + /** Handle a configured or broken managed proxy <b>mp</b>. */ static void handle_finished_proxy(managed_proxy_t *mp) @@ -745,6 +803,13 @@ handle_finished_proxy(managed_proxy_t *mp) managed_proxy_destroy(mp, 0); /* destroy it but don't terminate */ break; case PT_PROTO_CONFIGURED: /* if configured correctly: */ + if (mp->proxy_uri && !mp->proxy_supported) { + log_warn(LD_CONFIG, "Managed proxy '%s' did not configure the " + "specified outgoing proxy and will be terminated.", + mp->argv[0]); + managed_proxy_destroy(mp, 1); /* annihilate it. */ + break; + } register_proxy(mp); /* register its transports */ mp->conf_state = PT_PROTO_COMPLETED; /* and mark it as completed. */ break; @@ -862,6 +927,22 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) goto err; return; + } else if (!strcmpstart(line, PROTO_PROXY_DONE)) { + if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS) + goto err; + + if (mp->proxy_uri) { + mp->proxy_supported = 1; + return; + } + + /* No proxy was configured, this should log */ + } else if (!strcmpstart(line, PROTO_PROXY_ERROR)) { + if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS) + goto err; + + parse_proxy_error(line); + goto err; } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) { /* managed proxy launch failed: parse error message to learn why. */ int retval, child_state, saved_errno; @@ -1128,6 +1209,21 @@ parse_cmethod_line(const char *line, managed_proxy_t *mp) return r; } +/** Parses an PROXY-ERROR <b>line</b> and warns the user accordingly. */ +STATIC void +parse_proxy_error(const char *line) +{ + /* (Length of the protocol string) plus (a space) and (the first char of + the error message) */ + if (strlen(line) < (strlen(PROTO_PROXY_ERROR) + 2)) + log_notice(LD_CONFIG, "Managed proxy sent us an %s without an error " + "message.", PROTO_PROXY_ERROR); + + log_warn(LD_CONFIG, "Managed proxy failed to configure the " + "pluggable transport's outgoing proxy. (%s)", + line+strlen(PROTO_PROXY_ERROR)+1); +} + /** Return a newly allocated string that tor should place in * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server * manged proxy in <b>mp</b>. Return NULL if no such options are found. */ @@ -1292,6 +1388,14 @@ create_managed_proxy_environment(const managed_proxy_t *mp) } else { smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT="); } + } else { + /* If ClientTransportPlugin has a HTTPS/SOCKS proxy configured, set the + * TOR_PT_PROXY line. + */ + + if (mp->proxy_uri) { + smartlist_add_asprintf(envs, "TOR_PT_PROXY=%s", mp->proxy_uri); + } } SMARTLIST_FOREACH_BEGIN(envs, const char *, env_var) { @@ -1324,6 +1428,7 @@ managed_proxy_create(const smartlist_t *transport_list, mp->is_server = is_server; mp->argv = proxy_argv; mp->transports = smartlist_new(); + mp->proxy_uri = get_pt_proxy_uri(); mp->transports_to_launch = smartlist_new(); SMARTLIST_FOREACH(transport_list, const char *, transport, @@ -1349,9 +1454,9 @@ managed_proxy_create(const smartlist_t *transport_list, * Requires that proxy_argv be a NULL-terminated array of command-line * elements, containing at least one element. **/ -void -pt_kickstart_proxy(const smartlist_t *transport_list, - char **proxy_argv, int is_server) +MOCK_IMPL(void, +pt_kickstart_proxy, (const smartlist_t *transport_list, + char **proxy_argv, int is_server)) { managed_proxy_t *mp=NULL; transport_t *old_transport = NULL; @@ -1395,7 +1500,7 @@ pt_kickstart_proxy(const smartlist_t *transport_list, /** Frees the array of pointers in <b>arg</b> used as arguments to execve(2). */ -static INLINE void +STATIC void free_execve_args(char **arg) { char **tmp = arg; diff --git a/src/or/transports.h b/src/or/transports.h index 1365ead006..7c69941496 100644 --- a/src/or/transports.h +++ b/src/or/transports.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2013, The Tor Project, Inc. */ + * Copyright (c) 2007-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -32,14 +32,16 @@ typedef struct transport_t { void mark_transport_list(void); void sweep_transport_list(void); -int transport_add_from_config(const tor_addr_t *addr, uint16_t port, - const char *name, int socks_ver); +MOCK_DECL(int, transport_add_from_config, + (const tor_addr_t *addr, uint16_t port, + const char *name, int socks_ver)); void transport_free(transport_t *transport); transport_t *transport_get_by_name(const char *name); -void pt_kickstart_proxy(const smartlist_t *transport_list, char **proxy_argv, - int is_server); +MOCK_DECL(void, pt_kickstart_proxy, + (const smartlist_t *transport_list, char **proxy_argv, + int is_server)); #define pt_kickstart_client_proxy(tl, pa) \ pt_kickstart_proxy(tl, pa, 0) @@ -81,6 +83,9 @@ typedef struct { char **argv; /* the cli arguments of this proxy */ int conf_protocol; /* the configuration protocol version used */ + char *proxy_uri; /* the outgoing proxy in TOR_PT_PROXY URI format */ + unsigned int proxy_supported : 1; /* the proxy honors TOR_PT_PROXY */ + int is_server; /* is it a server proxy? */ /* A pointer to the process handle of this managed proxy. */ @@ -112,6 +117,7 @@ STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp); STATIC int parse_version(const char *line, managed_proxy_t *mp); STATIC void parse_env_error(const char *line); +STATIC void parse_proxy_error(const char *line); STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); @@ -123,6 +129,10 @@ STATIC managed_proxy_t *managed_proxy_create(const smartlist_t *transport_list, STATIC int configure_proxy(managed_proxy_t *mp); +STATIC char* get_pt_proxy_uri(void); + +STATIC void free_execve_args(char **arg); + #endif #endif |