diff options
Diffstat (limited to 'src')
39 files changed, 828 insertions, 349 deletions
diff --git a/src/app/config/config.c b/src/app/config/config.c index c7799ec1a2..2d4155e9bc 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -2466,6 +2466,7 @@ static const struct { .command=CMD_DUMP_CONFIG, .quiet=QUIET_SILENT }, { .name="--list-fingerprint", + .takes_argument=ARGUMENT_OPTIONAL, .command=CMD_LIST_FINGERPRINT }, { .name="--keygen", .command=CMD_KEYGEN }, @@ -2584,8 +2585,11 @@ config_parse_commandline(int argc, char **argv, int ignore_errors) parsed_cmdline_free(result); return NULL; } - } else if (want_arg == ARGUMENT_OPTIONAL && is_last) { + } else if (want_arg == ARGUMENT_OPTIONAL && + /* optional arguments may never start with '-'. */ + (is_last || argv[i+1][0] == '-')) { arg = tor_strdup(""); + want_arg = ARGUMENT_NONE; // prevent skipping the next flag. } else { arg = (want_arg != ARGUMENT_NONE) ? tor_strdup(argv[i+1]) : tor_strdup(""); @@ -3274,7 +3278,7 @@ options_validate_cb(const void *old_options_, void *options_, char **msg) } #else /* defined(HAVE_SYS_UN_H) */ if (options->ControlSocketsGroupWritable && !options->ControlSocket) { - *msg = tor_strdup("Setting ControlSocketGroupWritable without setting " + *msg = tor_strdup("Setting ControlSocketsGroupWritable without setting " "a ControlSocket makes no sense."); return -1; } @@ -4317,6 +4321,7 @@ find_torrc_filename(const config_line_t *cmd_arg, const config_line_t *p_index; const char *fname_opt = defaults_file ? "--defaults-torrc" : "-f"; const char *ignore_opt = defaults_file ? NULL : "--ignore-missing-torrc"; + const char *keygen_opt = "--keygen"; if (defaults_file) *ignore_missing_torrc = 1; @@ -4338,7 +4343,8 @@ find_torrc_filename(const config_line_t *cmd_arg, } *using_default_fname = 0; - } else if (ignore_opt && !strcmp(p_index->key,ignore_opt)) { + } else if ((ignore_opt && !strcmp(p_index->key, ignore_opt)) || + (keygen_opt && !strcmp(p_index->key, keygen_opt))) { *ignore_missing_torrc = 1; } } @@ -4485,6 +4491,25 @@ options_init_from_torrc(int argc, char **argv) if (config_line_find(cmdline_only_options, "--version")) { printf("Tor version %s.\n",get_version()); + printf("Tor is running on %s with Libevent %s, " + "%s %s, Zlib %s, Liblzma %s, Libzstd %s and %s %s as libc.\n", + get_uname(), + tor_libevent_get_version_str(), + crypto_get_library_name(), + crypto_get_library_version_string(), + tor_compress_supports_method(ZLIB_METHOD) ? + tor_compress_version_str(ZLIB_METHOD) : "N/A", + tor_compress_supports_method(LZMA_METHOD) ? + tor_compress_version_str(LZMA_METHOD) : "N/A", + tor_compress_supports_method(ZSTD_METHOD) ? + tor_compress_version_str(ZSTD_METHOD) : "N/A", + tor_libc_get_name() ? + tor_libc_get_name() : "Unknown", + tor_libc_get_version_str()); + printf("Tor compiled with %s version %s\n", + strcmp(COMPILER_VENDOR, "gnu") == 0? + COMPILER:COMPILER_VENDOR, COMPILER_VERSION); + return 1; } @@ -6030,7 +6055,7 @@ port_parse_config(smartlist_t *out, tor_free(addrtmp); } else { /* Try parsing integer port before address, because, who knows? - "9050" might be a valid address. */ + * "9050" might be a valid address. */ port = (int) tor_parse_long(addrport, 10, 0, 65535, &ok, NULL); if (ok) { tor_addr_copy(&addr, &default_addr); diff --git a/src/app/config/fallback_dirs.inc b/src/app/config/fallback_dirs.inc index a7ef39bb96..83834890ce 100644 --- a/src/app/config/fallback_dirs.inc +++ b/src/app/config/fallback_dirs.inc @@ -354,6 +354,7 @@ URL: https:onionoo.torproject.orguptime?typerelay&first_seen_days90-&last_seen_d /* ===== */ , "193.11.114.45:9031 orport=9002 id=80AAF8D5956A43C197104CEF2550CD42D165C6FB" +" ipv6=[2001:6b0:30:1000::100]:9050" /* nickname=mdfnet2 */ /* extrainfo=0 */ /* ===== */ @@ -552,6 +553,7 @@ URL: https:onionoo.torproject.orguptime?typerelay&first_seen_days90-&last_seen_d /* ===== */ , "193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E" +" ipv6=[2001:6b0:30:1000::101]:9050" /* nickname=mdfnet3 */ /* extrainfo=0 */ /* ===== */ diff --git a/src/app/include.am b/src/app/include.am index 8bb315fff1..2e2180deca 100644 --- a/src/app/include.am +++ b/src/app/include.am @@ -28,7 +28,7 @@ src_app_tor_cov_SOURCES = $(src_app_tor_SOURCES) src_app_tor_cov_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_app_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_app_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ - @TOR_LDFLAGS_libevent@ @TOR_STATIC_LDFALGS@ + @TOR_LDFLAGS_libevent@ @TOR_STATIC_LDFLAGS@ src_app_tor_cov_LDADD = src/test/libtor-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ \ diff --git a/src/app/main/main.c b/src/app/main/main.c index 589d365add..e7ffb31b4f 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -58,6 +58,7 @@ #include "feature/stats/rephist.h" #include "lib/compress/compress.h" #include "lib/buf/buffers.h" +#include "lib/crypt_ops/crypto_format.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" #include "lib/net/resolve.h" @@ -735,29 +736,52 @@ tor_remove_file(const char *filename) static int do_list_fingerprint(void) { - char buf[FINGERPRINT_LEN+1]; + const or_options_t *options = get_options(); + const char *arg = options->command_arg; + char rsa[FINGERPRINT_LEN + 1]; crypto_pk_t *k; - const char *nickname = get_options()->Nickname; + const ed25519_public_key_t *edkey; + const char *nickname = options->Nickname; sandbox_disable_getaddrinfo_cache(); - if (!server_mode(get_options())) { + + bool show_rsa = !strcmp(arg, "") || !strcmp(arg, "rsa"); + bool show_ed25519 = !strcmp(arg, "ed25519"); + if (!show_rsa && !show_ed25519) { + log_err(LD_GENERAL, + "If you give a key type, you must specify 'rsa' or 'ed25519'. Exiting."); + return -1; + } + + if (!server_mode(options)) { log_err(LD_GENERAL, "Clients don't have long-term identity keys. Exiting."); return -1; } tor_assert(nickname); if (init_keys() < 0) { - log_err(LD_GENERAL,"Error initializing keys; exiting."); + log_err(LD_GENERAL, "Error initializing keys; exiting."); return -1; } if (!(k = get_server_identity_key())) { - log_err(LD_GENERAL,"Error: missing identity key."); + log_err(LD_GENERAL, "Error: missing RSA identity key."); + return -1; + } + if (crypto_pk_get_fingerprint(k, rsa, 1) < 0) { + log_err(LD_BUG, "Error computing RSA fingerprint"); return -1; } - if (crypto_pk_get_fingerprint(k, buf, 1)<0) { - log_err(LD_BUG, "Error computing fingerprint"); + if (!(edkey = get_master_identity_key())) { + log_err(LD_GENERAL,"Error: missing ed25519 identity key."); return -1; } - printf("%s %s\n", nickname, buf); + if (show_rsa) { + printf("%s %s\n", nickname, rsa); + } + if (show_ed25519) { + char ed25519[ED25519_BASE64_LEN + 1]; + digest256_to_base64(ed25519, (const char *) edkey->pubkey); + printf("%s %s\n", nickname, ed25519); + } return 0; } @@ -1080,6 +1104,7 @@ sandbox_init_filter(void) OPEN_DATADIR2_SUFFIX("stats", "buffer-stats", ".tmp"); OPEN_DATADIR2_SUFFIX("stats", "conn-stats", ".tmp"); OPEN_DATADIR2_SUFFIX("stats", "hidserv-stats", ".tmp"); + OPEN_DATADIR2_SUFFIX("stats", "hidserv-v3-stats", ".tmp"); OPEN_DATADIR("approved-routers"); OPEN_DATADIR_SUFFIX("fingerprint", ".tmp"); @@ -1105,6 +1130,7 @@ sandbox_init_filter(void) RENAME_SUFFIX2("stats", "buffer-stats", ".tmp"); RENAME_SUFFIX2("stats", "conn-stats", ".tmp"); RENAME_SUFFIX2("stats", "hidserv-stats", ".tmp"); + RENAME_SUFFIX2("stats", "hidserv-v3-stats", ".tmp"); RENAME_SUFFIX("hashed-fingerprint", ".tmp"); RENAME_SUFFIX("router-stability", ".tmp"); diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 3d551c4ba8..df89145cd4 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -948,7 +948,6 @@ connection_free_minimal(connection_t *conn) connection_or_clear_identity(TO_OR_CONN(conn)); } if (conn->type == CONN_TYPE_OR || conn->type == CONN_TYPE_EXT_OR) { - connection_or_remove_from_ext_or_id_map(TO_OR_CONN(conn)); tor_free(TO_OR_CONN(conn)->ext_or_conn_id); tor_free(TO_OR_CONN(conn)->ext_or_auth_correct_client_hash); tor_free(TO_OR_CONN(conn)->ext_or_transport); @@ -1718,13 +1717,6 @@ connection_listener_new(const struct sockaddr *listensockaddr, } } - /* Force IPv4 and IPv6 traffic on for non-SOCKSPorts. - * Forcing options on isn't a good idea, see #32994 and #33607. */ - if (type != CONN_TYPE_AP_LISTENER) { - lis_conn->entry_cfg.ipv4_traffic = 1; - lis_conn->entry_cfg.ipv6_traffic = 1; - } - if (connection_add(conn) < 0) { /* no space, forget it */ log_warn(LD_NET,"connection_add for listener failed. Giving up."); goto err; @@ -3246,7 +3238,7 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) * we hit those, bail early so tor can stop. */ if (!new_conn) { log_warn(LD_NET, "Unable to create listener port: %s:%d", - fmt_addr(&r->new_port->addr), r->new_port->port); + fmt_and_decorate_addr(&r->new_port->addr), r->new_port->port); retval = -1; break; } @@ -3265,7 +3257,8 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) * any configured port. Kill 'em. */ SMARTLIST_FOREACH_BEGIN(listeners, connection_t *, conn) { log_notice(LD_NET, "Closing no-longer-configured %s on %s:%d", - conn_type_to_string(conn->type), conn->address, conn->port); + conn_type_to_string(conn->type), + fmt_and_decorate_addr(&conn->addr), conn->port); connection_close_immediate(conn); connection_mark_for_close(conn); } SMARTLIST_FOREACH_END(conn); @@ -5824,7 +5817,6 @@ connection_free_all(void) /* Unlink everything from the identity map. */ connection_or_clear_identity_map(); - connection_or_clear_ext_or_id_map(); /* Clear out our list of broken connections */ clear_broken_connection_map(0); @@ -5861,7 +5853,8 @@ clock_skew_warning, (const connection_t *conn, long apparent_skew, int trusted, char *ext_source = NULL, *warn = NULL; format_time_interval(dbuf, sizeof(dbuf), apparent_skew); if (conn) - tor_asprintf(&ext_source, "%s:%s:%d", source, conn->address, conn->port); + tor_asprintf(&ext_source, "%s:%s:%d", source, + fmt_and_decorate_addr(&conn->addr), conn->port); else ext_source = tor_strdup(source); log_fn(trusted ? LOG_WARN : LOG_INFO, domain, diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 77ab6f26c8..f30545eef0 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1224,7 +1224,7 @@ run_connection_housekeeping(int i, time_t now) * mark it now. */ log_info(LD_OR, "Expiring non-used OR connection to fd %d (%s:%d) [Too old].", - (int)conn->s, conn->address, conn->port); + (int)conn->s, fmt_and_decorate_addr(&conn->addr), conn->port); if (conn->state == OR_CONN_STATE_CONNECTING) connection_or_connect_failed(TO_OR_CONN(conn), END_OR_CONN_REASON_TIMEOUT, @@ -1234,7 +1234,7 @@ run_connection_housekeeping(int i, time_t now) if (past_keepalive) { /* We never managed to actually get this connection open and happy. */ log_info(LD_OR,"Expiring non-open OR connection to fd %d (%s:%d).", - (int)conn->s,conn->address, conn->port); + (int)conn->s, fmt_and_decorate_addr(&conn->addr), conn->port); connection_or_close_normally(TO_OR_CONN(conn), 0); } } else if (we_are_hibernating() && @@ -1244,7 +1244,7 @@ run_connection_housekeeping(int i, time_t now) * flush.*/ log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) " "[Hibernating or exiting].", - (int)conn->s,conn->address, conn->port); + (int)conn->s, fmt_and_decorate_addr(&conn->addr), conn->port); connection_or_close_normally(TO_OR_CONN(conn), 1); } else if (!have_any_circuits && now - or_conn->idle_timeout >= @@ -1252,7 +1252,7 @@ run_connection_housekeeping(int i, time_t now) log_info(LD_OR,"Expiring non-used OR connection %"PRIu64" to fd %d " "(%s:%d) [no circuits for %d; timeout %d; %scanonical].", (chan->global_identifier), - (int)conn->s, conn->address, conn->port, + (int)conn->s, fmt_and_decorate_addr(&conn->addr), conn->port, (int)(now - chan->timestamp_last_had_circuits), or_conn->idle_timeout, or_conn->is_canonical ? "" : "non"); @@ -1264,14 +1264,14 @@ run_connection_housekeeping(int i, time_t now) log_fn(LOG_PROTOCOL_WARN,LD_PROTOCOL, "Expiring stuck OR connection to fd %d (%s:%d). (%d bytes to " "flush; %d seconds since last write)", - (int)conn->s, conn->address, conn->port, + (int)conn->s, fmt_and_decorate_addr(&conn->addr), conn->port, (int)connection_get_outbuf_len(conn), (int)(now-conn->timestamp_last_write_allowed)); connection_or_close_normally(TO_OR_CONN(conn), 0); } else if (past_keepalive && !connection_get_outbuf_len(conn)) { /* send a padding cell */ log_fn(LOG_DEBUG,LD_OR,"Sending keepalive to (%s:%d)", - conn->address, conn->port); + fmt_and_decorate_addr(&conn->addr), conn->port); memset(&cell,0,sizeof(cell_t)); cell.command = CELL_PADDING; connection_or_write_cell_to_buf(&cell, or_conn); @@ -1937,7 +1937,11 @@ write_stats_file_callback(time_t now, const or_options_t *options) next_time_to_write_stats_files = next_write; } if (options->HiddenServiceStatistics) { - time_t next_write = rep_hist_hs_stats_write(now); + time_t next_write = rep_hist_hs_stats_write(now, false); + if (next_write && next_write < next_time_to_write_stats_files) + next_time_to_write_stats_files = next_write; + + next_write = rep_hist_hs_stats_write(now, true); if (next_write && next_write < next_time_to_write_stats_files) next_time_to_write_stats_files = next_write; } diff --git a/src/core/or/command.c b/src/core/or/command.c index 9226309ff7..e2bab87def 100644 --- a/src/core/or/command.c +++ b/src/core/or/command.c @@ -331,6 +331,13 @@ command_process_create_cell(cell_t *cell, channel_t *chan) return; } + /* Mark whether this circuit used TAP in case we need to use this + * information for onion service statistics later on. */ + if (create_cell->handshake_type == ONION_HANDSHAKE_TYPE_FAST || + create_cell->handshake_type == ONION_HANDSHAKE_TYPE_TAP) { + circ->used_legacy_circuit_handshake = true; + } + if (!channel_is_client(chan)) { /* remember create types we've seen, but don't remember them from * clients, to be extra conservative about client statistics. */ @@ -587,11 +594,27 @@ command_process_relay_cell(cell_t *cell, channel_t *chan) } /* If this is a cell in an RP circuit, count it as part of the - hidden service stats */ + onion service stats */ if (options->HiddenServiceStatistics && !CIRCUIT_IS_ORIGIN(circ) && - TO_OR_CIRCUIT(circ)->circuit_carries_hs_traffic_stats) { - rep_hist_seen_new_rp_cell(); + CONST_TO_OR_CIRCUIT(circ)->circuit_carries_hs_traffic_stats) { + /** We need to figure out of this is a v2 or v3 RP circuit to count it + * appropriately. v2 services always use the TAP legacy handshake to + * connect to the RP; we use this feature to distinguish between v2/v3. */ + bool is_v2 = false; + if (CONST_TO_OR_CIRCUIT(circ)->used_legacy_circuit_handshake) { + is_v2 = true; + } else if (CONST_TO_OR_CIRCUIT(circ)->rend_splice) { + /* If this is a client->RP circuit we need to check the spliced circuit + * (which is the service->RP circuit) to see if it was using TAP and + * hence if it's a v2 circuit. That's because client->RP circuits can + * still use ntor even on v2; but service->RP will always use TAP. */ + const or_circuit_t *splice = CONST_TO_OR_CIRCUIT(circ)->rend_splice; + if (splice->used_legacy_circuit_handshake) { + is_v2 = true; + } + } + rep_hist_seen_new_rp_cell(is_v2); } } diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index a33c64fe19..8adfd73e81 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -2209,7 +2209,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, } /* If this is a .exit hostname, strip off the .name.exit part, and - * see whether we're willing to connect there, and and otherwise handle the + * see whether we're willing to connect there, and otherwise handle the * .exit address. * * We'll set chosen_exit_name and/or close the connection as appropriate. diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index e3e81ed9cb..40c4441de6 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -686,6 +686,11 @@ connection_or_finished_flushing(or_connection_t *conn) /* PROXY_HAPROXY gets connected by receiving an ack. */ if (conn->proxy_type == PROXY_HAPROXY) { tor_assert(TO_CONN(conn)->proxy_state == PROXY_HAPROXY_WAIT_FOR_FLUSH); + IF_BUG_ONCE(buf_datalen(TO_CONN(conn)->inbuf) != 0) { + /* This should be impossible; we're not even reading. */ + connection_or_close_for_error(conn, 0); + return -1; + } TO_CONN(conn)->proxy_state = PROXY_CONNECTED; if (connection_tls_start_handshake(conn, 0) < 0) { diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h index 4e17b1c143..4da88889ce 100644 --- a/src/core/or/or_circuit_st.h +++ b/src/core/or/or_circuit_st.h @@ -63,6 +63,12 @@ struct or_circuit_t { * statistics. */ unsigned int circuit_carries_hs_traffic_stats : 1; + /** True iff this circuit was made with a CREATE_FAST cell, or a CREATE[2] + * cell with a TAP handshake. If this is the case and this is a rend circuit, + * this is a v2 circuit, otherwise if this is a rend circuit it's a v3 + * circuit. */ + bool used_legacy_circuit_handshake; + /** Number of cells that were removed from circuit queue; reset every * time when writing buffer stats to disk. */ uint32_t processed_cells; diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 232216c521..078024a9be 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -3853,7 +3853,7 @@ guards_retry_optimistic(const or_options_t *options) * Check if we are missing any crucial dirinfo for the guard subsystem to * work. Return NULL if everything went well, otherwise return a newly * allocated string with an informative error message. In the latter case, use - * the genreal descriptor information <b>using_mds</b>, <b>num_present</b> and + * the general descriptor information <b>using_mds</b>, <b>num_present</b> and * <b>num_usable</b> to improve the error message. */ char * guard_selection_get_err_str_if_dir_info_missing(guard_selection_t *gs, diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index fa4d919aa9..0703f43063 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -1757,26 +1757,14 @@ networkstatus_compute_consensus(smartlist_t *votes, } { - char *max_unmeasured_param = NULL; - /* XXXX Extract this code into a common function. Or don't! see #19011 */ - if (params) { - if (strcmpstart(params, "maxunmeasuredbw=") == 0) - max_unmeasured_param = params; - else - max_unmeasured_param = strstr(params, " maxunmeasuredbw="); - } - if (max_unmeasured_param) { - int ok = 0; - char *eq = strchr(max_unmeasured_param, '='); - if (eq) { - max_unmeasured_bw_kb = (uint32_t) - tor_parse_ulong(eq+1, 10, 1, UINT32_MAX, &ok, NULL); - if (!ok) { - log_warn(LD_DIR, "Bad element '%s' in max unmeasured bw param", - escaped(max_unmeasured_param)); - max_unmeasured_bw_kb = DEFAULT_MAX_UNMEASURED_BW_KB; - } - } + if (consensus_method < MIN_METHOD_FOR_CORRECT_BWWEIGHTSCALE) { + max_unmeasured_bw_kb = (int32_t) extract_param_buggy( + params, "maxunmeasuredbw", DEFAULT_MAX_UNMEASURED_BW_KB); + } else { + max_unmeasured_bw_kb = dirvote_get_intermediate_param_value( + param_list, "maxunmeasurdbw", DEFAULT_MAX_UNMEASURED_BW_KB); + if (max_unmeasured_bw_kb < 1) + max_unmeasured_bw_kb = 1; } } @@ -2326,38 +2314,16 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add_strdup(chunks, "directory-footer\n"); { - int64_t weight_scale = BW_WEIGHT_SCALE; - char *bw_weight_param = NULL; - - // Parse params, extract BW_WEIGHT_SCALE if present - // DO NOT use consensus_param_bw_weight_scale() in this code! - // The consensus is not formed yet! - /* XXXX Extract this code into a common function. Or not: #19011. */ - if (params) { - if (strcmpstart(params, "bwweightscale=") == 0) - bw_weight_param = params; - else - bw_weight_param = strstr(params, " bwweightscale="); - } - - if (bw_weight_param) { - int ok=0; - char *eq = strchr(bw_weight_param, '='); - if (eq) { - weight_scale = tor_parse_long(eq+1, 10, 1, INT32_MAX, &ok, - NULL); - if (!ok) { - log_warn(LD_DIR, "Bad element '%s' in bw weight param", - escaped(bw_weight_param)); - weight_scale = BW_WEIGHT_SCALE; - } - } else { - log_warn(LD_DIR, "Bad element '%s' in bw weight param", - escaped(bw_weight_param)); - weight_scale = BW_WEIGHT_SCALE; - } + int64_t weight_scale; + if (consensus_method < MIN_METHOD_FOR_CORRECT_BWWEIGHTSCALE) { + weight_scale = extract_param_buggy(params, "bwweightscale", + BW_WEIGHT_SCALE); + } else { + weight_scale = dirvote_get_intermediate_param_value( + param_list, "bwweightscale", BW_WEIGHT_SCALE); + if (weight_scale < 1) + weight_scale = 1; } - added_weights = networkstatus_compute_bw_weights_v10(chunks, G, M, E, D, T, weight_scale); } @@ -2459,6 +2425,53 @@ networkstatus_compute_consensus(smartlist_t *votes, return result; } +/** Extract the value of a parameter from a string encoding a list of + * parameters, badly. + * + * This is a deliberately buggy implementation, for backward compatibility + * with versions of Tor affected by #19011. Once all authorities have + * upgraded to consensus method 31 or later, then we can throw away this + * function. */ +STATIC int64_t +extract_param_buggy(const char *params, + const char *param_name, + int64_t default_value) +{ + int64_t value = default_value; + const char *param_str = NULL; + + if (params) { + char *prefix1 = NULL, *prefix2=NULL; + tor_asprintf(&prefix1, "%s=", param_name); + tor_asprintf(&prefix2, " %s=", param_name); + if (strcmpstart(params, prefix1) == 0) + param_str = params; + else + param_str = strstr(params, prefix2); + tor_free(prefix1); + tor_free(prefix2); + } + + if (param_str) { + int ok=0; + char *eq = strchr(param_str, '='); + if (eq) { + value = tor_parse_long(eq+1, 10, 1, INT32_MAX, &ok, NULL); + if (!ok) { + log_warn(LD_DIR, "Bad element '%s' in %s", + escaped(param_str), param_name); + value = default_value; + } + } else { + log_warn(LD_DIR, "Bad element '%s' in %s", + escaped(param_str), param_name); + value = default_value; + } + } + + return value; +} + /** Given a list of networkstatus_t for each vote, return a newly allocated * string containing the "package" lines for the vote. */ STATIC char * diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index f9441773a7..983b108e95 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -53,7 +53,7 @@ #define MIN_SUPPORTED_CONSENSUS_METHOD 28 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 30 +#define MAX_SUPPORTED_CONSENSUS_METHOD 31 /** * Lowest consensus method where microdescriptor lines are put in canonical @@ -65,6 +65,11 @@ * See #7869 */ #define MIN_METHOD_FOR_UNPADDED_NTOR_KEY 30 +/** Lowest consensus method for which we use the correct algorithm for + * extracting the bwweightscale= and maxunmeasuredbw= parameters. See #19011. + */ +#define MIN_METHOD_FOR_CORRECT_BWWEIGHTSCALE 31 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not * get confused with the above macros.) */ @@ -259,6 +264,9 @@ STATIC char *networkstatus_get_detached_signatures(smartlist_t *consensuses); STATIC microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method); +STATIC int64_t extract_param_buggy(const char *params, + const char *param_name, + int64_t default_value); /** The recommended relay protocols for this authority's votes. * Recommending a new protocol causes old tor versions to log a warning. diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c index c1334a7d27..765323df0d 100644 --- a/src/feature/hs/hs_cache.c +++ b/src/feature/hs/hs_cache.c @@ -20,6 +20,7 @@ #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/rend/rendcache.h" +#include "feature/stats/rephist.h" #include "feature/hs/hs_cache.h" @@ -175,7 +176,10 @@ cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc) * old HS protocol cache subsystem for which we are tied with. */ rend_cache_increment_allocation(cache_get_dir_entry_size(desc)); - /* XXX: Update HS statistics. We should have specific stats for v3. */ + /* Update HSv3 statistics */ + if (get_options()->HiddenServiceStatistics) { + rep_hist_hsdir_stored_maybe_new_v3_onion(desc->key); + } return 0; diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c index eaf99cf8b2..f0059a1a7c 100644 --- a/src/feature/hs/hs_circuit.c +++ b/src/feature/hs/hs_circuit.c @@ -1181,7 +1181,7 @@ hs_circ_send_introduce1(origin_circuit_t *intro_circ, /* We should never select an invalid rendezvous point in theory but if we * do, this function will fail to populate the introduce data. */ if (setup_introduce1_data(ip, exit_node, subcredential, &intro1_data) < 0) { - log_warn(LD_REND, "Unable to setup INTRODUCE1 data. The chosen rendezvous " + log_info(LD_REND, "Unable to setup INTRODUCE1 data. The chosen rendezvous " "point is unusable. Closing circuit."); goto close; } diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c index 4b4e268542..3b03bda1f5 100644 --- a/src/feature/hs/hs_client.c +++ b/src/feature/hs/hs_client.c @@ -1131,7 +1131,7 @@ handle_introduce_ack_success(origin_circuit_t *intro_circ) rend_circ = hs_circuitmap_get_established_rend_circ_client_side(rendezvous_cookie); if (rend_circ == NULL) { - log_warn(LD_REND, "Can't find any rendezvous circuit. Stopping"); + log_info(LD_REND, "Can't find any rendezvous circuit. Stopping"); goto end; } diff --git a/src/feature/hs_common/shared_random_client.c b/src/feature/hs_common/shared_random_client.c index 4e8a2942fc..b927e13a3b 100644 --- a/src/feature/hs_common/shared_random_client.c +++ b/src/feature/hs_common/shared_random_client.c @@ -34,12 +34,11 @@ srv_to_control_string(const sr_srv_t *srv) } /** - * If we have no consensus and we are not an authority, assume that this is - * the voting interval. We should never actually use this: only authorities - * should be trying to figure out the schedule when they don't have a - * consensus. - **/ + * If we have no consensus and we are not an authority, assume that this is the + * voting interval. This can be used while bootstrapping as a relay and we are + * asked to initialize HS stats (see rep_hist_hs_stats_init()) */ #define DEFAULT_NETWORK_VOTING_INTERVAL (3600) +#define TESTING_DEFAULT_NETWORK_VOTING_INTERVAL (20) /* This is an unpleasing workaround for tests. Our unit tests assume that we * are scheduling all of our shared random stuff as if we were a directory @@ -72,11 +71,13 @@ get_voting_interval(void) * It's better than falling back to the non-consensus case. */ interval = (int)(consensus->fresh_until - consensus->valid_after); } else { - /* We should never be reaching this point, since a client should never - * call this code unless they have some kind of a consensus. All we can - * do is hope that this network is using the default voting interval. */ - tor_assert_nonfatal_unreached_once(); - interval = DEFAULT_NETWORK_VOTING_INTERVAL; + /* We can reach this as a relay when bootstrapping and we are asked to + * initialize HS stats (see rep_hist_hs_stats_init()). */ + if (get_options()->TestingTorNetwork) { + interval = TESTING_DEFAULT_NETWORK_VOTING_INTERVAL; + } else { + interval = DEFAULT_NETWORK_VOTING_INTERVAL; + } } tor_assert(interval > 0); return interval; diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index ece3c9e059..5deec01f82 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -240,7 +240,7 @@ networkstatus_get_cache_fname,(int flav, } /** - * Read and and return the cached consensus of type <b>flavorname</b>. If + * Read and return the cached consensus of type <b>flavorname</b>. If * <b>unverified</b> is false, get the one we haven't verified. Return NULL if * the file isn't there. */ static tor_mmap_t * diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 03b158e68d..7387f0d1d3 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -1040,6 +1040,7 @@ nodelist_ensure_freshness(const networkstatus_t *ns) nodelist_set_consensus(ns); } } + /** Return a list of a node_t * for every node we know about. The caller * MUST NOT modify the list. (You can set and clear flags in the nodes if * you must, but you must not add or remove nodes.) */ diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c index 1bb8741e45..c45a0b463f 100644 --- a/src/feature/relay/ext_orport.c +++ b/src/feature/relay/ext_orport.c @@ -656,75 +656,17 @@ connection_ext_or_start_auth(or_connection_t *or_conn) return 0; } -/** Global map between Extended ORPort identifiers and OR - * connections. */ -static digestmap_t *orconn_ext_or_id_map = NULL; - -/** Remove the Extended ORPort identifier of <b>conn</b> from the - * global identifier list. Also, clear the identifier from the - * connection itself. */ -void -connection_or_remove_from_ext_or_id_map(or_connection_t *conn) -{ - or_connection_t *tmp; - if (!orconn_ext_or_id_map) - return; - if (!conn->ext_or_conn_id) - return; - - tmp = digestmap_remove(orconn_ext_or_id_map, conn->ext_or_conn_id); - if (!tor_digest_is_zero(conn->ext_or_conn_id)) - tor_assert(tmp == conn); - - memset(conn->ext_or_conn_id, 0, EXT_OR_CONN_ID_LEN); -} - -#ifdef TOR_UNIT_TESTS -/** Return the connection whose ext_or_id is <b>id</b>. Return NULL if no such - * connection is found. */ -or_connection_t * -connection_or_get_by_ext_or_id(const char *id) -{ - if (!orconn_ext_or_id_map) - return NULL; - return digestmap_get(orconn_ext_or_id_map, id); -} -#endif /* defined(TOR_UNIT_TESTS) */ - -/** Deallocate the global Extended ORPort identifier list */ -void -connection_or_clear_ext_or_id_map(void) -{ - digestmap_free(orconn_ext_or_id_map, NULL); - orconn_ext_or_id_map = NULL; -} - /** Creates an Extended ORPort identifier for <b>conn</b> and deposits * it into the global list of identifiers. */ void connection_or_set_ext_or_identifier(or_connection_t *conn) { char random_id[EXT_OR_CONN_ID_LEN]; - or_connection_t *tmp; - - if (!orconn_ext_or_id_map) - orconn_ext_or_id_map = digestmap_new(); - - /* Remove any previous identifiers: */ - if (conn->ext_or_conn_id && !tor_digest_is_zero(conn->ext_or_conn_id)) - connection_or_remove_from_ext_or_id_map(conn); - - do { - crypto_rand(random_id, sizeof(random_id)); - } while (digestmap_get(orconn_ext_or_id_map, random_id)); if (!conn->ext_or_conn_id) conn->ext_or_conn_id = tor_malloc_zero(EXT_OR_CONN_ID_LEN); memcpy(conn->ext_or_conn_id, random_id, EXT_OR_CONN_ID_LEN); - - tmp = digestmap_set(orconn_ext_or_id_map, random_id, conn); - tor_assert(!tmp); } /** Free any leftover allocated memory of the ext_orport.c subsystem. */ diff --git a/src/feature/relay/ext_orport.h b/src/feature/relay/ext_orport.h index 416c358397..b149f9eb1c 100644 --- a/src/feature/relay/ext_orport.h +++ b/src/feature/relay/ext_orport.h @@ -36,8 +36,6 @@ int connection_ext_or_start_auth(or_connection_t *or_conn); void connection_or_set_ext_or_identifier(or_connection_t *conn); -void connection_or_remove_from_ext_or_id_map(or_connection_t *conn); -void connection_or_clear_ext_or_id_map(void); int connection_ext_or_finished_flushing(or_connection_t *conn); int connection_ext_or_process_inbuf(or_connection_t *or_conn); char *get_ext_or_auth_cookie_file_name(void); @@ -71,10 +69,6 @@ connection_ext_or_process_inbuf(or_connection_t *conn) } #define connection_or_set_ext_or_identifier(conn) \ ((void)(conn)) -#define connection_or_remove_from_ext_or_id_map(conn) \ - ((void)(conn)) -#define connection_or_clear_ext_or_id_map() \ - STMT_NIL #define get_ext_or_auth_cookie_file_name() \ (NULL) @@ -94,7 +88,6 @@ STATIC int handle_client_auth_nonce(const char *client_nonce, #ifdef TOR_UNIT_TESTS extern uint8_t *ext_or_auth_cookie; extern int ext_or_auth_cookie_is_set; -or_connection_t *connection_or_get_by_ext_or_id(const char *id); #endif #endif /* defined(EXT_ORPORT_PRIVATE) */ diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 4bc71eb486..4fc970683b 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -831,6 +831,25 @@ router_initialize_tls_context(void) (unsigned int)lifetime); } +/** Announce URL to bridge status page. */ +STATIC void +router_announce_bridge_status_page(void) +{ + char fingerprint[FINGERPRINT_LEN + 1]; + + if (crypto_pk_get_hashed_fingerprint(get_server_identity_key(), + fingerprint) < 0) { + // LCOV_EXCL_START + log_err(LD_GENERAL, "Unable to compute bridge fingerprint"); + return; + // LCOV_EXCL_STOP + } + + log_notice(LD_GENERAL, "You can check the status of your bridge relay at " + "https://bridges.torproject.org/status?id=%s", + fingerprint); +} + /** Compute fingerprint (or hashed fingerprint if hashed is 1) and write * it to 'fingerprint' (or 'hashed-fingerprint'). Return 0 on success, or * -1 if Tor should die, @@ -1133,6 +1152,10 @@ init_keys(void) return -1; } + /* Display URL to bridge status page. */ + if (! public_server_mode(options)) + router_announce_bridge_status_page(); + if (!authdir_mode(options)) return 0; /* 6. [authdirserver only] load approved-routers file */ @@ -3311,6 +3334,11 @@ extrainfo_dump_to_string_stats_helper(smartlist_t *chunks, "hidserv-stats-end", now, &contents) > 0) { smartlist_add(chunks, contents); } + if (options->HiddenServiceStatistics && + load_stats_file("stats"PATH_SEPARATOR"hidserv-v3-stats", + "hidserv-v3-stats-end", now, &contents) > 0) { + smartlist_add(chunks, contents); + } if (options->EntryStatistics && load_stats_file("stats"PATH_SEPARATOR"entry-stats", "entry-stats-end", now, &contents) > 0) { diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index aa03c27142..9556a66e68 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -129,6 +129,7 @@ void router_free_all(void); STATIC void get_platform_str(char *platform, size_t len); STATIC int router_write_fingerprint(int hashed, int ed25519_identity); STATIC smartlist_t *get_my_declared_family(const or_options_t *options); +STATIC void router_announce_bridge_status_page(void); STATIC int load_stats_file(const char *filename, const char *ts_tag, time_t now, char **out); diff --git a/src/feature/relay/selftest.c b/src/feature/relay/selftest.c index 86b1533be1..137c478fef 100644 --- a/src/feature/relay/selftest.c +++ b/src/feature/relay/selftest.c @@ -277,7 +277,7 @@ router_do_orport_reachability_checks(const routerinfo_t *me, if (!orport_reachable) { /* Only log if we are actually doing a reachability test to learn if our * ORPort is reachable. Else, this prints a log notice if we are simply - * opening a bandwidth testing circuit even do we are reachable. */ + * opening a bandwidth testing circuit even though we are reachable. */ inform_testing_reachability(&ap->addr, ap->port, false); } diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c index 04f6390a7f..a471c8f463 100644 --- a/src/feature/rend/rendcache.c +++ b/src/feature/rend/rendcache.c @@ -718,7 +718,7 @@ rend_cache_store_v2_desc_as_dir(const char *desc) safe_str(desc_id_base32), (int)encoded_size); /* Statistics: Note down this potentially new HS. */ if (options->HiddenServiceStatistics) { - rep_hist_stored_maybe_new_hs(e->parsed->pk); + rep_hist_hsdir_stored_maybe_new_v2_onion(e->parsed->pk); } number_stored++; diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c index 3c22fda3b8..f8d7887e65 100644 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@ -1710,123 +1710,248 @@ rep_hist_log_circuit_handshake_stats(time_t now) /** Start of the current hidden service stats interval or 0 if we're * not collecting hidden service statistics. */ -static time_t start_of_hs_stats_interval; +static time_t start_of_hs_v2_stats_interval; -/** Carries the various hidden service statistics, and any other - * information needed. */ -typedef struct hs_stats_t { - /** How many relay cells have we seen as rendezvous points? */ - uint64_t rp_relay_cells_seen; +/** Our v2 statistics structure singleton. */ +static hs_v2_stats_t *hs_v2_stats = NULL; - /** Set of unique public key digests we've seen this stat period - * (could also be implemented as sorted smartlist). */ - digestmap_t *onions_seen_this_period; -} hs_stats_t; +/** HSv2 stats */ -/** Our statistics structure singleton. */ -static hs_stats_t *hs_stats = NULL; - -/** Allocate, initialize and return an hs_stats_t structure. */ -static hs_stats_t * -hs_stats_new(void) +/** Allocate, initialize and return an hs_v2_stats_t structure. */ +static hs_v2_stats_t * +hs_v2_stats_new(void) { - hs_stats_t *new_hs_stats = tor_malloc_zero(sizeof(hs_stats_t)); - new_hs_stats->onions_seen_this_period = digestmap_new(); + hs_v2_stats_t *new_hs_v2_stats = tor_malloc_zero(sizeof(hs_v2_stats_t)); + new_hs_v2_stats->v2_onions_seen_this_period = digestmap_new(); - return new_hs_stats; + return new_hs_v2_stats; } -#define hs_stats_free(val) \ - FREE_AND_NULL(hs_stats_t, hs_stats_free_, (val)) +#define hs_v2_stats_free(val) \ + FREE_AND_NULL(hs_v2_stats_t, hs_v2_stats_free_, (val)) -/** Free an hs_stats_t structure. */ +/** Free an hs_v2_stats_t structure. */ static void -hs_stats_free_(hs_stats_t *victim_hs_stats) +hs_v2_stats_free_(hs_v2_stats_t *victim_hs_v2_stats) { - if (!victim_hs_stats) { + if (!victim_hs_v2_stats) { return; } - digestmap_free(victim_hs_stats->onions_seen_this_period, NULL); - tor_free(victim_hs_stats); + digestmap_free(victim_hs_v2_stats->v2_onions_seen_this_period, NULL); + tor_free(victim_hs_v2_stats); } -/** Initialize hidden service statistics. */ +/** Clear history of hidden service statistics and set the measurement + * interval start to <b>now</b>. */ +static void +rep_hist_reset_hs_v2_stats(time_t now) +{ + if (!hs_v2_stats) { + hs_v2_stats = hs_v2_stats_new(); + } + + hs_v2_stats->rp_v2_relay_cells_seen = 0; + + digestmap_free(hs_v2_stats->v2_onions_seen_this_period, NULL); + hs_v2_stats->v2_onions_seen_this_period = digestmap_new(); + + start_of_hs_v2_stats_interval = now; +} + +/** As HSDirs, we saw another v2 onion with public key <b>pubkey</b>. Check + * whether we have counted it before, if not count it now! */ void -rep_hist_hs_stats_init(time_t now) +rep_hist_hsdir_stored_maybe_new_v2_onion(const crypto_pk_t *pubkey) +{ + char pubkey_hash[DIGEST_LEN]; + + if (!hs_v2_stats) { + return; // We're not collecting stats + } + + /* Get the digest of the pubkey which will be used to detect whether + we've seen this hidden service before or not. */ + if (crypto_pk_get_digest(pubkey, pubkey_hash) < 0) { + /* This fail should not happen; key has been validated by + descriptor parsing code first. */ + return; + } + + /* Check if this is the first time we've seen this hidden + service. If it is, count it as new. */ + if (!digestmap_get(hs_v2_stats->v2_onions_seen_this_period, + pubkey_hash)) { + digestmap_set(hs_v2_stats->v2_onions_seen_this_period, + pubkey_hash, (void*)(uintptr_t)1); + } +} + +/*** HSv3 stats ******/ + +/** Start of the current hidden service stats interval or 0 if we're not + * collecting hidden service statistics. + * + * This is particularly important for v3 statistics since this variable + * controls the start time of initial v3 stats collection. It's initialized by + * rep_hist_hs_stats_init() to the next time period start (i.e. 12:00UTC), and + * should_collect_v3_stats() ensures that functions that collect v3 stats do + * not do so sooner than that. + * + * Collecting stats from 12:00UTC to 12:00UTC is extremely important for v3 + * stats because rep_hist_hsdir_stored_maybe_new_v3_onion() uses the blinded + * key of each onion service as its double-counting index. Onion services + * rotate their descriptor at around 00:00UTC which means that their blinded + * key also changes around that time. However the precise time that onion + * services rotate their descriptors is actually when they fetch a new + * 00:00UTC consensus and that happens at a random time (e.g. it can even + * happen at 02:00UTC). This means that if we started keeping v3 stats at + * around 00:00UTC we wouldn't be able to tell when onion services change + * their blinded key and hence we would double count an unpredictable amount + * of them (for example, if an onion service fetches the 00:00UTC consensus at + * 01:00UTC it would upload to its old HSDir at 00:45UTC, and then to a + * different HSDir at 01:50UTC). + * + * For this reason, we start collecting statistics at 12:00UTC. This way we + * know that by the time we stop collecting statistics for that time period 24 + * hours later, all the onion services have switched to their new blinded + * key. This way we can predict much better how much double counting has been + * performed. + */ +static time_t start_of_hs_v3_stats_interval; + +/** Our v3 statistics structure singleton. */ +static hs_v3_stats_t *hs_v3_stats = NULL; + +/** Allocate, initialize and return an hs_v3_stats_t structure. */ +static hs_v3_stats_t * +hs_v3_stats_new(void) +{ + hs_v3_stats_t *new_hs_v3_stats = tor_malloc_zero(sizeof(hs_v3_stats_t)); + new_hs_v3_stats->v3_onions_seen_this_period = digest256map_new(); + + return new_hs_v3_stats; +} + +#define hs_v3_stats_free(val) \ + FREE_AND_NULL(hs_v3_stats_t, hs_v3_stats_free_, (val)) + +/** Free an hs_v3_stats_t structure. */ +static void +hs_v3_stats_free_(hs_v3_stats_t *victim_hs_v3_stats) { - if (!hs_stats) { - hs_stats = hs_stats_new(); + if (!victim_hs_v3_stats) { + return; } - start_of_hs_stats_interval = now; + digest256map_free(victim_hs_v3_stats->v3_onions_seen_this_period, NULL); + tor_free(victim_hs_v3_stats); } /** Clear history of hidden service statistics and set the measurement * interval start to <b>now</b>. */ static void -rep_hist_reset_hs_stats(time_t now) +rep_hist_reset_hs_v3_stats(time_t now) { - if (!hs_stats) { - hs_stats = hs_stats_new(); + if (!hs_v3_stats) { + hs_v3_stats = hs_v3_stats_new(); } - hs_stats->rp_relay_cells_seen = 0; + digest256map_free(hs_v3_stats->v3_onions_seen_this_period, NULL); + hs_v3_stats->v3_onions_seen_this_period = digest256map_new(); - digestmap_free(hs_stats->onions_seen_this_period, NULL); - hs_stats->onions_seen_this_period = digestmap_new(); + hs_v3_stats->rp_v3_relay_cells_seen = 0; - start_of_hs_stats_interval = now; + start_of_hs_v3_stats_interval = now; } -/** Stop collecting hidden service stats in a way that we can re-start - * doing so in rep_hist_buffer_stats_init(). */ -void -rep_hist_hs_stats_term(void) +/** Return true if it's a good time to collect v3 stats. + * + * v3 stats have a strict stats collection period (from 12:00UTC to 12:00UTC + * on the real network). We don't want to collect statistics if (for example) + * we just booted and it's 03:00UTC; we will wait until 12:00UTC before we + * start collecting statistics to make sure that the final result represents + * the whole collection period. This behavior is controlled by + * rep_hist_hs_stats_init(). + */ +MOCK_IMPL(STATIC bool, +should_collect_v3_stats,(void)) { - rep_hist_reset_hs_stats(0); + return start_of_hs_v3_stats_interval <= approx_time(); } -/** We saw a new HS relay cell, Count it! */ +/** We just received a new descriptor with <b>blinded_key</b>. See if we've + * seen this blinded key before, and if not add it to the stats. */ void -rep_hist_seen_new_rp_cell(void) +rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key) { - if (!hs_stats) { - return; // We're not collecting stats + /* Return early if we don't collect HSv3 stats, or if it's not yet the time + * to collect them. */ + if (!hs_v3_stats || !should_collect_v3_stats()) { + return; } - hs_stats->rp_relay_cells_seen++; + bool seen_before = + !!digest256map_get(hs_v3_stats->v3_onions_seen_this_period, + blinded_key); + + log_info(LD_GENERAL, "Considering v3 descriptor with %s (%sseen before)", + safe_str(hex_str((char*)blinded_key, 32)), + seen_before ? "" : "not "); + + /* Count it if we haven't seen it before. */ + if (!seen_before) { + digest256map_set(hs_v3_stats->v3_onions_seen_this_period, + blinded_key, (void*)(uintptr_t)1); + } } -/** As HSDirs, we saw another hidden service with public key - * <b>pubkey</b>. Check whether we have counted it before, if not - * count it now! */ +/** We saw a new HS relay cell: count it! + * If <b>is_v2</b> is set then it's a v2 RP cell, otherwise it's a v3. */ void -rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey) +rep_hist_seen_new_rp_cell(bool is_v2) { - char pubkey_hash[DIGEST_LEN]; + log_debug(LD_GENERAL, "New RP cell (%d)", is_v2); - if (!hs_stats) { - return; // We're not collecting stats + if (is_v2 && hs_v2_stats) { + hs_v2_stats->rp_v2_relay_cells_seen++; + } else if (!is_v2 && hs_v3_stats && should_collect_v3_stats()) { + hs_v3_stats->rp_v3_relay_cells_seen++; } +} - /* Get the digest of the pubkey which will be used to detect whether - we've seen this hidden service before or not. */ - if (crypto_pk_get_digest(pubkey, pubkey_hash) < 0) { - /* This fail should not happen; key has been validated by - descriptor parsing code first. */ - return; +/** Generic HS stats code */ + +/** Initialize v2 and v3 hidden service statistics. */ +void +rep_hist_hs_stats_init(time_t now) +{ + if (!hs_v2_stats) { + hs_v2_stats = hs_v2_stats_new(); } - /* Check if this is the first time we've seen this hidden - service. If it is, count it as new. */ - if (!digestmap_get(hs_stats->onions_seen_this_period, - pubkey_hash)) { - digestmap_set(hs_stats->onions_seen_this_period, - pubkey_hash, (void*)(uintptr_t)1); + /* Start collecting v2 stats straight away */ + start_of_hs_v2_stats_interval = now; + + if (!hs_v3_stats) { + hs_v3_stats = hs_v3_stats_new(); } + + /* Start collecting v3 stats at the next 12:00 UTC */ + start_of_hs_v3_stats_interval = hs_get_start_time_of_next_time_period(now); +} + +/** Stop collecting hidden service stats in a way that we can re-start + * doing so in rep_hist_buffer_stats_init(). */ +void +rep_hist_hs_stats_term(void) +{ + rep_hist_reset_hs_v2_stats(0); + rep_hist_reset_hs_v3_stats(0); } +/** Stats reporting code */ + /* The number of cells that are supposed to be hidden from the adversary * by adding noise from the Laplace distribution. This value, divided by * EPSILON, is Laplace parameter b. It must be greater than 0. */ @@ -1851,58 +1976,69 @@ rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey) #define ONIONS_SEEN_BIN_SIZE 8 /** Allocate and return a string containing hidden service stats that - * are meant to be placed in the extra-info descriptor. */ -static char * -rep_hist_format_hs_stats(time_t now) + * are meant to be placed in the extra-info descriptor. + * + * Function works for both v2 and v3 stats depending on <b>is_v3</b>. */ +STATIC char * +rep_hist_format_hs_stats(time_t now, bool is_v3) { char t[ISO_TIME_LEN+1]; char *hs_stats_string; - int64_t obfuscated_cells_seen; - int64_t obfuscated_onions_seen; + int64_t obfuscated_onions_seen, obfuscated_cells_seen; + + uint64_t rp_cells_seen = is_v3 ? + hs_v3_stats->rp_v3_relay_cells_seen : hs_v2_stats->rp_v2_relay_cells_seen; + size_t onions_seen = is_v3 ? + digest256map_size(hs_v3_stats->v3_onions_seen_this_period) : + digestmap_size(hs_v2_stats->v2_onions_seen_this_period); + time_t start_of_hs_stats_interval = is_v3 ? + start_of_hs_v3_stats_interval : start_of_hs_v2_stats_interval; uint64_t rounded_cells_seen - = round_uint64_to_next_multiple_of(hs_stats->rp_relay_cells_seen, - REND_CELLS_BIN_SIZE); + = round_uint64_to_next_multiple_of(rp_cells_seen, REND_CELLS_BIN_SIZE); rounded_cells_seen = MIN(rounded_cells_seen, INT64_MAX); obfuscated_cells_seen = add_laplace_noise((int64_t)rounded_cells_seen, crypto_rand_double(), REND_CELLS_DELTA_F, REND_CELLS_EPSILON); uint64_t rounded_onions_seen = - round_uint64_to_next_multiple_of((size_t)digestmap_size( - hs_stats->onions_seen_this_period), - ONIONS_SEEN_BIN_SIZE); + round_uint64_to_next_multiple_of(onions_seen, ONIONS_SEEN_BIN_SIZE); rounded_onions_seen = MIN(rounded_onions_seen, INT64_MAX); obfuscated_onions_seen = add_laplace_noise((int64_t)rounded_onions_seen, crypto_rand_double(), ONIONS_SEEN_DELTA_F, ONIONS_SEEN_EPSILON); format_iso_time(t, now); - tor_asprintf(&hs_stats_string, "hidserv-stats-end %s (%d s)\n" - "hidserv-rend-relayed-cells %"PRId64" delta_f=%d " - "epsilon=%.2f bin_size=%d\n" - "hidserv-dir-onions-seen %"PRId64" delta_f=%d " - "epsilon=%.2f bin_size=%d\n", + tor_asprintf(&hs_stats_string, "%s %s (%u s)\n" + "%s %"PRId64" delta_f=%d epsilon=%.2f bin_size=%d\n" + "%s %"PRId64" delta_f=%d epsilon=%.2f bin_size=%d\n", + is_v3 ? "hidserv-v3-stats-end" : "hidserv-stats-end", t, (unsigned) (now - start_of_hs_stats_interval), - (obfuscated_cells_seen), REND_CELLS_DELTA_F, + is_v3 ? + "hidserv-rend-v3-relayed-cells" : "hidserv-rend-relayed-cells", + obfuscated_cells_seen, REND_CELLS_DELTA_F, REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE, - (obfuscated_onions_seen), - ONIONS_SEEN_DELTA_F, + is_v3 ? "hidserv-dir-v3-onions-seen" :"hidserv-dir-onions-seen", + obfuscated_onions_seen, ONIONS_SEEN_DELTA_F, ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE); return hs_stats_string; } /** If 24 hours have passed since the beginning of the current HS - * stats period, write buffer stats to $DATADIR/stats/hidserv-stats + * stats period, write buffer stats to $DATADIR/stats/hidserv-v3-stats * (possibly overwriting an existing file) and reset counters. Return * when we would next want to write buffer stats or 0 if we never want to - * write. */ + * write. Function works for both v2 and v3 stats depending on <b>is_v3</b>. + */ time_t -rep_hist_hs_stats_write(time_t now) +rep_hist_hs_stats_write(time_t now, bool is_v3) { char *str = NULL; + time_t start_of_hs_stats_interval = is_v3 ? + start_of_hs_v3_stats_interval : start_of_hs_v2_stats_interval; + if (!start_of_hs_stats_interval) { return 0; /* Not initialized. */ } @@ -1912,15 +2048,20 @@ rep_hist_hs_stats_write(time_t now) } /* Generate history string. */ - str = rep_hist_format_hs_stats(now); + str = rep_hist_format_hs_stats(now, is_v3); /* Reset HS history. */ - rep_hist_reset_hs_stats(now); + if (is_v3) { + rep_hist_reset_hs_v3_stats(now); + } else { + rep_hist_reset_hs_v2_stats(now); + } /* Try to write to disk. */ if (!check_or_create_data_subdir("stats")) { - write_to_data_subdir("stats", "hidserv-stats", str, - "hidden service stats"); + write_to_data_subdir("stats", + is_v3 ? "hidserv-v3-stats" : "hidserv-stats", + str, "hidden service stats"); } done: @@ -2134,7 +2275,8 @@ rep_hist_log_link_protocol_counts(void) void rep_hist_free_all(void) { - hs_stats_free(hs_stats); + hs_v2_stats_free(hs_v2_stats); + hs_v3_stats_free(hs_v3_stats); digestmap_free(history_map, free_or_history); tor_free(exit_bytes_read); @@ -2155,3 +2297,19 @@ rep_hist_free_all(void) tor_assert_nonfatal(rephist_total_alloc == 0); tor_assert_nonfatal_once(rephist_total_num == 0); } + +#ifdef TOR_UNIT_TESTS +/* only exists for unit tests: get HSv2 stats object */ +const hs_v2_stats_t * +rep_hist_get_hs_v2_stats(void) +{ + return hs_v2_stats; +} + +/* only exists for unit tests: get HSv2 stats object */ +const hs_v3_stats_t * +rep_hist_get_hs_v3_stats(void) +{ + return hs_v3_stats; +} +#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h index c9ebc5c328..de27b16ae0 100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@ -65,10 +65,14 @@ MOCK_DECL(int, rep_hist_get_circuit_handshake_assigned, (uint16_t type)); void rep_hist_hs_stats_init(time_t now); void rep_hist_hs_stats_term(void); -time_t rep_hist_hs_stats_write(time_t now); -char *rep_hist_get_hs_stats_string(void); -void rep_hist_seen_new_rp_cell(void); -void rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey); +time_t rep_hist_hs_stats_write(time_t now, bool is_v3); + +char *rep_hist_get_hs_v2_stats_string(void); +void rep_hist_seen_new_rp_cell(bool is_v2); +void rep_hist_hsdir_stored_maybe_new_v2_onion(const crypto_pk_t *pubkey); + +char *rep_hist_get_hs_v3_stats_string(void); +void rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key); void rep_hist_free_all(void); @@ -83,6 +87,40 @@ extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; #endif +#ifdef REPHIST_PRIVATE +/** Carries the various hidden service statistics, and any other + * information needed. */ +typedef struct hs_v2_stats_t { + /** How many v2 relay cells have we seen as rendezvous points? */ + uint64_t rp_v2_relay_cells_seen; + + /** Set of unique public key digests we've seen this stat period + * (could also be implemented as sorted smartlist). */ + digestmap_t *v2_onions_seen_this_period; +} hs_v2_stats_t; + +/** Structure that contains the various statistics we keep about v3 + * services. + * + * Because of the time period logic of v3 services, v3 statistics are more + * sensitive to time than v2 stats. For this reason, we collect v3 + * statistics strictly from 12:00UTC to 12:00UTC as dictated by + * 'start_of_hs_v3_stats_interval'. + **/ +typedef struct hs_v3_stats_t { + /** How many v3 relay cells have we seen as a rendezvous point? */ + uint64_t rp_v3_relay_cells_seen; + + /* The number of unique v3 onion descriptors (actually, unique v3 blind keys) + * we've seen during the measurement period */ + digest256map_t *v3_onions_seen_this_period; +} hs_v3_stats_t; + +MOCK_DECL(STATIC bool, should_collect_v3_stats,(void)); + +STATIC char *rep_hist_format_hs_stats(time_t now, bool is_v3); +#endif /* defined(REPHIST_PRIVATE) */ + /** * Represents the type of a cell for padding accounting */ @@ -108,4 +146,11 @@ void rep_hist_reset_padding_counts(void); void rep_hist_prep_published_padding_counts(time_t now); void rep_hist_padding_count_timers(uint64_t num_timers); +#ifdef TOR_UNIT_TESTS +struct hs_v2_stats_t; +const struct hs_v2_stats_t *rep_hist_get_hs_v2_stats(void); +struct hs_v3_stats_t; +const struct hs_v3_stats_t *rep_hist_get_hs_v3_stats(void); +#endif + #endif /* !defined(TOR_REPHIST_H) */ diff --git a/src/lib/lock/compat_mutex.h b/src/lib/lock/compat_mutex.h index 5631993cc4..518ba96b53 100644 --- a/src/lib/lock/compat_mutex.h +++ b/src/lib/lock/compat_mutex.h @@ -39,8 +39,15 @@ /** A generic lock structure for multithreaded builds. */ typedef struct tor_mutex_t { #if defined(USE_WIN32_THREADS) - /** Windows-only: on windows, we implement locks with CRITICAL_SECTIONS. */ - CRITICAL_SECTION mutex; + /** Windows-only: on windows, we implement locks with SRW locks. */ + SRWLOCK mutex; + /** For recursive lock support (SRW locks are not recursive) */ + enum mutex_type_t { + NON_RECURSIVE = 0, + RECURSIVE + } type; + LONG lock_owner; // id of the thread that owns the lock + int lock_count; // number of times the lock is held recursively #elif defined(USE_PTHREADS) /** Pthreads-only: with pthreads, we implement locks with * pthread_mutex_t. */ diff --git a/src/lib/lock/compat_mutex_winthreads.c b/src/lib/lock/compat_mutex_winthreads.c index 5fe6870a93..151a7b80f7 100644 --- a/src/lib/lock/compat_mutex_winthreads.c +++ b/src/lib/lock/compat_mutex_winthreads.c @@ -9,6 +9,23 @@ * \brief Implement the tor_mutex API using CRITICAL_SECTION. **/ +#include "orconfig.h" + +/* For SRW locks support */ +#ifndef WINVER +#error "orconfig.h didn't define WINVER" +#endif +#ifndef _WIN32_WINNT +#error "orconfig.h didn't define _WIN32_WINNT" +#endif +#if WINVER < 0x0600 +#error "winver too low" +#endif +#if _WIN32_WINNT < 0x0600 +#error "winver too low" +#endif + +#include <windows.h> #include "lib/lock/compat_mutex.h" #include "lib/err/torerr.h" @@ -20,27 +37,78 @@ tor_locking_init(void) void tor_mutex_init(tor_mutex_t *m) { - InitializeCriticalSection(&m->mutex); + m->type = RECURSIVE; + m->lock_owner = 0; + m->lock_count = 0; + InitializeSRWLock(&m->mutex); } void tor_mutex_init_nonrecursive(tor_mutex_t *m) { - InitializeCriticalSection(&m->mutex); + m->type = NON_RECURSIVE; + InitializeSRWLock(&m->mutex); } void tor_mutex_uninit(tor_mutex_t *m) { - DeleteCriticalSection(&m->mutex); + (void) m; +} + +static void +tor_mutex_acquire_recursive(tor_mutex_t *m) +{ + LONG thread_id = GetCurrentThreadId(); + // use InterlockedCompareExchange to perform an atomic read + LONG lock_owner = InterlockedCompareExchange(&m->lock_owner, 0, 0); + if (thread_id == lock_owner) { + ++m->lock_count; + return; + } + AcquireSRWLockExclusive(&m->mutex); + InterlockedExchange(&m->lock_owner, thread_id); + m->lock_count = 1; +} + +static void +tor_mutex_acquire_nonrecursive(tor_mutex_t *m) +{ + AcquireSRWLockExclusive(&m->mutex); } + void tor_mutex_acquire(tor_mutex_t *m) { raw_assert(m); - EnterCriticalSection(&m->mutex); + if (m->type == NON_RECURSIVE) { + tor_mutex_acquire_nonrecursive(m); + } else { + tor_mutex_acquire_recursive(m); + } +} + +static void +tor_mutex_release_recursive(tor_mutex_t *m) +{ + if (--m->lock_count) { + return; + } + InterlockedExchange(&m->lock_owner, 0); + ReleaseSRWLockExclusive(&m->mutex); } + +static void +tor_mutex_release_nonrecursive(tor_mutex_t *m) +{ + ReleaseSRWLockExclusive(&m->mutex); +} + void tor_mutex_release(tor_mutex_t *m) { - LeaveCriticalSection(&m->mutex); + if (m->type == NON_RECURSIVE) { + tor_mutex_release_nonrecursive(m); + } else { + tor_mutex_release_recursive(m); + } } diff --git a/src/lib/thread/compat_winthreads.c b/src/lib/thread/compat_winthreads.c index fcc9c0279b..a6213aa46a 100644 --- a/src/lib/thread/compat_winthreads.c +++ b/src/lib/thread/compat_winthreads.c @@ -144,13 +144,17 @@ tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value) int tor_cond_wait(tor_cond_t *cond, tor_mutex_t *lock_, const struct timeval *tv) { - CRITICAL_SECTION *lock = &lock_->mutex; + // recursive SRW locks are not supported because they need extra logic for + // acquiring and releasing but SleepConditionVariableSRW will use the OS + // lock relase function which lacks our extra logic + tor_assert(lock_->type == NON_RECURSIVE); + SRWLOCK *lock = &lock_->mutex; DWORD ms = INFINITE; if (tv) { ms = tv->tv_sec*1000 + (tv->tv_usec+999)/1000; } - BOOL ok = SleepConditionVariableCS(&cond->cond, lock, ms); + BOOL ok = SleepConditionVariableSRW(&cond->cond, lock, ms, 0); if (!ok) { DWORD err = GetLastError(); if (err == ERROR_TIMEOUT) { diff --git a/src/test/hs_build_address.py b/src/test/hs_build_address.py index 91864eabcb..216b7626bf 100644 --- a/src/test/hs_build_address.py +++ b/src/test/hs_build_address.py @@ -10,17 +10,21 @@ import base64 # Python 3.6+, the SHA3 is available in hashlib natively. Else this requires # the pysha3 package (pip install pysha3). +TEST_INPUT = b"Hello World" if sys.version_info < (3, 6): import sha3 + m = sha3.sha3_256(TEST_INPUT) +else: + m = hashlib.sha3_256(TEST_INPUT) # Test vector to make sure the right sha3 version will be used. pysha3 < 1.0 # used the old Keccak implementation. During the finalization of SHA3, NIST # changed the delimiter suffix from 0x01 to 0x06. The Keccak sponge function # stayed the same. pysha3 1.0 provides the previous Keccak hash, too. TEST_VALUE = "e167f68d6563d75bb25f3aa49c29ef612d41352dc00606de7cbd630bb2665f51" -if TEST_VALUE != sha3.sha3_256(b"Hello World").hexdigest(): +if TEST_VALUE != m.hexdigest(): print("pysha3 version is < 1.0. Please install from:") - print("https://github.com/tiran/pysha3https://github.com/tiran/pysha3") + print("https://github.com/tiran/pysha3") sys.exit(1) # Checksum is built like so: @@ -28,7 +32,11 @@ if TEST_VALUE != sha3.sha3_256(b"Hello World").hexdigest(): PREFIX = ".onion checksum".encode() # 32 bytes ed25519 pubkey from first test vector of # https://tools.ietf.org/html/draft-josefsson-eddsa-ed25519-02#section-6 -PUBKEY = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a".decode('hex') +PUBKEY_STRING = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a" +if sys.version_info < (3, 0): + PUBKEY = PUBKEY_STRING.decode('hex') +else: + PUBKEY = bytes.fromhex(PUBKEY_STRING) # Version 3 is proposal224 VERSION = 3 diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c index e9aafa4760..e1ecf9fe56 100644 --- a/src/test/hs_test_helpers.c +++ b/src/test/hs_test_helpers.c @@ -134,7 +134,8 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now, * points are added. */ static hs_descriptor_t * hs_helper_build_hs_desc_impl(unsigned int no_ip, - const ed25519_keypair_t *signing_kp) + const ed25519_keypair_t *signing_kp, + uint64_t rev_counter) { int ret; int i; @@ -161,7 +162,7 @@ hs_helper_build_hs_desc_impl(unsigned int no_ip, &signing_kp->pubkey, now, 3600, CERT_FLAG_INCLUDE_SIGNING_KEY); tt_assert(desc->plaintext_data.signing_key_cert); - desc->plaintext_data.revision_counter = 42; + desc->plaintext_data.revision_counter = rev_counter; desc->plaintext_data.lifetime_sec = 3 * 60 * 60; hs_get_subcredential(&signing_kp->pubkey, &blinded_kp.pubkey, @@ -226,18 +227,26 @@ hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp, subcred_out); } +/* Build a descriptor with a specific rev counter. */ +hs_descriptor_t * +hs_helper_build_hs_desc_with_rev_counter(const ed25519_keypair_t *signing_kp, + uint64_t revision_counter) +{ + return hs_helper_build_hs_desc_impl(0, signing_kp, revision_counter); +} + /* Build a descriptor with introduction points. */ hs_descriptor_t * hs_helper_build_hs_desc_with_ip(const ed25519_keypair_t *signing_kp) { - return hs_helper_build_hs_desc_impl(0, signing_kp); + return hs_helper_build_hs_desc_impl(0, signing_kp, 42); } /* Build a descriptor without any introduction points. */ hs_descriptor_t * hs_helper_build_hs_desc_no_ip(const ed25519_keypair_t *signing_kp) { - return hs_helper_build_hs_desc_impl(1, signing_kp); + return hs_helper_build_hs_desc_impl(1, signing_kp, 42); } hs_descriptor_t * @@ -247,7 +256,7 @@ hs_helper_build_hs_desc_with_client_auth( const ed25519_keypair_t *signing_kp) { curve25519_keypair_t auth_ephemeral_kp; - hs_descriptor_t *desc = hs_helper_build_hs_desc_impl(0, signing_kp); + hs_descriptor_t *desc = hs_helper_build_hs_desc_impl(0, signing_kp, 42); hs_desc_authorized_client_t *desc_client; /* The number of client authorized auth has tobe a multiple of diff --git a/src/test/hs_test_helpers.h b/src/test/hs_test_helpers.h index 23d11f2a4a..e22295b660 100644 --- a/src/test/hs_test_helpers.h +++ b/src/test/hs_test_helpers.h @@ -17,6 +17,10 @@ hs_descriptor_t *hs_helper_build_hs_desc_no_ip( const ed25519_keypair_t *signing_kp); hs_descriptor_t *hs_helper_build_hs_desc_with_ip( const ed25519_keypair_t *signing_kp); +hs_descriptor_t * +hs_helper_build_hs_desc_with_rev_counter(const ed25519_keypair_t *signing_kp, + uint64_t revision_counter); + hs_descriptor_t *hs_helper_build_hs_desc_with_client_auth( const uint8_t *descriptor_cookie, const curve25519_public_key_t *client_pk, diff --git a/src/test/test_dirvote.c b/src/test/test_dirvote.c index b5e57ad071..d92d1aaf90 100644 --- a/src/test/test_dirvote.c +++ b/src/test/test_dirvote.c @@ -656,6 +656,30 @@ done: ROUTER_FREE(pppp); } +static void +test_dirvote_parse_param_buggy(void *arg) +{ + (void)arg; + + /* Tests for behavior with bug emulation to migrate away from bug 19011. */ + tt_i64_op(extract_param_buggy("blah blah", "bwweightscale", 10000), + OP_EQ, 10000); + tt_i64_op(extract_param_buggy("bwweightscale=7", "bwweightscale", 10000), + OP_EQ, 7); + tt_i64_op(extract_param_buggy("bwweightscale=7 foo=9", + "bwweightscale", 10000), + OP_EQ, 10000); + tt_i64_op(extract_param_buggy("foo=7 bwweightscale=777 bar=9", + "bwweightscale", 10000), + OP_EQ, 10000); + tt_i64_op(extract_param_buggy("foo=7 bwweightscale=1234", + "bwweightscale", 10000), + OP_EQ, 1234); + + done: + ; +} + #define NODE(name, flags) \ { \ #name, test_dirvote_##name, (flags), NULL, NULL \ @@ -668,4 +692,5 @@ struct testcase_t dirvote_tests[] = { NODE(get_sybil_by_ip_version_ipv4, TT_FORK), NODE(get_sybil_by_ip_version_ipv6, TT_FORK), NODE(get_all_possible_sybil, TT_FORK), + NODE(parse_param_buggy, 0), END_OF_TESTCASES}; diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index 7935530653..89a1aa90b3 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -24,60 +24,6 @@ #include <sys/stat.h> #endif -/* Test connection_or_remove_from_ext_or_id_map and - * connection_or_set_ext_or_identifier */ -static void -test_ext_or_id_map(void *arg) -{ - or_connection_t *c1 = NULL, *c2 = NULL, *c3 = NULL; - char *idp = NULL, *idp2 = NULL; - (void)arg; - - /* pre-initialization */ - tt_ptr_op(NULL, OP_EQ, - connection_or_get_by_ext_or_id("xxxxxxxxxxxxxxxxxxxx")); - - c1 = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); - c2 = or_connection_new(CONN_TYPE_EXT_OR, AF_INET); - c3 = or_connection_new(CONN_TYPE_OR, AF_INET); - - tt_ptr_op(c1->ext_or_conn_id, OP_NE, NULL); - tt_ptr_op(c2->ext_or_conn_id, OP_NE, NULL); - tt_ptr_op(c3->ext_or_conn_id, OP_EQ, NULL); - - tt_ptr_op(c1, OP_EQ, connection_or_get_by_ext_or_id(c1->ext_or_conn_id)); - tt_ptr_op(c2, OP_EQ, connection_or_get_by_ext_or_id(c2->ext_or_conn_id)); - tt_ptr_op(NULL, OP_EQ, - connection_or_get_by_ext_or_id("xxxxxxxxxxxxxxxxxxxx")); - - idp = tor_memdup(c2->ext_or_conn_id, EXT_OR_CONN_ID_LEN); - - /* Give c2 a new ID. */ - connection_or_set_ext_or_identifier(c2); - tt_mem_op(idp, OP_NE, c2->ext_or_conn_id, EXT_OR_CONN_ID_LEN); - idp2 = tor_memdup(c2->ext_or_conn_id, EXT_OR_CONN_ID_LEN); - tt_assert(!tor_digest_is_zero(idp2)); - - tt_ptr_op(NULL, OP_EQ, connection_or_get_by_ext_or_id(idp)); - tt_ptr_op(c2, OP_EQ, connection_or_get_by_ext_or_id(idp2)); - - /* Now remove it. */ - connection_or_remove_from_ext_or_id_map(c2); - tt_ptr_op(NULL, OP_EQ, connection_or_get_by_ext_or_id(idp)); - tt_ptr_op(NULL, OP_EQ, connection_or_get_by_ext_or_id(idp2)); - - done: - if (c1) - connection_free_minimal(TO_CONN(c1)); - if (c2) - connection_free_minimal(TO_CONN(c2)); - if (c3) - connection_free_minimal(TO_CONN(c3)); - tor_free(idp); - tor_free(idp2); - connection_or_clear_ext_or_id_map(); -} - /* Simple connection_write_to_buf_impl_ replacement that unconditionally * writes to outbuf. */ static void @@ -527,7 +473,7 @@ test_ext_or_handshake(void *arg) tt_int_op(handshake_start_called,OP_EQ,1); tt_int_op(TO_CONN(conn)->type, OP_EQ, CONN_TYPE_OR); tt_int_op(TO_CONN(conn)->state, OP_EQ, 0); - close_closeable_connections(); + connection_free_(TO_CONN(conn)); conn = NULL; /* Okay, this time let's succeed the handshake but fail the USERADDR @@ -581,7 +527,6 @@ test_ext_or_handshake(void *arg) } struct testcase_t extorport_tests[] = { - { "id_map", test_ext_or_id_map, TT_FORK, NULL, NULL }, { "write_command", test_ext_or_write_command, TT_FORK, NULL, NULL }, { "init_auth", test_ext_or_init_auth, TT_FORK, NULL, NULL }, { "cookie_auth", test_ext_or_cookie_auth, TT_FORK, NULL, NULL }, diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh index 1ba8179aa1..2e2745e0a3 100755 --- a/src/test/test_key_expiration.sh +++ b/src/test/test_key_expiration.sh @@ -107,7 +107,7 @@ TOR="${TOR_BINARY} --DisableNetwork 1 --ShutdownWaitLength 0 --ORPort 12345 --Ex # Step 1: Start Tor with --list-fingerprint --quiet. Make sure everything is there. echo "Setup step #1" -${TOR} --list-fingerprint ${SILENTLY} > /dev/null +${TOR} ${SILENTLY} --list-fingerprint > /dev/null check_dir "${DATA_DIR}/keys" check_file "${DATA_DIR}/keys/ed25519_master_id_public_key" diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh index 6812f8883d..be1fde9e32 100755 --- a/src/test/test_keygen.sh +++ b/src/test/test_keygen.sh @@ -120,7 +120,7 @@ TOR="${TOR_BINARY} ${QUIETLY} --DisableNetwork 1 --ShutdownWaitLength 0 --ORPort # Step 1: Start Tor with --list-fingerprint --quiet. Make sure everything is there. mkdir "${DATA_DIR}/orig" -${TOR} --DataDirectory "${DATA_DIR}/orig" --list-fingerprint ${SILENTLY} > /dev/null +${TOR} --DataDirectory "${DATA_DIR}/orig" ${SILENTLY} --list-fingerprint > /dev/null check_dir "${DATA_DIR}/orig/keys" check_file "${DATA_DIR}/orig/keys/ed25519_master_id_public_key" @@ -206,7 +206,7 @@ SRC="${DATA_DIR}/orig" mkdir -p "${ME}/keys" cp "${SRC}/keys/ed25519_master_id_"* "${ME}/keys/" -${TOR} --DataDirectory "${ME}" --list-fingerprint ${SILENTLY} >/dev/null || die "Tor failed when starting with only master key" +${TOR} --DataDirectory "${ME}" ${SILENTLY} --list-fingerprint >/dev/null || die "Tor failed when starting with only master key" check_files_eq "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/ed25519_master_id_public_key" check_files_eq "${SRC}/keys/ed25519_master_id_secret_key" "${ME}/keys/ed25519_master_id_secret_key" check_file "${ME}/keys/ed25519_signing_cert" @@ -264,11 +264,11 @@ SRC="${DATA_DIR}/orig" mkdir -p "${ME}/keys" cp "${SRC}/keys/ed25519_master_id_secret_key" "${ME}/keys/" -${TOR} --DataDirectory "${ME}" --list-fingerprint ${SILENTLY} > "${ME}/fp1" || die "Tor wouldn't start with only unencrypted secret key" +${TOR} --DataDirectory "${ME}" ${SILENTLY} --list-fingerprint > "${ME}/fp1" || die "Tor wouldn't start with only unencrypted secret key" check_file "${ME}/keys/ed25519_master_id_public_key" check_file "${ME}/keys/ed25519_signing_cert" check_file "${ME}/keys/ed25519_signing_secret_key" -${TOR} --DataDirectory "${ME}" --list-fingerprint ${SILENTLY} > "${ME}/fp2" || die "Tor wouldn't start again after starting once with only unencrypted secret key." +${TOR} --DataDirectory "${ME}" ${SILENTLY} --list-fingerprint > "${ME}/fp2" || die "Tor wouldn't start again after starting once with only unencrypted secret key." check_files_eq "${ME}/fp1" "${ME}/fp2" @@ -330,7 +330,7 @@ cp "${SRC}/keys/ed25519_master_id_secret_key" "${ME}/keys/" cp "${SRC}/keys/ed25519_signing_cert" "${ME}/keys/" cp "${SRC}/keys/ed25519_signing_secret_key" "${ME}/keys/" -${TOR} --DataDirectory "${ME}" --list-fingerprint ${SILENTLY} >/dev/null || die "Failed when starting with missing public key" +${TOR} --DataDirectory "${ME}" ${SILENTLY} --list-fingerprint >/dev/null || die "Failed when starting with missing public key" check_keys_eq ed25519_master_id_secret_key check_keys_eq ed25519_master_id_public_key check_keys_eq ed25519_signing_secret_key @@ -352,7 +352,7 @@ cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/" cp "${SRC}/keys/ed25519_signing_cert" "${ME}/keys/" cp "${SRC}/keys/ed25519_signing_secret_key" "${ME}/keys/" -${TOR} --DataDirectory "${ME}" --list-fingerprint ${SILENTLY} >/dev/null || die "Failed when starting with offline secret key" +${TOR} --DataDirectory "${ME}" ${SILENTLY} --list-fingerprint >/dev/null || die "Failed when starting with offline secret key" check_no_file "${ME}/keys/ed25519_master_id_secret_key" check_keys_eq ed25519_master_id_public_key check_keys_eq ed25519_signing_secret_key @@ -373,7 +373,7 @@ mkdir -p "${ME}/keys" cp "${SRC}/keys/ed25519_signing_cert" "${ME}/keys/" cp "${SRC}/keys/ed25519_signing_secret_key" "${ME}/keys/" -${TOR} --DataDirectory "${ME}" --list-fingerprint ${SILENTLY} >/dev/null || die "Failed when starting with only signing material" +${TOR} --DataDirectory "${ME}" ${SILENTLY} --list-fingerprint >/dev/null || die "Failed when starting with only signing material" check_no_file "${ME}/keys/ed25519_master_id_secret_key" check_file "${ME}/keys/ed25519_master_id_public_key" check_keys_eq ed25519_signing_secret_key diff --git a/src/test/test_stats.c b/src/test/test_stats.c index d45afc7b15..617a36faba 100644 --- a/src/test/test_stats.c +++ b/src/test/test_stats.c @@ -12,6 +12,8 @@ #include "lib/crypt_ops/crypto_rand.h" #include "app/config/or_state_st.h" #include "test/rng_test_helpers.h" +#include "feature/hs/hs_cache.h" +#include "test/hs_test_helpers.h" #include <stdio.h> @@ -31,6 +33,7 @@ #define MAINLOOP_PRIVATE #define STATEFILE_PRIVATE #define BWHIST_PRIVATE +#define REPHIST_PRIVATE #define ROUTER_PRIVATE #include "core/or/or.h" @@ -495,6 +498,133 @@ test_get_bandwidth_lines(void *arg) bwhist_free_all(); } +static bool +mock_should_collect_v3_stats(void) +{ + return true; +} + +/* Test v3 metrics */ +static void +test_rephist_v3_onions(void *arg) +{ + int ret; + + char *stats_string = NULL; + char *desc1_str = NULL; + ed25519_keypair_t signing_kp1; + hs_descriptor_t *desc1 = NULL; + + const hs_v3_stats_t *hs_v3_stats = NULL; + + (void) arg; + + MOCK(should_collect_v3_stats, mock_should_collect_v3_stats); + + get_options_mutable()->HiddenServiceStatistics = 1; + + /* Initialize the subsystems */ + hs_cache_init(); + rep_hist_hs_stats_init(0); + + /* Change time to 03-01-2002 23:36 UTC */ + update_approx_time(1010101010); + + /* HS stats should be zero here */ + hs_v3_stats = rep_hist_get_hs_v3_stats(); + tt_int_op(digest256map_size(hs_v3_stats->v3_onions_seen_this_period), + OP_EQ, 0); + + /* Generate a valid descriptor */ + ret = ed25519_keypair_generate(&signing_kp1, 0); + tt_int_op(ret, OP_EQ, 0); + desc1 = hs_helper_build_hs_desc_with_rev_counter(&signing_kp1, 42); + tt_assert(desc1); + ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &desc1_str); + tt_int_op(ret, OP_EQ, 0); + + /* Store descriptor and check that stats got updated */ + ret = hs_cache_store_as_dir(desc1_str); + tt_int_op(ret, OP_EQ, 0); + hs_v3_stats = rep_hist_get_hs_v3_stats(); + tt_int_op(digest256map_size(hs_v3_stats->v3_onions_seen_this_period), + OP_EQ, 1); + + /* cleanup */ + hs_descriptor_free(desc1); + tor_free(desc1_str); + + /* Generate another valid descriptor */ + ret = ed25519_keypair_generate(&signing_kp1, 0); + tt_int_op(ret, OP_EQ, 0); + desc1 = hs_helper_build_hs_desc_with_rev_counter(&signing_kp1, 42); + tt_assert(desc1); + ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &desc1_str); + tt_int_op(ret, OP_EQ, 0); + + /* Store descriptor and check that stats are updated */ + ret = hs_cache_store_as_dir(desc1_str); + tt_int_op(ret, OP_EQ, 0); + hs_v3_stats = rep_hist_get_hs_v3_stats(); + tt_int_op(digest256map_size(hs_v3_stats->v3_onions_seen_this_period), + OP_EQ, 2); + + /* Check that storing the same descriptor twice does not work */ + ret = hs_cache_store_as_dir(desc1_str); + tt_int_op(ret, OP_EQ, -1); + + /* cleanup */ + hs_descriptor_free(desc1); + tor_free(desc1_str); + + /* Create a descriptor with the same identity key but diff rev counter and + same blinded key */ + desc1 = hs_helper_build_hs_desc_with_rev_counter(&signing_kp1, 43); + tt_assert(desc1); + ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &desc1_str); + tt_int_op(ret, OP_EQ, 0); + + /* Store descriptor and check that stats are updated */ + ret = hs_cache_store_as_dir(desc1_str); + tt_int_op(ret, OP_EQ, 0); + tt_int_op(digest256map_size(hs_v3_stats->v3_onions_seen_this_period), + OP_EQ, 2); + + /* cleanup */ + hs_descriptor_free(desc1); + tor_free(desc1_str); + + /* Now let's skip to four days forward so that the blinded key rolls + forward */ + update_approx_time(approx_time() + 345600); + + /* Now create a descriptor with the same identity key but diff rev counter + and different blinded key */ + desc1 = hs_helper_build_hs_desc_with_rev_counter(&signing_kp1, 44); + tt_assert(desc1); + ret = hs_desc_encode_descriptor(desc1, &signing_kp1, NULL, &desc1_str); + tt_int_op(ret, OP_EQ, 0); + + /* Store descriptor and check that stats are updated */ + ret = hs_cache_store_as_dir(desc1_str); + tt_int_op(ret, OP_EQ, 0); + tt_int_op(digest256map_size(hs_v3_stats->v3_onions_seen_this_period), + OP_EQ, 3); + + /* cleanup */ + hs_descriptor_free(desc1); + tor_free(desc1_str); + + /* Because of differential privacy we can't actually check the stat value, + but let's just check that it's formatted correctly. */ + stats_string = rep_hist_format_hs_stats(approx_time(), true); + tt_assert(strstr(stats_string, "hidserv-dir-v3-onions-seen")); + + done: + UNMOCK(should_collect_v3_stats); + tor_free(stats_string); +} + static void test_load_stats_file(void *arg) { @@ -586,6 +716,7 @@ struct testcase_t stats_tests[] = { FORK(add_obs), FORK(fill_bandwidth_history), FORK(get_bandwidth_lines), + FORK(rephist_v3_onions), FORK(load_stats_file), END_OF_TESTCASES diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 022b05fe25..908ea08d3a 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -217,7 +217,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.5.5-rc-dev" +#define VERSION "0.4.6.0-alpha-dev" #define HAVE_STRUCT_SOCKADDR_IN6 #define HAVE_STRUCT_IN6_ADDR |