diff options
Diffstat (limited to 'src')
190 files changed, 8553 insertions, 5421 deletions
diff --git a/src/app/config/config.c b/src/app/config/config.c index 2877bc1e6a..8df5275cc6 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -193,6 +193,7 @@ static const config_abbrev_t option_abbrevs_[] = { PLURAL(AuthDirBadDirCC), PLURAL(AuthDirBadExitCC), PLURAL(AuthDirInvalidCC), + PLURAL(AuthDirMiddleOnlyCC), PLURAL(AuthDirRejectCC), PLURAL(EntryNode), PLURAL(ExcludeNode), @@ -331,6 +332,8 @@ static const config_var_t option_vars_[] = { V(AuthDirBadExitCCs, CSV, ""), V(AuthDirInvalid, LINELIST, NULL), V(AuthDirInvalidCCs, CSV, ""), + V(AuthDirMiddleOnly, LINELIST, NULL), + V(AuthDirMiddleOnlyCCs, CSV, ""), V(AuthDirReject, LINELIST, NULL), V(AuthDirRejectCCs, CSV, ""), OBSOLETE("AuthDirRejectUnlisted"), @@ -616,6 +619,7 @@ static const config_var_t option_vars_[] = { V(ConnectionPadding, AUTOBOOL, "auto"), V(RefuseUnknownExits, AUTOBOOL, "auto"), V(CircuitPadding, BOOL, "1"), + V(ReconfigDropsBridgeDescs, BOOL, "0"), V(ReducedCircuitPadding, BOOL, "0"), V(RejectPlaintextPorts, CSV, ""), V(RelayBandwidthBurst, MEMUNIT, "0"), @@ -668,6 +672,7 @@ static const config_var_t option_vars_[] = { VAR("UseEntryGuards", BOOL, UseEntryGuards_option, "1"), OBSOLETE("UseEntryGuardsAsDirGuards"), V(UseGuardFraction, AUTOBOOL, "auto"), + V(VanguardsLiteEnabled, AUTOBOOL, "auto"), V(UseMicrodescriptors, AUTOBOOL, "auto"), OBSOLETE("UseNTorHandshake"), V_IMMUTABLE(User, STRING, NULL), @@ -2309,6 +2314,8 @@ options_act,(const or_options_t *old_options)) } if (transition_affects_guards) { + if (options->ReconfigDropsBridgeDescs) + routerlist_drop_bridge_descriptors(); if (guards_update_all()) { abandon_circuits = 1; } @@ -5468,6 +5475,77 @@ pt_parse_transport_line(const or_options_t *options, return r; } +/** + * Parse a flag describing an extra dirport for a directory authority. + * + * Right now, the supported format is exactly: + * `{upload,download,voting}=http://[IP:PORT]/`. + * Other URL schemes, and other suffixes, might be supported in the future. + * + * Only call this function if `flag` starts with one of the above strings. + * + * Return 0 on success, and -1 on failure. + * + * If `ds` is provided, then add any parsed dirport to `ds`. If `ds` is NULL, + * take no action other than parsing. + **/ +static int +parse_dirauth_dirport(dir_server_t *ds, const char *flag) +{ + tor_assert(flag); + + auth_dirport_usage_t usage; + + if (!strcasecmpstart(flag, "upload=")) { + usage = AUTH_USAGE_UPLOAD; + } else if (!strcasecmpstart(flag, "download=")) { + usage = AUTH_USAGE_DOWNLOAD; + } else if (!strcasecmpstart(flag, "vote=")) { + usage = AUTH_USAGE_VOTING; + } else { + // We shouldn't get called with a flag that we don't recognize. + tor_assert_nonfatal_unreached(); + return -1; + } + + const char *eq = strchr(flag, '='); + tor_assert(eq); + const char *target = eq + 1; + + // Find the part inside the http://{....}/ + if (strcmpstart(target, "http://")) { + log_warn(LD_CONFIG, "Unsupported URL scheme in authority flag %s", flag); + return -1; + } + const char *addr = target + strlen("http://"); + + const char *eos = strchr(addr, '/'); + size_t addr_len; + if (eos && strcmp(eos, "/")) { + log_warn(LD_CONFIG, "Unsupported URL prefix in authority flag %s", flag); + return -1; + } else if (eos) { + addr_len = eos - addr; + } else { + addr_len = strlen(addr); + } + + // Finally, parse the addr:port part. + char *addr_string = tor_strndup(addr, addr_len); + tor_addr_port_t dirport; + memset(&dirport, 0, sizeof(dirport)); + int rv = tor_addr_port_parse(LOG_WARN, addr_string, + &dirport.addr, &dirport.port, -1); + if (ds != NULL && rv == 0) { + trusted_dir_server_add_dirport(ds, usage, &dirport); + } else if (rv == -1) { + log_warn(LD_CONFIG, "Unable to parse address in authority flag %s",flag); + } + + tor_free(addr_string); + return rv; +} + /** Read the contents of a DirAuthority line from <b>line</b>. If * <b>validate_only</b> is 0, and the line is well-formed, and it * shares any bits with <b>required_type</b> or <b>required_type</b> @@ -5488,6 +5566,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, char v3_digest[DIGEST_LEN]; dirinfo_type_t type = 0; double weight = 1.0; + smartlist_t *extra_dirports = smartlist_new(); memset(v3_digest, 0, sizeof(v3_digest)); @@ -5556,6 +5635,12 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, } ipv6_addrport_ptr = &ipv6_addrport; } + } else if (!strcasecmpstart(flag, "upload=") || + !strcasecmpstart(flag, "download=") || + !strcasecmpstart(flag, "vote=")) { + // We'll handle these after creating the authority object. + smartlist_add(extra_dirports, flag); + flag = NULL; // prevent double-free. } else { log_warn(LD_CONFIG, "Unrecognized flag '%s' on DirAuthority line", flag); @@ -5599,6 +5684,13 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, goto err; } + if (validate_only) { + SMARTLIST_FOREACH_BEGIN(extra_dirports, const char *, cp) { + if (parse_dirauth_dirport(NULL, cp) < 0) + goto err; + } SMARTLIST_FOREACH_END(cp); + } + if (!validate_only && (!required_type || required_type & type)) { dir_server_t *ds; if (required_type) @@ -5610,16 +5702,23 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, ipv6_addrport_ptr, digest, v3_digest, type, weight))) goto err; + + SMARTLIST_FOREACH_BEGIN(extra_dirports, const char *, cp) { + if (parse_dirauth_dirport(ds, cp) < 0) + goto err; + } SMARTLIST_FOREACH_END(cp); dir_server_add(ds); } r = 0; goto done; - err: + err: r = -1; - done: + done: + SMARTLIST_FOREACH(extra_dirports, char*, s, tor_free(s)); + smartlist_free(extra_dirports); SMARTLIST_FOREACH(items, char*, s, tor_free(s)); smartlist_free(items); tor_free(addrport); diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 151b77c457..3a1acad044 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -293,6 +293,13 @@ struct or_options_t { * disabled. */ int CircuitPadding; + /** Boolean: if true, then this client will discard cached bridge + * descriptors on a setconf or other config change that impacts guards + * or bridges (see options_transition_affects_guards() for exactly which + * config changes trigger it). Useful for tools that test bridge + * reachability by fetching fresh descriptors. */ + int ReconfigDropsBridgeDescs; + /** Boolean: if true, then this client will only use circuit padding * algorithms that are known to use a low amount of overhead. If false, * we will use all available circuit padding algorithms. @@ -492,6 +499,9 @@ struct or_options_t { struct smartlist_t *NodeFamilySets; struct config_line_t *AuthDirBadExit; /**< Address policy for descriptors to * mark as bad exits. */ + /** Address policy for descriptors to mark as only suitable for the + * middle position in circuits. */ + struct config_line_t *AuthDirMiddleOnly; struct config_line_t *AuthDirReject; /**< Address policy for descriptors to * reject. */ struct config_line_t *AuthDirInvalid; /**< Address policy for descriptors to @@ -505,6 +515,7 @@ struct or_options_t { */ struct smartlist_t *AuthDirBadExitCCs; struct smartlist_t *AuthDirInvalidCCs; + struct smartlist_t *AuthDirMiddleOnlyCCs; struct smartlist_t *AuthDirRejectCCs; /**@}*/ @@ -587,6 +598,9 @@ struct or_options_t { * If 0, use value from NumEntryGuards. */ int NumPrimaryGuards; /**< How many primary guards do we want? */ + /** Boolean: Switch to toggle the vanguards-lite subsystem */ + int VanguardsLiteEnabled; + int RephistTrackTime; /**< How many seconds do we keep rephist info? */ /** Should we always fetch our dir info on the mirror schedule (which * means directly from the authorities) no matter our other config? */ diff --git a/src/app/include.am b/src/app/include.am index 2e2180deca..5494d904a3 100644 --- a/src/app/include.am +++ b/src/app/include.am @@ -17,7 +17,6 @@ src_app_tor_SOURCES = src/app/main/tor_main.c src_app_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ @TOR_LDFLAGS_libevent@ @TOR_STATIC_LDFLAGS@ src_app_tor_LDADD = libtor.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ diff --git a/src/app/main/main.c b/src/app/main/main.c index 89564490e6..f7b532f0ab 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -27,6 +27,8 @@ #include "core/or/channel.h" #include "core/or/channelpadding.h" #include "core/or/circuitpadding.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_flow.h" #include "core/or/circuitlist.h" #include "core/or/command.h" #include "core/or/connection_or.h" @@ -100,12 +102,6 @@ #include <systemd/sd-daemon.h> #endif /* defined(HAVE_SYSTEMD) */ -#ifdef HAVE_RUST -// helper function defined in Rust to output a log message indicating if tor is -// running with Rust enabled. See src/rust/tor_util -void rust_log_welcome_string(void); -#endif - /********* PROTOTYPES **********/ static void dumpmemusage(int severity); @@ -609,10 +605,6 @@ tor_init(int argc, char *argv[]) tor_compress_log_init_warnings(); } -#ifdef HAVE_RUST - rust_log_welcome_string(); -#endif /* defined(HAVE_RUST) */ - /* Warn _if_ the tracing subsystem is built in. */ tracing_log_warning(); @@ -630,6 +622,8 @@ tor_init(int argc, char *argv[]) * until we get a consensus */ channelpadding_new_consensus_params(NULL); circpad_new_consensus_params(NULL); + congestion_control_new_consensus_params(NULL); + flow_control_new_consensus_params(NULL); /* Initialize circuit padding to defaults+torrc until we get a consensus */ circpad_machines_init(); @@ -1343,6 +1337,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) pubsub_connect(); if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) { +#ifdef ENABLE_FRAGILE_HARDENING + log_warn(LD_CONFIG, "Sandbox is enabled but this Tor was built using " + "fragile compiler hardening. The sandbox may be unable to filter " + "requests to open files and directories and its overall " + "effectiveness will be reduced."); +#endif + sandbox_cfg_t* cfg = sandbox_init_filter(); if (sandbox_init(cfg)) { diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging index 7f43cd324e..667ab294b4 100644 --- a/src/config/torrc.minimal.in-staging +++ b/src/config/torrc.minimal.in-staging @@ -224,4 +224,4 @@ ## mechanisms like https://bridges.torproject.org/. If you want to run ## a private bridge, for example because you'll give out your bridge ## address manually to your friends, uncomment this line: -#PublishServerDescriptor 0 +#BridgeDistribution none diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index 5d593871dd..edc30d043c 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -239,7 +239,7 @@ ## mechanisms like https://bridges.torproject.org/. If you want to run ## a private bridge, for example because you'll give out your bridge ## address manually to your friends, uncomment this line: -#PublishServerDescriptor 0 +#BridgeDistribution none ## Configuration options can be imported from files or folders using the %include ## option with the value being a path. This path can have wildcards. Wildcards are diff --git a/src/core/crypto/include.am b/src/core/crypto/include.am index 28b7e22905..2d53b3cb0b 100644 --- a/src/core/crypto/include.am +++ b/src/core/crypto/include.am @@ -5,6 +5,7 @@ LIBTOR_APP_A_SOURCES += \ src/core/crypto/onion_crypto.c \ src/core/crypto/onion_fast.c \ src/core/crypto/onion_ntor.c \ + src/core/crypto/onion_ntor_v3.c \ src/core/crypto/onion_tap.c \ src/core/crypto/relay_crypto.c @@ -14,5 +15,6 @@ noinst_HEADERS += \ src/core/crypto/onion_crypto.h \ src/core/crypto/onion_fast.h \ src/core/crypto/onion_ntor.h \ + src/core/crypto/onion_ntor_v3.h \ src/core/crypto/onion_tap.h \ src/core/crypto/relay_crypto.h diff --git a/src/core/crypto/onion_ntor_v3.c b/src/core/crypto/onion_ntor_v3.c new file mode 100644 index 0000000000..491c69cf8d --- /dev/null +++ b/src/core/crypto/onion_ntor_v3.c @@ -0,0 +1,760 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file onion_ntor_v3.c + * @brief Implements the version 3 ntor handshake as first specified in + * proposal 332. + * + * The v3 ntor handshake differs from the earlier versions (ntor and hs-ntor) + * primarily in that it allows the client to send an authenticated encrypted + * message as part of its onion skin, and allows the relay to send and + * encrypted authenticated reply as part of its response. + * + * It also takes a "verification string" -- the handshake cannot succeed + * unless both parties use the same value for their verification stream. + **/ + +#define ONION_NTOR_V3_PRIVATE + +#include "orconfig.h" +#include "core/crypto/onion_ntor_v3.h" + +#include "lib/arch/bytes.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/ctime/di_ops.h" +#include "lib/log/util_bug.h" + +#include <string.h> + +/* Parameters used to keep the outputs of this handshake from colliding with + * others. These are defined in the specification. */ +#define PROTOID "ntor3-curve25519-sha3_256-1" +#define TWEAK(A) (PROTOID ":" A) + +#define T_MSGKDF TWEAK("kdf_phase1") +#define T_MSGMAC TWEAK("msg_mac") +#define T_KEY_SEED TWEAK("key_seed") +#define T_VERIFY TWEAK("verify") +#define T_FINAL TWEAK("kdf_final") +#define T_AUTH TWEAK("auth_final") + +/** + * Add @a len bytes of @a data as input to the provided @a xof. + * + * (This is provided just for abbreviation). + **/ +#define xof_add(xof, data, len) crypto_xof_add_bytes((xof), (data), (len)) +/** + * Add @a len bytes of @a data as input to the provided @a xof, + * prefixed with an encoding of the length. + * + * This is equivalent to ENCAP(data) in the spec. + **/ +static void +xof_add_encap(crypto_xof_t *xof, const uint8_t *data, size_t len) +{ + uint64_t len64 = tor_htonll(len); + xof_add(xof, (uint8_t *)(&len64), 8); + xof_add(xof, data, len); +} +/** + * Add an encapsulated tweak to the provided xof. + **/ +#define xof_add_tweak(d, s) xof_add_encap((d), (const uint8_t *)(s), strlen(s)) + +/** + * Add @a len bytes of @a data to the provided @a digest. + * + * This is provided as an abbreviation, and to get the types right. + **/ +static void +d_add(crypto_digest_t *digest, const uint8_t *data, size_t len) +{ + crypto_digest_add_bytes(digest, (const char *)data, len); +} +/** + * Add @a len bytes of @a data to the provided @a digest, prefixed + * with the encoded length. + * + * This is equivalent to ENCAP(data) from the spec. + **/ +static void +d_add_encap(crypto_digest_t *digest, const uint8_t *data, size_t len) +{ + uint64_t len64 = tor_htonll(len); + d_add(digest, (const uint8_t *)(&len64), 8); + d_add(digest, data, len); +} +/** + * Add an encapsulated tweak to the provided digest. + **/ +#define d_add_tweak(d, s) d_add_encap((d), (const uint8_t *)(s), strlen(s)) + +/** + * Helper: copy @a len bytes of @a data onto *@a ptr, and advance @a ptr + * forward by @a len bytes. + * + * Asserts that @a ptr will not be advanced beyond @a endptr. + **/ +static void +push(uint8_t **ptr, const uint8_t *endptr, const uint8_t *data, size_t len) +{ + size_t remaining = endptr - *ptr; + tor_assert(len <= remaining); + memcpy(*ptr, data, len); + *ptr += len; +} + +/** + * Helper: Drop storage held by @a state, after wiping it. + **/ +void +ntor3_handshake_state_free_(ntor3_handshake_state_t *state) +{ + if (!state) + return; + + memwipe(state, 0, sizeof(*state)); + tor_free(state); +} + +/** + * Perform a client-side v3 ntor handshake with a given relay. + * + * As inputs this function takes the relay's Ed25519 identity (@a relay_id), + * the relay's current ntor onion key (@a relay_key), a verification string + * (@a verification_len bytes at @a verification), and a message to send + * as part of the handshake (@a message_len bytes at @a message). + * + * The message will be encrypted and authenticated to the relay, but will not + * receive the same forward secrecy as the rest of the handshake. We should + * not put any super-confidential data in it. + * + * The handshake will only succeed if the relay uses the same verification + * string as we are using. + * + * As outputs, this function returns 0 on success and -1 on failure. On + * success, it sets @a onion_skin_out and @a onion_skin_len_out to a newly + * allocated handshake message that the client can send as part of its CREATE2 + * or EXTEND2 cell. It also sets it sets @a handshake_state_out to a newly + * allocated handshake state object; the client needs to use this object to + * process the relay's eventual reply. + **/ +int +onion_skin_ntor3_create(const ed25519_public_key_t *relay_id, + const curve25519_public_key_t *relay_key, + const uint8_t *verification, + const size_t verification_len, + const uint8_t *message, + const size_t message_len, + ntor3_handshake_state_t **handshake_state_out, + uint8_t **onion_skin_out, + size_t *onion_skin_len_out) +{ + curve25519_keypair_t client_keypair; + if (curve25519_keypair_generate(&client_keypair, 0) < 0) { + return -1; + } + int r = onion_skin_ntor3_create_nokeygen( + &client_keypair, + relay_id, + relay_key, + verification, + verification_len, + message, + message_len, + handshake_state_out, + onion_skin_out, + onion_skin_len_out); + memwipe(&client_keypair, 0, sizeof(client_keypair)); + return r; +} + +/** + * Like onion_skin_ntor3_create, but do not generate a new ephemeral keypair. + * Instead, take the ephemeral keypair (x,X) from @a client_keypair. + * + * (Having a separate function for this lets us test the code for correct + * behavior.) + **/ +STATIC int +onion_skin_ntor3_create_nokeygen( + const curve25519_keypair_t *client_keypair, + const ed25519_public_key_t *relay_id, + const curve25519_public_key_t *relay_key, + const uint8_t *verification, + const size_t verification_len, + const uint8_t *message, + const size_t message_len, + ntor3_handshake_state_t **handshake_state_out, + uint8_t **onion_skin_out, + size_t *onion_skin_len_out) +{ + *handshake_state_out = NULL; + *onion_skin_out = NULL; + *onion_skin_len_out = 0; + + // Set up the handshake state object. + *handshake_state_out = tor_malloc_zero(sizeof(ntor3_handshake_state_t)); + memcpy(&(*handshake_state_out)->client_keypair, client_keypair, + sizeof(*client_keypair)); + memcpy(&(*handshake_state_out)->relay_id, relay_id, sizeof(*relay_id)); + memcpy(&(*handshake_state_out)->relay_key, relay_key, sizeof(*relay_key)); + + // Perform the first DH handshake. + curve25519_handshake((*handshake_state_out)->bx, + &client_keypair->seckey, relay_key); + if (safe_mem_is_zero((*handshake_state_out)->bx, CURVE25519_OUTPUT_LEN)) { + // Okay to return early here, since our behavior here doesn't + // cause a visible timing sidechannel. + return -1; + } + + // Compute phase1_keys. + uint8_t enc_key[CIPHER256_KEY_LEN]; + uint8_t mac_key[DIGEST256_LEN]; + { + crypto_xof_t *xof = crypto_xof_new(); + // secret_input_phase1 = Bx | ID | X | B | PROTOID | ENCAP(VER) + xof_add_tweak(xof, T_MSGKDF); + xof_add(xof, (*handshake_state_out)->bx, CURVE25519_OUTPUT_LEN); + xof_add(xof, relay_id->pubkey, ED25519_PUBKEY_LEN); + xof_add(xof, client_keypair->pubkey.public_key, CURVE25519_PUBKEY_LEN); + xof_add(xof, relay_key->public_key, CURVE25519_PUBKEY_LEN); + xof_add(xof, (const uint8_t *)PROTOID, strlen(PROTOID)); + xof_add_encap(xof, verification, verification_len); + crypto_xof_squeeze_bytes(xof, enc_key, sizeof(enc_key)); + crypto_xof_squeeze_bytes(xof, mac_key, sizeof(mac_key)); + crypto_xof_free(xof); + } + + // Compute encrypted message. + uint8_t *encrypted_message = tor_memdup(message, message_len); + { + crypto_cipher_t *c = + crypto_cipher_new_with_bits((const char *)enc_key, 256); + crypto_cipher_crypt_inplace(c, (char *)encrypted_message, message_len); + crypto_cipher_free(c); + } + + // Compute the MAC value. + { + crypto_digest_t *m = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(m, T_MSGMAC); + d_add_encap(m, mac_key, sizeof(mac_key)); + d_add(m, relay_id->pubkey, ED25519_PUBKEY_LEN); + d_add(m, relay_key->public_key, CURVE25519_PUBKEY_LEN); + d_add(m, client_keypair->pubkey.public_key, CURVE25519_PUBKEY_LEN); + d_add(m, encrypted_message, message_len); + crypto_digest_get_digest(m, + (char *)(*handshake_state_out)->msg_mac, + DIGEST256_LEN); + crypto_digest_free(m); + } + + // Build the onionskin. + *onion_skin_len_out = (ED25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN*2 + + DIGEST256_LEN + message_len); + *onion_skin_out = tor_malloc(*onion_skin_len_out); + { + uint8_t *ptr = *onion_skin_out, *end = ptr + *onion_skin_len_out; + + push(&ptr, end, relay_id->pubkey, ED25519_PUBKEY_LEN); + push(&ptr, end, relay_key->public_key, CURVE25519_PUBKEY_LEN); + push(&ptr, end, client_keypair->pubkey.public_key, CURVE25519_PUBKEY_LEN); + push(&ptr, end, encrypted_message, message_len); + push(&ptr, end, (*handshake_state_out)->msg_mac, DIGEST256_LEN); + tor_assert(ptr == end); + } + + memwipe(&enc_key, 0, sizeof(enc_key)); + memwipe(&mac_key, 0, sizeof(mac_key)); + memwipe(encrypted_message, 0, message_len); + tor_free(encrypted_message); + + return 0; +} + +/** + * Complete a client-side v3 ntor handshake. + * + * Takes a @a handshake_state returned earlier by `onion_skin_ntor3_create()`, + * and the relay's reply to that handshake (@a reply_len bytes at @a + * handshake_reply). Also takes a verification string (@a verification_len + * bytes at @a verification). + * + * Returns 0 on success and -1 on failure. On success, generates @a key_len + * bytes of key material into the provided @a keys_out buffer, and sets @a + * message_out to the message that the relay sent in reply to our message (and + * sets @a message_out_len to that message's length). + **/ +int +onion_ntor3_client_handshake(const ntor3_handshake_state_t *handshake_state, + const uint8_t *handshake_reply, + size_t reply_len, + const uint8_t *verification, + size_t verification_len, + uint8_t *keys_out, + size_t keys_out_len, + uint8_t **message_out, + size_t *message_len_out) +{ + *message_out = NULL; + *message_len_out = 0; + + int problems = 0; + + // Parse the relay's message. + curve25519_public_key_t relay_Y; + uint8_t relay_auth[DIGEST256_LEN]; + size_t encrypted_msg_len; + const uint8_t *encrypted_msg; + { + if (reply_len < CURVE25519_PUBKEY_LEN + DIGEST256_LEN) { + // Okay to return early here, since the message is completely + // ill-formed, so we can't leak anything. + ++problems; + goto done; + } + encrypted_msg_len = reply_len - (CURVE25519_PUBKEY_LEN + DIGEST256_LEN); + + memcpy(&relay_Y.public_key, handshake_reply, CURVE25519_PUBKEY_LEN); + handshake_reply += CURVE25519_PUBKEY_LEN; + memcpy(&relay_auth, handshake_reply, DIGEST256_LEN); + handshake_reply += DIGEST256_LEN; + encrypted_msg = handshake_reply; + } + + // Finish the second diffie hellman handshake. + uint8_t yx[CURVE25519_OUTPUT_LEN]; + curve25519_handshake(yx, &handshake_state->client_keypair.seckey, &relay_Y); + problems |= safe_mem_is_zero(yx, sizeof(yx)); + + // Compute two tweaked hashes of secret_input. + uint8_t key_seed[DIGEST256_LEN], verify[DIGEST256_LEN]; + { + crypto_digest_t *ks = crypto_digest256_new(DIGEST_SHA3_256); + crypto_digest_t *v = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(ks, T_KEY_SEED); + d_add_tweak(v, T_VERIFY); +#define ADD2(s,len) STMT_BEGIN { \ + d_add(ks, (s),(len)); d_add(v, (s), (len)); \ + } STMT_END +#define ADD2_ENCAP(s,len) STMT_BEGIN { \ + d_add_encap(ks, (s),(len)); d_add_encap(v, (s), (len)); \ + } STMT_END + + ADD2(yx, sizeof(yx)); + ADD2(handshake_state->bx, sizeof(handshake_state->bx)); + ADD2(handshake_state->relay_id.pubkey, ED25519_PUBKEY_LEN); + ADD2(handshake_state->relay_key.public_key, CURVE25519_PUBKEY_LEN); + ADD2(handshake_state->client_keypair.pubkey.public_key, + CURVE25519_PUBKEY_LEN); + ADD2(relay_Y.public_key, CURVE25519_PUBKEY_LEN); + ADD2((const uint8_t *)PROTOID, strlen(PROTOID)); + ADD2_ENCAP(verification, verification_len); + + crypto_digest_get_digest(ks, (char*) key_seed, DIGEST256_LEN); + crypto_digest_get_digest(v, (char*) verify, DIGEST256_LEN); + crypto_digest_free(ks); + crypto_digest_free(v); + } + + // compute expected auth value. + uint8_t auth_computed[DIGEST256_LEN]; + { + crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(d, T_AUTH); + d_add(d, verify, sizeof(verify)); + d_add(d, handshake_state->relay_id.pubkey, ED25519_PUBKEY_LEN); + d_add(d, handshake_state->relay_key.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, relay_Y.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, handshake_state->client_keypair.pubkey.public_key, + CURVE25519_PUBKEY_LEN); + d_add(d, handshake_state->msg_mac, DIGEST256_LEN); + d_add_encap(d, encrypted_msg, encrypted_msg_len); + d_add(d, (const uint8_t*)PROTOID, strlen(PROTOID)); + d_add(d, (const uint8_t*)"Server", strlen("Server")); + crypto_digest_get_digest(d, (char *)auth_computed, DIGEST256_LEN); + crypto_digest_free(d); + } + + // Check authentication value. + problems |= tor_memneq(auth_computed, relay_auth, DIGEST256_LEN); + + // Compute keystream, decrypt message, and return. + *message_out = tor_malloc(encrypted_msg_len); + *message_len_out = encrypted_msg_len; + uint8_t enc_key[CIPHER256_KEY_LEN]; + { + crypto_xof_t *xof = crypto_xof_new(); + xof_add_tweak(xof, T_FINAL); + xof_add(xof, key_seed, sizeof(key_seed)); + crypto_xof_squeeze_bytes(xof, enc_key, sizeof(enc_key)); + crypto_xof_squeeze_bytes(xof, (uint8_t *)keys_out, keys_out_len); + crypto_xof_free(xof); + + crypto_cipher_t *c = + crypto_cipher_new_with_bits((const char *)enc_key, 256); + crypto_cipher_decrypt(c, (char *)*message_out, + (const char *)encrypted_msg, encrypted_msg_len); + crypto_cipher_free(c); + } + + done: + memwipe(&relay_Y, 0, sizeof(relay_Y)); + memwipe(&relay_auth, 0, sizeof(relay_auth)); + memwipe(&yx, 0, sizeof(yx)); + memwipe(key_seed, 0, sizeof(key_seed)); + memwipe(verify, 0, sizeof(verify)); + memwipe(enc_key, 0, sizeof(enc_key)); + if (problems) { + if (*message_out) { + memwipe(*message_out, 0, *message_len_out); + tor_free(*message_out); // Sets it to NULL. + } + *message_len_out = 0; + crypto_rand((char*)keys_out, keys_out_len); // In case bad code uses it. + return -1; + } + + return 0; +} + +/** + * Wipe a server handshake state, and release the storage it holds. + **/ +void +ntor3_server_handshake_state_free_(ntor3_server_handshake_state_t *state) +{ + if (state == NULL) + return; + + memwipe(state, 0, sizeof(ntor3_server_handshake_state_t)); + tor_free(state); +} + +/** + * As a relay, start handling a client's v3 ntor handshake. + * + * This function performs the _first half_ of the handshake, up to the point + * where the client's message is decoded. After calling it, the relay should + * decide how and whether to reply to the client's message, compose its reply, + * and call `onion_skin_ntor3_server_handshake_part2`. + * + * It takes as input a map of the relay's known onion keys in @a private_keys, + * along with a fake @a junk_key to use if there is a complete mismatch. It + * takes the relay's ed25519 identity in @a my_id, along with the client's + * handshake message (@a client_handshake_len bytes in @a client_handshake), + * and a verification string (@a verification_len bytes in @a verification). + * + * Return 0 on success, and -1 on failure. On success, sets @a + * client_message_out to a newly allocated string holding the plaintext of the + * message that the client sent as part of its handshake, and @a + * client_message_out_len to its length. Also sets @a state_out to a newly + * allocated state object holding the intermediate computation for this + * handshake. + **/ +int +onion_skin_ntor3_server_handshake_part1( + const di_digest256_map_t *private_keys, + const curve25519_keypair_t *junk_key, + const ed25519_public_key_t *my_id, + const uint8_t *client_handshake, + size_t client_handshake_len, + const uint8_t *verification, + size_t verification_len, + uint8_t **client_message_out, + size_t *client_message_len_out, + ntor3_server_handshake_state_t **state_out) +{ + *client_message_out = NULL; + *client_message_len_out = 0; + *state_out = NULL; + + int problems = 0; + + // Initialize state. + (*state_out) = tor_malloc_zero(sizeof(ntor3_server_handshake_state_t)); + memcpy(&(*state_out)->my_id, my_id, sizeof(*my_id)); + + const uint8_t *wanted_id; // [ED25519_PUBKEY_LEN] + const uint8_t *wanted_key; // [CURVE25519_PUBKEY_LEN] + const uint8_t *encrypted_message; + size_t encrypted_message_len; + // Unpack the client handshake. + { + const uint8_t *ptr = client_handshake; + const uint8_t *end = ptr + client_handshake_len; + + if (client_handshake_len < + ED25519_PUBKEY_LEN + CURVE25519_PUBKEY_LEN * 2 + DIGEST256_LEN) { + // Okay to end early; the client knows this is unparseable already. + ++problems; + goto done; + } + wanted_id = ptr; + ptr += ED25519_PUBKEY_LEN; + wanted_key = ptr; + ptr += CURVE25519_PUBKEY_LEN; + memcpy((*state_out)->client_key.public_key, ptr, CURVE25519_PUBKEY_LEN); + ptr += CURVE25519_PUBKEY_LEN; + size_t remaining = (end-ptr); + if (BUG(remaining < DIGEST256_LEN)) { + // Okay to end early; this is a bug. + ++problems; + goto done; + } + encrypted_message = ptr; + encrypted_message_len = remaining - DIGEST256_LEN; + ptr += encrypted_message_len; + remaining = (end-ptr); + tor_assert(remaining == DIGEST256_LEN); + memcpy((*state_out)->msg_mac, ptr, DIGEST256_LEN); + } + + // Check the identity. + problems |= tor_memneq(my_id->pubkey, wanted_id, ED25519_PUBKEY_LEN); + + // Find the correct keypair. + const curve25519_keypair_t *keypair = + dimap_search(private_keys, wanted_key, (void *)junk_key); + tor_assert(keypair); + memcpy(&(*state_out)->my_key, &keypair->pubkey, + sizeof(curve25519_public_key_t)); + + // Do the first diffie hellman handshake. + curve25519_handshake((*state_out)->xb, + &keypair->seckey, &(*state_out)->client_key); + problems |= safe_mem_is_zero((*state_out)->xb, CURVE25519_OUTPUT_LEN); + + // Derive the encryption and mac keys + uint8_t enc_key[CIPHER256_KEY_LEN], mac_key[DIGEST256_LEN]; + { + crypto_xof_t *xof = crypto_xof_new(); + xof_add_tweak(xof, T_MSGKDF); + xof_add(xof, (*state_out)->xb, CURVE25519_OUTPUT_LEN); + xof_add(xof, wanted_id, ED25519_PUBKEY_LEN); + xof_add(xof, (*state_out)->client_key.public_key, CURVE25519_PUBKEY_LEN); + xof_add(xof, keypair->pubkey.public_key, CURVE25519_PUBKEY_LEN); + xof_add(xof, (const uint8_t *)PROTOID, strlen(PROTOID)); + xof_add_encap(xof, verification, verification_len); + crypto_xof_squeeze_bytes(xof, enc_key, sizeof(enc_key)); + crypto_xof_squeeze_bytes(xof, mac_key, sizeof(mac_key)); + crypto_xof_free(xof); + } + + // Check the MAC. + uint8_t computed_mac[DIGEST256_LEN]; + { + crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(d, T_MSGMAC); + d_add_encap(d, mac_key, sizeof(mac_key)); + d_add(d, my_id->pubkey, ED25519_PUBKEY_LEN); + d_add(d, keypair->pubkey.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, (*state_out)->client_key.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, encrypted_message, encrypted_message_len); + crypto_digest_get_digest(d, (char *)computed_mac, DIGEST256_LEN); + crypto_digest_free(d); + } + + problems |= tor_memneq((*state_out)->msg_mac, computed_mac, DIGEST256_LEN); + + // Decrypt the message. + *client_message_out = tor_malloc(encrypted_message_len); + *client_message_len_out = encrypted_message_len; + { + crypto_cipher_t *c = + crypto_cipher_new_with_bits((const char *)enc_key, 256); + crypto_cipher_decrypt(c, (char *)*client_message_out, + (const char *)encrypted_message, + encrypted_message_len); + crypto_cipher_free(c); + } + + done: + memwipe(enc_key, 0, sizeof(enc_key)); + memwipe(mac_key, 0, sizeof(mac_key)); + memwipe(computed_mac, 0, sizeof(computed_mac)); + if (problems) { + if (*client_message_out) { + memwipe(*client_message_out, 0, *client_message_len_out); + tor_free(*client_message_out); // Sets it to NULL. + } + *client_message_len_out = 0; + ntor3_server_handshake_state_free(*state_out); + return -1; + } + + return 0; +} + +/** + * Finish the relay side of an ntor v3 handshake. + * + * The relay calls this function after it has decided to respond to the + * client's original encrypted message. This function receives the relay's + * message in @a server_message and its length in @a server_message_len, and + * completes the handshake. + * + * Returns 0 on success and -1 on failure. On success, stores the newly + * allocated handshake for the relay to send in @a handshake_out, and its + * length in @a handshake_len_out. Stores @a keys_out_len bytes of generated + * keys in the provided buffer at @a keys_out. + **/ +int +onion_skin_ntor3_server_handshake_part2( + const ntor3_server_handshake_state_t *state, + const uint8_t *verification, + size_t verification_len, + const uint8_t *server_message, + size_t server_message_len, + uint8_t **handshake_out, + size_t *handshake_len_out, + uint8_t *keys_out, + size_t keys_out_len) +{ + curve25519_keypair_t relay_keypair; + if (curve25519_keypair_generate(&relay_keypair, 0) < 0) { + return -1; + } + int r = onion_skin_ntor3_server_handshake_part2_nokeygen( + &relay_keypair, + state, + verification, + verification_len, + server_message, + server_message_len, + handshake_out, + handshake_len_out, + keys_out, + keys_out_len); + memwipe(&relay_keypair, 0, sizeof(relay_keypair)); + return r; +} + +/** + * Like `onion_skin_ntor3_server_handshake_part2`, but do not generate + * an ephemeral (y,Y) keypair. + * + * Instead, this function takes that keypair as @a relay_keypair_y. + * + * (Having a separate function for this lets us test the code for correct + * behavior.) + **/ +STATIC int +onion_skin_ntor3_server_handshake_part2_nokeygen( + const curve25519_keypair_t *relay_keypair_y, + const ntor3_server_handshake_state_t *state, + const uint8_t *verification, + size_t verification_len, + const uint8_t *server_message, + size_t server_message_len, + uint8_t **handshake_out, + size_t *handshake_len_out, + uint8_t *keys_out, + size_t keys_out_len) +{ + *handshake_out = NULL; + *handshake_len_out = 0; + + int problems = 0; + + // Second diffie-hellman handshake. + uint8_t xy[CURVE25519_OUTPUT_LEN]; + curve25519_handshake(xy, &relay_keypair_y->seckey, &state->client_key); + problems |= safe_mem_is_zero(xy, sizeof(xy)); + + // Compute two tweaked hashes of secret_input. + uint8_t key_seed[DIGEST256_LEN], verify[DIGEST256_LEN]; + { + crypto_digest_t *ks = crypto_digest256_new(DIGEST_SHA3_256); + crypto_digest_t *v = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(ks, T_KEY_SEED); + d_add_tweak(v, T_VERIFY); + ADD2(xy, sizeof(xy)); + ADD2(state->xb, sizeof(state->xb)); + ADD2(state->my_id.pubkey, ED25519_PUBKEY_LEN); + ADD2(state->my_key.public_key, CURVE25519_PUBKEY_LEN); + ADD2(state->client_key.public_key, CURVE25519_PUBKEY_LEN); + ADD2(relay_keypair_y->pubkey.public_key, CURVE25519_PUBKEY_LEN); + ADD2((const uint8_t *)PROTOID, strlen(PROTOID)); + ADD2_ENCAP(verification, verification_len); + crypto_digest_get_digest(ks, (char*) key_seed, DIGEST256_LEN); + crypto_digest_get_digest(v, (char*) verify, DIGEST256_LEN); + crypto_digest_free(ks); + crypto_digest_free(v); + } + + // Compute enc_key and keystream. + uint8_t enc_key[CIPHER256_KEY_LEN]; + { + crypto_xof_t *xof = crypto_xof_new(); + xof_add_tweak(xof, T_FINAL); + xof_add(xof, key_seed, sizeof(key_seed)); + crypto_xof_squeeze_bytes(xof, enc_key, sizeof(enc_key)); + crypto_xof_squeeze_bytes(xof, keys_out, keys_out_len); + crypto_xof_free(xof); + } + + // Encrypt message. + uint8_t *encrypted_message = tor_memdup(server_message, server_message_len); + { + crypto_cipher_t *c = + crypto_cipher_new_with_bits((const char *)enc_key, 256); + crypto_cipher_crypt_inplace( + c, (char *)encrypted_message, server_message_len); + crypto_cipher_free(c); + } + + // Compute AUTH digest. + uint8_t auth[DIGEST256_LEN]; + { + crypto_digest_t *d = crypto_digest256_new(DIGEST_SHA3_256); + d_add_tweak(d, T_AUTH); + d_add(d, verify, sizeof(verify)); + d_add(d, state->my_id.pubkey, ED25519_PUBKEY_LEN); + d_add(d, state->my_key.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, relay_keypair_y->pubkey.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, state->client_key.public_key, CURVE25519_PUBKEY_LEN); + d_add(d, state->msg_mac, DIGEST256_LEN); + d_add_encap(d, encrypted_message, server_message_len); + d_add(d, (const uint8_t*)PROTOID, strlen(PROTOID)); + d_add(d, (const uint8_t*)"Server", strlen("Server")); + crypto_digest_get_digest(d, (char *)auth, DIGEST256_LEN); + crypto_digest_free(d); + } + + // Compose the reply. + *handshake_len_out = CURVE25519_PUBKEY_LEN + DIGEST256_LEN + + server_message_len; + *handshake_out = tor_malloc(*handshake_len_out); + uint8_t *ptr = *handshake_out, *end = ptr + *handshake_len_out; + push(&ptr, end, relay_keypair_y->pubkey.public_key, CURVE25519_PUBKEY_LEN); + push(&ptr, end, auth, sizeof(auth)); + push(&ptr, end, encrypted_message, server_message_len); + tor_assert(ptr == end); + + // Clean up and return. + memwipe(xy, 0, sizeof(xy)); + memwipe(key_seed, 0, sizeof(key_seed)); + memwipe(verify, 0, sizeof(verify)); + memwipe(enc_key, 0, sizeof(enc_key)); + memwipe(encrypted_message, 0, server_message_len); + tor_free(encrypted_message); + + if (problems) { + memwipe(*handshake_out, 0, *handshake_len_out); + tor_free(*handshake_out); // Sets it to NULL. + *handshake_len_out = 0; + crypto_rand((char*)keys_out, keys_out_len); // In case bad code uses it. + return -1; + } + return 0; +} diff --git a/src/core/crypto/onion_ntor_v3.h b/src/core/crypto/onion_ntor_v3.h new file mode 100644 index 0000000000..4449eb237d --- /dev/null +++ b/src/core/crypto/onion_ntor_v3.h @@ -0,0 +1,140 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file onion_ntor_v3.h + * @brief Header for core/crypto/onion_ntor_v3.c + **/ + +#ifndef TOR_CORE_CRYPTO_ONION_NTOR_V3_H +#define TOR_CORE_CRYPTO_ONION_NTOR_V3_H + +#include "lib/cc/torint.h" +#include "lib/testsupport/testsupport.h" +#include "lib/crypt_ops/crypto_cipher.h" +#include "lib/crypt_ops/crypto_curve25519.h" +#include "lib/crypt_ops/crypto_ed25519.h" +#include "lib/malloc/malloc.h" + +/** + * Client-side state held while an ntor v3 handshake is in progress. + **/ +typedef struct ntor3_handshake_state_t ntor3_handshake_state_t; + +/** + * Server-side state held while the relay is handling a client's + * encapsulated message, before replying to the v3 handshake. + **/ +typedef struct ntor3_server_handshake_state_t ntor3_server_handshake_state_t; + +void ntor3_handshake_state_free_(ntor3_handshake_state_t *st); +#define ntor3_handshake_state_free(ptr) \ + FREE_AND_NULL(ntor3_handshake_state_t, ntor3_handshake_state_free_, (ptr)) +void ntor3_server_handshake_state_free_(ntor3_server_handshake_state_t *st); +#define ntor3_server_handshake_state_free(ptr) \ + FREE_AND_NULL(ntor3_server_handshake_state_t, \ + ntor3_server_handshake_state_free_, (ptr)) + +int onion_skin_ntor3_create(const ed25519_public_key_t *relay_id, + const curve25519_public_key_t *relay_key, + const uint8_t *verification, + const size_t verification_len, + const uint8_t *message, + const size_t message_len, + ntor3_handshake_state_t **handshake_state_out, + uint8_t **onion_skin_out, + size_t *onion_skin_len_out); + +int onion_ntor3_client_handshake( + const ntor3_handshake_state_t *handshake_state, + const uint8_t *handshake_reply, + size_t reply_len, + const uint8_t *verification, + size_t verification_len, + uint8_t *keys_out, + size_t keys_out_len, + uint8_t **message_out, + size_t *message_len_out); + +struct di_digest256_map_t; +int onion_skin_ntor3_server_handshake_part1( + const struct di_digest256_map_t *private_keys, + const curve25519_keypair_t *junk_key, + const ed25519_public_key_t *my_id, + const uint8_t *client_handshake, + size_t client_handshake_len, + const uint8_t *verification, + size_t verification_len, + uint8_t **client_message_out, + size_t *client_message_len_out, + ntor3_server_handshake_state_t **state_out); + +int onion_skin_ntor3_server_handshake_part2( + const ntor3_server_handshake_state_t *state, + const uint8_t *verification, + size_t verification_len, + const uint8_t *server_message, + size_t server_message_len, + uint8_t **handshake_out, + size_t *handshake_len_out, + uint8_t *keys_out, + size_t keys_out_len); + +#ifdef ONION_NTOR_V3_PRIVATE +struct ntor3_handshake_state_t { + /** Ephemeral (x,X) keypair. */ + curve25519_keypair_t client_keypair; + /** Relay's ed25519 identity key (ID) */ + ed25519_public_key_t relay_id; + /** Relay's public key (B) */ + curve25519_public_key_t relay_key; + /** Shared secret (Bx). */ + uint8_t bx[CURVE25519_OUTPUT_LEN]; + /** MAC of the client's encrypted message data (MAC) */ + uint8_t msg_mac[DIGEST256_LEN]; +}; + +struct ntor3_server_handshake_state_t { + /** Relay's ed25519 identity key (ID) */ + ed25519_public_key_t my_id; + /** Relay's public key (B) */ + curve25519_public_key_t my_key; + /** Client's public ephemeral key (X). */ + curve25519_public_key_t client_key; + + /** Shared secret (Xb) */ + uint8_t xb[CURVE25519_OUTPUT_LEN]; + /** MAC of the client's encrypted message data */ + uint8_t msg_mac[DIGEST256_LEN]; +}; + +STATIC int onion_skin_ntor3_create_nokeygen( + const curve25519_keypair_t *client_keypair, + const ed25519_public_key_t *relay_id, + const curve25519_public_key_t *relay_key, + const uint8_t *verification, + const size_t verification_len, + const uint8_t *message, + const size_t message_len, + ntor3_handshake_state_t **handshake_state_out, + uint8_t **onion_skin_out, + size_t *onion_skin_len_out); + +STATIC int onion_skin_ntor3_server_handshake_part2_nokeygen( + const curve25519_keypair_t *relay_keypair_y, + const ntor3_server_handshake_state_t *state, + const uint8_t *verification, + size_t verification_len, + const uint8_t *server_message, + size_t server_message_len, + uint8_t **handshake_out, + size_t *handshake_len_out, + uint8_t *keys_out, + size_t keys_out_len); + +#endif + +#endif /* !defined(TOR_CORE_CRYPTO_ONION_NTOR_V3_H) */ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index b17d7bf2bd..9271a70914 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -117,6 +117,7 @@ #include "lib/cc/ctassert.h" #include "lib/sandbox/sandbox.h" #include "lib/net/buffers_net.h" +#include "lib/net/address.h" #include "lib/tls/tortls.h" #include "lib/evloop/compat_libevent.h" #include "lib/compress/compress.h" @@ -146,6 +147,8 @@ #include "feature/nodelist/routerinfo_st.h" #include "core/or/socks_request_st.h" +#include "core/or/congestion_control_flow.h" + /** * On Windows and Linux we cannot reliably bind() a socket to an * address and port if: 1) There's already a socket bound to wildcard @@ -250,13 +253,13 @@ CONST_TO_LISTENER_CONN(const connection_t *c) } size_t -connection_get_inbuf_len(connection_t *conn) +connection_get_inbuf_len(const connection_t *conn) { return conn->inbuf ? buf_datalen(conn->inbuf) : 0; } size_t -connection_get_outbuf_len(connection_t *conn) +connection_get_outbuf_len(const connection_t *conn) { return conn->outbuf ? buf_datalen(conn->outbuf) : 0; } @@ -612,6 +615,11 @@ entry_connection_new(int type, int socket_family) entry_conn->entry_cfg.ipv4_traffic = 1; else if (socket_family == AF_INET6) entry_conn->entry_cfg.ipv6_traffic = 1; + + /* Initialize the read token bucket to the maximum value which is the same as + * no rate limiting. */ + token_bucket_rw_init(&ENTRY_TO_EDGE_CONN(entry_conn)->bucket, INT32_MAX, + INT32_MAX, monotime_coarse_get_stamp()); return entry_conn; } @@ -623,6 +631,10 @@ edge_connection_new(int type, int socket_family) edge_connection_t *edge_conn = tor_malloc_zero(sizeof(edge_connection_t)); tor_assert(type == CONN_TYPE_EXIT); connection_init(time(NULL), TO_CONN(edge_conn), type, socket_family); + /* Initialize the read token bucket to the maximum value which is the same as + * no rate limiting. */ + token_bucket_rw_init(&edge_conn->bucket, INT32_MAX, INT32_MAX, + monotime_coarse_get_stamp()); return edge_conn; } @@ -1261,7 +1273,7 @@ socket_failed_from_resource_exhaustion(void) */ if (get_max_sockets() > 65535) { /* TCP port exhaustion */ - rep_hist_note_overload(OVERLOAD_GENERAL); + rep_hist_note_tcp_exhaustion(); } else { /* File descriptor exhaustion */ rep_hist_note_overload(OVERLOAD_FD_EXHAUSTED); @@ -3457,6 +3469,19 @@ connection_bucket_read_limit(connection_t *conn, time_t now) base = get_cell_network_size(or_conn->wide_circ_ids); } + /* Edge connection have their own read bucket due to flow control being able + * to set a rate limit for them. However, for exit connections, we still need + * to honor the global bucket as well. */ + if (CONN_IS_EDGE(conn)) { + const edge_connection_t *edge_conn = CONST_TO_EDGE_CONN(conn); + conn_bucket = token_bucket_rw_get_read(&edge_conn->bucket); + if (conn->type == CONN_TYPE_EXIT) { + /* Decide between our limit and the global one. */ + goto end; + } + return conn_bucket; + } + if (!connection_is_rate_limited(conn)) { /* be willing to read on local conns even if our buckets are empty */ return conn_bucket>=0 ? conn_bucket : 1<<14; @@ -3467,6 +3492,7 @@ connection_bucket_read_limit(connection_t *conn, time_t now) global_bucket_val = MIN(global_bucket_val, relayed); } + end: return connection_bucket_get_share(base, priority, global_bucket_val, conn_bucket); } @@ -3644,6 +3670,13 @@ connection_buckets_decrement(connection_t *conn, time_t now, record_num_bytes_transferred_impl(conn, now, num_read, num_written); + /* Edge connection need to decrement the read side of the bucket used by our + * congestion control. */ + if (CONN_IS_EDGE(conn) && num_read > 0) { + edge_connection_t *edge_conn = TO_EDGE_CONN(conn); + token_bucket_rw_dec(&edge_conn->bucket, num_read, 0); + } + if (!connection_is_rate_limited(conn)) return; /* local IPs are free */ @@ -3697,14 +3730,16 @@ connection_write_bw_exhausted(connection_t *conn, bool is_global_bw) void connection_consider_empty_read_buckets(connection_t *conn) { + int is_global = 1; const char *reason; - if (!connection_is_rate_limited(conn)) + if (CONN_IS_EDGE(conn) && + token_bucket_rw_get_read(&TO_EDGE_CONN(conn)->bucket) <= 0) { + reason = "edge connection read bucket exhausted. Pausing."; + is_global = false; + } else if (!connection_is_rate_limited(conn)) { return; /* Always okay. */ - - int is_global = 1; - - if (token_bucket_rw_get_read(&global_bucket) <= 0) { + } else if (token_bucket_rw_get_read(&global_bucket) <= 0) { reason = "global read bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && token_bucket_rw_get_read(&global_relayed_bucket) <= 0) { @@ -3714,8 +3749,9 @@ connection_consider_empty_read_buckets(connection_t *conn) token_bucket_rw_get_read(&TO_OR_CONN(conn)->bucket) <= 0) { reason = "connection read bucket exhausted. Pausing."; is_global = false; - } else + } else { return; /* all good, no need to stop it */ + } LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); connection_read_bw_exhausted(conn, is_global); @@ -3819,6 +3855,10 @@ connection_bucket_refill_single(connection_t *conn, uint32_t now_ts) or_connection_t *or_conn = TO_OR_CONN(conn); token_bucket_rw_refill(&or_conn->bucket, now_ts); } + + if (CONN_IS_EDGE(conn)) { + token_bucket_rw_refill(&TO_EDGE_CONN(conn)->bucket, now_ts); + } } /** @@ -4556,9 +4596,9 @@ connection_handle_write_impl(connection_t *conn, int force) !dont_stop_writing) { /* it's done flushing */ if (connection_finished_flushing(conn) < 0) { /* already marked */ - return -1; + goto err; } - return 0; + goto done; } /* Call even if result is 0, since the global write bucket may @@ -4568,7 +4608,17 @@ connection_handle_write_impl(connection_t *conn, int force) if (n_read > 0 && connection_is_reading(conn)) connection_consider_empty_read_buckets(conn); + done: + /* If this is an edge connection with congestion control, check to see + * if it is time to send an xon */ + if (conn_uses_flow_control(conn)) { + flow_control_decide_xon(TO_EDGE_CONN(conn), n_written); + } + return 0; + + err: + return -1; } /* DOCDOC connection_handle_write */ diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index 36c94d6570..8b378b15a4 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -274,8 +274,8 @@ void connection_buf_add_compress(const char *string, size_t len, struct dir_connection_t *conn, int done); void connection_buf_add_buf(struct connection_t *conn, struct buf_t *buf); -size_t connection_get_inbuf_len(struct connection_t *conn); -size_t connection_get_outbuf_len(struct connection_t *conn); +size_t connection_get_inbuf_len(const struct connection_t *conn); +size_t connection_get_outbuf_len(const struct connection_t *conn); struct connection_t *connection_get_by_global_id(uint64_t id); struct connection_t *connection_get_by_type(int type); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 69606c0d53..cd57dea3d4 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -641,6 +641,13 @@ connection_start_reading,(connection_t *conn)) if (connection_should_read_from_linked_conn(conn)) connection_start_reading_from_linked_conn(conn); } else { + if (CONN_IS_EDGE(conn) && TO_EDGE_CONN(conn)->xoff_received) { + /* We should not get called here if we're waiting for an XON, but + * belt-and-suspenders */ + log_notice(LD_NET, + "Request to start reading on an edgeconn blocked with XOFF"); + return; + } if (event_add(conn->read_event, NULL)) log_warn(LD_NET, "Error from libevent setting read event state for %d " "to watched: %s", @@ -1293,6 +1300,7 @@ signewnym_impl(time_t now) circuit_mark_all_dirty_circs_as_unusable(); addressmap_clear_transient(); hs_client_purge_state(); + purge_vanguards_lite(); time_of_last_signewnym = now; signewnym_is_pending = 0; @@ -1370,6 +1378,7 @@ CALLBACK(save_state); CALLBACK(write_stats_file); CALLBACK(control_per_second_events); CALLBACK(second_elapsed); +CALLBACK(manage_vglite); #undef CALLBACK @@ -1392,6 +1401,9 @@ STATIC periodic_event_item_t mainloop_periodic_events[] = { CALLBACK(second_elapsed, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + /* Update vanguards-lite once per hour, if we have networking */ + CALLBACK(manage_vglite, NET_PARTICIPANT, FL(NEED_NET)), + /* XXXX Do we have a reason to do this on a callback? Does it do any good at * all? For now, if we're dormant, we can let our listeners decay. */ CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)), @@ -1662,6 +1674,21 @@ mainloop_schedule_shutdown(int delay_sec) mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv); } +/** + * Update vanguards-lite layer2 nodes, once every 15 minutes + */ +static int +manage_vglite_callback(time_t now, const or_options_t *options) +{ + (void)now; + (void)options; +#define VANGUARDS_LITE_INTERVAL (15*60) + + maintain_layer2_guards(); + + return VANGUARDS_LITE_INTERVAL; +} + /** Perform regular maintenance tasks. This function gets run once per * second. */ diff --git a/src/core/or/channel.c b/src/core/or/channel.c index c4f3e76fc8..c46fa93e58 100644 --- a/src/core/or/channel.c +++ b/src/core/or/channel.c @@ -2629,24 +2629,42 @@ channel_dump_statistics, (channel_t *chan, int severity)) circuitmux_num_circuits(chan->cmux) : 0); /* Describe timestamps */ - tor_log(severity, LD_GENERAL, - " * Channel %"PRIu64 " was last used by a " - "client at %"PRIu64 " (%"PRIu64 " seconds ago)", - (chan->global_identifier), - (uint64_t)(chan->timestamp_client), - (uint64_t)(now - chan->timestamp_client)); - tor_log(severity, LD_GENERAL, - " * Channel %"PRIu64 " last received a cell " - "at %"PRIu64 " (%"PRIu64 " seconds ago)", - (chan->global_identifier), - (uint64_t)(chan->timestamp_recv), - (uint64_t)(now - chan->timestamp_recv)); - tor_log(severity, LD_GENERAL, - " * Channel %"PRIu64 " last transmitted a cell " - "at %"PRIu64 " (%"PRIu64 " seconds ago)", - (chan->global_identifier), - (uint64_t)(chan->timestamp_xmit), - (uint64_t)(now - chan->timestamp_xmit)); + if (chan->timestamp_client == 0) { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " was never used by a " + "client", (chan->global_identifier)); + } else { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " was last used by a " + "client at %"PRIu64 " (%"PRIu64 " seconds ago)", + (chan->global_identifier), + (uint64_t)(chan->timestamp_client), + (uint64_t)(now - chan->timestamp_client)); + } + if (chan->timestamp_recv == 0) { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " never received a cell", + (chan->global_identifier)); + } else { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " last received a cell " + "at %"PRIu64 " (%"PRIu64 " seconds ago)", + (chan->global_identifier), + (uint64_t)(chan->timestamp_recv), + (uint64_t)(now - chan->timestamp_recv)); + } + if (chan->timestamp_xmit == 0) { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " never transmitted a cell", + (chan->global_identifier)); + } else { + tor_log(severity, LD_GENERAL, + " * Channel %"PRIu64 " last transmitted a cell " + "at %"PRIu64 " (%"PRIu64 " seconds ago)", + (chan->global_identifier), + (uint64_t)(chan->timestamp_xmit), + (uint64_t)(now - chan->timestamp_xmit)); + } /* Describe counters and rates */ tor_log(severity, LD_GENERAL, diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 481dafef91..9db8e2392d 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -64,6 +64,7 @@ #include "trunnel/netinfo.h" #include "core/or/channelpadding.h" #include "core/or/extendinfo.h" +#include "core/or/congestion_control_common.h" #include "core/or/cell_st.h" #include "core/or/cell_queue_st.h" @@ -793,7 +794,7 @@ channel_tls_num_cells_writeable_method(channel_t *chan) cell_network_size = get_cell_network_size(tlschan->conn->wide_circ_ids); outbuf_len = connection_get_outbuf_len(TO_CONN(tlschan->conn)); /* Get the number of cells */ - n = CEIL_DIV(OR_CONN_HIGHWATER - outbuf_len, cell_network_size); + n = CEIL_DIV(or_conn_highwatermark() - outbuf_len, cell_network_size); if (n < 0) n = 0; #if SIZEOF_SIZE_T > SIZEOF_INT if (n > INT_MAX) n = INT_MAX; diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index 870bcbf7cf..be6429438a 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -22,6 +22,7 @@ struct hs_token_t; struct circpad_machine_spec_t; struct circpad_machine_runtime_t; +struct congestion_control_t; /** Number of padding state machines on a circuit. */ #define CIRCPAD_MAX_MACHINES (2) @@ -244,6 +245,9 @@ struct circuit_t { * that STOP commands actually correspond to the current machine, * and not a previous one. */ uint32_t padding_machine_ctr; + + /** Congestion control fields */ + struct congestion_control_t *ccontrol; }; #endif /* !defined(CIRCUIT_ST_H) */ diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 2bcc642a97..31e3868b65 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -1359,7 +1359,9 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) int routelen = DEFAULT_ROUTE_LEN; int known_purpose = 0; - if (circuit_should_use_vanguards(purpose)) { + /* If we're using L3 vanguards, we need longer paths for onion services */ + if (circuit_purpose_is_hidden_service(purpose) && + get_options()->HSLayer3Nodes) { /* Clients want an extra hop for rends to avoid linkability. * Services want it for intro points to avoid publishing their * layer3 guards. They want it for hsdir posts to use @@ -1374,14 +1376,6 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) return routelen+1; - /* If we only have Layer2 vanguards, then we do not need - * the extra hop for linkabilty reasons (see below). - * This means all hops can be of the form: - * S/C - G - L2 - M - R/HSDir/I - */ - if (get_options()->HSLayer2Nodes && !get_options()->HSLayer3Nodes) - return routelen+1; - /* For connections to hsdirs, clients want two extra hops * when using layer3 guards, to avoid linkability. * Same goes for intro points. Note that the route len @@ -1400,16 +1394,14 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) return routelen; switch (purpose) { - /* These two purposes connect to a router that we chose, so - * DEFAULT_ROUTE_LEN is safe. */ - case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: - /* hidden service connecting to introduction point */ + /* These purposes connect to a router that we chose, so DEFAULT_ROUTE_LEN + * is safe: */ case CIRCUIT_PURPOSE_TESTING: /* router reachability testing */ known_purpose = 1; break; - /* These three purposes connect to a router that someone else + /* These purposes connect to a router that someone else * might have chosen, so add an extra hop to protect anonymity. */ case CIRCUIT_PURPOSE_C_GENERAL: case CIRCUIT_PURPOSE_C_HSDIR_GET: @@ -1419,6 +1411,9 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) /* client connecting to introduction point */ case CIRCUIT_PURPOSE_S_CONNECT_REND: /* hidden service connecting to rendezvous point */ + case CIRCUIT_PURPOSE_S_ESTABLISH_INTRO: + /* hidden service connecting to intro point. In this case we want an extra + hop to avoid linkability attacks by the introduction point. */ known_purpose = 1; routelen++; break; @@ -2019,7 +2014,7 @@ cpath_build_state_to_crn_ipv6_extend_flag(const cpath_build_state_t *state, } /** Decide a suitable length for circ's cpath, and pick an exit - * router (or use <b>exit</b> if provided). Store these in the + * router (or use <b>exit_ei</b> if provided). Store these in the * cpath. * * If <b>is_hs_v3_rp_circuit</b> is set, then this exit should be suitable to @@ -2072,7 +2067,7 @@ onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei, return 0; } -/** Give <b>circ</b> a new exit destination to <b>exit</b>, and add a +/** Give <b>circ</b> a new exit destination to <b>exit_ei</b>, and add a * hop to the cpath reflecting this. Don't send the next extend cell -- * the caller will do this if it wants to. */ @@ -2114,8 +2109,6 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) return -1; } - // XXX: Should cannibalized circuits be dirty or not? Not easy to say.. - return 0; } @@ -2261,8 +2254,14 @@ middle_node_must_be_vanguard(const or_options_t *options, return 0; } - /* If we have sticky L2 nodes, and this is an L2 pick, use vanguards */ - if (options->HSLayer2Nodes && cur_len == 1) { + /* Don't even bother if the feature is disabled */ + if (!vanguards_lite_is_enabled()) { + return 0; + } + + /* If we are a hidden service circuit, always use either vanguards-lite + * or HSLayer2Nodes for 2nd hop. */ + if (cur_len == 1) { return 1; } @@ -2286,7 +2285,8 @@ pick_vanguard_middle_node(const or_options_t *options, /* Pick the right routerset based on the current hop */ if (cur_len == 1) { - vanguard_routerset = options->HSLayer2Nodes; + vanguard_routerset = options->HSLayer2Nodes ? + options->HSLayer2Nodes : get_layer2_guards(); } else if (cur_len == 2) { vanguard_routerset = options->HSLayer3Nodes; } else { @@ -2295,6 +2295,10 @@ pick_vanguard_middle_node(const or_options_t *options, return NULL; } + if (BUG(!vanguard_routerset)) { + return NULL; + } + node = pick_restricted_middle_node(flags, vanguard_routerset, options->ExcludeNodes, excluded, cur_len+1); diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 4f62284e29..4dbf4d4549 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -64,6 +64,7 @@ #include "core/or/circuitpadding.h" #include "core/or/crypt_path.h" #include "core/or/extendinfo.h" +#include "core/or/status.h" #include "core/or/trace_probes_circuit.h" #include "core/mainloop/connection.h" #include "app/config/config.h" @@ -100,6 +101,7 @@ #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/buf/buffers.h" +#include "core/or/congestion_control_common.h" #include "core/or/ocirc_event.h" @@ -1143,6 +1145,8 @@ circuit_free_(circuit_t *circ) * hs identifier is freed. */ hs_circ_cleanup_on_free(circ); + congestion_control_free(circ->ccontrol); + if (CIRCUIT_IS_ORIGIN(circ)) { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); mem = ocirc; @@ -2343,6 +2347,12 @@ circuit_about_to_free(circuit_t *circ) circuitmux_detach_circuit(or_circ->p_chan->cmux, circ); circuit_set_p_circid_chan(or_circ, 0, NULL); } + + if (or_circ->n_cells_discarded_at_end) { + time_t age = approx_time() - circ->timestamp_created.tv_sec; + note_circ_closed_for_unrecognized_cells( + age, or_circ->n_cells_discarded_at_end); + } } else { origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); edge_connection_t *conn; @@ -2586,8 +2596,10 @@ conns_compare_by_buffer_age_(const void **a_, const void **b_) /** We're out of memory for cells, having allocated <b>current_allocation</b> * bytes' worth. Kill the 'worst' circuits until we're under - * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. */ -void + * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. + * + * Return the number of bytes removed. */ +size_t circuits_handle_oom(size_t current_allocation) { smartlist_t *circlist; @@ -2597,6 +2609,7 @@ circuits_handle_oom(size_t current_allocation) size_t mem_recovered=0; int n_circuits_killed=0; int n_dirconns_killed=0; + int n_edgeconns_killed = 0; uint32_t now_ts; log_notice(LD_GENERAL, "We're low on memory (cell queues total alloc:" " %"TOR_PRIuSZ" buffer total alloc: %" TOR_PRIuSZ "," @@ -2613,12 +2626,11 @@ circuits_handle_oom(size_t current_allocation) tor_zstd_get_total_allocation(), tor_lzma_get_total_allocation(), hs_cache_get_total_allocation()); - { size_t mem_target = (size_t)(get_options()->MaxMemInQueues * FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) - return; + return 0; mem_to_recover = current_allocation - mem_target; } @@ -2664,12 +2676,19 @@ circuits_handle_oom(size_t current_allocation) if (conn_age < circ->age_tmp) { break; } - if (conn->type == CONN_TYPE_DIR && conn->linked_conn == NULL) { + /* Also consider edge connections so we don't accumulate bytes on the + * outbuf due to a malicious destination holding off the read on us. */ + if ((conn->type == CONN_TYPE_DIR && conn->linked_conn == NULL) || + CONN_IS_EDGE(conn)) { if (!conn->marked_for_close) connection_mark_for_close(conn); mem_recovered += single_conn_free_bytes(conn); - ++n_dirconns_killed; + if (conn->type == CONN_TYPE_DIR) { + ++n_dirconns_killed; + } else { + ++n_edgeconns_killed; + } if (mem_recovered >= mem_to_recover) goto done_recovering_mem; @@ -2697,14 +2716,16 @@ circuits_handle_oom(size_t current_allocation) } SMARTLIST_FOREACH_END(circ); done_recovering_mem: - log_notice(LD_GENERAL, "Removed %"TOR_PRIuSZ" bytes by killing %d circuits; " "%d circuits remain alive. Also killed %d non-linked directory " - "connections.", + "connections. Killed %d edge connections", mem_recovered, n_circuits_killed, smartlist_len(circlist) - n_circuits_killed, - n_dirconns_killed); + n_dirconns_killed, + n_edgeconns_killed); + + return mem_recovered; } /** Verify that circuit <b>c</b> has all of its invariants diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index f5791d7c12..147e2cb2f8 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -232,7 +232,7 @@ int circuit_count_pending_on_channel(channel_t *chan); MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c)); void circuit_free_all(void); -void circuits_handle_oom(size_t current_allocation); +size_t circuits_handle_oom(size_t current_allocation); void circuit_clear_testing_cell_stats(circuit_t *circ); diff --git a/src/core/or/circuitmux_ewma.c b/src/core/or/circuitmux_ewma.c index 0382e62f75..adf256ab05 100644 --- a/src/core/or/circuitmux_ewma.c +++ b/src/core/or/circuitmux_ewma.c @@ -45,7 +45,10 @@ /*** EWMA parameter #defines ***/ /** How long does a tick last (seconds)? */ -#define EWMA_TICK_LEN 10 +#define EWMA_TICK_LEN_DEFAULT 10 +#define EWMA_TICK_LEN_MIN 1 +#define EWMA_TICK_LEN_MAX 600 +static int ewma_tick_len = EWMA_TICK_LEN_DEFAULT; /** The default per-tick scale factor, if it hasn't been overridden by a * consensus or a configuration setting. zero means "disabled". */ @@ -148,7 +151,7 @@ cell_ewma_get_tick(void) monotime_coarse_get(&now); int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick, &now); - return current_tick_num + msec_diff / (1000*EWMA_TICK_LEN); + return current_tick_num + msec_diff / (1000*ewma_tick_len); } /** @@ -527,15 +530,15 @@ cell_ewma_get_current_tick_and_fraction(double *remainder_out) monotime_coarse_get(&now); int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick, &now); - if (msec_diff > (1000*EWMA_TICK_LEN)) { - unsigned ticks_difference = msec_diff / (1000*EWMA_TICK_LEN); + if (msec_diff > (1000*ewma_tick_len)) { + unsigned ticks_difference = msec_diff / (1000*ewma_tick_len); monotime_coarse_add_msec(&start_of_current_tick, &start_of_current_tick, - ticks_difference * 1000 * EWMA_TICK_LEN); + ticks_difference * 1000 * ewma_tick_len); current_tick_num += ticks_difference; - msec_diff %= 1000*EWMA_TICK_LEN; + msec_diff %= 1000*ewma_tick_len; } - *remainder_out = ((double)msec_diff) / (1.0e3 * EWMA_TICK_LEN); + *remainder_out = ((double)msec_diff) / (1.0e3 * ewma_tick_len); return current_tick_num; } @@ -605,15 +608,20 @@ cmux_ewma_set_options(const or_options_t *options, /* Both options and consensus can be NULL. This assures us to either get a * valid configured value or the default one. */ halflife = get_circuit_priority_halflife(options, consensus, &source); + ewma_tick_len = networkstatus_get_param(consensus, + "CircuitPriorityTickSecs", + EWMA_TICK_LEN_DEFAULT, + EWMA_TICK_LEN_MIN, + EWMA_TICK_LEN_MAX); /* convert halflife into halflife-per-tick. */ - halflife /= EWMA_TICK_LEN; + halflife /= ewma_tick_len; /* compute per-tick scale factor. */ ewma_scale_factor = exp(LOG_ONEHALF / halflife); log_info(LD_OR, "Enabled cell_ewma algorithm because of value in %s; " "scale factor is %f per %d seconds", - source, ewma_scale_factor, EWMA_TICK_LEN); + source, ewma_scale_factor, ewma_tick_len); } /** Return the multiplier necessary to convert the value of a cell sent in diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c index 6dfe94de01..99dc5f9d83 100644 --- a/src/core/or/circuitpadding.c +++ b/src/core/or/circuitpadding.c @@ -2967,6 +2967,8 @@ signed_error_t circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) { int retval = 0; + /* Should we send back a STOP cell? */ + bool respond_with_stop = true; circpad_negotiate_t *negotiate; if (CIRCUIT_IS_ORIGIN(circ)) { @@ -2992,6 +2994,12 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) negotiate->machine_type, negotiate->machine_ctr); goto done; } + + /* If we reached this point we received a STOP command from an old or + unknown machine. Don't reply with our own STOP since there is no one to + handle it on the other end */ + respond_with_stop = false; + if (negotiate->machine_ctr <= circ->padding_machine_ctr) { log_info(LD_CIRC, "Received STOP command for old machine %u, ctr %u", negotiate->machine_type, negotiate->machine_ctr); @@ -3023,10 +3031,13 @@ circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) retval = -1; done: - circpad_padding_negotiated(circ, negotiate->machine_type, - negotiate->command, - (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR, - negotiate->machine_ctr); + if (respond_with_stop) { + circpad_padding_negotiated(circ, negotiate->machine_type, + negotiate->command, + (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR, + negotiate->machine_ctr); + } + circpad_negotiate_free(negotiate); return retval; diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 044b30b8b3..2ec391eca0 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -1204,25 +1204,6 @@ needs_circuits_for_build(int num) return 0; } -/** - * Launch the appropriate type of predicted circuit for hidden - * services, depending on our options. - */ -static void -circuit_launch_predicted_hs_circ(int flags) -{ - /* K.I.S.S. implementation of bug #23101: If we are using - * vanguards or pinned middles, pre-build a specific purpose - * for HS circs. */ - if (circuit_should_use_vanguards(CIRCUIT_PURPOSE_HS_VANGUARDS)) { - circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags); - } else { - /* If no vanguards, then no HS-specific prebuilt circuits are needed. - * Normal GENERAL circs are fine */ - circuit_launch(CIRCUIT_PURPOSE_C_GENERAL, flags); - } -} - /** Determine how many circuits we have open that are clean, * Make sure it's enough for all the upcoming behaviors we predict we'll have. * But put an upper bound on the total number of circuits. @@ -1276,7 +1257,7 @@ circuit_predict_and_launch_new(void) "Have %d clean circs (%d internal), need another internal " "circ for my hidden service.", num, num_internal); - circuit_launch_predicted_hs_circ(flags); + circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags); return; } @@ -1295,7 +1276,10 @@ circuit_predict_and_launch_new(void) " another hidden service circ.", num, num_uptime_internal, num_internal); - circuit_launch_predicted_hs_circ(flags); + /* Always launch vanguards purpose circuits for HS clients, + * for vanguards-lite. This prevents us from cannibalizing + * to build these circuits (and thus not use vanguards). */ + circuit_launch(CIRCUIT_PURPOSE_HS_VANGUARDS, flags); return; } @@ -2022,16 +2006,12 @@ circuit_is_hs_v3(const circuit_t *circ) int circuit_should_use_vanguards(uint8_t purpose) { - const or_options_t *options = get_options(); - - /* Only hidden service circuits use vanguards */ - if (!circuit_purpose_is_hidden_service(purpose)) - return 0; - - /* Pinned middles are effectively vanguards */ - if (options->HSLayer2Nodes || options->HSLayer3Nodes) + /* All hidden service circuits use either vanguards or + * vanguards-lite. */ + if (circuit_purpose_is_hidden_service(purpose)) return 1; + /* Everything else is a normal circuit */ return 0; } @@ -2069,13 +2049,11 @@ circuit_should_cannibalize_to_build(uint8_t purpose_to_build, return 0; } - /* For vanguards, the server-side intro circ is not cannibalized - * because we pre-build 4 hop HS circuits, and it only needs a 3 hop - * circuit. It is also long-lived, so it is more important that - * it have lower latency than get built fast. + /* The server-side intro circ is not cannibalized because it only + * needs a 3 hop circuit. It is also long-lived, so it is more + * important that it have lower latency than get built fast. */ - if (circuit_should_use_vanguards(purpose_to_build) && - purpose_to_build == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) { + if (purpose_to_build == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO) { return 0; } diff --git a/src/core/or/command.c b/src/core/or/command.c index 622217a78e..40eb1554c0 100644 --- a/src/core/or/command.c +++ b/src/core/or/command.c @@ -563,7 +563,7 @@ command_process_relay_cell(cell_t *cell, channel_t *chan) } if ((reason = circuit_receive_relay_cell(cell, circ, direction)) < 0) { - log_fn(LOG_PROTOCOL_WARN,LD_PROTOCOL,"circuit_receive_relay_cell " + log_fn(LOG_DEBUG,LD_PROTOCOL,"circuit_receive_relay_cell " "(%s) failed. Closing.", direction==CELL_DIRECTION_OUT?"forward":"backward"); /* Always emit a bandwidth event for closed circs */ diff --git a/src/core/or/congestion_control_common.c b/src/core/or/congestion_control_common.c new file mode 100644 index 0000000000..0919f037db --- /dev/null +++ b/src/core/or/congestion_control_common.c @@ -0,0 +1,1038 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_common.c + * \brief Common code used by all congestion control algorithms. + */ + +#define TOR_CONGESTION_CONTROL_COMMON_PRIVATE + +#include "core/or/or.h" + +#include "core/or/circuitlist.h" +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "core/mainloop/connection.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_vegas.h" +#include "core/or/congestion_control_nola.h" +#include "core/or/congestion_control_westwood.h" +#include "core/or/congestion_control_st.h" +#include "core/or/trace_probes_cc.h" +#include "lib/time/compat_time.h" +#include "feature/nodelist/networkstatus.h" + +/* Consensus parameter defaults. + * + * More details for each of the parameters can be found in proposal 324, + * section 6.5 including tuning notes. */ +#define CIRCWINDOW_INIT (500) +#define SENDME_INC_DFLT (50) + +#define CWND_INC_DFLT (50) +#define CWND_INC_PCT_SS_DFLT (100) +#define CWND_INC_RATE_DFLT (1) +#define CWND_MAX_DFLT (INT32_MAX) +#define CWND_MIN_DFLT (MAX(100, SENDME_INC_DFLT)) + +#define BWE_SENDME_MIN_DFLT (5) +#define EWMA_CWND_COUNT_DFLT (2) + +/* BDP algorithms for each congestion control algorithms use the piecewise + * estimattor. See section 3.1.4 of proposal 324. */ +#define WESTWOOD_BDP_ALG BDP_ALG_PIECEWISE +#define VEGAS_BDP_MIX_ALG BDP_ALG_PIECEWISE +#define NOLA_BDP_ALG BDP_ALG_PIECEWISE + +/* Indicate OR connection buffer limitations used to stop or start accepting + * cells in its outbuf. + * + * These watermarks are historical to tor in a sense that they've been used + * almost from the genesis point. And were likely defined to fit the bounds of + * TLS records of 16KB which would be around 32 cells. + * + * These are defaults of the consensus parameter "orconn_high" and "orconn_low" + * values. */ +#define OR_CONN_HIGHWATER_DFLT (32*1024) +#define OR_CONN_LOWWATER_DFLT (16*1024) + +/* Low and high values of circuit cell queue sizes. They are used to tell when + * to start or stop reading on the streams attached on the circuit. + * + * These are defaults of the consensus parameters "cellq_high" and "cellq_low". + */ +#define CELL_QUEUE_LOW_DFLT (10) +#define CELL_QUEUE_HIGH_DFLT (256) + +static uint64_t congestion_control_update_circuit_rtt(congestion_control_t *, + uint64_t); +static bool congestion_control_update_circuit_bdp(congestion_control_t *, + const circuit_t *, + const crypt_path_t *, + uint64_t, uint64_t); + +/* Consensus parameters cached. The non static ones are extern. */ +static uint32_t cwnd_max = CWND_MAX_DFLT; +int32_t cell_queue_high = CELL_QUEUE_HIGH_DFLT; +int32_t cell_queue_low = CELL_QUEUE_LOW_DFLT; +uint32_t or_conn_highwater = OR_CONN_HIGHWATER_DFLT; +uint32_t or_conn_lowwater = OR_CONN_LOWWATER_DFLT; + +/** + * Update global congestion control related consensus parameter values, + * every consensus update. + */ +void +congestion_control_new_consensus_params(const networkstatus_t *ns) +{ +#define CELL_QUEUE_HIGH_MIN (1) +#define CELL_QUEUE_HIGH_MAX (1000) + cell_queue_high = networkstatus_get_param(ns, "cellq_high", + CELL_QUEUE_HIGH_DFLT, + CELL_QUEUE_HIGH_MIN, + CELL_QUEUE_HIGH_MAX); + +#define CELL_QUEUE_LOW_MIN (1) +#define CELL_QUEUE_LOW_MAX (1000) + cell_queue_low = networkstatus_get_param(ns, "cellq_low", + CELL_QUEUE_LOW_DFLT, + CELL_QUEUE_LOW_MIN, + CELL_QUEUE_LOW_MAX); + +#define OR_CONN_HIGHWATER_MIN (CELL_PAYLOAD_SIZE) +#define OR_CONN_HIGHWATER_MAX (INT32_MAX) + or_conn_highwater = + networkstatus_get_param(ns, "orconn_high", + OR_CONN_HIGHWATER_DFLT, + OR_CONN_HIGHWATER_MIN, + OR_CONN_HIGHWATER_MAX); + +#define OR_CONN_LOWWATER_MIN (CELL_PAYLOAD_SIZE) +#define OR_CONN_LOWWATER_MAX (INT32_MAX) + or_conn_lowwater = + networkstatus_get_param(ns, "orconn_low", + OR_CONN_LOWWATER_DFLT, + OR_CONN_LOWWATER_MIN, + OR_CONN_LOWWATER_MAX); + +#define CWND_MAX_MIN 500 +#define CWND_MAX_MAX (INT32_MAX) + cwnd_max = + networkstatus_get_param(NULL, "cc_cwnd_max", + CWND_MAX_DFLT, + CWND_MAX_MIN, + CWND_MAX_MAX); +} + +/** + * Set congestion control parameters on a circuit's congestion + * control object based on values from the consensus. + * + * cc_alg is the negotiated congestion control algorithm. + * + * sendme_inc is the number of packaged cells that a sendme cell + * acks. This parameter will come from circuit negotiation. + */ +static void +congestion_control_init_params(congestion_control_t *cc, + cc_alg_t cc_alg, + int sendme_inc) +{ +#define CWND_INIT_MIN 100 +#define CWND_INIT_MAX (10000) + cc->cwnd = + networkstatus_get_param(NULL, "cc_cwnd_init", + CIRCWINDOW_INIT, + CWND_INIT_MIN, + CWND_INIT_MAX); + +#define CWND_INC_PCT_SS_MIN 1 +#define CWND_INC_PCT_SS_MAX (500) + cc->cwnd_inc_pct_ss = + networkstatus_get_param(NULL, "cc_cwnd_inc_pct_ss", + CWND_INC_PCT_SS_DFLT, + CWND_INC_PCT_SS_MIN, + CWND_INC_PCT_SS_MAX); + +#define CWND_INC_MIN 1 +#define CWND_INC_MAX (1000) + cc->cwnd_inc = + networkstatus_get_param(NULL, "cc_cwnd_inc", + CWND_INC_DFLT, + CWND_INC_MIN, + CWND_INC_MAX); + +#define CWND_INC_RATE_MIN 1 +#define CWND_INC_RATE_MAX (250) + cc->cwnd_inc_rate = + networkstatus_get_param(NULL, "cc_cwnd_inc_rate", + CWND_INC_RATE_DFLT, + CWND_INC_RATE_MIN, + CWND_INC_RATE_MAX); + +#define SENDME_INC_MIN 10 +#define SENDME_INC_MAX (1000) + cc->sendme_inc = + networkstatus_get_param(NULL, "cc_sendme_inc", + sendme_inc, + SENDME_INC_MIN, + SENDME_INC_MAX); + + // XXX: this min needs to abide by sendme_inc range rules somehow +#define CWND_MIN_MIN sendme_inc +#define CWND_MIN_MAX (1000) + cc->cwnd_min = + networkstatus_get_param(NULL, "cc_cwnd_min", + CWND_MIN_DFLT, + CWND_MIN_MIN, + CWND_MIN_MAX); + +#define EWMA_CWND_COUNT_MIN 1 +#define EWMA_CWND_COUNT_MAX (100) + cc->ewma_cwnd_cnt = + networkstatus_get_param(NULL, "cc_ewma_cwnd_cnt", + EWMA_CWND_COUNT_DFLT, + EWMA_CWND_COUNT_MIN, + EWMA_CWND_COUNT_MAX); + +#define BWE_SENDME_MIN_MIN 2 +#define BWE_SENDME_MIN_MAX (20) + cc->bwe_sendme_min = + networkstatus_get_param(NULL, "cc_bwe_min", + BWE_SENDME_MIN_DFLT, + BWE_SENDME_MIN_MIN, + BWE_SENDME_MIN_MAX); + +#define CC_ALG_MIN 0 +#define CC_ALG_MAX (NUM_CC_ALGS-1) + cc->cc_alg = + networkstatus_get_param(NULL, "cc_alg", + cc_alg, + CC_ALG_MIN, + CC_ALG_MAX); + + bdp_alg_t default_bdp_alg = 0; + + switch (cc->cc_alg) { + case CC_ALG_WESTWOOD: + default_bdp_alg = WESTWOOD_BDP_ALG; + break; + case CC_ALG_VEGAS: + default_bdp_alg = VEGAS_BDP_MIX_ALG; + break; + case CC_ALG_NOLA: + default_bdp_alg = NOLA_BDP_ALG; + break; + case CC_ALG_SENDME: + default: + tor_fragile_assert(); + return; // No alg-specific params + } + + cc->bdp_alg = + networkstatus_get_param(NULL, "cc_bdp_alg", + default_bdp_alg, + 0, + NUM_BDP_ALGS-1); + + /* Algorithm-specific parameters */ + if (cc->cc_alg == CC_ALG_WESTWOOD) { + congestion_control_westwood_set_params(cc); + } else if (cc->cc_alg == CC_ALG_VEGAS) { + congestion_control_vegas_set_params(cc); + } else if (cc->cc_alg == CC_ALG_NOLA) { + congestion_control_nola_set_params(cc); + } +} + +/** + * Allocate and initialize fields in congestion control object. + * + * cc_alg is the negotiated congestion control algorithm. + * + * sendme_inc is the number of packaged cells that a sendme cell + * acks. This parameter will come from circuit negotiation. + */ +static void +congestion_control_init(congestion_control_t *cc, cc_alg_t cc_alg, + int sendme_inc) +{ + cc->sendme_pending_timestamps = smartlist_new(); + cc->sendme_arrival_timestamps = smartlist_new(); + + cc->in_slow_start = 1; + congestion_control_init_params(cc, cc_alg, sendme_inc); + + cc->next_cc_event = CWND_UPDATE_RATE(cc); +} + +/** Allocate and initialize a new congestion control object */ +congestion_control_t * +congestion_control_new(void) +{ + congestion_control_t *cc = tor_malloc_zero(sizeof(congestion_control_t)); + + // XXX: the alg and the sendme_inc need to be negotiated during + // circuit handshake + congestion_control_init(cc, CC_ALG_VEGAS, SENDME_INC_DFLT); + + return cc; +} + +/** + * Free a congestion control object and its asssociated state. + */ +void +congestion_control_free_(congestion_control_t *cc) +{ + if (!cc) + return; + + SMARTLIST_FOREACH(cc->sendme_pending_timestamps, uint64_t *, t, tor_free(t)); + SMARTLIST_FOREACH(cc->sendme_arrival_timestamps, uint64_t *, t, tor_free(t)); + smartlist_free(cc->sendme_pending_timestamps); + smartlist_free(cc->sendme_arrival_timestamps); + + tor_free(cc); +} + +/** + * Enqueue a u64 timestamp to the end of a queue of timestamps. + */ +static inline void +enqueue_timestamp(smartlist_t *timestamps_u64, uint64_t timestamp_usec) +{ + uint64_t *timestamp_ptr = tor_malloc(sizeof(uint64_t)); + *timestamp_ptr = timestamp_usec; + + smartlist_add(timestamps_u64, timestamp_ptr); +} + +/** + * Peek at the head of a smartlist queue of u64 timestamps. + */ +static inline uint64_t +peek_timestamp(const smartlist_t *timestamps_u64_usecs) +{ + uint64_t *timestamp_ptr = smartlist_get(timestamps_u64_usecs, 0); + + if (BUG(!timestamp_ptr)) { + log_err(LD_CIRC, "Congestion control timestamp list became empty!"); + return 0; + } + + return *timestamp_ptr; +} + +/** + * Dequeue a u64 monotime usec timestamp from the front of a + * smartlist of pointers to 64. + */ +static inline uint64_t +dequeue_timestamp(smartlist_t *timestamps_u64_usecs) +{ + uint64_t *timestamp_ptr = smartlist_get(timestamps_u64_usecs, 0); + uint64_t timestamp_u64; + + if (BUG(!timestamp_ptr)) { + log_err(LD_CIRC, "Congestion control timestamp list became empty!"); + return 0; + } + + timestamp_u64 = *timestamp_ptr; + smartlist_del_keeporder(timestamps_u64_usecs, 0); + tor_free(timestamp_ptr); + + return timestamp_u64; +} + +/** + * Returns the number of sendme acks that will be recieved in the + * current congestion window size, rounded to nearest int. + */ +static inline uint64_t +sendme_acks_per_cwnd(const congestion_control_t *cc) +{ + /* We add half a sendme_inc to cwnd to round to the nearest int */ + return ((cc->cwnd + cc->sendme_inc/2)/cc->sendme_inc); +} + +/** + * Get a package window from either old sendme logic, or congestion control. + * + * A package window is how many cells you can still send. + */ +int +congestion_control_get_package_window(const circuit_t *circ, + const crypt_path_t *cpath) +{ + int package_window; + congestion_control_t *cc; + + tor_assert(circ); + + if (cpath) { + package_window = cpath->package_window; + cc = cpath->ccontrol; + } else { + package_window = circ->package_window; + cc = circ->ccontrol; + } + + if (!cc) { + return package_window; + } else { + /* Inflight can be above cwnd if cwnd was just reduced */ + if (cc->inflight > cc->cwnd) + return 0; + /* In the extremely unlikely event that cwnd-inflight is larger than + * INT32_MAX, just return that cap, so old code doesn't explode. */ + else if (cc->cwnd - cc->inflight > INT32_MAX) + return INT32_MAX; + else + return (int)(cc->cwnd - cc->inflight); + } +} + +/** + * Returns the number of cells that are acked by every sendme. + */ +int +sendme_get_inc_count(const circuit_t *circ, const crypt_path_t *layer_hint) +{ + int sendme_inc = CIRCWINDOW_INCREMENT; + congestion_control_t *cc = NULL; + + if (layer_hint) { + cc = layer_hint->ccontrol; + } else { + cc = circ->ccontrol; + } + + if (cc) { + sendme_inc = cc->sendme_inc; + } + + return sendme_inc; +} + +/** Return true iff the next cell we send will result in the other endpoint + * sending a SENDME. + * + * We are able to know that because the package or inflight window value minus + * one cell (the possible SENDME cell) should be a multiple of the + * cells-per-sendme increment value (set via consensus parameter, negotiated + * for the circuit, and passed in as sendme_inc). + * + * This function is used when recording a cell digest and this is done quite + * low in the stack when decrypting or encrypting a cell. The window is only + * updated once the cell is actually put in the outbuf. + */ +bool +circuit_sent_cell_for_sendme(const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + congestion_control_t *cc; + int window; + + tor_assert(circ); + + if (layer_hint) { + window = layer_hint->package_window; + cc = layer_hint->ccontrol; + } else { + window = circ->package_window; + cc = circ->ccontrol; + } + + /* If we are using congestion control and the alg is not + * old-school 'fixed', then use cc->inflight to determine + * when sendmes will be sent */ + if (cc) { + if (!cc->inflight) + return false; + + /* This check must be +1 because this function is called *before* + * inflight is incremented for the sent cell */ + if ((cc->inflight+1) % cc->sendme_inc != 0) + return false; + + return true; + } + + /* At the start of the window, no SENDME will be expected. */ + if (window == CIRCWINDOW_START) { + return false; + } + + /* Are we at the limit of the increment and if not, we don't expect next + * cell is a SENDME. + * + * We test against the window minus 1 because when we are looking if the + * next cell is a SENDME, the window (either package or deliver) hasn't been + * decremented just yet so when this is called, we are currently processing + * the "window - 1" cell. + */ + if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) { + return false; + } + + /* Next cell is expected to be a SENDME. */ + return true; +} + +/** + * Call-in to tell congestion control code that this circuit sent a cell. + * + * This updates the 'inflight' counter, and if this is a cell that will + * cause the other end to send a SENDME, record the current time in a list + * of pending timestamps, so that we can later compute the circuit RTT when + * the SENDME comes back. */ +void +congestion_control_note_cell_sent(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *cpath) +{ + tor_assert(circ); + tor_assert(cc); + + /* Is this the last cell before a SENDME? The idea is that if the + * package_window reaches a multiple of the increment, after this cell, we + * should expect a SENDME. Note that this function must be called *before* + * we account for the sent cell. */ + if (!circuit_sent_cell_for_sendme(circ, cpath)) { + cc->inflight++; + return; + } + + cc->inflight++; + + /* Record this cell time for RTT computation when SENDME arrives */ + enqueue_timestamp(cc->sendme_pending_timestamps, + monotime_absolute_usec()); +} + +/** + * Returns true if any edge connections are active. + * + * We need to know this so that we can stop computing BDP if the + * edges are not sending on the circuit. + */ +static int +circuit_has_active_streams(const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + const edge_connection_t *streams; + + if (CIRCUIT_IS_ORIGIN(circ)) { + streams = CONST_TO_ORIGIN_CIRCUIT(circ)->p_streams; + } else { + streams = CONST_TO_OR_CIRCUIT(circ)->n_streams; + } + + /* Check linked list of streams */ + for (const edge_connection_t *conn = streams; conn != NULL; + conn = conn->next_stream) { + if (conn->base_.marked_for_close) + continue; + + if (!layer_hint || conn->cpath_layer == layer_hint) { + if (connection_get_inbuf_len(TO_CONN(conn)) > 0) { + log_info(LD_CIRC, "CC: More in edge inbuf..."); + return 1; + } + + /* If we did not reach EOF on this read, there's more */ + if (!TO_CONN(conn)->inbuf_reached_eof) { + log_info(LD_CIRC, "CC: More on edge conn..."); + return 1; + } + + if (TO_CONN(conn)->linked_conn) { + if (connection_get_inbuf_len(TO_CONN(conn)->linked_conn) > 0) { + log_info(LD_CIRC, "CC: More in linked inbuf..."); + return 1; + } + + /* If there is a linked conn, and *it* did not each EOF, + * there's more */ + if (!TO_CONN(conn)->linked_conn->inbuf_reached_eof) { + log_info(LD_CIRC, "CC: More on linked conn..."); + return 1; + } + } + } + } + + return 0; +} + +/** + * Upon receipt of a SENDME, pop the oldest timestamp off the timestamp + * list, and use this to update RTT. + * + * Returns true if circuit estimates were successfully updated, false + * otherwise. + */ +bool +congestion_control_update_circuit_estimates(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + uint64_t now_usec = monotime_absolute_usec(); + + /* Update RTT first, then BDP. BDP needs fresh RTT */ + uint64_t curr_rtt_usec = congestion_control_update_circuit_rtt(cc, now_usec); + return congestion_control_update_circuit_bdp(cc, circ, layer_hint, now_usec, + curr_rtt_usec); +} + +/** + * Returns true if we have enough time data to use heuristics + * to compare RTT to a baseline. + */ +static bool +time_delta_should_use_heuristics(const congestion_control_t *cc) +{ + + /* If we have exited slow start, we should have processed at least + * a cwnd worth of RTTs */ + if (!cc->in_slow_start) { + return true; + } + + /* If we managed to get enough acks to estimate a SENDME BDP, then + * we have enough to estimate clock jumps relative to a baseline, + * too. (This is at least 'cc_bwe_min' acks). */ + if (cc->bdp[BDP_ALG_SENDME_RATE]) { + return true; + } + + /* Not enough data to estimate clock jumps */ + return false; +} + +static bool is_monotime_clock_broken = false; + +/** + * Returns true if the monotime delta is 0, or is significantly + * different than the previous delta. Either case indicates + * that the monotime time source stalled or jumped. + * + * Also caches the clock state in the is_monotime_clock_broken flag, + * so we can also provide a is_monotime_clock_reliable() function, + * used by flow control rate timing. + */ +static bool +time_delta_stalled_or_jumped(const congestion_control_t *cc, + uint64_t old_delta, uint64_t new_delta) +{ +#define DELTA_DISCREPENCY_RATIO_MAX 100 + /* If we have a 0 new_delta, that is definitely a monotime stall */ + if (new_delta == 0) { + static ratelim_t stall_info_limit = RATELIM_INIT(60); + log_fn_ratelim(&stall_info_limit, LOG_INFO, LD_CIRC, + "Congestion control cannot measure RTT due to monotime stall."); + + /* If delta is every 0, the monotime clock has stalled, and we should + * not use it anywhere. */ + is_monotime_clock_broken = true; + + return is_monotime_clock_broken; + } + + /* If the old_delta is 0, we have no previous values on this circuit. + * + * So, return the global monotime status from other circuits, and + * do not update. + */ + if (old_delta == 0) { + return is_monotime_clock_broken; + } + + /* + * For the heuristic cases, we need at least a few timestamps, + * to average out any previous partial stalls or jumps. So until + * than point, let's just use the cached status from other circuits. + */ + if (!time_delta_should_use_heuristics(cc)) { + return is_monotime_clock_broken; + } + + /* If old_delta is significantly larger than new_delta, then + * this means that the monotime clock recently stopped moving + * forward. */ + if (old_delta > new_delta * DELTA_DISCREPENCY_RATIO_MAX) { + static ratelim_t dec_notice_limit = RATELIM_INIT(300); + log_fn_ratelim(&dec_notice_limit, LOG_NOTICE, LD_CIRC, + "Sudden decrease in circuit RTT (%"PRIu64" vs %"PRIu64 + "), likely due to clock jump.", + new_delta/1000, old_delta/1000); + + is_monotime_clock_broken = true; + + return is_monotime_clock_broken; + } + + /* If new_delta is significantly larger than old_delta, then + * this means that the monotime clock suddenly jumped forward. */ + if (new_delta > old_delta * DELTA_DISCREPENCY_RATIO_MAX) { + static ratelim_t dec_notice_limit = RATELIM_INIT(300); + log_fn_ratelim(&dec_notice_limit, LOG_NOTICE, LD_CIRC, + "Sudden increase in circuit RTT (%"PRIu64" vs %"PRIu64 + "), likely due to clock jump.", + new_delta/1000, old_delta/1000); + + is_monotime_clock_broken = true; + + return is_monotime_clock_broken; + } + + /* All good! Update cached status, too */ + is_monotime_clock_broken = false; + + return is_monotime_clock_broken; +} + +/** + * Is the monotime clock stalled according to any circuits? + */ +bool +is_monotime_clock_reliable(void) +{ + return !is_monotime_clock_broken; +} + +/** + * Called when we get a SENDME. Updates circuit RTT by pulling off a + * timestamp of when we sent the CIRCWINDOW_INCREMENT-th cell from + * the queue of such timestamps, and comparing that to current time. + * + * Also updates min, max, and EWMA of RTT. + * + * Returns the current circuit RTT in usecs, or 0 if it could not be + * measured (due to clock jump, stall, etc). + */ +static uint64_t +congestion_control_update_circuit_rtt(congestion_control_t *cc, + uint64_t now_usec) +{ + uint64_t rtt, ewma_cnt; + uint64_t sent_at_timestamp; + + tor_assert(cc); + + /* Get the time that we sent the cell that resulted in the other + * end sending this sendme. Use this to calculate RTT */ + sent_at_timestamp = dequeue_timestamp(cc->sendme_pending_timestamps); + + rtt = now_usec - sent_at_timestamp; + + /* Do not update RTT at all if it looks fishy */ + if (time_delta_stalled_or_jumped(cc, cc->ewma_rtt_usec, rtt)) { + return 0; + } + + ewma_cnt = cc->ewma_cwnd_cnt*sendme_acks_per_cwnd(cc); + ewma_cnt = MAX(ewma_cnt, 2); // Use at least 2 + + cc->ewma_rtt_usec = n_count_ewma(rtt, cc->ewma_rtt_usec, ewma_cnt); + + if (rtt > cc->max_rtt_usec) { + cc->max_rtt_usec = rtt; + } + + if (cc->min_rtt_usec == 0 || rtt < cc->min_rtt_usec) { + cc->min_rtt_usec = rtt; + } + + return rtt; +} + +/** + * Called when we get a SENDME. Updates the bandwidth-delay-product (BDP) + * estimates of a circuit. Several methods of computing BDP are used, + * depending on scenario. While some congestion control algorithms only + * use one of these methods, we update them all because it's quick and easy. + * + * - now_usec is the current monotime in usecs. + * - curr_rtt_usec is the current circuit RTT in usecs. It may be 0 if no + * RTT could bemeasured. + * + * Returns true if we were able to update BDP, false otherwise. + */ +static bool +congestion_control_update_circuit_bdp(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint, + uint64_t now_usec, + uint64_t curr_rtt_usec) +{ + int chan_q = 0; + unsigned int blocked_on_chan = 0; + uint64_t timestamp_usec; + uint64_t sendme_rate_bdp = 0; + + tor_assert(cc); + + if (CIRCUIT_IS_ORIGIN(circ)) { + /* origin circs use n_chan */ + chan_q = circ->n_chan_cells.n; + blocked_on_chan = circ->streams_blocked_on_n_chan; + } else { + /* Both onion services and exits use or_circuit and p_chan */ + chan_q = CONST_TO_OR_CIRCUIT(circ)->p_chan_cells.n; + blocked_on_chan = circ->streams_blocked_on_p_chan; + } + + /* If we have no EWMA RTT, it is because monotime has been stalled + * or messed up the entire time so far. Set our BDP estimates directly + * to current cwnd */ + if (!cc->ewma_rtt_usec) { + uint64_t cwnd = cc->cwnd; + + /* If the channel is blocked, keep subtracting off the chan_q + * until we hit the min cwnd. */ + if (blocked_on_chan) { + cwnd = MAX(cwnd - chan_q, cc->cwnd_min); + cc->blocked_chan = 1; + } else { + cc->blocked_chan = 0; + } + + cc->bdp[BDP_ALG_CWND_RTT] = cwnd; + cc->bdp[BDP_ALG_INFLIGHT_RTT] = cwnd; + cc->bdp[BDP_ALG_SENDME_RATE] = cwnd; + cc->bdp[BDP_ALG_PIECEWISE] = cwnd; + + static ratelim_t dec_notice_limit = RATELIM_INIT(300); + log_fn_ratelim(&dec_notice_limit, LOG_NOTICE, LD_CIRC, + "Our clock has been stalled for the entire lifetime of a circuit. " + "Performance may be sub-optimal."); + + return blocked_on_chan; + } + + /* Congestion window based BDP will respond to changes in RTT only, and is + * relative to cwnd growth. It is useful for correcting for BDP + * overestimation, but if BDP is higher than the current cwnd, it will + * underestimate it. + * + * We multiply here first to avoid precision issues from min_RTT being + * close to ewma RTT. Since all fields are u64, there is plenty of + * room here to multiply first. + */ + cc->bdp[BDP_ALG_CWND_RTT] = cc->cwnd*cc->min_rtt_usec/cc->ewma_rtt_usec; + + /* + * If we have no pending streams, we do not have enough data to fill + * the BDP, so preserve our old estimates but do not make any more. + */ + if (!blocked_on_chan && !circuit_has_active_streams(circ, layer_hint)) { + log_info(LD_CIRC, + "CC: Streams drained. Spare package window: %"PRIu64 + ", no BDP update", cc->cwnd - cc->inflight); + + /* Clear SENDME timestamps; they will be wrong with intermittent data */ + SMARTLIST_FOREACH(cc->sendme_arrival_timestamps, uint64_t *, t, + tor_free(t)); + smartlist_clear(cc->sendme_arrival_timestamps); + } else if (curr_rtt_usec && is_monotime_clock_reliable()) { + /* Sendme-based BDP will quickly measure BDP in much less than + * a cwnd worth of data when in use (in 2-10 SENDMEs). + * + * But if the link goes idle, it will be vastly lower than true BDP. Hence + * we only compute it if we have either pending stream data, or streams + * are still blocked on the channel queued data. + * + * We also do not compute it if we do not have a current RTT passed in, + * because that means that monotime is currently stalled or just jumped. + */ + enqueue_timestamp(cc->sendme_arrival_timestamps, now_usec); + + if (smartlist_len(cc->sendme_arrival_timestamps) >= cc->bwe_sendme_min) { + /* If we have more sendmes than fit in a cwnd, trim the list. + * Those are not acurrately measuring throughput, if cwnd is + * currently smaller than BDP */ + while (smartlist_len(cc->sendme_arrival_timestamps) > + cc->bwe_sendme_min && + (uint64_t)smartlist_len(cc->sendme_arrival_timestamps) > + sendme_acks_per_cwnd(cc)) { + (void)dequeue_timestamp(cc->sendme_arrival_timestamps); + } + int sendme_cnt = smartlist_len(cc->sendme_arrival_timestamps); + + /* Calculate SENDME_BWE_COUNT pure average */ + timestamp_usec = peek_timestamp(cc->sendme_arrival_timestamps); + uint64_t delta = now_usec - timestamp_usec; + + /* The acked data is in sendme_cnt-1 chunks, because we are counting the + * data that is processed by the other endpoint *between* all of these + * sendmes. There's one less gap between the sendmes than the number + * of sendmes. */ + uint64_t cells = (sendme_cnt-1)*cc->sendme_inc; + + /* The bandwidth estimate is cells/delta, which when multiplied + * by min RTT obtains the BDP. However, we multiply first to + * avoid precision issues with the RTT being close to delta in size. */ + sendme_rate_bdp = cells*cc->min_rtt_usec/delta; + + /* Calculate BDP_EWMA_COUNT N-EWMA */ + cc->bdp[BDP_ALG_SENDME_RATE] = + n_count_ewma(sendme_rate_bdp, cc->bdp[BDP_ALG_SENDME_RATE], + cc->ewma_cwnd_cnt*sendme_acks_per_cwnd(cc)); + } + + /* In-flight BDP will cause the cwnd to drift down when underutilized. + * It is most useful when the local OR conn is blocked, so we only + * compute it if we're utilized. */ + cc->bdp[BDP_ALG_INFLIGHT_RTT] = + (cc->inflight - chan_q)*cc->min_rtt_usec/ + MAX(cc->ewma_rtt_usec, curr_rtt_usec); + } else { + /* We can still update inflight with just an EWMA RTT, but only + * if there is data flowing */ + cc->bdp[BDP_ALG_INFLIGHT_RTT] = + (cc->inflight - chan_q)*cc->min_rtt_usec/cc->ewma_rtt_usec; + } + + /* The orconn is blocked; use smaller of inflight vs SENDME */ + if (blocked_on_chan) { + log_info(LD_CIRC, "CC: Streams blocked on circ channel. Chanq: %d", + chan_q); + + /* A blocked channel is an immediate congestion signal, but it still + * happens only once per cwnd */ + if (!cc->blocked_chan) { + cc->next_cc_event = 0; + cc->blocked_chan = 1; + } + + if (cc->bdp[BDP_ALG_SENDME_RATE]) { + cc->bdp[BDP_ALG_PIECEWISE] = MIN(cc->bdp[BDP_ALG_INFLIGHT_RTT], + cc->bdp[BDP_ALG_SENDME_RATE]); + } else { + cc->bdp[BDP_ALG_PIECEWISE] = cc->bdp[BDP_ALG_INFLIGHT_RTT]; + } + } else { + /* If we were previously blocked, emit a new congestion event + * now that we are unblocked, to re-evaluate cwnd */ + if (cc->blocked_chan) { + cc->blocked_chan = 0; + cc->next_cc_event = 0; + log_info(LD_CIRC, "CC: Streams un-blocked on circ channel. Chanq: %d", + chan_q); + } + + cc->bdp[BDP_ALG_PIECEWISE] = MAX(cc->bdp[BDP_ALG_SENDME_RATE], + cc->bdp[BDP_ALG_CWND_RTT]); + } + + /* We can end up with no piecewise value if we didn't have either + * a SENDME estimate or enough data for an inflight estimate. + * It also happens on the very first sendme, since we need two + * to get a BDP. In these cases, use the cwnd method. */ + if (!cc->bdp[BDP_ALG_PIECEWISE]) { + cc->bdp[BDP_ALG_PIECEWISE] = cc->bdp[BDP_ALG_CWND_RTT]; + log_info(LD_CIRC, "CC: No piecewise BDP. Using %"PRIu64, + cc->bdp[BDP_ALG_PIECEWISE]); + } + + if (cc->next_cc_event == 0) { + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC: Circuit %d " + "SENDME RTT: %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", " + "BDP estimates: " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64". ", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->min_rtt_usec/1000, + curr_rtt_usec/1000, + cc->ewma_rtt_usec/1000, + cc->max_rtt_usec/1000, + cc->bdp[BDP_ALG_INFLIGHT_RTT], + cc->bdp[BDP_ALG_CWND_RTT], + sendme_rate_bdp, + cc->bdp[BDP_ALG_SENDME_RATE], + cc->bdp[BDP_ALG_PIECEWISE] + ); + } else { + log_info(LD_CIRC, + "CC: Circuit %"PRIu64":%d " + "SENDME RTT: %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64", " + "%"PRIu64". ", + // XXX: actually, is this p_chan here? This is + // an or_circuit (exit or onion) + circ->n_chan->global_identifier, circ->n_circ_id, + cc->min_rtt_usec/1000, + curr_rtt_usec/1000, + cc->ewma_rtt_usec/1000, + cc->max_rtt_usec/1000, + cc->bdp[BDP_ALG_INFLIGHT_RTT], + cc->bdp[BDP_ALG_CWND_RTT], + sendme_rate_bdp, + cc->bdp[BDP_ALG_SENDME_RATE], + cc->bdp[BDP_ALG_PIECEWISE] + ); + } + } + + /* We updated BDP this round if either we had a blocked channel, or + * the curr_rtt_usec was not 0. */ + bool ret = (blocked_on_chan || curr_rtt_usec != 0); + if (ret) { + tor_trace(TR_SUBSYS(cc), TR_EV(bdp_update), circ, cc, curr_rtt_usec, + sendme_rate_bdp); + } + return ret; +} + +/** + * Dispatch the sendme to the appropriate congestion control algorithm. + */ +int +congestion_control_dispatch_cc_alg(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + int ret = -END_CIRC_REASON_INTERNAL; + switch (cc->cc_alg) { + case CC_ALG_WESTWOOD: + ret = congestion_control_westwood_process_sendme(cc, circ, layer_hint); + break; + + case CC_ALG_VEGAS: + ret = congestion_control_vegas_process_sendme(cc, circ, layer_hint); + break; + + case CC_ALG_NOLA: + ret = congestion_control_nola_process_sendme(cc, circ, layer_hint); + break; + + case CC_ALG_SENDME: + default: + tor_assert(0); + } + + if (cc->cwnd > cwnd_max) { + static ratelim_t cwnd_limit = RATELIM_INIT(60); + log_fn_ratelim(&cwnd_limit, LOG_NOTICE, LD_CIRC, + "Congestion control cwnd %"PRIu64" exceeds max %d, clamping.", + cc->cwnd, cwnd_max); + cc->cwnd = cwnd_max; + } + + return ret; +} diff --git a/src/core/or/congestion_control_common.h b/src/core/or/congestion_control_common.h new file mode 100644 index 0000000000..01dbc1ceb4 --- /dev/null +++ b/src/core/or/congestion_control_common.h @@ -0,0 +1,113 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_common.h + * \brief Public APIs for congestion control + **/ + +#ifndef TOR_CONGESTION_CONTROL_COMMON_H +#define TOR_CONGESTION_CONTROL_COMMON_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +typedef struct congestion_control_t congestion_control_t; + +/** Wrapper for the free function, set the CC pointer to NULL after free */ +#define congestion_control_free(cc) \ + FREE_AND_NULL(congestion_control_t, congestion_control_free_, cc) + +void congestion_control_free_(congestion_control_t *cc); + +congestion_control_t *congestion_control_new(void); + +int congestion_control_dispatch_cc_alg(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); + +void congestion_control_note_cell_sent(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *cpath); + +bool congestion_control_update_circuit_estimates(congestion_control_t *, + const circuit_t *, + const crypt_path_t *); + +int congestion_control_get_package_window(const circuit_t *, + const crypt_path_t *); + +int sendme_get_inc_count(const circuit_t *, const crypt_path_t *); +bool circuit_sent_cell_for_sendme(const circuit_t *, const crypt_path_t *); +bool is_monotime_clock_reliable(void); + +void congestion_control_new_consensus_params(const networkstatus_t *ns); + +/* Ugh, C.. these four are private. Use the getter instead, when + * external to the congestion control code. */ +extern uint32_t or_conn_highwater; +extern uint32_t or_conn_lowwater; +extern int32_t cell_queue_high; +extern int32_t cell_queue_low; + +/** Stop writing on an orconn when its outbuf is this large */ +static inline uint32_t +or_conn_highwatermark(void) +{ + return or_conn_highwater; +} + +/** Resume writing on an orconn when its outbuf is less than this */ +static inline uint32_t +or_conn_lowwatermark(void) +{ + return or_conn_lowwater; +} + +/** Stop reading on edge connections when we have this many cells + * waiting on the appropriate queue. */ +static inline int32_t +cell_queue_highwatermark(void) +{ + return cell_queue_high; +} + +/** Start reading from edge connections again when we get down to this many + * cells. */ +static inline int32_t +cell_queue_lowwatermark(void) +{ + return cell_queue_low; +} + +/** + * Compute an N-count EWMA, aka N-EWMA. N-EWMA is defined as: + * EWMA = alpha*value + (1-alpha)*EWMA_prev + * with alpha = 2/(N+1). + * + * This works out to: + * EWMA = value*2/(N+1) + EMA_prev*(N-1)/(N+1) + * = (value*2 + EWMA_prev*(N-1))/(N+1) + */ +static inline uint64_t +n_count_ewma(uint64_t curr, uint64_t prev, uint64_t N) +{ + if (prev == 0) + return curr; + else + return (2*curr + (N-1)*prev)/(N+1); +} + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_COMMON_H) */ diff --git a/src/core/or/congestion_control_flow.c b/src/core/or/congestion_control_flow.c new file mode 100644 index 0000000000..805654664c --- /dev/null +++ b/src/core/or/congestion_control_flow.c @@ -0,0 +1,710 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_flow.c + * \brief Code that implements flow control for congestion controlled + * circuits. + */ + +#define TOR_CONGESTION_CONTROL_FLOW_PRIVATE + +#include "core/or/or.h" + +#include "core/or/relay.h" +#include "core/mainloop/connection.h" +#include "core/or/connection_edge.h" +#include "core/mainloop/mainloop.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_flow.h" +#include "core/or/congestion_control_st.h" +#include "core/or/circuitlist.h" +#include "core/or/trace_probes_cc.h" +#include "feature/nodelist/networkstatus.h" +#include "trunnel/flow_control_cells.h" + +#include "core/or/connection_st.h" +#include "core/or/cell_st.h" +#include "app/config/config.h" + +/** Cache consensus parameters */ +static uint32_t xoff_client; +static uint32_t xoff_exit; + +static uint32_t xon_change_pct; +static uint32_t xon_ewma_cnt; +static uint32_t xon_rate_bytes; + +/* In normal operation, we can get a burst of up to 32 cells before returning + * to libevent to flush the outbuf. This is a heuristic from hardcoded values + * and strange logic in connection_bucket_get_share(). */ +#define MAX_EXPECTED_CELL_BURST 32 + +/* The following three are for dropmark rate limiting. They define when we + * scale down our XON, XOFF, and xmit byte counts. Early scaling is beneficial + * because it limits the ability of spurious XON/XOFF to be sent after large + * amounts of data without XON/XOFF. At these limits, after 10MB of data (or + * more), an adversary can only inject (log2(10MB)-log2(200*500))*100 ~= 1000 + * cells of fake XOFF/XON before the xmit byte count will be halved enough to + * triggering a limit. */ +#define XON_COUNT_SCALE_AT 200 +#define XOFF_COUNT_SCALE_AT 200 +#define ONE_MEGABYTE (UINT64_C(1) << 20) +#define TOTAL_XMIT_SCALE_AT (10 * ONE_MEGABYTE) + +/** + * Return the congestion control object of the given edge connection. + * + * Returns NULL if the edge connection doesn't have a cpath_layer or not + * attached to a circuit. But also if the cpath_layer or circuit doesn't have a + * congestion control object. + */ +static inline const congestion_control_t * +edge_get_ccontrol(const edge_connection_t *edge) +{ + if (edge->cpath_layer) + return edge->cpath_layer->ccontrol; + else if (edge->on_circuit) + return edge->on_circuit->ccontrol; + else + return NULL; +} + +/** + * Update global congestion control related consensus parameter values, every + * consensus update. + * + * More details for each of the parameters can be found in proposal 324, + * section 6.5 including tuning notes. + */ +void +flow_control_new_consensus_params(const networkstatus_t *ns) +{ +#define CC_XOFF_CLIENT_DFLT 500 +#define CC_XOFF_CLIENT_MIN 1 +#define CC_XOFF_CLIENT_MAX 10000 + xoff_client = networkstatus_get_param(ns, "cc_xoff_client", + CC_XOFF_CLIENT_DFLT, + CC_XOFF_CLIENT_MIN, + CC_XOFF_CLIENT_MAX)*RELAY_PAYLOAD_SIZE; + +#define CC_XOFF_EXIT_DFLT 500 +#define CC_XOFF_EXIT_MIN 1 +#define CC_XOFF_EXIT_MAX 10000 + xoff_exit = networkstatus_get_param(ns, "cc_xoff_exit", + CC_XOFF_EXIT_DFLT, + CC_XOFF_EXIT_MIN, + CC_XOFF_EXIT_MAX)*RELAY_PAYLOAD_SIZE; + +#define CC_XON_CHANGE_PCT_DFLT 25 +#define CC_XON_CHANGE_PCT_MIN 1 +#define CC_XON_CHANGE_PCT_MAX 99 + xon_change_pct = networkstatus_get_param(ns, "cc_xon_change_pct", + CC_XON_CHANGE_PCT_DFLT, + CC_XON_CHANGE_PCT_MIN, + CC_XON_CHANGE_PCT_MAX); + +#define CC_XON_RATE_BYTES_DFLT (500) +#define CC_XON_RATE_BYTES_MIN (1) +#define CC_XON_RATE_BYTES_MAX (5000) + xon_rate_bytes = networkstatus_get_param(ns, "cc_xon_rate", + CC_XON_RATE_BYTES_DFLT, + CC_XON_RATE_BYTES_MIN, + CC_XON_RATE_BYTES_MAX)*RELAY_PAYLOAD_SIZE; + +#define CC_XON_EWMA_CNT_DFLT (2) +#define CC_XON_EWMA_CNT_MIN (1) +#define CC_XON_EWMA_CNT_MAX (100) + xon_ewma_cnt = networkstatus_get_param(ns, "cc_xon_ewma_cnt", + CC_XON_EWMA_CNT_DFLT, + CC_XON_EWMA_CNT_MIN, + CC_XON_EWMA_CNT_MAX); +} + +/** + * Send an XOFF for this stream, and note that we sent one + */ +static void +circuit_send_stream_xoff(edge_connection_t *stream) +{ + xoff_cell_t xoff; + uint8_t payload[CELL_PAYLOAD_SIZE]; + ssize_t xoff_size; + + memset(&xoff, 0, sizeof(xoff)); + memset(payload, 0, sizeof(payload)); + + xoff_cell_set_version(&xoff, 0); + + if ((xoff_size = xoff_cell_encode(payload, CELL_PAYLOAD_SIZE, &xoff)) < 0) { + log_warn(LD_BUG, "Failed to encode xon cell"); + return; + } + + if (connection_edge_send_command(stream, RELAY_COMMAND_XOFF, + (char*)payload, (size_t)xoff_size) == 0) { + stream->xoff_sent = true; + } +} + +/** + * Compute the recent drain rate (write rate) for this edge + * connection and return it, in KB/sec (1000 bytes/sec). + * + * Returns 0 if the monotime clock is busted. + */ +static inline uint32_t +compute_drain_rate(const edge_connection_t *stream) +{ + if (BUG(!is_monotime_clock_reliable())) { + log_warn(LD_BUG, "Computing drain rate with stalled monotime clock"); + return 0; + } + + uint64_t delta = monotime_absolute_usec() - stream->drain_start_usec; + + if (delta == 0) { + log_warn(LD_BUG, "Computing stream drain rate with zero time delta"); + return 0; + } + + /* Overflow checks */ + if (stream->prev_drained_bytes > INT32_MAX/1000 || /* Intermediate */ + stream->prev_drained_bytes/delta > INT32_MAX/1000) { /* full value */ + return INT32_MAX; + } + + /* kb/sec = bytes/usec * 1000 usec/msec * 1000 msec/sec * kb/1000bytes */ + return MAX(1, (uint32_t)(stream->prev_drained_bytes * 1000)/delta); +} + +/** + * Send an XON for this stream, with appropriate advisory rate information. + * + * Reverts the xoff sent status, and stores the rate information we sent, + * in case it changes. + */ +static void +circuit_send_stream_xon(edge_connection_t *stream) +{ + xon_cell_t xon; + uint8_t payload[CELL_PAYLOAD_SIZE]; + ssize_t xon_size; + + memset(&xon, 0, sizeof(xon)); + memset(payload, 0, sizeof(payload)); + + xon_cell_set_version(&xon, 0); + xon_cell_set_kbps_ewma(&xon, stream->ewma_drain_rate); + + if ((xon_size = xon_cell_encode(payload, CELL_PAYLOAD_SIZE, &xon)) < 0) { + log_warn(LD_BUG, "Failed to encode xon cell"); + return; + } + + /* Store the advisory rate information, to send advisory updates if + * it changes */ + stream->ewma_rate_last_sent = stream->ewma_drain_rate; + + if (connection_edge_send_command(stream, RELAY_COMMAND_XON, (char*)payload, + (size_t)xon_size) == 0) { + /* Revert the xoff sent status, so we can send another one if need be */ + stream->xoff_sent = false; + } +} + +/** + * Process a stream XOFF, parsing it, and then stopping reading on + * the edge connection. + * + * Record that we have recieved an xoff, so we know not to resume + * reading on this edge conn until we get an XON. + * + * Returns false if the XOFF did not validate; true if it does. + */ +bool +circuit_process_stream_xoff(edge_connection_t *conn, + const crypt_path_t *layer_hint, + const cell_t *cell) +{ + (void)cell; + bool retval = true; + + if (BUG(!conn)) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XOFF on invalid stream?"); + return false; + } + + /* Make sure this XOFF came from the right hop */ + if (layer_hint && layer_hint != conn->cpath_layer) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XOFF from wrong hop."); + return false; + } + + if (edge_get_ccontrol(conn) == NULL) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XOFF for non-congestion control circuit"); + return false; + } + + if (conn->xoff_received) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got multiple XOFF on connection"); + return false; + } + + /* If we are near the max, scale everything down */ + if (conn->num_xoff_recv == XOFF_COUNT_SCALE_AT) { + log_info(LD_EDGE, "Scaling down for XOFF count: %d %d %d", + conn->total_bytes_xmit, + conn->num_xoff_recv, + conn->num_xon_recv); + conn->total_bytes_xmit /= 2; + conn->num_xoff_recv /= 2; + conn->num_xon_recv /= 2; + } + + conn->num_xoff_recv++; + + /* Client-side check to make sure that XOFF is not sent too early, + * for dropmark attacks. The main sidechannel risk is early cells, + * but we also check to make sure that we have not received more XOFFs + * than could have been generated by the bytes we sent. + */ + if (TO_CONN(conn)->type == CONN_TYPE_AP || conn->hs_ident != NULL) { + uint32_t limit = 0; + + /* TODO: This limit technically needs to come from negotiation, + * and be bounds checked for sanity, because the other endpoint + * may have a different consensus */ + if (conn->hs_ident) + limit = xoff_client; + else + limit = xoff_exit; + + if (conn->total_bytes_xmit < limit*conn->num_xoff_recv) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got extra XOFF for bytes sent. Got %d, expected max %d", + conn->num_xoff_recv, conn->total_bytes_xmit/limit); + /* We still process this, because the only dropmark defenses + * in C tor are via the vanguards addon's use of the read valid + * cells. So just signal that we think this is not valid protocol + * data and proceed. */ + retval = false; + } + } + + // TODO: Count how many xoffs we have; log if "too many", for shadow + // analysis of chatter. Possibly add to extra-info? + + log_info(LD_EDGE, "Got XOFF!"); + connection_stop_reading(TO_CONN(conn)); + conn->xoff_received = true; + + return retval; +} + +/** + * Process a stream XON, and if it validates, clear the xoff + * flag and resume reading on this edge connection. + * + * Also, use provided rate information to rate limit + * reading on this edge (or packagaing from it onto + * the circuit), to avoid XON/XOFF chatter. + * + * Returns true if the XON validates, false otherwise. + */ +bool +circuit_process_stream_xon(edge_connection_t *conn, + const crypt_path_t *layer_hint, + const cell_t *cell) +{ + xon_cell_t *xon; + bool retval = true; + + if (BUG(!conn)) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XON on invalid stream?"); + return false; + } + + /* Make sure this XON came from the right hop */ + if (layer_hint && layer_hint != conn->cpath_layer) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XON from wrong hop."); + return false; + } + + if (edge_get_ccontrol(conn) == NULL) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got XON for non-congestion control circuit"); + return false; + } + + if (xon_cell_parse(&xon, cell->payload+RELAY_HEADER_SIZE, + CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Received malformed XON cell."); + return false; + } + + /* If we are near the max, scale everything down */ + if (conn->num_xon_recv == XON_COUNT_SCALE_AT) { + log_info(LD_EDGE, "Scaling down for XON count: %d %d %d", + conn->total_bytes_xmit, + conn->num_xoff_recv, + conn->num_xon_recv); + conn->total_bytes_xmit /= 2; + conn->num_xoff_recv /= 2; + conn->num_xon_recv /= 2; + } + + conn->num_xon_recv++; + + /* Client-side check to make sure that XON is not sent too early, + * for dropmark attacks. The main sidechannel risk is early cells, + * but we also check to see that we did not get more XONs than make + * sense for the number of bytes we sent. + */ + if (TO_CONN(conn)->type == CONN_TYPE_AP || conn->hs_ident != NULL) { + uint32_t limit = 0; + + /* TODO: This limit technically needs to come from negotiation, + * and be bounds checked for sanity, because the other endpoint + * may have a different consensus */ + if (conn->hs_ident) + limit = MIN(xoff_client, xon_rate_bytes); + else + limit = MIN(xoff_exit, xon_rate_bytes); + + if (conn->total_bytes_xmit < limit*conn->num_xon_recv) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Got extra XON for bytes sent. Got %d, expected max %d", + conn->num_xon_recv, conn->total_bytes_xmit/limit); + + /* We still process this, because the only dropmark defenses + * in C tor are via the vanguards addon's use of the read valid + * cells. So just signal that we think this is not valid protocol + * data and proceed. */ + retval = false; + } + } + + log_info(LD_EDGE, "Got XON: %d", xon->kbps_ewma); + + /* Adjust the token bucket of this edge connection with the drain rate in + * the XON. Rate is in bytes from kilobit (kpbs). */ + uint64_t rate = ((uint64_t) xon_cell_get_kbps_ewma(xon) * 1000); + if (rate == 0 || INT32_MAX < rate) { + /* No rate. */ + rate = INT32_MAX; + } + token_bucket_rw_adjust(&conn->bucket, (uint32_t) rate, (uint32_t) rate); + + if (conn->xoff_received) { + /* Clear the fact that we got an XOFF, so that this edge can + * start and stop reading normally */ + conn->xoff_received = false; + connection_start_reading(TO_CONN(conn)); + } + + xon_cell_free(xon); + + return retval; +} + +/** + * Called from sendme_stream_data_received(), when data arrives + * from a circuit to our edge's outbuf, to decide if we need to send + * an XOFF. + * + * Returns the amount of cells remaining until the buffer is full, at + * which point it sends an XOFF, and returns 0. + * + * Returns less than 0 if we have queued more than a congestion window + * worth of data and need to close the circuit. + */ +int +flow_control_decide_xoff(edge_connection_t *stream) +{ + size_t total_buffered = connection_get_outbuf_len(TO_CONN(stream)); + uint32_t buffer_limit_xoff = 0; + + if (BUG(edge_get_ccontrol(stream) == NULL)) { + log_err(LD_BUG, "Flow control called for non-congestion control circuit"); + return -1; + } + + /* Onion services and clients are typically localhost edges, so they + * need different buffering limits than exits do */ + if (TO_CONN(stream)->type == CONN_TYPE_AP || stream->hs_ident != NULL) { + buffer_limit_xoff = xoff_client; + } else { + buffer_limit_xoff = xoff_exit; + } + + if (total_buffered > buffer_limit_xoff) { + if (!stream->xoff_sent) { + log_info(LD_EDGE, "Sending XOFF: %"TOR_PRIuSZ" %d", + total_buffered, buffer_limit_xoff); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xoff_sending), stream); + + circuit_send_stream_xoff(stream); + + /* Clear the drain rate. It is considered wrong if we + * got all the way to XOFF */ + stream->ewma_drain_rate = 0; + } + } + + /* If the outbuf has accumulated more than the expected burst limit of + * cells, then assume it is not draining, and call decide_xon. We must + * do this because writes only happen when the socket unblocks, so + * may not otherwise notice accumulation of data in the outbuf for + * advisory XONs. */ + if (total_buffered > MAX_EXPECTED_CELL_BURST*RELAY_PAYLOAD_SIZE) { + flow_control_decide_xon(stream, 0); + } + + /* Flow control always takes more data; we rely on the oomkiller to + * handle misbehavior. */ + return 0; +} + +/** + * Returns true if the stream's drain rate has changed significantly. + * + * Returns false if the monotime clock is stalled, or if we have + * no previous drain rate information. + */ +static bool +stream_drain_rate_changed(const edge_connection_t *stream) +{ + if (!is_monotime_clock_reliable()) { + return false; + } + + if (!stream->ewma_rate_last_sent) { + return false; + } + + if (stream->ewma_drain_rate > + (100+(uint64_t)xon_change_pct)*stream->ewma_rate_last_sent/100) { + return true; + } + + if (stream->ewma_drain_rate < + (100-(uint64_t)xon_change_pct)*stream->ewma_rate_last_sent/100) { + return true; + } + + return false; +} + +/** + * Called whenever we drain an edge connection outbuf by writing on + * its socket, to decide if it is time to send an xon. + * + * The n_written parameter tells us how many bytes we have written + * this time, which is used to compute the advisory drain rate fields. + */ +void +flow_control_decide_xon(edge_connection_t *stream, size_t n_written) +{ + size_t total_buffered = connection_get_outbuf_len(TO_CONN(stream)); + + /* Bounds check the number of drained bytes, and scale */ + if (stream->drained_bytes >= UINT32_MAX - n_written) { + /* Cut the bytes in half, and move the start time up halfway to now + * (if we have one). */ + stream->drained_bytes /= 2; + + if (stream->drain_start_usec) { + uint64_t now = monotime_absolute_usec(); + + stream->drain_start_usec = now - (now-stream->drain_start_usec)/2; + } + } + + /* Accumulate drained bytes since last rate computation */ + stream->drained_bytes += n_written; + + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon), stream, n_written); + + /* Check for bad monotime clock and bytecount wrap */ + if (!is_monotime_clock_reliable()) { + /* If the monotime clock ever goes wrong, the safest thing to do + * is just clear our short-term rate info and wait for the clock to + * become reliable again.. */ + stream->drain_start_usec = 0; + stream->drained_bytes = 0; + } else { + /* If we have no drain start timestamp, and we still have + * remaining buffer, start the buffering counter */ + if (!stream->drain_start_usec && total_buffered > 0) { + log_debug(LD_EDGE, "Began edge buffering: %d %d %"TOR_PRIuSZ, + stream->ewma_rate_last_sent, + stream->ewma_drain_rate, + total_buffered); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon_drain_start), + stream); + stream->drain_start_usec = monotime_absolute_usec(); + stream->drained_bytes = 0; + } + } + + if (stream->drain_start_usec) { + /* If we have spent enough time in a queued state, update our drain + * rate. */ + if (stream->drained_bytes > xon_rate_bytes) { + /* No previous drained bytes means it is the first time we are computing + * it so use the value we just drained onto the socket as a baseline. It + * won't be accurate but it will be a start towards the right value. + * + * We have to do this in order to have a drain rate else we could be + * sending a drain rate of 0 in an XON which would be undesirable and + * basically like sending an XOFF. */ + if (stream->prev_drained_bytes == 0) { + stream->prev_drained_bytes = stream->drained_bytes; + } + uint32_t drain_rate = compute_drain_rate(stream); + /* Once the drain rate has been computed, note how many bytes we just + * drained so it can be used at the next calculation. We do this here + * because it gets reset once the rate is changed. */ + stream->prev_drained_bytes = stream->drained_bytes; + + if (drain_rate) { + stream->ewma_drain_rate = + (uint32_t)n_count_ewma(drain_rate, + stream->ewma_drain_rate, + xon_ewma_cnt); + log_debug(LD_EDGE, "Updating drain rate: %d %d %"TOR_PRIuSZ, + drain_rate, + stream->ewma_drain_rate, + total_buffered); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon_drain_update), + stream, drain_rate); + /* Reset recent byte counts. This prevents us from sending advisory + * XONs more frequent than every xon_rate_bytes. */ + stream->drained_bytes = 0; + stream->drain_start_usec = 0; + } + } + } + + /* If we don't have an XOFF outstanding, consider updating an + * old rate */ + if (!stream->xoff_sent) { + if (stream_drain_rate_changed(stream)) { + /* If we are still buffering and the rate changed, update + * advisory XON */ + log_info(LD_EDGE, "Sending rate-change XON: %d %d %"TOR_PRIuSZ, + stream->ewma_rate_last_sent, + stream->ewma_drain_rate, + total_buffered); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon_rate_change), stream); + circuit_send_stream_xon(stream); + } + } else if (total_buffered == 0) { + log_info(LD_EDGE, "Sending XON: %d %d %"TOR_PRIuSZ, + stream->ewma_rate_last_sent, + stream->ewma_drain_rate, + total_buffered); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon_partial_drain), stream); + circuit_send_stream_xon(stream); + } + + /* If the buffer has fully emptied, clear the drain timestamp, + * so we can total only bytes drained while outbuf is 0. */ + if (total_buffered == 0) { + stream->drain_start_usec = 0; + + /* After we've spent 'xon_rate_bytes' with the queue fully drained, + * double any rate we sent. */ + if (stream->drained_bytes >= xon_rate_bytes && + stream->ewma_rate_last_sent) { + stream->ewma_drain_rate = MIN(INT32_MAX, 2*stream->ewma_drain_rate); + + log_debug(LD_EDGE, + "Queue empty for xon_rate_limit bytes: %d %d", + stream->ewma_rate_last_sent, + stream->ewma_drain_rate); + tor_trace(TR_SUBSYS(cc), TR_EV(flow_decide_xon_drain_doubled), stream); + /* Resetting the drained bytes count. We need to keep its value as a + * previous so the drain rate calculation takes into account what was + * actually drain the last time. */ + stream->prev_drained_bytes = stream->drained_bytes; + stream->drained_bytes = 0; + } + } + + return; +} + +/** + * Note that we packaged some data on this stream. Used to enforce + * client-side dropmark limits + */ +void +flow_control_note_sent_data(edge_connection_t *stream, size_t len) +{ + /* If we are near the max, scale everything down */ + if (stream->total_bytes_xmit >= TOTAL_XMIT_SCALE_AT-len) { + log_info(LD_EDGE, "Scaling down for flow control xmit bytes:: %d %d %d", + stream->total_bytes_xmit, + stream->num_xoff_recv, + stream->num_xon_recv); + + stream->total_bytes_xmit /= 2; + stream->num_xoff_recv /= 2; + stream->num_xon_recv /= 2; + } + + stream->total_bytes_xmit += len; +} + +/** Returns true if an edge connection uses flow control */ +bool +edge_uses_flow_control(const edge_connection_t *stream) +{ + bool ret = (stream->on_circuit && stream->on_circuit->ccontrol) || + (stream->cpath_layer && stream->cpath_layer->ccontrol); + + /* All circuits with congestion control use flow control */ + return ret; +} + +/** + * Returns the max RTT for the circuit that carries this stream, + * as observed by congestion control. + */ +uint64_t +edge_get_max_rtt(const edge_connection_t *stream) +{ + if (stream->on_circuit && stream->on_circuit->ccontrol) + return stream->on_circuit->ccontrol->max_rtt_usec; + else if (stream->cpath_layer && stream->cpath_layer->ccontrol) + return stream->cpath_layer->ccontrol->max_rtt_usec; + + return 0; +} + +/** Returns true if a connection is an edge conn that uses flow control */ +bool +conn_uses_flow_control(connection_t *conn) +{ + bool ret = false; + + if (CONN_IS_EDGE(conn)) { + edge_connection_t *edge = TO_EDGE_CONN(conn); + + if (edge_uses_flow_control(edge)) { + ret = true; + } + } + + return ret; +} + diff --git a/src/core/or/congestion_control_flow.h b/src/core/or/congestion_control_flow.h new file mode 100644 index 0000000000..6c318027ea --- /dev/null +++ b/src/core/or/congestion_control_flow.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_flow.h + * \brief APIs for stream flow control on congestion controlled circuits. + **/ + +#ifndef TOR_CONGESTION_CONTROL_FLOW_H +#define TOR_CONGESTION_CONTROL_FLOW_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" +#include "core/or/edge_connection_st.h" + +void flow_control_new_consensus_params(const struct networkstatus_t *); + +bool circuit_process_stream_xoff(edge_connection_t *conn, + const crypt_path_t *layer_hint, + const cell_t *cell); +bool circuit_process_stream_xon(edge_connection_t *conn, + const crypt_path_t *layer_hint, + const cell_t *cell); + +int flow_control_decide_xoff(edge_connection_t *stream); +void flow_control_decide_xon(edge_connection_t *stream, size_t n_written); + +void flow_control_note_sent_data(edge_connection_t *stream, size_t len); + +bool edge_uses_flow_control(const edge_connection_t *stream); + +bool conn_uses_flow_control(connection_t *stream); + +uint64_t edge_get_max_rtt(const edge_connection_t *); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_FLOW_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_FLOW_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_FLOW_H) */ diff --git a/src/core/or/congestion_control_nola.c b/src/core/or/congestion_control_nola.c new file mode 100644 index 0000000000..09f88d4699 --- /dev/null +++ b/src/core/or/congestion_control_nola.c @@ -0,0 +1,126 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_nola.c + * \brief Code that implements the TOR_NOLA congestion control algorithm + * from Proposal #324. + */ + +#define TOR_CONGESTION_CONTROL_NOLA_PRIVATE + +#include "core/or/or.h" + +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_st.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_nola.h" +#include "core/or/circuituse.h" +#include "core/or/circuitlist.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "feature/nodelist/networkstatus.h" + +#define NOLA_BDP_OVERSHOOT 100 + +/** + * Cache NOLA consensus parameters. + */ +void +congestion_control_nola_set_params(congestion_control_t *cc) +{ + tor_assert(cc->cc_alg == CC_ALG_NOLA); + + cc->nola_params.bdp_overshoot = + networkstatus_get_param(NULL, "cc_nola_overshoot", + NOLA_BDP_OVERSHOOT, + 0, + 1000); +} + +/** +* Process a SENDME and update the congestion window according to the +* rules specified in TOR_NOLA of Proposal #324. +* +* TOR_NOLA updates the congestion window to match the current +* BDP estimate, every sendme. Because this can result in downward +* drift, a fixed overhead is added to the BDP estimate. This will +* cause some queuing, but ensures that the algorithm always uses +* the full BDP. +* +* To handle the case where the local orconn blocks, TOR_NOLA uses +* the 'piecewise' BDP estimate, which uses more a conservative BDP +* estimate method when blocking occurrs, but a more aggressive BDP +* estimate when there is no local blocking. This minimizes local +* client queues. +*/ +int +congestion_control_nola_process_sendme(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + tor_assert(cc && cc->cc_alg == CC_ALG_NOLA); + tor_assert(circ); + + if (cc->next_cc_event) + cc->next_cc_event--; + + /* If we get a congestion event, the only thing NOLA + * does is note this as if we exited slow-start + * (which for NOLA just means we finished our ICW). */ + if (cc->next_cc_event == 0) + cc->in_slow_start = 0; + + /* If we did not successfully update BDP, we must return. Otherwise, + * NOLA can drift downwards */ + if (!congestion_control_update_circuit_estimates(cc, circ, layer_hint)) { + cc->inflight = cc->inflight - cc->sendme_inc; + return 0; + } + + /* We overshoot the BDP by the cwnd_inc param amount, because BDP + * may otherwise drift down. This helps us probe for more capacity. + * But there is no sense to do it if the local channel is blocked. */ + if (cc->blocked_chan) + cc->cwnd = cc->bdp[cc->bdp_alg]; + else + cc->cwnd = cc->bdp[cc->bdp_alg] + cc->nola_params.bdp_overshoot; + + /* cwnd can never fall below 1 increment */ + cc->cwnd = MAX(cc->cwnd, cc->cwnd_min); + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC TOR_NOLA: Circuit %d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->in_slow_start + ); + } else { + log_info(LD_CIRC, + "CC TOR_NOLA: Circuit %"PRIu64":%d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + circ->n_chan->global_identifier, circ->n_circ_id, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->in_slow_start + ); + } + + /* Update inflight with ack */ + cc->inflight = cc->inflight - cc->sendme_inc; + + return 0; +} diff --git a/src/core/or/congestion_control_nola.h b/src/core/or/congestion_control_nola.h new file mode 100644 index 0000000000..9c7d6e0635 --- /dev/null +++ b/src/core/or/congestion_control_nola.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_nola.h + * \brief Private-ish APIs for the TOR_NOLA congestion control algorithm + **/ + +#ifndef TOR_CONGESTION_CONTROL_NOLA_H +#define TOR_CONGESTION_CONTROL_NOLA_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/* Processing SENDME cell. */ +int congestion_control_nola_process_sendme(struct congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); +void congestion_control_nola_set_params(struct congestion_control_t *cc); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_NOLA_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_NOLA_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_NOLA_H) */ diff --git a/src/core/or/congestion_control_st.h b/src/core/or/congestion_control_st.h new file mode 100644 index 0000000000..251ebd82e3 --- /dev/null +++ b/src/core/or/congestion_control_st.h @@ -0,0 +1,257 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_st.h + * \brief Structure definitions for congestion control. + **/ + +#ifndef CONGESTION_CONTROL_ST_H +#define CONGESTION_CONTROL_ST_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/** Signifies which sendme algorithm to use */ +typedef enum { + /** OG Tor fixed-sized circ and stream windows. It sucks, but it is important + * to make sure that the new algs can compete with the old garbage. */ + CC_ALG_SENDME = 0, + + /** + * Prop#324 TOR_WESTWOOD - Deliberately agressive. Westwood may not even + * converge to fairness in some cases because max RTT will also increase + * on congesgtion, which boosts the Westwood RTT congestion threshhold. So it + * can cause runaway queue bloat, which may or may not lead to a robot + * uprising... Ok that's Westworld, not Westwood. Still, we need to test + * Vegas and NOLA against something more agressive to ensure they do not + * starve in the presence of cheaters. We also need to make sure cheaters + * trigger the oomkiller in those cases. + */ + CC_ALG_WESTWOOD = 1, + + /** + * Prop#324 TOR_VEGAS - TCP Vegas-style BDP tracker. Because Vegas backs off + * whenever it detects queue delay, it can be beaten out by more agressive + * algs. However, in live network testing, it seems to do just fine against + * current SENDMEs. It outperforms Westwood and does not stall. */ + CC_ALG_VEGAS = 2, + + /** + * Prop#324: TOR_NOLA - NOLA looks the BDP right in the eye and uses it + * immediately as CWND. No slow start, no other congestion signals, no delay, + * no bullshit. Like TOR_VEGAS, it also uses agressive BDP estimates, to + * avoid out-competition. It seems a bit better throughput than Vegas, + * but its agressive BDP and rapid updates may lead to more queue latency. */ + CC_ALG_NOLA = 3, +} cc_alg_t; + +/* Total number of CC algs in cc_alg_t enum */ +#define NUM_CC_ALGS (CC_ALG_NOLA+1) + +/** Signifies how we estimate circuit BDP */ +typedef enum { + /* CWND-based BDP will respond to changes in RTT only, and is relative + * to cwnd growth. So in slow-start, this will under-estimate BDP */ + BDP_ALG_CWND_RTT = 0, + + /* Sendme-based BDP will quickly measure BDP in less than + * a cwnd worth of data when in use. So it should be good for slow-start. + * But if the link goes idle, it will be vastly lower than true BDP. Thus, + * this estimate gets reset when the cwnd is not fully utilized. */ + BDP_ALG_SENDME_RATE = 1, + + /* Inflight BDP is similar to the cwnd estimator, except it uses + * packets inflight minus local circuit queues instead of current cwnd. + * Because it is strictly less than or equal to the cwnd, it will cause + * the cwnd to drift downward. It is only used if the local OR connection + * is blocked. */ + BDP_ALG_INFLIGHT_RTT = 2, + + /* The Piecewise BDP estimator uses the CWND estimator before there + * are sufficient SENDMEs to calculate the SENDME estimator. At that + * point, it uses the SENDME estimator, unless the local OR connection + * becomes blocked. In that case, it switches to the inflight estimator. */ + BDP_ALG_PIECEWISE = 3, + +} bdp_alg_t; + +/** Total number of BDP algs in bdp_alg_t enum */ +#define NUM_BDP_ALGS (BDP_ALG_PIECEWISE+1) + +/** Westwood algorithm parameters */ +struct westwood_params_t { + /** Cwnd backoff multiplier upon congestion (as percent) */ + uint8_t cwnd_backoff_m; + /** Max RTT backoff multiplier upon congestion (as percent) */ + uint8_t rtt_backoff_m; + + /** Threshold between min and max RTT, to signal congestion (percent) */ + uint8_t rtt_thresh; + + /** + * If true, use minimum of BDP and backoff multiplication in backoff. + * If false, use maximum of BDP and backoff multiplication in backoff. */ + bool min_backoff; +}; + +/** Vegas algorithm parameters. */ +struct vegas_params_t { + /** The queue use allowed before we exit slow start */ + uint16_t gamma; + /** The queue use below which we increment cwnd */ + uint16_t alpha; + /** The queue use above which we decrement cwnd */ + uint16_t beta; + /** Weighted average (percent) between cwnd estimator and + * piecewise estimator. */ + uint8_t bdp_mix_pct; +}; + +/** NOLA consensus params */ +struct nola_params_t { + /** How many cells to add to BDP estimate to obtain cwnd */ + uint16_t bdp_overshoot; +}; + +/** Fields common to all congestion control algorithms */ +typedef struct congestion_control_t { + /** + * Smartlist of uint64_t monotime usec timestamps of when we sent a data + * cell that is pending a sendme. FIFO queue that is managed similar to + * sendme_last_digests. */ + smartlist_t *sendme_pending_timestamps; + + /** + * Smartlist of uint64_t monotime timestamp of when sendme's arrived. + * FIFO queue that is managed similar to sendme_last_digests. + * Used to estimate circuitbandwidth and BDP. */ + smartlist_t *sendme_arrival_timestamps; + + /** RTT time data for congestion control. */ + uint64_t ewma_rtt_usec; + uint64_t min_rtt_usec; + uint64_t max_rtt_usec; + + /* BDP estimates by algorithm */ + uint64_t bdp[NUM_BDP_ALGS]; + + /** Congestion window */ + uint64_t cwnd; + + /** Number of cells in-flight (sent but awaiting SENDME ack). */ + uint64_t inflight; + + /** + * For steady-state: the number of sendme acks until we will acknowledge + * a congestion event again. It starts out as the number of sendme acks + * in a congestion windowm and is decremented each ack. When this reaches + * 0, it means we should examine our congestion algorithm conditions. + * In this way, we only react to one congestion event per congestion window. + * + * It is also reset to 0 immediately whenever the circuit's orconn is + * blocked, and when a previously blocked orconn is unblocked. + */ + uint64_t next_cc_event; + + /** Are we in slow start? */ + bool in_slow_start; + + /** Is the local channel blocked on us? That's a congestion signal */ + bool blocked_chan; + + /* The following parameters are cached from consensus values upon + * circuit setup. */ + + /** Percent of cwnd to increment by during slow start */ + uint16_t cwnd_inc_pct_ss; + + /** Number of cells to increment cwnd by during steady state */ + uint16_t cwnd_inc; + + /** Minimum congestion window (must be at least sendme_inc) */ + uint16_t cwnd_min; + + /** + * Number of times per congestion window to update based on congestion + * signals */ + uint8_t cwnd_inc_rate; + + /** + * Number of cwnd worth of sendme acks to smooth RTT and BDP with, + * using N_EWMA */ + uint8_t ewma_cwnd_cnt; + + /** + * Minimum number of sendmes before we begin BDP estimates + */ + uint8_t bwe_sendme_min; + + /** + * Number of cells to ack with every sendme. Taken from consensus parameter + * and negotiation during circuit setup. */ + uint8_t sendme_inc; + + /** Which congestion control algorithm to use. Taken from + * consensus parameter and negotiation during circuit setup. */ + cc_alg_t cc_alg; + + /** Which algorithm to estimate circuit bandwidth with. Taken from + * consensus parameter during circuit setup. */ + bdp_alg_t bdp_alg; + + /** Algorithm-specific parameters. The specific struct that is used + * depends upon the algoritghm selected by the cc_alg parameter. + * These should not be accessed anywhere other than the algorithm-specific + * files. */ + union { + struct westwood_params_t westwood_params; + struct vegas_params_t vegas_params; + struct nola_params_t nola_params; + }; +} congestion_control_t; + +/** + * Returns the number of sendme acks we will recieve before we update cwnd. + * + * Congestion control literature recommends only one update of cwnd per + * cwnd worth of acks. However, we can also tune this to be more frequent + * by increasing the 'cc_cwnd_inc_rate' consensus parameter. + * + * If this returns 0 due to high cwnd_inc_rate, the calling code will + * update every sendme ack. + */ +static inline uint64_t CWND_UPDATE_RATE(const congestion_control_t *cc) +{ + /* We add cwnd_inc_rate*sendme_inc/2 to round to nearest integer number + * of acks */ + return ((cc->cwnd + cc->cwnd_inc_rate*cc->sendme_inc/2) + / (cc->cwnd_inc_rate*cc->sendme_inc)); +} + +/** + * Returns the amount to increment the congestion window each update, + * during slow start. + * + * Congestion control literature recommends either doubling the cwnd + * every cwnd during slow start, or some similar exponential growth + * (such as 50% more every cwnd, for Vegas). + * + * This is controlled by a consensus parameter 'cwnd_inc_pct_ss', which + * allows us to specify the percent of the current consensus window + * to update by. + */ +static inline uint64_t CWND_INC_SS(const congestion_control_t *cc) +{ + return (cc->cwnd_inc_pct_ss*cc->cwnd/100); +} + +/** + * Returns the amount to increment (and for Vegas, also decrement) the + * congestion window by, every update period. + * + * This is controlled by the cc_cwnd_inc consensus parameter. + */ +#define CWND_INC(cc) ((cc)->cwnd_inc) + +#endif /* !defined(CONGESTION_CONTROL_ST_H) */ diff --git a/src/core/or/congestion_control_vegas.c b/src/core/or/congestion_control_vegas.c new file mode 100644 index 0000000000..3206821f4c --- /dev/null +++ b/src/core/or/congestion_control_vegas.c @@ -0,0 +1,200 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_vegas.c + * \brief Code that implements the TOR_VEGAS congestion control algorithm + * from Proposal #324. + */ + +#define TOR_CONGESTION_CONTROL_VEGAS_PRIVATE + +#include "core/or/or.h" + +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_st.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_vegas.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "feature/nodelist/networkstatus.h" + +#define VEGAS_GAMMA(cc) (6*(cc)->sendme_inc) +#define VEGAS_ALPHA(cc) (3*(cc)->sendme_inc) +#define VEGAS_BETA(cc) (6*(cc)->sendme_inc) + +#define VEGAS_BDP_MIX_PCT 0 + +/** + * The original TCP Vegas used only a congestion window BDP estimator. We + * believe that the piecewise estimator is likely to perform better, but + * for purposes of experimentation, we might as well have a way to blend + * them. It also lets us set Vegas to its original estimator while other + * algorithms on the same network use piecewise (by setting the + * 'vegas_bdp_mix_pct' consensus parameter to 100, while leaving the + * 'cc_bdp_alg' parameter set to piecewise). + * + * Returns a percentage weighted average between the CWND estimator and + * the specified consensus BDP estimator. + */ +static inline uint64_t +vegas_bdp_mix(const congestion_control_t *cc) +{ + return cc->vegas_params.bdp_mix_pct*cc->bdp[BDP_ALG_CWND_RTT]/100 + + (100-cc->vegas_params.bdp_mix_pct)*cc->bdp[cc->bdp_alg]/100; +} + +/** + * Cache Vegas consensus parameters. + */ +void +congestion_control_vegas_set_params(congestion_control_t *cc) +{ + tor_assert(cc->cc_alg == CC_ALG_VEGAS); + + cc->vegas_params.gamma = + networkstatus_get_param(NULL, "cc_vegas_gamma", + VEGAS_GAMMA(cc), + 0, + 1000); + + cc->vegas_params.alpha = + networkstatus_get_param(NULL, "cc_vegas_alpha", + VEGAS_ALPHA(cc), + 0, + 1000); + + cc->vegas_params.beta = + networkstatus_get_param(NULL, "cc_vegas_beta", + VEGAS_BETA(cc), + 0, + 1000); + + cc->vegas_params.bdp_mix_pct = + networkstatus_get_param(NULL, "cc_vegas_bdp_mix", + VEGAS_BDP_MIX_PCT, + 0, + 100); +} + +/** + * Process a SENDME and update the congestion window according to the + * rules specified in TOR_VEGAS of Proposal #324. + * + * Essentially, this algorithm attempts to measure queue lengths on + * the circuit by subtracting the bandwidth-delay-product estimate + * from the current congestion window. + * + * If the congestion window is larger than the bandwidth-delay-product, + * then data is assumed to be queuing. We reduce the congestion window + * in that case. + * + * If the congestion window is smaller than the bandwidth-delay-product, + * then there is spare bandwidth capacity on the circuit. We increase the + * congestion window in that case. + * + * The congestion window is updated only once every congestion window worth of + * packets, even if the signal persists. It is also updated whenever the + * upstream orcon blocks, or unblocks. This minimizes local client queues. + */ +int +congestion_control_vegas_process_sendme(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + uint64_t queue_use; + + tor_assert(cc && cc->cc_alg == CC_ALG_VEGAS); + tor_assert(circ); + + /* Update ack counter until next congestion signal event is allowed */ + if (cc->next_cc_event) + cc->next_cc_event--; + + /* Compute BDP and RTT. If we did not update, don't run the alg */ + if (!congestion_control_update_circuit_estimates(cc, circ, layer_hint)) { + cc->inflight = cc->inflight - cc->sendme_inc; + return 0; + } + + /* We only update anything once per window */ + if (cc->next_cc_event == 0) { + /* The queue use is the amount in which our cwnd is above BDP; + * if it is below, then 0 queue use. */ + if (vegas_bdp_mix(cc) > cc->cwnd) + queue_use = 0; + else + queue_use = cc->cwnd - vegas_bdp_mix(cc); + + if (cc->in_slow_start) { + if (queue_use < cc->vegas_params.gamma && !cc->blocked_chan) { + /* Grow to BDP immediately, then exponential growth until + * congestion signal */ + cc->cwnd = MAX(cc->cwnd + CWND_INC_SS(cc), + vegas_bdp_mix(cc)); + } else { + /* Congestion signal: Fall back to Vegas equilibrium (BDP) */ + cc->cwnd = vegas_bdp_mix(cc); + cc->in_slow_start = 0; + log_info(LD_CIRC, "CC: TOR_VEGAS exiting slow start"); + } + } else { + if (queue_use > cc->vegas_params.beta || cc->blocked_chan) { + cc->cwnd -= CWND_INC(cc); + } else if (queue_use < cc->vegas_params.alpha) { + cc->cwnd += CWND_INC(cc); + } + } + + /* cwnd can never fall below 1 increment */ + cc->cwnd = MAX(cc->cwnd, cc->cwnd_min); + + /* Schedule next update */ + cc->next_cc_event = CWND_UPDATE_RATE(cc); + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC: TOR_VEGAS Circuit %d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "VBDP: %"PRIu64", " + "QUSE: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->cwnd, + cc->inflight, + vegas_bdp_mix(cc), + queue_use, + cc->next_cc_event, + cc->in_slow_start + ); + } else { + log_info(LD_CIRC, + "CC: TOR_VEGAS Circuit %"PRIu64":%d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "VBDP: %"PRIu64", " + "QUSE: %"PRIu64", " + "NCCE: %"PRIu64", " + "SS: %d", + circ->n_chan->global_identifier, circ->n_circ_id, + cc->cwnd, + cc->inflight, + vegas_bdp_mix(cc), + queue_use, + cc->next_cc_event, + cc->in_slow_start + ); + } + } + + /* Update inflight with ack */ + cc->inflight = cc->inflight - cc->sendme_inc; + + return 0; +} diff --git a/src/core/or/congestion_control_vegas.h b/src/core/or/congestion_control_vegas.h new file mode 100644 index 0000000000..111345081c --- /dev/null +++ b/src/core/or/congestion_control_vegas.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_vegas.h + * \brief Private-ish APIs for the TOR_VEGAS congestion control algorithm + **/ + +#ifndef TOR_CONGESTION_CONTROL_VEGAS_H +#define TOR_CONGESTION_CONTROL_VEGAS_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/* Processing SENDME cell. */ +int congestion_control_vegas_process_sendme(struct congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); +void congestion_control_vegas_set_params(struct congestion_control_t *cc); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_VEGAS_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_VEGAS_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_VEGAS_H) */ diff --git a/src/core/or/congestion_control_westwood.c b/src/core/or/congestion_control_westwood.c new file mode 100644 index 0000000000..4b24234212 --- /dev/null +++ b/src/core/or/congestion_control_westwood.c @@ -0,0 +1,231 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_westwood.c + * \brief Code that implements the TOR_WESTWOOD congestion control algorithm + * from Proposal #324. + */ + +#define TOR_CONGESTION_CONTROL_WESTWOOD_PRIVATE + +#include "core/or/or.h" + +#include "core/or/crypt_path.h" +#include "core/or/or_circuit_st.h" +#include "core/or/sendme.h" +#include "core/or/congestion_control_st.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_westwood.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/channel.h" +#include "feature/nodelist/networkstatus.h" + +#define USEC_ONE_MS (1000) + +#define WESTWOOD_CWND_BACKOFF_M 75 +#define WESTWOOD_RTT_BACKOFF_M 100 +#define WESTWOOD_RTT_THRESH 33 +#define WESTWOOD_MIN_BACKOFF 0 + +/** + * Cache westwood consensus parameters. + */ +void +congestion_control_westwood_set_params(congestion_control_t *cc) +{ + tor_assert(cc->cc_alg == CC_ALG_WESTWOOD); + + cc->westwood_params.cwnd_backoff_m = + networkstatus_get_param(NULL, "cc_westwood_cwnd_m", + WESTWOOD_CWND_BACKOFF_M, + 0, + 100); + + cc->westwood_params.rtt_backoff_m = + networkstatus_get_param(NULL, "cc_westwood_rtt_m", + WESTWOOD_RTT_BACKOFF_M, + 50, + 100); + + cc->westwood_params.rtt_thresh = + networkstatus_get_param(NULL, "cc_westwood_rtt_thresh", + WESTWOOD_RTT_THRESH, + 0, + 100); + + cc->westwood_params.min_backoff = + networkstatus_get_param(NULL, "cc_westwood_min_backoff", + WESTWOOD_MIN_BACKOFF, + 0, + 1); +} + +/** + * Return the RTT threshhold that signals congestion. + * + * Computed from the threshold parameter that specifies a + * percent between the min and max RTT obseved so far. + */ +static inline uint64_t +westwood_rtt_signal(const congestion_control_t *cc) +{ + return ((100 - cc->westwood_params.rtt_thresh)*cc->min_rtt_usec + + cc->westwood_params.rtt_thresh*(cc)->max_rtt_usec)/100; +} + +/** + * Compute a backoff to reduce the max RTT. + * + * This may be necessary to ensure that westwood does not have + * a runaway condition where congestion inflates the max RTT, which + * inflates the congestion threshold. That cannot happen with one + * Westwood instance, but it may happen in aggregate. Hence, this is + * a safety parameter, in case we need it. + */ +static inline uint64_t +westwood_rtt_max_backoff(const congestion_control_t *cc) +{ + return cc->min_rtt_usec + + (cc->westwood_params.rtt_backoff_m * + (cc->max_rtt_usec - cc->min_rtt_usec))/100; +} + +/** + * Returns true if the circuit is experiencing congestion, as per + * TOR_WESTWOOD rules. + */ +static inline bool +westwood_is_congested(const congestion_control_t *cc) +{ + /* If the local channel is blocked, that is always congestion */ + if (cc->blocked_chan) + return true; + + /* If the min RTT is within 1ms of the signal, then there is not enough + * range in RTTs to signify congestion. Treat that as not congested. */ + if (westwood_rtt_signal(cc) < cc->min_rtt_usec || + westwood_rtt_signal(cc) - cc->min_rtt_usec < USEC_ONE_MS) + return false; + + /* If the EWMA-smoothed RTT exceeds the westwood RTT threshhold, + * then it is congestion. */ + if (cc->ewma_rtt_usec > westwood_rtt_signal(cc)) + return true; + + return false; +} + +/** + * Process a SENDME and update the congestion window according to the + * rules specified in TOR_WESTWOOD of Proposal #324. + * + * Essentially, this algorithm uses a threshhold of 'rtt_thresh', which + * is a midpoint between the min and max RTT. If the RTT exceeds this + * threshhold, then queue delay due to congestion is assumed to be present, + * and the algirithm reduces the congestion window. If the RTT is below the + * threshhold, the circuit is not congested (ie: queue delay is low), and we + * increase the congestion window. + * + * The congestion window is updated only once every congestion window worth of + * packets, even if the signal persists. It is also updated whenever the + * upstream orcon blocks, or unblocks. This minimizes local client queues. + */ +int +congestion_control_westwood_process_sendme(congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint) +{ + tor_assert(cc && cc->cc_alg == CC_ALG_WESTWOOD); + tor_assert(circ); + + /* Update ack counter until next congestion signal event is allowed */ + if (cc->next_cc_event) + cc->next_cc_event--; + + /* If we were unable to update our circuit estimates, Westwood must + * *not* update its cwnd, otherwise it could run to infinity, or to 0. + * Just update inflight from the sendme and return. */ + if (!congestion_control_update_circuit_estimates(cc, circ, layer_hint)) { + cc->inflight = cc->inflight - cc->sendme_inc; + return 0; + } + + /* We only update anything once per window */ + if (cc->next_cc_event == 0) { + if (!westwood_is_congested(cc)) { + if (cc->in_slow_start) { + cc->cwnd = MAX(cc->cwnd + CWND_INC_SS(cc), + cc->bdp[cc->bdp_alg]); + } else { + cc->cwnd = cc->cwnd + CWND_INC(cc); + } + } else { + if (cc->westwood_params.min_backoff) + cc->cwnd = MIN(cc->cwnd*cc->westwood_params.cwnd_backoff_m/100, + cc->bdp[cc->bdp_alg]); + else + cc->cwnd = MAX(cc->cwnd*cc->westwood_params.cwnd_backoff_m/100, + cc->bdp[cc->bdp_alg]); + + cc->in_slow_start = 0; + + // Because Westwood's congestion can runaway and boost max rtt, + // which increases its congestion signal, we backoff the max rtt + // too. + cc->max_rtt_usec = westwood_rtt_max_backoff(cc); + + log_info(LD_CIRC, "CC: TOR_WESTWOOD congestion. New max RTT: %"PRIu64, + cc->max_rtt_usec/1000); + } + + /* cwnd can never fall below 1 increment */ + cc->cwnd = MAX(cc->cwnd, cc->cwnd_min); + + /* Schedule next update */ + cc->next_cc_event = CWND_UPDATE_RATE(cc); + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_info(LD_CIRC, + "CC: TOR_WESTWOOD Circuit %d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "WRTT: %"PRIu64", " + "WSIG: %"PRIu64", " + "SS: %d", + CONST_TO_ORIGIN_CIRCUIT(circ)->global_identifier, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->ewma_rtt_usec/1000, + westwood_rtt_signal(cc)/1000, + cc->in_slow_start + ); + } else { + log_info(LD_CIRC, + "CC: TOR_WESTWOOD Circuit %"PRIu64":%d " + "CWND: %"PRIu64", " + "INFL: %"PRIu64", " + "NCCE: %"PRIu64", " + "WRTT: %"PRIu64", " + "WSIG: %"PRIu64", " + "SS: %d", + circ->n_chan->global_identifier, circ->n_circ_id, + cc->cwnd, + cc->inflight, + cc->next_cc_event, + cc->ewma_rtt_usec/1000, + westwood_rtt_signal(cc)/1000, + cc->in_slow_start + ); + } + } + + /* Update inflight with ack */ + cc->inflight = cc->inflight - cc->sendme_inc; + + return 0; +} diff --git a/src/core/or/congestion_control_westwood.h b/src/core/or/congestion_control_westwood.h new file mode 100644 index 0000000000..c6fd596df4 --- /dev/null +++ b/src/core/or/congestion_control_westwood.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2019-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file congestion_control_westwood.h + * \brief Private-ish APIs for the TOR_WESTWOOD congestion control algorithm + **/ + +#ifndef TOR_CONGESTION_CONTROL_WESTWOOD_H +#define TOR_CONGESTION_CONTROL_WESTWOOD_H + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" + +/* Processing SENDME cell. */ +int congestion_control_westwood_process_sendme(struct congestion_control_t *cc, + const circuit_t *circ, + const crypt_path_t *layer_hint); +void congestion_control_westwood_set_params(struct congestion_control_t *cc); + +/* Private section starts. */ +#ifdef TOR_CONGESTION_CONTROL_WESTWOOD_PRIVATE + +/* + * Unit tests declaractions. + */ +#ifdef TOR_UNIT_TESTS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* defined(TOR_CONGESTION_CONTROL_WESTWOOD_PRIVATE) */ + +#endif /* !defined(TOR_CONGESTION_CONTROL_WESTWOOD_H) */ diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index d3979b3a7e..ea4bf00735 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -69,6 +69,8 @@ #include "core/or/circuituse.h" #include "core/or/circuitpadding.h" #include "core/or/connection_edge.h" +#include "core/or/congestion_control_flow.h" +#include "core/or/circuitstats.h" #include "core/or/connection_or.h" #include "core/or/extendinfo.h" #include "core/or/policies.h" @@ -614,20 +616,39 @@ connection_half_edge_add(const edge_connection_t *conn, half_conn->stream_id = conn->stream_id; - // How many sendme's should I expect? - half_conn->sendmes_pending = - (STREAMWINDOW_START-conn->package_window)/STREAMWINDOW_INCREMENT; - // Is there a connected cell pending? half_conn->connected_pending = conn->base_.state == AP_CONN_STATE_CONNECT_WAIT; - /* Data should only arrive if we're not waiting on a resolved cell. - * It can arrive after waiting on connected, because of optimistic - * data. */ - if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { - // How many more data cells can arrive on this id? - half_conn->data_pending = conn->deliver_window; + if (edge_uses_flow_control(conn)) { + /* If the edge uses the new congestion control flow control, we must use + * time-based limits on half-edge activity. */ + uint64_t timeout_usec = (uint64_t)(get_circuit_build_timeout_ms()*1000); + half_conn->used_ccontrol = 1; + + /* If this is an onion service circuit, double the CBT as an approximate + * value for the other half of the circuit */ + if (conn->hs_ident) { + timeout_usec *= 2; + } + + /* The stream should stop seeing any use after the larger of the circuit + * RTT and the overall circuit build timeout */ + half_conn->end_ack_expected_usec = MAX(timeout_usec, + edge_get_max_rtt(conn)) + + monotime_absolute_usec(); + } else { + // How many sendme's should I expect? + half_conn->sendmes_pending = + (STREAMWINDOW_START-conn->package_window)/STREAMWINDOW_INCREMENT; + + /* Data should only arrive if we're not waiting on a resolved cell. + * It can arrive after waiting on connected, because of optimistic + * data. */ + if (conn->base_.state != AP_CONN_STATE_RESOLVE_WAIT) { + // How many more data cells can arrive on this id? + half_conn->data_pending = conn->deliver_window; + } } insert_at = smartlist_bsearch_idx(circ->half_streams, &half_conn->stream_id, @@ -688,6 +709,12 @@ connection_half_edge_is_valid_data(const smartlist_t *half_conns, if (!half) return 0; + if (half->used_ccontrol) { + if (monotime_absolute_usec() > half->end_ack_expected_usec) + return 0; + return 1; + } + if (half->data_pending > 0) { half->data_pending--; return 1; @@ -740,6 +767,10 @@ connection_half_edge_is_valid_sendme(const smartlist_t *half_conns, if (!half) return 0; + /* congestion control edges don't use sendmes */ + if (half->used_ccontrol) + return 0; + if (half->sendmes_pending > 0) { half->sendmes_pending--; return 1; @@ -1269,15 +1300,6 @@ connection_ap_rescan_and_attach_pending(void) connection_ap_attach_pending(1); } -#ifdef DEBUGGING_17659 -#define UNMARK() do { \ - entry_conn->marked_pending_circ_line = 0; \ - entry_conn->marked_pending_circ_file = 0; \ - } while (0) -#else /* !defined(DEBUGGING_17659) */ -#define UNMARK() do { } while (0) -#endif /* defined(DEBUGGING_17659) */ - /** Tell any AP streams that are listed as waiting for a new circuit to try * again. If there is an available circuit for a stream, attach it. Otherwise, * launch a new circuit. @@ -1306,21 +1328,18 @@ connection_ap_attach_pending(int retry) connection_t *conn = ENTRY_TO_CONN(entry_conn); tor_assert(conn && entry_conn); if (conn->marked_for_close) { - UNMARK(); continue; } if (conn->magic != ENTRY_CONNECTION_MAGIC) { log_warn(LD_BUG, "%p has impossible magic value %u.", entry_conn, (unsigned)conn->magic); - UNMARK(); continue; } if (conn->state != AP_CONN_STATE_CIRCUIT_WAIT) { - log_warn(LD_BUG, "%p is no longer in circuit_wait. Its current state " - "is %s. Why is it on pending_entry_connections?", - entry_conn, - conn_state_to_string(conn->type, conn->state)); - UNMARK(); + /* The connection_ap_handshake_attach_circuit() call, for onion service, + * can lead to more than one connections in the "pending" list to change + * state and so it is OK to get here. Ignore it because this connection + * won't be in pending_entry_connections list after this point. */ continue; } @@ -1345,7 +1364,6 @@ connection_ap_attach_pending(int retry) /* If we got here, then we either closed the connection, or * we attached it. */ - UNMARK(); } SMARTLIST_FOREACH_END(entry_conn); smartlist_free(pending); @@ -1416,7 +1434,6 @@ connection_ap_mark_as_non_pending_circuit(entry_connection_t *entry_conn) { if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) return; - UNMARK(); smartlist_remove(pending_entry_connections, entry_conn); } @@ -1612,23 +1629,6 @@ consider_plaintext_ports(entry_connection_t *conn, uint16_t port) return 0; } -/** Return true iff <b>query</b> is a syntactically valid service ID (as - * generated by rend_get_service_id). */ -static int -rend_valid_v2_service_id(const char *query) -{ - /** Length of 'y' portion of 'y.onion' URL. */ -#define REND_SERVICE_ID_LEN_BASE32 16 - - if (strlen(query) != REND_SERVICE_ID_LEN_BASE32) - return 0; - - if (strspn(query, BASE32_CHARS) != REND_SERVICE_ID_LEN_BASE32) - return 0; - - return 1; -} - /** Parse the given hostname in address. Returns true if the parsing was * successful and type_out contains the type of the hostname. Else, false is * returned which means it was not recognized and type_out is set to @@ -1692,14 +1692,6 @@ parse_extended_hostname(char *address, hostname_type_t *type_out) if (q != address) { memmove(address, q, strlen(q) + 1 /* also get \0 */); } - /* v2 onion address check. */ - if (strlen(query) == REND_SERVICE_ID_LEN_BASE32) { - *type_out = ONION_V2_HOSTNAME; - if (rend_valid_v2_service_id(query)) { - goto success; - } - goto failed; - } /* v3 onion address check. */ if (strlen(query) == HS_SERVICE_ADDR_LEN_BASE32) { @@ -1719,8 +1711,7 @@ parse_extended_hostname(char *address, hostname_type_t *type_out) failed: /* otherwise, return to previous state and return 0 */ *s = '.'; - const bool is_onion = (*type_out == ONION_V2_HOSTNAME) || - (*type_out == ONION_V3_HOSTNAME); + const bool is_onion = (*type_out == ONION_V3_HOSTNAME); log_warn(LD_APP, "Invalid %shostname %s; rejecting", is_onion ? "onion " : "", safe_str_client(address)); @@ -2242,7 +2233,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, } /* Now, we handle everything that isn't a .onion address. */ - if (addresstype != ONION_V3_HOSTNAME && addresstype != ONION_V2_HOSTNAME) { + if (addresstype != ONION_V3_HOSTNAME) { /* Not a hidden-service request. It's either a hostname or an IP, * possibly with a .exit that we stripped off. We're going to check * if we're allowed to connect/resolve there, and then launch the @@ -2527,28 +2518,6 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, return 0; } else { /* If we get here, it's a request for a .onion address! */ - - /* We don't support v2 onions anymore. Log a warning and bail. */ - if (addresstype == ONION_V2_HOSTNAME) { - static bool log_once = false; - if (!log_once) { - log_warn(LD_PROTOCOL, "Tried to connect to a v2 onion address, but " - "this version of Tor no longer supports them. Please " - "encourage the site operator to upgrade. For more " - "information see " - "https://blog.torproject.org/v2-deprecation-timeline."); - log_once = true; - } - control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s", - escaped(socks->address)); - /* Send back the 0xF6 extended code indicating a bad hostname. This is - * mostly so Tor Browser can make a proper UX with regards to v2 - * addresses. */ - conn->socks_request->socks_extended_error_code = SOCKS5_HS_BAD_ADDRESS; - connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); - return -1; - } - tor_assert(addresstype == ONION_V3_HOSTNAME); tor_assert(!automap); return connection_ap_handle_onion(conn, socks, circ); diff --git a/src/core/or/connection_edge.h b/src/core/or/connection_edge.h index 72869f348b..966a9391d8 100644 --- a/src/core/or/connection_edge.h +++ b/src/core/or/connection_edge.h @@ -80,7 +80,6 @@ typedef enum hostname_type_t { BAD_HOSTNAME, EXIT_HOSTNAME, NORMAL_HOSTNAME, - ONION_V2_HOSTNAME, ONION_V3_HOSTNAME, } hostname_type_t; diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index dd31638eb3..db9f93e6f6 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -65,6 +65,7 @@ #include "core/or/scheduler.h" #include "feature/nodelist/torcert.h" #include "core/or/channelpadding.h" +#include "core/or/congestion_control_common.h" #include "feature/dirauth/authmode.h" #include "feature/hs/hs_service.h" @@ -636,7 +637,7 @@ connection_or_flushed_some(or_connection_t *conn) /* If we're under the low water mark, add cells until we're just over the * high water mark. */ datalen = connection_get_outbuf_len(TO_CONN(conn)); - if (datalen < OR_CONN_LOWWATER) { + if (datalen < or_conn_lowwatermark()) { /* Let the scheduler know */ scheduler_channel_wants_writes(TLS_CHAN_TO_BASE(conn->chan)); } @@ -660,9 +661,9 @@ connection_or_num_cells_writeable(or_connection_t *conn) * used to trigger when to start writing after we've stopped. */ datalen = connection_get_outbuf_len(TO_CONN(conn)); - if (datalen < OR_CONN_HIGHWATER) { + if (datalen < or_conn_highwatermark()) { cell_network_size = get_cell_network_size(conn->wide_circ_ids); - n = CEIL_DIV(OR_CONN_HIGHWATER - datalen, cell_network_size); + n = CEIL_DIV(or_conn_highwatermark() - datalen, cell_network_size); } return n; diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c index 29356d7c2a..7673bc306f 100644 --- a/src/core/or/crypt_path.c +++ b/src/core/or/crypt_path.c @@ -27,6 +27,7 @@ #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/extendinfo.h" +#include "core/or/congestion_control_common.h" #include "lib/crypt_ops/crypto_dh.h" #include "lib/crypt_ops/crypto_util.h" @@ -165,6 +166,7 @@ cpath_free(crypt_path_t *victim) onion_handshake_state_release(&victim->handshake_state); crypto_dh_free(victim->rend_dh_handshake_state); extend_info_free(victim->extend_info); + congestion_control_free(victim->ccontrol); memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */ tor_free(victim); diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h index 2529b6ee41..ddc85eec14 100644 --- a/src/core/or/crypt_path_st.h +++ b/src/core/or/crypt_path_st.h @@ -29,6 +29,8 @@ struct onion_handshake_state_t { } u; }; +struct congestion_control_t; + /** Macro to encapsulate private members of a struct. * * Renames 'x' to 'x_crypt_path_private_field'. @@ -80,6 +82,9 @@ struct crypt_path_t { int deliver_window; /**< How many cells are we willing to deliver originating * at this step? */ + /** Congestion control info */ + struct congestion_control_t *ccontrol; + /*********************** Private members ****************************/ /** Private member: Cryptographic state used for encrypting and diff --git a/src/core/or/edge_connection_st.h b/src/core/or/edge_connection_st.h index 0120c3df25..dab32fc8d0 100644 --- a/src/core/or/edge_connection_st.h +++ b/src/core/or/edge_connection_st.h @@ -15,6 +15,7 @@ #include "core/or/or.h" #include "core/or/connection_st.h" +#include "lib/evloop/token_bucket.h" /** Subtype of connection_t for an "edge connection" -- that is, an entry (ap) * connection, or an exit. */ @@ -73,6 +74,60 @@ struct edge_connection_t { * that's going away and being used on channels instead. We still tag * edge connections with dirreq_id from circuits, so it's copied here. */ uint64_t dirreq_id; + + /* The following are flow control fields */ + + /** Used for rate limiting the read side of this edge connection when + * congestion control is enabled on its circuit. The XON cell ewma_drain_rate + * parameter is used to set the bucket limits. */ + token_bucket_rw_t bucket; + + /** + * Monotime timestamp of the last time we sent a flow control message + * for this edge, used to compute advisory rates */ + uint64_t drain_start_usec; + + /** + * Number of bytes written since we either emptied our buffers, + * or sent an advisory drate rate. Can wrap, buf if so, + * we must reset the usec timestamp above. (Or make this u64, idk). + */ + uint32_t drained_bytes; + uint32_t prev_drained_bytes; + + /** + * N_EWMA of the drain rate of writes on this edge conn + * while buffers were present. + */ + uint32_t ewma_drain_rate; + + /** + * The ewma drain rate the last time we sent an xon. + */ + uint32_t ewma_rate_last_sent; + + /** + * The following fields are used to count the total bytes sent on this + * stream, and compare them to the number of XON and XOFFs recieved, so + * that clients can check rate limits of XOFF/XON to prevent dropmark + * attacks. */ + uint32_t total_bytes_xmit; + + /** Number of XOFFs received */ + uint8_t num_xoff_recv; + + /** Number of XONs received */ + uint8_t num_xon_recv; + + /** + * Flag that tells us if an XOFF has been sent; cleared when we send an XON. + * Used to avoid sending multiple */ + uint8_t xoff_sent : 1; + + /** Flag that tells us if an XOFF has been received; cleared when we get + * an XON. Used to ensure that this edge keeps reads on its edge socket + * disabled. */ + uint8_t xoff_received : 1; }; #endif /* !defined(EDGE_CONNECTION_ST_H) */ diff --git a/src/core/or/half_edge_st.h b/src/core/or/half_edge_st.h index c956c7434a..ac97eb19f1 100644 --- a/src/core/or/half_edge_st.h +++ b/src/core/or/half_edge_st.h @@ -31,6 +31,18 @@ typedef struct half_edge_t { * our deliver window */ int data_pending; + /** + * Monotime timestamp of when the other end should have successfuly + * shut down the stream and stop sending data, based on the larger + * of circuit RTT and CBT. Used if 'used_ccontrol' is true, to expire + * the half_edge at this monotime timestamp. */ + uint64_t end_ack_expected_usec; + + /** + * Did this edge use congestion control? If so, use + * timer instead of pending data approach */ + int used_ccontrol : 1; + /** Is there a connected cell pending? */ int connected_pending : 1; } half_edge_t; diff --git a/src/core/or/include.am b/src/core/or/include.am index b578b75673..b08f8509cc 100644 --- a/src/core/or/include.am +++ b/src/core/or/include.am @@ -28,13 +28,17 @@ LIBTOR_APP_A_SOURCES += \ src/core/or/orconn_event.c \ src/core/or/policies.c \ src/core/or/protover.c \ - src/core/or/protover_rust.c \ src/core/or/reasons.c \ src/core/or/relay.c \ src/core/or/scheduler.c \ src/core/or/scheduler_kist.c \ src/core/or/scheduler_vanilla.c \ src/core/or/sendme.c \ + src/core/or/congestion_control_common.c \ + src/core/or/congestion_control_vegas.c \ + src/core/or/congestion_control_nola.c \ + src/core/or/congestion_control_westwood.c \ + src/core/or/congestion_control_flow.c \ src/core/or/status.c \ src/core/or/versions.c @@ -57,6 +61,7 @@ noinst_HEADERS += \ src/core/or/circuitpadding_machines.h \ src/core/or/circuituse.h \ src/core/or/command.h \ + src/core/or/congestion_control_st.h \ src/core/or/connection_edge.h \ src/core/or/connection_or.h \ src/core/or/connection_st.h \ @@ -77,6 +82,7 @@ noinst_HEADERS += \ src/core/or/entry_port_cfg_st.h \ src/core/or/extend_info_st.h \ src/core/or/listener_connection_st.h \ + src/core/or/lttng_cc.inc \ src/core/or/lttng_circuit.inc \ src/core/or/onion.h \ src/core/or/or.h \ @@ -97,6 +103,11 @@ noinst_HEADERS += \ src/core/or/relay_crypto_st.h \ src/core/or/scheduler.h \ src/core/or/sendme.h \ + src/core/or/congestion_control_flow.h \ + src/core/or/congestion_control_common.h \ + src/core/or/congestion_control_vegas.h \ + src/core/or/congestion_control_nola.h \ + src/core/or/congestion_control_westwood.h \ src/core/or/server_port_cfg_st.h \ src/core/or/socks_request_st.h \ src/core/or/status.h \ @@ -106,7 +117,9 @@ noinst_HEADERS += \ if USE_TRACING_INSTRUMENTATION_LTTNG LIBTOR_APP_A_SOURCES += \ + src/core/or/trace_probes_cc.c \ src/core/or/trace_probes_circuit.c noinst_HEADERS += \ + src/core/or/trace_probes_cc.h \ src/core/or/trace_probes_circuit.h endif diff --git a/src/core/or/lttng_cc.inc b/src/core/or/lttng_cc.inc new file mode 100644 index 0000000000..b7bf58e196 --- /dev/null +++ b/src/core/or/lttng_cc.inc @@ -0,0 +1,166 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file lttng_cc.inc + * \brief LTTng tracing probe declaration for the congestion control subsystem. + * It is in this .inc file due to the non C standard syntax and the way + * we guard the header with the LTTng specific + * TRACEPOINT_HEADER_MULTI_READ. + **/ + +#include "orconfig.h" + +/* We only build the following if LTTng instrumentation has been enabled. */ +#ifdef USE_TRACING_INSTRUMENTATION_LTTNG + +/* The following defines are LTTng-UST specific. */ +#undef TRACEPOINT_PROVIDER +#define TRACEPOINT_PROVIDER tor_cc + +#undef TRACEPOINT_INCLUDE +#define TRACEPOINT_INCLUDE "./src/core/or/lttng_cc.inc" + +#if !defined(LTTNG_CC_INC) || defined(TRACEPOINT_HEADER_MULTI_READ) +#define LTTNG_CC_INC + +#include <lttng/tracepoint.h> + +/* + * Flow Control + */ + +/* Emitted everytime the flow_control_decide_xon() function is called. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon, + TP_ARGS(const edge_connection_t *, stream, size_t, n_written), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(size_t, written_bytes, n_written) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* Emitted when flow control starts measuring the drain rate. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon_drain_start, + TP_ARGS(const edge_connection_t *, stream), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* Emitted when the drain rate is updated. The new_drain_rate value is what was + * just computed. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon_drain_update, + TP_ARGS(const edge_connection_t *, stream, uint32_t, drain_rate), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, new_drain_rate, drain_rate) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* Emitted when an XON cell is sent due to a notice in a drain rate change. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon_rate_change, + TP_ARGS(const edge_connection_t *, stream), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* Emitted when an XON cell is sent because we partially or fully drained the + * edge connection buffer. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon_partial_drain, + TP_ARGS(const edge_connection_t *, stream), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* Emitted when we double the drain rate which is an attempt to see if we can + * speed things up. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xon_drain_doubled, + TP_ARGS(const edge_connection_t *, stream), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* XOFF */ + +/* Emitted when we send an XOFF cell. */ +TRACEPOINT_EVENT(tor_cc, flow_decide_xoff_sending, + TP_ARGS(const edge_connection_t *, stream), + TP_FIELDS( + ctf_integer(uint64_t, stream_id, TO_CONN(stream)->global_identifier) + ctf_integer(uint32_t, drained_bytes_current, stream->drained_bytes) + ctf_integer(uint32_t, drained_bytes_previous, stream->prev_drained_bytes) + ctf_integer(uint32_t, ewma_drain_rate_last, stream->ewma_rate_last_sent) + ctf_integer(uint32_t, ewma_drain_rate_current, stream->ewma_drain_rate) + ctf_integer(size_t, outbuf_len, + connection_get_outbuf_len(TO_CONN(stream))) + ) +) + +/* + * Congestion Control + */ + +/* Emitted when the BDP value has been updated. */ +TRACEPOINT_EVENT(tor_cc, bdp_update, + TP_ARGS(const circuit_t *, circ, const congestion_control_t *, cc, + uint64_t, curr_rtt_usec, uint64_t, sendme_rate_bdp), + TP_FIELDS( + ctf_integer(uint64_t, circuit_ptr, circ) + ctf_integer(uint32_t, n_circ_id, circ->n_circ_id) + ctf_integer(uint64_t, min_rtt_usec, cc->min_rtt_usec) + ctf_integer(uint64_t, curr_rtt_usec, curr_rtt_usec) + ctf_integer(uint64_t, ewma_rtt_usec, cc->ewma_rtt_usec) + ctf_integer(uint64_t, max_rtt_usec, cc->max_rtt_usec) + ctf_integer(uint64_t, bdp_inflight_rtt, cc->bdp[BDP_ALG_INFLIGHT_RTT]) + ctf_integer(uint64_t, bdp_cwnd_rtt, cc->bdp[BDP_ALG_CWND_RTT]) + ctf_integer(uint64_t, bdp_sendme_rate, cc->bdp[BDP_ALG_SENDME_RATE]) + ctf_integer(uint64_t, bdp_piecewise, cc->bdp[BDP_ALG_PIECEWISE]) + ctf_integer(uint64_t, sendme_rate_bdp, sendme_rate_bdp) + ) +) + +#endif /* LTTNG_CC_INC || TRACEPOINT_HEADER_MULTI_READ */ + +/* Must be included after the probes declaration. */ +#include <lttng/tracepoint-event.h> + +#endif /* USE_TRACING_INSTRUMENTATION_LTTNG */ diff --git a/src/core/or/or.h b/src/core/or/or.h index 99948f26e2..392a848ee7 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -210,6 +210,9 @@ struct curve25519_public_key_t; #define RELAY_COMMAND_PADDING_NEGOTIATE 41 #define RELAY_COMMAND_PADDING_NEGOTIATED 42 +#define RELAY_COMMAND_XOFF 43 +#define RELAY_COMMAND_XON 44 + /* Reasons why an OR connection is closed. */ #define END_OR_CONN_REASON_DONE 1 #define END_OR_CONN_REASON_REFUSED 2 /* connection refused */ @@ -591,18 +594,6 @@ typedef struct or_handshake_state_t or_handshake_state_t; /** Length of Extended ORPort connection identifier. */ #define EXT_OR_CONN_ID_LEN DIGEST_LEN /* 20 */ -/* - * OR_CONN_HIGHWATER and OR_CONN_LOWWATER moved from connection_or.c so - * channeltls.c can see them too. - */ - -/** When adding cells to an OR connection's outbuf, keep adding until the - * outbuf is at least this long, or we run out of cells. */ -#define OR_CONN_HIGHWATER (32*1024) - -/** Add cells to an OR connection's outbuf whenever the outbuf's data length - * drops below this size. */ -#define OR_CONN_LOWWATER (16*1024) typedef struct connection_t connection_t; typedef struct control_connection_t control_connection_t; diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h index b8fbf9658e..11695ec301 100644 --- a/src/core/or/or_circuit_st.h +++ b/src/core/or/or_circuit_st.h @@ -52,6 +52,10 @@ struct or_circuit_t { /** Stores KH for the handshake. */ char rend_circ_nonce[DIGEST_LEN];/* KH in tor-spec.txt */ + /** Number of cells which we have discarded because of having no next hop, + * despite not recognizing the cell. */ + uint32_t n_cells_discarded_at_end; + /** How many more relay_early cells can we send on this circuit, according * to the specification? */ unsigned int remaining_relay_early_cells : 4; @@ -93,4 +97,3 @@ struct or_circuit_t { }; #endif /* !defined(OR_CIRCUIT_ST_H) */ - diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h index 9264077c50..6c86a56000 100644 --- a/src/core/or/origin_circuit_st.h +++ b/src/core/or/origin_circuit_st.h @@ -180,6 +180,12 @@ struct origin_circuit_t { unsigned first_hop_from_controller : 1; /** + * If true, this circuit's path has been chosen, in full or in part, + * by the controller API, and it's okay to ignore checks that we'd + * usually do on the path as whole. */ + unsigned int any_hop_from_controller : 1; + + /** * Tristate variable to guard against pathbias miscounting * due to circuit purpose transitions changing the decision * of pathbias_should_count(). This variable is informational diff --git a/src/core/or/policies.c b/src/core/or/policies.c index f91c23ad31..a53849b4d0 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -59,6 +59,9 @@ static smartlist_t *authdir_invalid_policy = NULL; /** Policy that addresses for incoming router descriptors must <b>not</b> * match in order to not be marked as BadExit. */ static smartlist_t *authdir_badexit_policy = NULL; +/** Policy that addresses for incoming router descriptors must <b>not</b> + * match in order to not be marked as MiddleOnly. */ +static smartlist_t *authdir_middleonly_policy = NULL; /** Parsed addr_policy_t describing which addresses we believe we can start * circuits at. */ @@ -1119,6 +1122,17 @@ authdir_policy_badexit_address(const tor_addr_t *addr, uint16_t port) return addr_is_in_cc_list(addr, get_options()->AuthDirBadExitCCs); } +/** Return 1 if <b>addr</b>:<b>port</b> should be marked as MiddleOnly, + * based on <b>authdir_middleonly_policy</b>. Else return 0. + */ +int +authdir_policy_middleonly_address(const tor_addr_t *addr, uint16_t port) +{ + if (!addr_policy_permits_tor_addr(addr, port, authdir_middleonly_policy)) + return 1; + return addr_is_in_cc_list(addr, get_options()->AuthDirMiddleOnlyCCs); +} + #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); goto err; STMT_END @@ -1173,6 +1187,9 @@ validate_addr_policies(const or_options_t *options, char **msg) if (parse_addr_policy(options->AuthDirBadExit, &addr_policy, ADDR_POLICY_REJECT)) REJECT("Error in AuthDirBadExit entry."); + if (parse_addr_policy(options->AuthDirMiddleOnly, &addr_policy, + ADDR_POLICY_REJECT)) + REJECT("Error in AuthDirMiddleOnly entry."); if (parse_addr_policy(options->ReachableAddresses, &addr_policy, ADDR_POLICY_ACCEPT)) @@ -1266,6 +1283,9 @@ policies_parse_from_options(const or_options_t *options) if (load_policy_from_option(options->AuthDirBadExit, "AuthDirBadExit", &authdir_badexit_policy, ADDR_POLICY_REJECT) < 0) ret = -1; + if (load_policy_from_option(options->AuthDirMiddleOnly, "AuthDirMiddleOnly", + &authdir_middleonly_policy, ADDR_POLICY_REJECT) < 0) + ret = -1; if (parse_metrics_port_policy(options) < 0) { ret = -1; } @@ -3112,6 +3132,8 @@ policies_free_all(void) authdir_invalid_policy = NULL; addr_policy_list_free(authdir_badexit_policy); authdir_badexit_policy = NULL; + addr_policy_list_free(authdir_middleonly_policy); + authdir_middleonly_policy = NULL; if (!HT_EMPTY(&policy_root)) { policy_map_ent_t **ent; diff --git a/src/core/or/policies.h b/src/core/or/policies.h index a32f50ab1d..e11e1d0ff5 100644 --- a/src/core/or/policies.h +++ b/src/core/or/policies.h @@ -106,6 +106,7 @@ int metrics_policy_permits_address(const tor_addr_t *addr); int authdir_policy_permits_address(const tor_addr_t *addr, uint16_t port); int authdir_policy_valid_address(const tor_addr_t *addr, uint16_t port); int authdir_policy_badexit_address(const tor_addr_t *addr, uint16_t port); +int authdir_policy_middleonly_address(const tor_addr_t *addr, uint16_t port); int validate_addr_policies(const or_options_t *options, char **msg); void policy_expand_private(smartlist_t **policy); diff --git a/src/core/or/protover.c b/src/core/or/protover.c index 04df8aeeb8..199fc830a0 100644 --- a/src/core/or/protover.c +++ b/src/core/or/protover.c @@ -28,8 +28,6 @@ #include "core/or/versions.h" #include "lib/tls/tortls.h" -#ifndef HAVE_RUST - static const smartlist_t *get_supported_protocol_list(void); static int protocol_list_contains(const smartlist_t *protos, protocol_type_t pr, uint32_t ver); @@ -855,5 +853,3 @@ protover_free_all(void) supported_protocol_list = NULL; } } - -#endif /* !defined(HAVE_RUST) */ diff --git a/src/core/or/protover.h b/src/core/or/protover.h index c0739a092e..ae258d74a5 100644 --- a/src/core/or/protover.h +++ b/src/core/or/protover.h @@ -103,13 +103,13 @@ typedef struct proto_entry_t { uint64_t bitmask; } proto_entry_t; -#if !defined(HAVE_RUST) && defined(TOR_UNIT_TESTS) +#if defined(TOR_UNIT_TESTS) STATIC struct smartlist_t *parse_protocol_list(const char *s); STATIC char *encode_protocol_list(const struct smartlist_t *sl); STATIC const char *protocol_type_to_str(protocol_type_t pr); STATIC int str_to_protocol_type(const char *s, protocol_type_t *pr_out); STATIC void proto_entry_free_(proto_entry_t *entry); -#endif /* !defined(HAVE_RUST) && defined(TOR_UNIT_TESTS) */ +#endif /* defined(TOR_UNIT_TESTS) */ #define proto_entry_free(entry) \ FREE_AND_NULL(proto_entry_t, proto_entry_free_, (entry)) diff --git a/src/core/or/protover_rust.c b/src/core/or/protover_rust.c deleted file mode 100644 index 31ddfa1bdf..0000000000 --- a/src/core/or/protover_rust.c +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright (c) 2016-2021, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/* - * \file protover_rust.c - * \brief Provide a C wrapper for functions exposed in /src/rust/protover, - * and safe translation/handling between the Rust/C boundary. - */ - -#include "core/or/or.h" -#include "core/or/protover.h" - -#ifdef HAVE_RUST - -/* Define for compatibility, used in main.c */ -void -protover_free_all(void) -{ -} - -int protover_contains_long_protocol_names_(const char *s); - -/** - * Return true if the unparsed protover in <b>s</b> would contain a protocol - * name longer than MAX_PROTOCOL_NAME_LENGTH, and false otherwise. - */ -bool -protover_list_is_invalid(const char *s) -{ - return protover_contains_long_protocol_names_(s) != 0; -} - -#endif /* defined(HAVE_RUST) */ - diff --git a/src/core/or/relay.c b/src/core/or/relay.c index f5a9e73856..68fddd1ae7 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -97,6 +97,8 @@ #include "feature/nodelist/routerinfo_st.h" #include "core/or/socks_request_st.h" #include "core/or/sendme.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_flow.h" static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, @@ -115,13 +117,6 @@ static void adjust_exit_policy_from_exitpolicy_failure(origin_circuit_t *circ, node_t *node, const tor_addr_t *addr); -/** Stop reading on edge connections when we have this many cells - * waiting on the appropriate queue. */ -#define CELL_QUEUE_HIGHWATER_SIZE 256 -/** Start reading from edge connections again when we get down to this many - * cells. */ -#define CELL_QUEUE_LOWWATER_SIZE 64 - /** Stats: how many relay cells have originated at this hop, or have * been relayed onward (not recognized at this hop)? */ @@ -338,8 +333,17 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, } return 0; } - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Didn't recognize cell, but circ stops here! Closing circ."); + if (BUG(CIRCUIT_IS_ORIGIN(circ))) { + /* Should be impossible at this point. */ + return -END_CIRC_REASON_TORPROTOCOL; + } + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + if (++or_circ->n_cells_discarded_at_end == 1) { + time_t seconds_open = approx_time() - circ->timestamp_created.tv_sec; + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Didn't recognize a cell, but circ stops here! Closing circuit. " + "It was created %ld seconds ago.", (long)seconds_open); + } return -END_CIRC_REASON_TORPROTOCOL; } @@ -1574,6 +1578,7 @@ process_sendme_cell(const relay_header_t *rh, const cell_t *cell, } /* Stream level SENDME cell. */ + // TODO: Turn this off for cc_alg=1,2,3; use XON/XOFF instead ret = sendme_process_stream_level(conn, circ, rh->length); if (ret < 0) { /* Means we need to close the circuit with reason ret. */ @@ -1738,6 +1743,44 @@ handle_relay_cell_command(cell_t *cell, circuit_t *circ, } return 0; + case RELAY_COMMAND_XOFF: + if (!conn) { + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + if (relay_crypt_from_last_hop(ocirc, layer_hint) && + connection_half_edge_is_valid_data(ocirc->half_streams, + rh->stream_id)) { + circuit_read_valid_data(ocirc, rh->length); + } + } + return 0; + } + + if (circuit_process_stream_xoff(conn, layer_hint, cell)) { + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); + } + } + return 0; + case RELAY_COMMAND_XON: + if (!conn) { + if (CIRCUIT_IS_ORIGIN(circ)) { + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + if (relay_crypt_from_last_hop(ocirc, layer_hint) && + connection_half_edge_is_valid_data(ocirc->half_streams, + rh->stream_id)) { + circuit_read_valid_data(ocirc, rh->length); + } + } + return 0; + } + + if (circuit_process_stream_xon(conn, layer_hint, cell)) { + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); + } + } + return 0; case RELAY_COMMAND_END: reason = rh->length > 0 ? get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC; @@ -2091,6 +2134,7 @@ void circuit_reset_sendme_randomness(circuit_t *circ) { circ->have_sent_sufficiently_random_cell = 0; + // XXX: do we need to change this check for congestion control? circ->send_randomness_after_n_cells = CIRCWINDOW_INCREMENT / 2 + crypto_fast_rng_get_uint(get_thread_fast_rng(), CIRCWINDOW_INCREMENT / 2); } @@ -2284,7 +2328,7 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial, } /* Handle the stream-level SENDME package window. */ - if (sendme_note_stream_data_packaged(conn) < 0) { + if (sendme_note_stream_data_packaged(conn, length) < 0) { connection_stop_reading(TO_CONN(conn)); log_debug(domain,"conn->package_window reached 0."); circuit_consider_stop_edge_reading(circ, cpath_layer); @@ -2350,15 +2394,16 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn, /* How many cells do we have space for? It will be the minimum of * the number needed to exhaust the package window, and the minimum * needed to fill the cell queue. */ - max_to_package = circ->package_window; + + max_to_package = congestion_control_get_package_window(circ, layer_hint); if (CIRCUIT_IS_ORIGIN(circ)) { cells_on_queue = circ->n_chan_cells.n; } else { or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); cells_on_queue = or_circ->p_chan_cells.n; } - if (CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue < max_to_package) - max_to_package = CELL_QUEUE_HIGHWATER_SIZE - cells_on_queue; + if (cell_queue_highwatermark() - cells_on_queue < max_to_package) + max_to_package = cell_queue_highwatermark() - cells_on_queue; /* Once we used to start listening on the streams in the order they * appeared in the linked list. That leads to starvation on the @@ -2398,7 +2443,8 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn, /* Activate reading starting from the chosen stream */ for (conn=chosen_stream; conn; conn = conn->next_stream) { /* Start reading for the streams starting from here */ - if (conn->base_.marked_for_close || conn->package_window <= 0) + if (conn->base_.marked_for_close || conn->package_window <= 0 || + conn->xoff_received) continue; if (!layer_hint || conn->cpath_layer == layer_hint) { connection_start_reading(TO_CONN(conn)); @@ -2409,7 +2455,8 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn, } /* Go back and do the ones we skipped, circular-style */ for (conn = first_conn; conn != chosen_stream; conn = conn->next_stream) { - if (conn->base_.marked_for_close || conn->package_window <= 0) + if (conn->base_.marked_for_close || conn->package_window <= 0 || + conn->xoff_received) continue; if (!layer_hint || conn->cpath_layer == layer_hint) { connection_start_reading(TO_CONN(conn)); @@ -2495,7 +2542,7 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); log_debug(domain,"considering circ->package_window %d", circ->package_window); - if (circ->package_window <= 0) { + if (congestion_control_get_package_window(circ, layer_hint) <= 0) { log_debug(domain,"yes, not-at-origin. stopped."); for (conn = or_circ->n_streams; conn; conn=conn->next_stream) connection_stop_reading(TO_CONN(conn)); @@ -2506,7 +2553,7 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint) /* else, layer hint is defined, use it */ log_debug(domain,"considering layer_hint->package_window %d", layer_hint->package_window); - if (layer_hint->package_window <= 0) { + if (congestion_control_get_package_window(circ, layer_hint) <= 0) { log_debug(domain,"yes, at-origin. stopped."); for (conn = TO_ORIGIN_CIRCUIT(circ)->p_streams; conn; conn=conn->next_stream) { @@ -2722,11 +2769,18 @@ cell_queues_get_total_allocation(void) /** The time at which we were last low on memory. */ static time_t last_time_under_memory_pressure = 0; +/** Statistics on how many bytes were removed by the OOM per type. */ +uint64_t oom_stats_n_bytes_removed_dns = 0; +uint64_t oom_stats_n_bytes_removed_cell = 0; +uint64_t oom_stats_n_bytes_removed_geoip = 0; +uint64_t oom_stats_n_bytes_removed_hsdir = 0; + /** Check whether we've got too much space used for cells. If so, * call the OOM handler and return 1. Otherwise, return 0. */ STATIC int cell_queues_check_size(void) { + size_t removed = 0; time_t now = time(NULL); size_t alloc = cell_queues_get_total_allocation(); alloc += half_streams_get_total_allocation(); @@ -2751,20 +2805,27 @@ cell_queues_check_size(void) if (hs_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = hs_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= hs_cache_handle_oom(now, bytes_to_remove); + removed = hs_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_hsdir += removed; + alloc -= removed; } if (geoip_client_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = geoip_client_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= geoip_client_cache_handle_oom(now, bytes_to_remove); + removed = geoip_client_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_geoip += removed; + alloc -= removed; } if (dns_cache_total > get_options()->MaxMemInQueues / 5) { const size_t bytes_to_remove = dns_cache_total - (size_t)(get_options()->MaxMemInQueues / 10); - alloc -= dns_cache_handle_oom(now, bytes_to_remove); + removed = dns_cache_handle_oom(now, bytes_to_remove); + oom_stats_n_bytes_removed_dns += removed; + alloc -= removed; } - circuits_handle_oom(alloc); + removed = circuits_handle_oom(alloc); + oom_stats_n_bytes_removed_cell += removed; return 1; } } @@ -3062,7 +3123,7 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max)) /* Is the cell queue low enough to unblock all the streams that are waiting * to write to this circuit? */ - if (streams_blocked && queue->n <= CELL_QUEUE_LOWWATER_SIZE) + if (streams_blocked && queue->n <= cell_queue_lowwatermark()) set_streams_blocked_on_circ(circ, chan, 0, 0); /* unblock streams */ /* If n_flushed < max still, loop around and pick another circuit */ @@ -3180,7 +3241,7 @@ append_cell_to_circuit_queue(circuit_t *circ, channel_t *chan, /* If we have too many cells on the circuit, we should stop reading from * the edge streams for a while. */ - if (!streams_blocked && queue->n >= CELL_QUEUE_HIGHWATER_SIZE) + if (!streams_blocked && queue->n >= cell_queue_highwatermark()) set_streams_blocked_on_circ(circ, chan, 1, 0); /* block streams */ if (streams_blocked && fromstream) { diff --git a/src/core/or/relay.h b/src/core/or/relay.h index 2f337d5d16..eac920f491 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -49,6 +49,11 @@ extern uint64_t stats_n_data_bytes_packaged; extern uint64_t stats_n_data_cells_received; extern uint64_t stats_n_data_bytes_received; +extern uint64_t oom_stats_n_bytes_removed_dns; +extern uint64_t oom_stats_n_bytes_removed_cell; +extern uint64_t oom_stats_n_bytes_removed_geoip; +extern uint64_t oom_stats_n_bytes_removed_hsdir; + void dump_cell_pool_usage(int severity); size_t packed_cell_mem_cost(void); diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c index ce3385ae98..ee670f9d51 100644 --- a/src/core/or/sendme.c +++ b/src/core/or/sendme.c @@ -21,6 +21,8 @@ #include "core/or/or_circuit_st.h" #include "core/or/relay.h" #include "core/or/sendme.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_flow.h" #include "feature/nodelist/networkstatus.h" #include "lib/ctime/di_ops.h" #include "trunnel/sendme_cell.h" @@ -64,13 +66,6 @@ pop_first_cell_digest(const circuit_t *circ) return NULL; } - /* More cell digest than the SENDME window is never suppose to happen. The - * cell should have been rejected before reaching this point due to its - * package_window down to 0 leading to a circuit close. Scream loudly but - * still pop the element so we don't memory leak. */ - tor_assert_nonfatal(smartlist_len(circ->sendme_last_digests) <= - CIRCWINDOW_START_MAX / CIRCWINDOW_INCREMENT); - circ_digest = smartlist_get(circ->sendme_last_digests, 0); smartlist_del_keeporder(circ->sendme_last_digests, 0); return circ_digest; @@ -334,17 +329,18 @@ record_cell_digest_on_circ(circuit_t *circ, const uint8_t *sendme_digest) /** Return true iff the next cell for the given cell window is expected to be * a SENDME. * - * We are able to know that because the package or deliver window value minus - * one cell (the possible SENDME cell) should be a multiple of the increment - * window value. */ + * We are able to know that because the package or inflight window value minus + * one cell (the possible SENDME cell) should be a multiple of the + * cells-per-sendme increment value (set via consensus parameter, negotiated + * for the circuit, and passed in as sendme_inc). + * + * This function is used when recording a cell digest and this is done quite + * low in the stack when decrypting or encrypting a cell. The window is only + * updated once the cell is actually put in the outbuf. + */ static bool -circuit_sendme_cell_is_next(int window) +circuit_sendme_cell_is_next(int window, int sendme_inc) { - /* At the start of the window, no SENDME will be expected. */ - if (window == CIRCWINDOW_START) { - return false; - } - /* Are we at the limit of the increment and if not, we don't expect next * cell is a SENDME. * @@ -352,11 +348,8 @@ circuit_sendme_cell_is_next(int window) * next cell is a SENDME, the window (either package or deliver) hasn't been * decremented just yet so when this is called, we are currently processing * the "window - 1" cell. - * - * This function is used when recording a cell digest and this is done quite - * low in the stack when decrypting or encrypting a cell. The window is only - * updated once the cell is actually put in the outbuf. */ - if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) { + */ + if (((window - 1) % sendme_inc) != 0) { return false; } @@ -378,6 +371,10 @@ sendme_connection_edge_consider_sending(edge_connection_t *conn) int log_domain = TO_CONN(conn)->type == CONN_TYPE_AP ? LD_APP : LD_EXIT; + /* If we use flow control, we do not send stream sendmes */ + if (edge_uses_flow_control(conn)) + goto end; + /* Don't send it if we still have data to deliver. */ if (connection_outbuf_too_full(TO_CONN(conn))) { goto end; @@ -419,15 +416,16 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) { bool sent_one_sendme = false; const uint8_t *digest; + int sendme_inc = sendme_get_inc_count(circ, layer_hint); while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <= - CIRCWINDOW_START - CIRCWINDOW_INCREMENT) { + CIRCWINDOW_START - sendme_inc) { log_debug(LD_CIRC,"Queuing circuit sendme."); if (layer_hint) { - layer_hint->deliver_window += CIRCWINDOW_INCREMENT; + layer_hint->deliver_window += sendme_inc; digest = cpath_get_sendme_digest(layer_hint); } else { - circ->deliver_window += CIRCWINDOW_INCREMENT; + circ->deliver_window += sendme_inc; digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto); } if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) { @@ -448,6 +446,9 @@ sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint) * the length of the SENDME cell payload (excluding the header). The * cell_payload is the payload. * + * This function validates the SENDME's digest, and then dispatches to + * the appropriate congestion control algorithm in use on the circuit. + * * Return 0 on success (the SENDME is valid and the package window has * been updated properly). * @@ -460,6 +461,7 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, { tor_assert(circ); tor_assert(cell_payload); + congestion_control_t *cc; /* Validate the SENDME cell. Depending on the version, different validation * can be done. An invalid SENDME requires us to close the circuit. */ @@ -467,6 +469,34 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, return -END_CIRC_REASON_TORPROTOCOL; } + // Get CC + if (layer_hint) { + cc = layer_hint->ccontrol; + + /* origin circuits need to count valid sendmes as valid protocol data */ + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len); + } else { + cc = circ->ccontrol; + } + + /* If there is no CC object, assume fixed alg */ + if (!cc) { + return sendme_process_circuit_level_impl(layer_hint, circ); + } + + return congestion_control_dispatch_cc_alg(cc, circ, layer_hint); +} + +/** + * Process a SENDME for Tor's original fixed window circuit-level flow control. + * Updates the package_window and ensures that it does not exceed the max. + * + * Returns -END_CIRC_REASON_TORPROTOCOL if the max is exceeded, otherwise + * returns 0. + */ +int +sendme_process_circuit_level_impl(crypt_path_t *layer_hint, circuit_t *circ) +{ /* If we are the origin of the circuit, we are the Client so we use the * layer hint (the Exit hop) for the package window tracking. */ if (CIRCUIT_IS_ORIGIN(circ)) { @@ -486,10 +516,6 @@ sendme_process_circuit_level(crypt_path_t *layer_hint, layer_hint->package_window += CIRCWINDOW_INCREMENT; log_debug(LD_APP, "circ-level sendme at origin, packagewindow %d.", layer_hint->package_window); - - /* We count circuit-level sendme's as valid delivered data because they - * are rate limited. */ - circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len); } else { /* We aren't the origin of this circuit so we are the Exit and thus we * track the package window with the circuit object. */ @@ -525,6 +551,12 @@ sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, tor_assert(conn); tor_assert(circ); + if (edge_uses_flow_control(conn)) { + log_fn(LOG_PROTOCOL_WARN, LD_EDGE, + "Congestion control got stream sendme"); + return -END_CIRC_REASON_TORPROTOCOL; + } + /* Don't allow the other endpoint to request more than our maximum (i.e. * initial) stream SENDME window worth of data. Well-behaved stock clients * will not request more than this max (as per the check in the while loop @@ -582,7 +614,12 @@ int sendme_stream_data_received(edge_connection_t *conn) { tor_assert(conn); - return --conn->deliver_window; + + if (edge_uses_flow_control(conn)) { + return flow_control_decide_xoff(conn); + } else { + return --conn->deliver_window; + } } /* Called when a relay DATA cell is packaged on the given circuit. If @@ -592,34 +629,56 @@ int sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint) { int package_window, domain; + congestion_control_t *cc; tor_assert(circ); - if (CIRCUIT_IS_ORIGIN(circ)) { - /* Client side. */ - tor_assert(layer_hint); - --layer_hint->package_window; - package_window = layer_hint->package_window; + if (layer_hint) { + cc = layer_hint->ccontrol; domain = LD_APP; } else { - /* Exit side. */ - tor_assert(!layer_hint); - --circ->package_window; - package_window = circ->package_window; + cc = circ->ccontrol; domain = LD_EXIT; } - log_debug(domain, "Circuit package_window now %d.", package_window); - return package_window; + if (cc) { + congestion_control_note_cell_sent(cc, circ, layer_hint); + } else { + /* Fixed alg uses package_window and must update it */ + + if (CIRCUIT_IS_ORIGIN(circ)) { + /* Client side. */ + tor_assert(layer_hint); + --layer_hint->package_window; + package_window = layer_hint->package_window; + } else { + /* Exit side. */ + tor_assert(!layer_hint); + --circ->package_window; + package_window = circ->package_window; + } + log_debug(domain, "Circuit package_window now %d.", package_window); + } + + /* Return appropriate number designating how many cells can still be sent */ + return congestion_control_get_package_window(circ, layer_hint); } /* Called when a relay DATA cell is packaged for the given edge connection * conn. Update the package window and return its new value. */ int -sendme_note_stream_data_packaged(edge_connection_t *conn) +sendme_note_stream_data_packaged(edge_connection_t *conn, size_t len) { tor_assert(conn); + if (edge_uses_flow_control(conn)) { + flow_control_note_sent_data(conn, len); + if (conn->xoff_received) + return -1; + else + return 1; + } + --conn->package_window; log_debug(LD_APP, "Stream package_window now %d.", conn->package_window); return conn->package_window; @@ -631,20 +690,14 @@ sendme_note_stream_data_packaged(edge_connection_t *conn) void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath) { - int package_window; uint8_t *sendme_digest; tor_assert(circ); - package_window = circ->package_window; - if (cpath) { - package_window = cpath->package_window; - } - /* Is this the last cell before a SENDME? The idea is that if the * package_window reaches a multiple of the increment, after this cell, we * should expect a SENDME. */ - if (!circuit_sendme_cell_is_next(package_window)) { + if (!circuit_sent_cell_for_sendme(circ, cpath)) { return; } @@ -670,7 +723,8 @@ sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath) /* Only record if the next cell is expected to be a SENDME. */ if (!circuit_sendme_cell_is_next(cpath ? cpath->deliver_window : - circ->deliver_window)) { + circ->deliver_window, + sendme_get_inc_count(circ, cpath))) { return; } @@ -692,8 +746,7 @@ sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath) tor_assert(circ); /* Only record if the next cell is expected to be a SENDME. */ - if (!circuit_sendme_cell_is_next(cpath ? cpath->package_window : - circ->package_window)) { + if (!circuit_sent_cell_for_sendme(circ, cpath)) { goto end; } diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h index a008940905..2abec91a91 100644 --- a/src/core/or/sendme.h +++ b/src/core/or/sendme.h @@ -22,6 +22,7 @@ void sendme_circuit_consider_sending(circuit_t *circ, int sendme_process_circuit_level(crypt_path_t *layer_hint, circuit_t *circ, const uint8_t *cell_payload, uint16_t cell_payload_len); +int sendme_process_circuit_level_impl(crypt_path_t *, circuit_t *); int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ, uint16_t cell_body_len); @@ -32,7 +33,7 @@ int sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint); /* Update package window functions. */ int sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint); -int sendme_note_stream_data_packaged(edge_connection_t *conn); +int sendme_note_stream_data_packaged(edge_connection_t *conn, size_t len); /* Record cell digest on circuit. */ void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath); diff --git a/src/core/or/status.c b/src/core/or/status.c index 9e7ae70535..1e599aafb3 100644 --- a/src/core/or/status.c +++ b/src/core/or/status.c @@ -147,6 +147,32 @@ note_connection(bool inbound, int family) } } +/** + * @name Counters for unrecognized cells + * + * Track cells that we drop because they are unrecognized and we have + * nobody to send them to. + **/ +/**@{*/ +static unsigned n_circs_closed_for_unrecognized_cells; +static uint64_t n_unrecognized_cells_discarded; +static uint64_t n_secs_on_circs_with_unrecognized_cells; +/**@}*/ + +/** + * Note that a circuit has closed @a n_seconds after having been created, + * because of one or more unrecognized cells. Also note the number of + * unrecognized cells @a n_cells. + */ +void +note_circ_closed_for_unrecognized_cells(time_t n_seconds, uint32_t n_cells) +{ + ++n_circs_closed_for_unrecognized_cells; + n_unrecognized_cells_discarded += n_cells; + if (n_seconds >= 0) + n_secs_on_circs_with_unrecognized_cells += (uint64_t) n_seconds; +} + /** Log a "heartbeat" message describing Tor's status and history so that the * user can know that there is indeed a running Tor. Return 0 on success and * -1 on failure. */ @@ -240,6 +266,23 @@ log_heartbeat(time_t now) (main_loop_idle_count)); } + if (n_circs_closed_for_unrecognized_cells) { + double avg_time_alive = ((double) n_secs_on_circs_with_unrecognized_cells) + / n_circs_closed_for_unrecognized_cells; + double avg_cells = ((double) n_unrecognized_cells_discarded) + / n_circs_closed_for_unrecognized_cells; + log_fn(LOG_NOTICE, LD_HEARTBEAT, + "Since our last heartbeat, %u circuits were closed because of " + "unrecognized cells while we were the last hop. On average, each " + "one was alive for %lf seconds, and had %lf unrecognized cells.", + n_circs_closed_for_unrecognized_cells, + avg_time_alive, + avg_cells); + n_circs_closed_for_unrecognized_cells = 0; + n_unrecognized_cells_discarded = 0; + n_secs_on_circs_with_unrecognized_cells = 0; + } + /** Now, if we are an HS service, log some stats about our usage */ log_onion_service_stats(); diff --git a/src/core/or/status.h b/src/core/or/status.h index 927df9a192..57e28002fc 100644 --- a/src/core/or/status.h +++ b/src/core/or/status.h @@ -12,6 +12,9 @@ #include "lib/testsupport/testsupport.h" void note_connection(bool inbound, int family); +void note_circ_closed_for_unrecognized_cells(time_t n_seconds, + uint32_t n_cells); + int log_heartbeat(time_t now); #ifdef STATUS_PRIVATE diff --git a/src/core/or/trace_probes_cc.c b/src/core/or/trace_probes_cc.c new file mode 100644 index 0000000000..d52646da4f --- /dev/null +++ b/src/core/or/trace_probes_cc.c @@ -0,0 +1,33 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file trace_probes_cc.c + * \brief Tracepoint provider source file for the cc subsystem. Probes + * are generated within this C file for LTTng-UST + **/ + +#include "orconfig.h" + +/* + * Following section is specific to LTTng-UST. + */ +#ifdef USE_TRACING_INSTRUMENTATION_LTTNG + +/* Header files that the probes need. */ +#include "core/or/or.h" +#include "core/or/channel.h" +#include "core/or/circuit_st.h" +#include "core/or/circuitlist.h" +#include "core/or/congestion_control_st.h" +#include "core/or/connection_st.h" +#include "core/or/edge_connection_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/origin_circuit_st.h" + +#define TRACEPOINT_DEFINE +#define TRACEPOINT_CREATE_PROBES + +#include "core/or/trace_probes_cc.h" + +#endif /* defined(USE_TRACING_INSTRUMENTATION_LTTNG) */ diff --git a/src/core/or/trace_probes_cc.h b/src/core/or/trace_probes_cc.h new file mode 100644 index 0000000000..1f87528723 --- /dev/null +++ b/src/core/or/trace_probes_cc.h @@ -0,0 +1,22 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file trace_probes_cc.c + * \brief The tracing probes for the congestion control subsystem. + * Currently, only LTTng-UST probes are available. + **/ + +#ifndef TOR_TRACE_PROBES_CC_H +#define TOR_TRACE_PROBES_CC_H + +#include "lib/trace/events.h" + +/* We only build the following if LTTng instrumentation has been enabled. */ +#ifdef USE_TRACING_INSTRUMENTATION_LTTNG + +#include "core/or/lttng_cc.inc" + +#endif /* USE_TRACING_INSTRUMENTATION_LTTNG */ + +#endif /* !defined(TOR_TRACE_PROBES_CC_H) */ diff --git a/src/ext/rust b/src/ext/rust deleted file mode 160000 -Subproject aa37fb84fb829902e83ca11a7244bbc6b86b809 diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c index 051be50b3a..88e91ebfd5 100644 --- a/src/feature/api/tor_api.c +++ b/src/feature/api/tor_api.c @@ -18,9 +18,9 @@ // Include this after the above headers, to insure that they don't // depend on anything else. #include "orconfig.h" +#include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "feature/api/tor_api_internal.h" -#include "lib/cc/compat_compiler.h" #include <stdio.h> #include <stdlib.h> @@ -39,7 +39,9 @@ #include "lib/net/socketpair.h" #define raw_socketpair tor_ersatz_socketpair #define raw_closesocket closesocket +#if !defined(HAVE_SNPRINTF) #define snprintf _snprintf +#endif #else /* !defined(_WIN32) */ #define raw_socketpair socketpair #define raw_closesocket close diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index d40bcc6c8e..9e36d26929 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -943,9 +943,17 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } /** We just learned a descriptor for a bridge. See if that - * digest is in our entry guard list, and add it if not. */ + * digest is in our entry guard list, and add it if not. Schedule the + * next fetch for a long time from now, and initiate any follow-up + * activities like continuing to bootstrap. + * + * <b>from_cache</b> * tells us whether we fetched it from disk (else + * the network) + * + * <b>desc_is_new</b> tells us if we preferred it to the old version we + * had, if any. */ void -learned_bridge_descriptor(routerinfo_t *ri, int from_cache) +learned_bridge_descriptor(routerinfo_t *ri, int from_cache, int desc_is_new) { tor_assert(ri); tor_assert(ri->purpose == ROUTER_PURPOSE_BRIDGE); @@ -961,12 +969,14 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) if (bridge) { /* if we actually want to use this one */ node_t *node; - /* it's here; schedule its re-fetch for a long time from now. */ if (!from_cache) { /* This schedules the re-fetch at a constant interval, which produces * a pattern of bridge traffic. But it's better than trying all * configured bridges several times in the first few minutes. */ download_status_reset(&bridge->fetch_status); + /* it's here; schedule its re-fetch for a long time from now. */ + bridge->fetch_status.next_attempt_at += + get_options()->TestingBridgeDownloadInitialDelay; } node = node_get_mutable_by_id(ri->cache_info.identity_digest); @@ -982,8 +992,10 @@ learned_bridge_descriptor(routerinfo_t *ri, int from_cache) entry_guard_learned_bridge_identity(&bridge->addrport_configured, (const uint8_t*)ri->cache_info.identity_digest); - log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", ri->nickname, - from_cache ? "cached" : "fresh", router_describe(ri)); + if (desc_is_new) + log_notice(LD_DIR, "new bridge descriptor '%s' (%s): %s", + ri->nickname, + from_cache ? "cached" : "fresh", router_describe(ri)); /* If we didn't have a reachable bridge before this one, try directory * documents again. */ if (first) { diff --git a/src/feature/client/bridges.h b/src/feature/client/bridges.h index a42363f683..dd3e498a0a 100644 --- a/src/feature/client/bridges.h +++ b/src/feature/client/bridges.h @@ -46,7 +46,8 @@ void learned_router_identity(const tor_addr_t *addr, uint16_t port, void bridge_add_from_config(struct bridge_line_t *bridge_line); void retry_bridge_descriptor_fetch_directly(const char *digest); void fetch_bridge_descriptors(const or_options_t *options, time_t now); -void learned_bridge_descriptor(routerinfo_t *ri, int from_cache); +void learned_bridge_descriptor(routerinfo_t *ri, + int from_cache, int desc_is_new); const smartlist_t *get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port); diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c index 9c0ecc56ad..ff9e05a645 100644 --- a/src/feature/client/circpathbias.c +++ b/src/feature/client/circpathbias.c @@ -363,6 +363,17 @@ pathbias_should_count(origin_circuit_t *circ) return 0; } + /* Ignore circuits where the controller helped choose the path. When + * this happens, we can't be sure whether the path was chosen randomly + * or not. */ + if (circ->any_hop_from_controller) { + /* (In this case, we _don't_ check to see if shouldcount is changing, + * since it's possible that an already-created circuit later gets extended + * by the controller. */ + circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_IGNORED; + return 0; + } + /* Completely ignore one hop circuits */ if (circ->build_state->onehop_tunnel || circ->build_state->desired_path_len == 1) { diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c index 502cb99690..32ecb4f705 100644 --- a/src/feature/client/entrynodes.c +++ b/src/feature/client/entrynodes.c @@ -132,6 +132,7 @@ #include "feature/client/entrynodes.h" #include "feature/client/transports.h" #include "feature/control/control_events.h" +#include "feature/dirclient/dlstatus.h" #include "feature/dircommon/directory.h" #include "feature/nodelist/describe.h" #include "feature/nodelist/microdesc.h" @@ -559,7 +560,7 @@ get_extreme_restriction_threshold(void) int32_t pct = networkstatus_get_param(NULL, "guard-extreme-restriction-percent", DFLT_EXTREME_RESTRICTION_PERCENT, - 1, INT32_MAX); + 1, 100); return pct / 100.0; } @@ -576,6 +577,18 @@ mark_guard_maybe_reachable(entry_guard_t *guard) guard->is_reachable = GUARD_REACHABLE_MAYBE; if (guard->is_filtered_guard) guard->is_usable_filtered_guard = 1; + + /* Check if it is a bridge and we don't have its descriptor yet */ + if (guard->bridge_addr && !guard_has_descriptor(guard)) { + /* Reset the descriptor fetch retry schedule, so it gives it another + * go soon. It's important to keep any "REACHABLE_MAYBE" bridges in + * sync with the descriptor fetch schedule, since we will refuse to + * use the network until our first primary bridges are either + * known-usable or known-unusable. See bug 40396. */ + download_status_t *dl = get_bridge_dl_status_by_id(guard->identity); + if (dl) + download_status_reset(dl); + } } /** @@ -2046,6 +2059,14 @@ entry_guard_consider_retry(entry_guard_t *guard) get_retry_schedule(guard->failing_since, now, guard->is_primary); const time_t last_attempt = guard->last_tried_to_connect; + /* Check if it is a bridge and we don't have its descriptor yet */ + if (guard->bridge_addr && !guard_has_descriptor(guard)) { + /* We want to leave the retry schedule to fetch_bridge_descriptors(), + * so we don't have two retry schedules clobbering each other. See + * bugs 40396 and 40497 for details of why we need this exception. */ + return; + } + if (BUG(last_attempt == 0) || now >= last_attempt + delay) { /* We should mark this retriable. */ @@ -2271,6 +2292,13 @@ entry_guards_note_guard_failure(guard_selection_t *gs, guard->is_primary?"primary ":"", guard->confirmed_idx>=0?"confirmed ":"", entry_guard_describe(guard)); + + /* Schedule a re-assessment of whether we have enough dir info to + * use the network. Counterintuitively, *losing* a bridge might actually + * be just what we need to *resume* using the network, if we had it in + * state GUARD_REACHABLE_MAYBE and we were stalling to learn this + * outcome. See bug 40396 for more details. */ + router_dir_info_changed(); } /** @@ -2295,6 +2323,12 @@ entry_guards_note_guard_success(guard_selection_t *gs, /* If guard was not already marked as reachable, send a GUARD UP signal */ if (guard->is_reachable != GUARD_REACHABLE_YES) { control_event_guard(guard->nickname, guard->identity, "UP"); + + /* Schedule a re-assessment of whether we have enough dir info to + * use the network. One of our guards has just moved to + * GUARD_REACHABLE_YES, so maybe we can resume using the network + * now. */ + router_dir_info_changed(); } guard->is_reachable = GUARD_REACHABLE_YES; @@ -3538,6 +3572,11 @@ entry_guards_changed_for_guard_selection(guard_selection_t *gs) entry_guards_update_guards_in_state() */ or_state_mark_dirty(get_or_state(), when); + + /* Schedule a re-assessment of whether we have enough dir info to + * use the network. When we add or remove or disable or enable a + * guard, the decision could shift. */ + router_dir_info_changed(); } /** Our list of entry guards has changed for the default guard selection @@ -3930,6 +3969,253 @@ guard_selection_free_(guard_selection_t *gs) tor_free(gs); } +/**********************************************************************/ + +/** Layer2 guard subsystem (vanguards-lite) used for onion service circuits */ + +/** A simple representation of a layer2 guard. We just need its identity so + * that we feed it into a routerset, and a sampled timestamp to do expiration + * checks. */ +typedef struct layer2_guard_t { + /** Identity of the guard */ + char identity[DIGEST_LEN]; + /** When does this guard expire? (randomized timestamp) */ + time_t expire_on_date; +} layer2_guard_t; + +#define layer2_guard_free(val) \ + FREE_AND_NULL(layer2_guard_t, layer2_guard_free_, (val)) + +/** Return true if the vanguards-lite subsystem is enabled */ +bool +vanguards_lite_is_enabled(void) +{ + /* First check torrc option and then maybe also the consensus parameter. */ + const or_options_t *options = get_options(); + + /* If the option is explicitly disabled, that's the final word here */ + if (options->VanguardsLiteEnabled == 0) { + return false; + } + + /* If the option is set to auto, then check the consensus parameter */ + if (options->VanguardsLiteEnabled == -1) { + return networkstatus_get_param(NULL, "vanguards-lite-enabled", + 1, /* default to "on" */ + 0, 1); + } + + /* else it's enabled */ + tor_assert_nonfatal(options->VanguardsLiteEnabled == 1); + return options->VanguardsLiteEnabled; +} + +static void +layer2_guard_free_(layer2_guard_t *l2) +{ + if (!l2) { + return; + } + + tor_free(l2); +} + +/** Global list and routerset of L2 guards. They are both synced and they get + * updated periodically. We need both the list and the routerset: we use the + * smartlist to keep track of expiration times and the routerset is what we + * return to the users of this subsystem. */ +static smartlist_t *layer2_guards = NULL; +static routerset_t *layer2_routerset = NULL; + +/** Number of L2 guards */ +#define NUMBER_SECOND_GUARDS 4 +/** Make sure that the number of L2 guards is less than the number of + * MAX_SANE_RESTRICTED_NODES */ +CTASSERT(NUMBER_SECOND_GUARDS < 20); + +/** Lifetime of L2 guards: + * 1 to 12 days, for an average of a week using the max(x,x) distribution */ +#define MIN_SECOND_GUARD_LIFETIME (3600*24) +#define MAX_SECOND_GUARD_LIFETIME (3600*24*12) + +/** Return the number of guards our L2 guardset should have */ +static int +get_number_of_layer2_hs_guards(void) +{ + return (int) networkstatus_get_param(NULL, + "guard-hs-l2-number", + NUMBER_SECOND_GUARDS, + 1, 19); +} + +/** Return the minimum lifetime of L2 guards */ +static int +get_min_lifetime_of_layer2_hs_guards(void) +{ + return (int) networkstatus_get_param(NULL, + "guard-hs-l2-lifetime-min", + MIN_SECOND_GUARD_LIFETIME, + 1, INT32_MAX); +} + +/** Return the maximum lifetime of L2 guards */ +static int +get_max_lifetime_of_layer2_hs_guards(void) +{ + return (int) networkstatus_get_param(NULL, + "guard-hs-l2-lifetime-max", + MAX_SECOND_GUARD_LIFETIME, + 1, INT32_MAX); +} + +/** + * Sample and return a lifetime for an L2 guard. + * + * Lifetime randomized uniformly between min and max consensus params. + */ +static int +get_layer2_hs_guard_lifetime(void) +{ + int min = get_min_lifetime_of_layer2_hs_guards(); + int max = get_max_lifetime_of_layer2_hs_guards(); + + if (BUG(min >= max)) { + return min; + } + + return crypto_rand_int_range(min, max); +} + +/** Maintain the L2 guard list. Make sure the list contains enough guards, do + * expirations as necessary, and keep all the data structures of this + * subsystem synchronized */ +void +maintain_layer2_guards(void) +{ + if (!router_have_minimum_dir_info()) { + return; + } + + /* Create the list if it doesn't exist */ + if (!layer2_guards) { + layer2_guards = smartlist_new(); + } + + /* Go through the list and perform any needed expirations */ + SMARTLIST_FOREACH_BEGIN(layer2_guards, layer2_guard_t *, g) { + /* Expire based on expiration date */ + if (g->expire_on_date <= approx_time()) { + log_info(LD_GENERAL, "Removing expired Layer2 guard %s", + safe_str_client(hex_str(g->identity, DIGEST_LEN))); + // Nickname may be gone from consensus and doesn't matter anyway + control_event_guard("None", g->identity, "BAD_L2"); + layer2_guard_free(g); + SMARTLIST_DEL_CURRENT_KEEPORDER(layer2_guards, g); + continue; + } + + /* Expire if relay has left consensus */ + if (router_get_consensus_status_by_id(g->identity) == NULL) { + log_info(LD_GENERAL, "Removing missing Layer2 guard %s", + safe_str_client(hex_str(g->identity, DIGEST_LEN))); + // Nickname may be gone from consensus and doesn't matter anyway + control_event_guard("None", g->identity, "BAD_L2"); + layer2_guard_free(g); + SMARTLIST_DEL_CURRENT_KEEPORDER(layer2_guards, g); + continue; + } + } SMARTLIST_FOREACH_END(g); + + /* Find out how many guards we need to add */ + int new_guards_needed_n = + get_number_of_layer2_hs_guards() - smartlist_len(layer2_guards); + if (new_guards_needed_n <= 0) { + return; + } + + log_info(LD_GENERAL, "Adding %d guards to Layer2 routerset", + new_guards_needed_n); + + /* Add required guards to the list */ + smartlist_t *excluded = smartlist_new(); + for (int i = 0; i < new_guards_needed_n; i++) { + const node_t *choice = NULL; + const or_options_t *options = get_options(); + /* Pick Stable nodes */ + router_crn_flags_t flags = CRN_NEED_DESC|CRN_NEED_UPTIME; + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); + if (!choice) { + break; + } + + /* We found our node: create an L2 guard out of it */ + layer2_guard_t *layer2_guard = tor_malloc_zero(sizeof(layer2_guard_t)); + memcpy(layer2_guard->identity, choice->identity, DIGEST_LEN); + layer2_guard->expire_on_date = approx_time() + + get_layer2_hs_guard_lifetime(); + smartlist_add(layer2_guards, layer2_guard); + log_info(LD_GENERAL, "Adding Layer2 guard %s", + safe_str_client(hex_str(layer2_guard->identity, DIGEST_LEN))); + // Nickname can also be None here because it is looked up later + control_event_guard("None", layer2_guard->identity, + "GOOD_L2"); + /* Exclude this node and its family so that we don't double-pick. */ + nodelist_add_node_and_family(excluded, choice); + } + + /* Some cleanup */ + smartlist_free(excluded); + + /* Now that the list is up to date, synchronize the routerset */ + routerset_free(layer2_routerset); + layer2_routerset = routerset_new(); + + SMARTLIST_FOREACH_BEGIN (layer2_guards, layer2_guard_t *, g) { + routerset_parse(layer2_routerset, + hex_str(g->identity, DIGEST_LEN), + "l2 guards"); + } SMARTLIST_FOREACH_END(g); +} + +/** + * Reset vanguards-lite list(s). + * + * Used for SIGNAL NEWNYM. + */ +void +purge_vanguards_lite(void) +{ + if (!layer2_guards) + return; + + /* Go through the list and perform any needed expirations */ + SMARTLIST_FOREACH_BEGIN(layer2_guards, layer2_guard_t *, g) { + layer2_guard_free(g); + } SMARTLIST_FOREACH_END(g); + + smartlist_clear(layer2_guards); + + /* Pick new l2 guards */ + maintain_layer2_guards(); +} + +/** Return a routerset containing the L2 guards or NULL if it's not yet + * initialized. Callers must not free the routerset. Designed for use in + * pick_vanguard_middle_node() and should not be used anywhere else. Do not + * store this pointer -- any future calls to maintain_layer2_guards() and + * purge_vanguards_lite() can invalidate it. */ +const routerset_t * +get_layer2_guards(void) +{ + if (!layer2_guards) { + maintain_layer2_guards(); + } + + return layer2_routerset; +} + +/*****************************************************************************/ + /** Release all storage held by the list of entry guards and related * memory structs. */ void @@ -3946,4 +4232,15 @@ entry_guards_free_all(void) guard_contexts = NULL; } circuit_build_times_free_timeouts(get_circuit_build_times_mutable()); + + if (!layer2_guards) { + return; + } + + SMARTLIST_FOREACH_BEGIN(layer2_guards, layer2_guard_t *, g) { + layer2_guard_free(g); + } SMARTLIST_FOREACH_END(g); + + smartlist_free(layer2_guards); + routerset_free(layer2_routerset); } diff --git a/src/feature/client/entrynodes.h b/src/feature/client/entrynodes.h index 88ed8f649e..08fd7cf745 100644 --- a/src/feature/client/entrynodes.h +++ b/src/feature/client/entrynodes.h @@ -651,4 +651,9 @@ guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw, int orig_bandwidth, uint32_t guardfraction_percentage); +bool vanguards_lite_is_enabled(void); +const routerset_t *get_layer2_guards(void); +void maintain_layer2_guards(void); +void purge_vanguards_lite(void); + #endif /* !defined(TOR_ENTRYNODES_H) */ diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 167beb96c6..80903ac9e5 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -843,7 +843,7 @@ handle_methods_done(const managed_proxy_t *mp) tor_assert(mp->transports); if (smartlist_len(mp->transports) == 0) - log_notice(LD_GENERAL, "Managed proxy '%s' was spawned successfully, " + log_warn(LD_GENERAL, "Managed proxy '%s' was spawned successfully, " "but it didn't launch any pluggable transport listeners!", mp->argv[0]); @@ -903,14 +903,22 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS) goto err; + /* Log the error but do not kill the managed proxy. + * A proxy may contain several transports and if one + * of them is misconfigured, we still want to use + * the other transports. A managed proxy with no usable + * transports will log a warning. + * See https://gitlab.torproject.org/tpo/core/tor/-/issues/7362 + * */ parse_client_method_error(line); - goto err; + return; } else if (!strcmpstart(line, PROTO_SMETHOD_ERROR)) { if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS) goto err; + /* Log the error but do not kill the managed proxy */ parse_server_method_error(line); - goto err; + return; } else if (!strcmpstart(line, PROTO_CMETHOD)) { if (mp->conf_state != PT_PROTO_ACCEPTING_METHODS) goto err; diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c index bd0d41d29e..b19935e456 100644 --- a/src/feature/control/control_cmd.c +++ b/src/feature/control/control_cmd.c @@ -817,6 +817,8 @@ handle_control_extendcircuit(control_connection_t *conn, circ->first_hop_from_controller = 1; } + circ->any_hop_from_controller = 1; + /* now circ refers to something that is ready to be extended */ first_node = zero_circ; SMARTLIST_FOREACH(nodes, const node_t *, node, @@ -1075,7 +1077,7 @@ static const control_cmd_syntax_t redirectstream_syntax = { .max_args = UINT_MAX, // XXX should be 3. }; -/** Called when we receive a REDIRECTSTERAM command. Try to change the target +/** Called when we receive a REDIRECTSTREAM command. Try to change the target * address of the named AP stream, and report success or failure. */ static int handle_control_redirectstream(control_connection_t *conn, diff --git a/src/feature/dirauth/dirauth_options.inc b/src/feature/dirauth/dirauth_options.inc index 05726b8c2f..4fd07a8859 100644 --- a/src/feature/dirauth/dirauth_options.inc +++ b/src/feature/dirauth/dirauth_options.inc @@ -27,6 +27,10 @@ CONF_VAR(AuthDirHasIPv6Connectivity, BOOL, 0, "0") * good. */ CONF_VAR(AuthDirListBadExits, BOOL, 0, "0") +/** True iff we should list middle-only relays, and vote for all other + * relays as possibly suitable for other positions. */ +CONF_VAR(AuthDirListMiddleOnly, BOOL, 0, "0") + /** Do not permit more than this number of servers per IP address. */ CONF_VAR(AuthDirMaxServersPerAddr, POSINT, 0, "2") @@ -109,4 +113,7 @@ CONF_VAR(VersioningAuthoritativeDirectory, BOOL, 0, "0") * pressure or not. */ CONF_VAR(AuthDirRejectRequestsUnderLoad, BOOL, 0, "1") +/** Boolean: Should we not give bandwidth weight measurements to dirauths? */ +CONF_VAR(AuthDirDontVoteOnDirAuthBandwidth, BOOL, 0, "1") + END_CONF_STRUCT(dirauth_options_t) diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index fa906c0c3c..cdd2c132ef 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -1479,6 +1479,21 @@ compute_nth_protocol_set(int n, int n_voters, const smartlist_t *votes) return result; } +/** Helper: Takes a smartlist of `const char *` flags, and a flag to remove. + * + * Removes that flag if it is present in the list. Doesn't free it. + */ +static void +remove_flag(smartlist_t *sl, const char *flag) +{ + /* We can't use smartlist_string_remove() here, since that doesn't preserve + * order, and since it frees elements from the string. */ + + int idx = smartlist_string_pos(sl, flag); + if (idx >= 0) + smartlist_del_keeporder(sl, idx); +} + /** Given a list of vote networkstatus_t in <b>votes</b>, our public * authority <b>identity_key</b>, our private authority <b>signing_key</b>, * and the number of <b>total_authorities</b> that we believe exist in our @@ -1633,6 +1648,9 @@ networkstatus_compute_consensus(smartlist_t *votes, tor_free(votesec_list); tor_free(distsec_list); } + // True if anybody is voting on the BadExit flag. + const bool badexit_flag_is_listed = + smartlist_contains_string(flags, "BadExit"); chunks = smartlist_new(); @@ -1924,7 +1942,7 @@ networkstatus_compute_consensus(smartlist_t *votes, const char *chosen_name = NULL; int exitsummary_disagreement = 0; int is_named = 0, is_unnamed = 0, is_running = 0, is_valid = 0; - int is_guard = 0, is_exit = 0, is_bad_exit = 0; + int is_guard = 0, is_exit = 0, is_bad_exit = 0, is_middle_only = 0; int naming_conflict = 0; int n_listing = 0; char microdesc_digest[DIGEST256_LEN]; @@ -2055,7 +2073,6 @@ networkstatus_compute_consensus(smartlist_t *votes, } /* Set the flags. */ - smartlist_add(chosen_flags, (char*)"s"); /* for the start of the line. */ SMARTLIST_FOREACH_BEGIN(flags, const char *, fl) { if (!strcmp(fl, "Named")) { if (is_named) @@ -2077,6 +2094,8 @@ networkstatus_compute_consensus(smartlist_t *votes, is_running = 1; else if (!strcmp(fl, "BadExit")) is_bad_exit = 1; + else if (!strcmp(fl, "MiddleOnly")) + is_middle_only = 1; else if (!strcmp(fl, "Valid")) is_valid = 1; } @@ -2093,6 +2112,22 @@ networkstatus_compute_consensus(smartlist_t *votes, if (!is_valid) continue; + /* Starting with consensus method 32, we handle the middle-only + * flag specially: when it is present, we clear some flags, and + * set others. */ + if (is_middle_only && consensus_method >= MIN_METHOD_FOR_MIDDLEONLY) { + remove_flag(chosen_flags, "Exit"); + remove_flag(chosen_flags, "V2Dir"); + remove_flag(chosen_flags, "Guard"); + remove_flag(chosen_flags, "HSDir"); + is_exit = is_guard = 0; + if (! is_bad_exit && badexit_flag_is_listed) { + is_bad_exit = 1; + smartlist_add(chosen_flags, (char *)"BadExit"); + smartlist_sort_strings(chosen_flags); // restore order. + } + } + /* Pick the version. */ if (smartlist_len(versions)) { sort_version_list(versions, 0); @@ -2253,6 +2288,8 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add_asprintf(chunks, "m %s\n", m); } /* Next line is all flags. The "\n" is missing. */ + smartlist_add_asprintf(chunks, "s%s", + smartlist_len(chosen_flags)?" ":""); smartlist_add(chunks, smartlist_join_strings(chosen_flags, " ", 0, NULL)); /* Now the version line. */ @@ -2265,7 +2302,8 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add_asprintf(chunks, "pr %s\n", chosen_protocol_list); } /* Now the weight line. */ - if (rs_out.has_bandwidth) { + if (rs_out.has_bandwidth && (!rs_out.is_authority || + !dirauth_get_options()->AuthDirDontVoteOnDirAuthBandwidth)) { char *guardfraction_str = NULL; int unmeasured = rs_out.bw_is_unmeasured; @@ -4581,6 +4619,7 @@ const char DIRVOTE_UNIVERSAL_FLAGS[] = * depending on our configuration. */ const char DIRVOTE_OPTIONAL_FLAGS[] = "BadExit " + "MiddleOnly " "Running"; /** Return a new networkstatus_t* containing our current opinion. (For v3 @@ -4598,7 +4637,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, smartlist_t *routers, *routerstatuses; char identity_digest[DIGEST_LEN]; char signing_key_digest[DIGEST_LEN]; - const int listbadexits = d_options->AuthDirListBadExits; + const int list_bad_exits = d_options->AuthDirListBadExits; + const int list_middle_only = d_options->AuthDirListMiddleOnly; routerlist_t *rl = router_get_routerlist(); time_t now = time(NULL); time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; @@ -4703,7 +4743,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); rs = &vrs->status; dirauth_set_routerstatus_from_routerinfo(rs, node, ri, now, - listbadexits); + list_bad_exits, + list_middle_only); if (ri->cache_info.signing_key_cert) { memcpy(vrs->ed25519_id, @@ -4825,8 +4866,10 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add_strdup(v3_out->known_flags, "Running"); - if (listbadexits) + if (list_bad_exits) smartlist_add_strdup(v3_out->known_flags, "BadExit"); + if (list_middle_only) + smartlist_add_strdup(v3_out->known_flags, "MiddleOnly"); smartlist_sort_strings(v3_out->known_flags); if (d_options->ConsensusParams) { diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index 3420098315..64aaec116e 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -53,7 +53,7 @@ #define MIN_SUPPORTED_CONSENSUS_METHOD 28 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 31 +#define MAX_SUPPORTED_CONSENSUS_METHOD 32 /** * Lowest consensus method where microdescriptor lines are put in canonical @@ -70,6 +70,10 @@ */ #define MIN_METHOD_FOR_CORRECT_BWWEIGHTSCALE 31 +/** Lowest consensus method for which we handle the MiddleOnly flag specially. + */ +#define MIN_METHOD_FOR_MIDDLEONLY 32 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not * get confused with the above macros.) */ diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index eca987b8b5..a75f516dca 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -226,6 +226,8 @@ dirserv_load_fingerprint_file(void) add_status = RTR_BADEXIT; } else if (!strcasecmp(nickname, "!invalid")) { add_status = RTR_INVALID; + } else if (!strcasecmp(nickname, "!middleonly")) { + add_status = RTR_MIDDLEONLY; } /* Check if fingerprint is RSA or ed25519 by verifying it. */ @@ -412,11 +414,11 @@ dirserv_rejects_tor_version(const char *platform, return true; } - /* Series between Tor 0.3.6 and 0.4.1 inclusive are unsupported. Reject - * them. 0.3.6.0-alpha-dev only existed for a short time, before it was - * renamed to 0.4.0.0-alpha-dev. */ + /* Series between Tor 0.3.6.x and 0.4.5.5-rc inclusive are unsupported. + * Reject them. 0.3.6.0-alpha-dev only existed for a short time, before it + * was renamed to 0.4.0.0-alpha-dev. */ if (tor_version_as_new_as(platform,"0.3.6.0-alpha-dev") && - !tor_version_as_new_as(platform,"0.4.2.1-alpha")) { + !tor_version_as_new_as(platform,"0.4.5.6")) { if (msg) { *msg = please_upgrade_string; } @@ -496,6 +498,13 @@ dirserv_get_status_impl(const char *id_digest, result |= RTR_BADEXIT; } + if (authdir_policy_middleonly_address(ipv4_addr, ipv4_orport)) { + log_fn(severity, LD_DIRSERV, + "Marking '%s' as middle-only because of address '%s'", + nickname, fmt_addr(ipv4_addr)); + result |= RTR_MIDDLEONLY; + } + if (!authdir_policy_permits_address(ipv4_addr, ipv4_orport)) { log_fn(severity, LD_DIRSERV, "Rejecting '%s' because of address '%s'", nickname, fmt_addr(ipv4_addr)); @@ -630,6 +639,7 @@ dirserv_set_node_flags_from_authoritative_status(node_t *node, { node->is_valid = (authstatus & RTR_INVALID) ? 0 : 1; node->is_bad_exit = (authstatus & RTR_BADEXIT) ? 1 : 0; + node->is_middle_only = (authstatus & RTR_MIDDLEONLY) ? 1 : 0; } /** True iff <b>a</b> is more severe than <b>b</b>. */ @@ -963,6 +973,11 @@ directory_remove_invalid(void) (r & RTR_BADEXIT) ? "bad" : "good"); node->is_bad_exit = (r&RTR_BADEXIT) ? 1: 0; } + if (bool_neq((r & RTR_MIDDLEONLY), node->is_middle_only)) { + log_info(LD_DIRSERV, "Router '%s' is now %smiddle-only", description, + (r & RTR_MIDDLEONLY) ? "" : "not"); + node->is_middle_only = (r&RTR_MIDDLEONLY) ? 1: 0; + } } SMARTLIST_FOREACH_END(node); routerlist_assert_ok(rl); diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index 6c056d11dd..a509eb1fbe 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -45,7 +45,8 @@ typedef struct authdir_config_t { #define RTR_REJECT 4 /**< We will not publish this router. */ /* 8 Historically used to avoid using this as a dir. */ #define RTR_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */ -/* 32 Historically used to indicade Unnamed */ +/** We'll vote to only use this router as a midpoint. */ +#define RTR_MIDDLEONLY 32 #endif /* defined(PROCESS_DESCS_PRIVATE) || defined(TOR_UNIT_TESTS) */ diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index d755a270be..05c19ff501 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -565,7 +565,8 @@ dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, const routerinfo_t *ri, time_t now, - int listbadexits) + int listbadexits, + int listmiddleonly) { const or_options_t *options = get_options(); uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri); @@ -597,6 +598,14 @@ dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs, /* Override rs->is_bad_exit */ rs->is_bad_exit = listbadexits && node->is_bad_exit; + /* Override rs->is_middle_only and related flags. */ + rs->is_middle_only = listmiddleonly && node->is_middle_only; + if (rs->is_middle_only) { + if (listbadexits) + rs->is_bad_exit = 1; + rs->is_exit = rs->is_possible_guard = rs->is_hs_dir = rs->is_v2_dir = 0; + } + /* Set rs->is_staledesc. */ rs->is_staledesc = (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now; diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index 818a0bafd2..8371f1c315 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -22,7 +22,8 @@ void dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, const routerinfo_t *ri, time_t now, - int listbadexits); + int listbadexits, + int listmiddleonly); void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); #endif /* defined(HAVE_MODULE_DIRAUTH) */ diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index 7fdb1bc70f..7319b96caf 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -1569,6 +1569,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, char *url = NULL; const or_options_t *options = get_options(); + (void) body_len; + log_debug(LD_DIRSERV,"Received POST command."); conn->base_.state = DIR_CONN_STATE_SERVER_WRITING; diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h index ed6b00647e..ac45f3787b 100644 --- a/src/feature/dirclient/dir_server_st.h +++ b/src/feature/dirclient/dir_server_st.h @@ -16,6 +16,8 @@ #include "core/or/or.h" #include "feature/nodelist/routerstatus_st.h" +struct smartlist_t; + /** Represents information about a single trusted or fallback directory * server. */ struct dir_server_t { @@ -48,6 +50,10 @@ struct dir_server_t { time_t addr_current_at; /**< When was the document that we derived the * address information from published? */ + /** Authority only. Can be null. If present, a list of auth_dirport_t + * representing HTTP dirports for this authority. */ + struct smartlist_t *auth_dirports; + routerstatus_t fake_status; /**< Used when we need to pass this trusted * dir_server_t to * directory_request_set_routerstatus. diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index 0b6a8101a5..4e9c8e2f45 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -1134,6 +1134,7 @@ directory_request_set_routerstatus(directory_request_t *req, { req->routerstatus = status; } + /** * Helper: update the addresses, ports, and identities in <b>req</b> * from the routerstatus object in <b>req</b>. Return 0 on success. @@ -1176,7 +1177,7 @@ directory_request_set_dir_from_routerstatus(directory_request_t *req) return -1; } - /* At this point, if we are a client making a direct connection to a + /* At this point, if we are a client making a direct connection to a * directory server, we have selected a server that has at least one address * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if @@ -1191,6 +1192,37 @@ directory_request_set_dir_from_routerstatus(directory_request_t *req) return -1; } + /* One last thing: If we're talking to an authority, we might want to use + * a special HTTP port for it based on our purpose. + */ + if (req->indirection == DIRIND_DIRECT_CONN && status->is_authority) { + const dir_server_t *ds = router_get_trusteddirserver_by_digest( + status->identity_digest); + if (ds) { + const tor_addr_port_t *v4 = NULL; + if (authdir_mode_v3(get_options())) { + // An authority connecting to another authority should always + // prefer the VOTING usage, if one is specifically configured. + v4 = trusted_dir_server_get_dirport_exact( + ds, AUTH_USAGE_VOTING, AF_INET); + } + if (! v4) { + // Everybody else should prefer a usage dependent on their + // the dir_purpose. + auth_dirport_usage_t usage = + auth_dirport_usage_for_purpose(req->dir_purpose); + v4 = trusted_dir_server_get_dirport(ds, usage, AF_INET); + } + tor_assert_nonfatal(v4); + if (v4) { + // XXXX We could, if we wanted, also select a v6 address. But a v4 + // address must exist here, and we as a relay are required to support + // ipv4. So we just that. + tor_addr_port_copy(&use_dir_ap, v4); + } + } + } + directory_request_set_or_addr_port(req, &use_or_ap); directory_request_set_dir_addr_port(req, &use_dir_ap); directory_request_set_directory_id_digest(req, status->identity_digest); @@ -1209,7 +1241,7 @@ directory_initiate_request,(directory_request_t *request)) tor_assert_nonfatal( ! directory_request_dir_contact_info_specified(request)); if (directory_request_set_dir_from_routerstatus(request) < 0) { - return; + return; // or here XXXX } } @@ -1324,6 +1356,8 @@ directory_initiate_request,(directory_request_t *request)) entry_guard_cancel(&guard_state); } + // XXXX This is the case where we replace. + switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr, port, &socket_error)) { case -1: diff --git a/src/feature/dirclient/dlstatus.c b/src/feature/dirclient/dlstatus.c index 8be2983a5d..c21dd113b4 100644 --- a/src/feature/dirclient/dlstatus.c +++ b/src/feature/dirclient/dlstatus.c @@ -73,15 +73,14 @@ find_dl_min_delay(const download_status_t *dls, const or_options_t *options) } } case DL_SCHED_BRIDGE: - if (options->UseBridges && num_bridges_usable(0) > 0) { - /* A bridge client that is sure that one or more of its bridges are - * running can afford to wait longer to update bridge descriptors. */ - return options->TestingBridgeDownloadInitialDelay; - } else { - /* A bridge client which might have no running bridges, must try to - * get bridge descriptors straight away. */ - return options->TestingBridgeBootstrapDownloadInitialDelay; - } + /* Be conservative here: always return the 'during bootstrap' delay + * value, so we never delay while trying to fetch descriptors + * for new bridges. Once we do succeed at fetching a descriptor + * for our bridge, we will adjust its next_attempt_at based on + * the longer "TestingBridgeDownloadInitialDelay" value. See + * learned_bridge_descriptor() for details. + */ + return options->TestingBridgeBootstrapDownloadInitialDelay; default: tor_assert(0); } diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c index c877227adc..323f2bd576 100644 --- a/src/feature/dircommon/consdiff.c +++ b/src/feature/dircommon/consdiff.c @@ -1128,7 +1128,7 @@ consdiff_get_digests(const smartlist_t *diff, { const cdline_t *line2 = smartlist_get(diff, 1); char *h = tor_memdup_nulterm(line2->s, line2->len); - smartlist_split_string(hash_words, h, " ", 0, 0); + smartlist_split_string(hash_words, h, " ", 0, 4); tor_free(h); } diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c index 947b3810a4..cd3e2731be 100644 --- a/src/feature/dirparse/ns_parse.c +++ b/src/feature/dirparse/ns_parse.c @@ -434,6 +434,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_possible_guard = 1; else if (!strcmp(tok->args[i], "BadExit")) rs->is_bad_exit = 1; + else if (!strcmp(tok->args[i], "MiddleOnly")) + rs->is_middle_only = 1; else if (!strcmp(tok->args[i], "Authority")) rs->is_authority = 1; else if (!strcmp(tok->args[i], "Unnamed") && diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c index 73f9176186..a76893fe1a 100644 --- a/src/feature/hs/hs_config.c +++ b/src/feature/hs/hs_config.c @@ -548,15 +548,19 @@ config_service(config_line_t *line, const or_options_t *options, tor_assert(service->config.version <= HS_VERSION_MAX); - /* Check permission on service directory that was just parsed. And this must - * be done regardless of the service version. Do not ask for the directory - * to be created, this is done when the keys are loaded because we could be - * in validation mode right now. */ - if (hs_check_service_private_dir(options->User, - service->config.directory_path, - service->config.dir_group_readable, - 0) < 0) { - goto err; + /* If we're running with TestingTorNetwork enabled, we relax the permissions + * check on the hs directory. */ + if (!options->TestingTorNetwork) { + /* Check permission on service directory that was just parsed. And this + * must be done regardless of the service version. Do not ask for the + * directory to be created, this is done when the keys are loaded because + * we could be in validation mode right now. */ + if (hs_check_service_private_dir(options->User, + service->config.directory_path, + service->config.dir_group_readable, + 0) < 0) { + goto err; + } } /* We'll try to learn the service version here by loading the key(s) if @@ -640,6 +644,7 @@ hs_config_service_all(const or_options_t *options, int validate_only) int rv = config_service(section, options, new_service_list); config_free_lines(section); if (rv < 0) { + config_free_lines(remaining); goto err; } } diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index 70ff4e9690..a37eab5b5d 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1607,8 +1607,8 @@ decrypt_desc_layer,(const hs_descriptor_t *desc, * put in decrypted_out which contains the superencrypted layer of the * descriptor. Return the length of decrypted_out on success else 0 is * returned and decrypted_out is set to NULL. */ -static size_t -desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out) +MOCK_IMPL(STATIC size_t, +desc_decrypt_superencrypted,(const hs_descriptor_t *desc,char **decrypted_out)) { size_t superencrypted_len = 0; char *superencrypted_plaintext = NULL; @@ -1639,10 +1639,10 @@ desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out) * decrypted_out which contains the encrypted layer of the descriptor. * Return the length of decrypted_out on success else 0 is returned and * decrypted_out is set to NULL. */ -static size_t -desc_decrypt_encrypted(const hs_descriptor_t *desc, - const curve25519_secret_key_t *client_auth_sk, - char **decrypted_out) +MOCK_IMPL(STATIC size_t, +desc_decrypt_encrypted,(const hs_descriptor_t *desc, + const curve25519_secret_key_t *client_auth_sk, + char **decrypted_out)) { size_t encrypted_len = 0; char *encrypted_plaintext = NULL; @@ -2145,7 +2145,7 @@ desc_decode_plaintext_v3(smartlist_t *tokens, /** Decode the version 3 superencrypted section of the given descriptor desc. * The desc_superencrypted_out will be populated with the decoded data. */ -static hs_desc_decode_status_t +STATIC hs_desc_decode_status_t desc_decode_superencrypted_v3(const hs_descriptor_t *desc, hs_desc_superencrypted_data_t * desc_superencrypted_out) @@ -2259,7 +2259,7 @@ desc_decode_superencrypted_v3(const hs_descriptor_t *desc, /** Decode the version 3 encrypted section of the given descriptor desc. The * desc_encrypted_out will be populated with the decoded data. */ -static hs_desc_decode_status_t +STATIC hs_desc_decode_status_t desc_decode_encrypted_v3(const hs_descriptor_t *desc, const curve25519_secret_key_t *client_auth_sk, hs_desc_encrypted_data_t *desc_encrypted_out) diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h index 7e437faeb8..d959431369 100644 --- a/src/feature/hs/hs_descriptor.h +++ b/src/feature/hs/hs_descriptor.h @@ -339,6 +339,25 @@ MOCK_DECL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc, bool is_superencrypted_layer, char **decrypted_out)); +STATIC hs_desc_decode_status_t desc_decode_encrypted_v3( + const hs_descriptor_t *desc, + const curve25519_secret_key_t *client_auth_sk, + hs_desc_encrypted_data_t *desc_encrypted_out); + +STATIC hs_desc_decode_status_t +desc_decode_superencrypted_v3(const hs_descriptor_t *desc, + hs_desc_superencrypted_data_t * + desc_superencrypted_out); + +MOCK_DECL(STATIC size_t, desc_decrypt_encrypted,( + const hs_descriptor_t *desc, + const curve25519_secret_key_t *client_auth_sk, + char **decrypted_out)); + +MOCK_DECL(STATIC size_t, desc_decrypt_superencrypted,( + const hs_descriptor_t *desc, + char **decrypted_out)); + #endif /* defined(HS_DESCRIPTOR_PRIVATE) */ #endif /* !defined(TOR_HS_DESCRIPTOR_H) */ diff --git a/src/feature/hs/hs_metrics.c b/src/feature/hs/hs_metrics.c index e023eab90c..0f1824c51c 100644 --- a/src/feature/hs/hs_metrics.c +++ b/src/feature/hs/hs_metrics.c @@ -29,18 +29,6 @@ port_to_str(const uint16_t port) return buf; } -/** Return a static buffer pointer that contains a formatted label on the form - * of key=value. - * - * Subsequent call to this function invalidates the previous buffer. */ -static const char * -format_label(const char *key, const char *value) -{ - static char buf[128]; - tor_snprintf(buf, sizeof(buf), "%s=%s", key, value); - return buf; -} - /** Initialize a metrics store for the given service. * * Essentially, this goes over the base_metrics array and adds them all to the @@ -61,12 +49,12 @@ init_store(hs_service_t *service) /* Add labels to the entry. */ metrics_store_entry_add_label(entry, - format_label("onion", service->onion_address)); + metrics_format_label("onion", service->onion_address)); if (base_metrics[i].port_as_label && service->config.ports) { SMARTLIST_FOREACH_BEGIN(service->config.ports, const hs_port_config_t *, p) { metrics_store_entry_add_label(entry, - format_label("port", port_to_str(p->virtual_port))); + metrics_format_label("port", port_to_str(p->virtual_port))); } SMARTLIST_FOREACH_END(p); } } @@ -96,7 +84,7 @@ hs_metrics_update_by_service(const hs_metrics_key_t key, SMARTLIST_FOREACH_BEGIN(entries, metrics_store_entry_t *, entry) { if (port == 0 || metrics_store_entry_has_label(entry, - format_label("port", port_to_str(port)))) { + metrics_format_label("port", port_to_str(port)))) { metrics_store_entry_update(entry, n); break; } diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c index 1f18bd71a2..1f1ac4d106 100644 --- a/src/feature/nodelist/dirlist.c +++ b/src/feature/nodelist/dirlist.c @@ -43,6 +43,14 @@ #include "feature/dirclient/dir_server_st.h" #include "feature/nodelist/node_st.h" +/** Information about an (HTTP) dirport for a directory authority. */ +struct auth_dirport_t { + /** What is the intended usage for this dirport? One of AUTH_USAGE_* */ + auth_dirport_usage_t usage; + /** What is the correct address/port ? */ + tor_addr_port_t dirport; +}; + /** Global list of a dir_server_t object for each directory * authority. */ static smartlist_t *trusted_dir_servers = NULL; @@ -66,6 +74,11 @@ add_trusted_dir_to_nodelist_addr_set(const dir_server_t *dir) /* IPv6 DirPort is not a thing yet for authorities. */ nodelist_add_addr_to_address_set(&dir->ipv6_addr, dir->ipv6_orport, 0); } + if (dir->auth_dirports) { + SMARTLIST_FOREACH_BEGIN(dir->auth_dirports, const auth_dirport_t *, p) { + nodelist_add_addr_to_address_set(&p->dirport.addr, 0, p->dirport.port); + } SMARTLIST_FOREACH_END(p); + } } /** Go over the trusted directory server list and add their address(es) to the @@ -256,7 +269,10 @@ MOCK_IMPL(int, router_digest_is_trusted_dir_type, /** Return true iff the given address matches a trusted directory that matches * at least one bit of type. * - * If type is NO_DIRINFO or ALL_DIRINFO, any authority is matched. */ + * If type is NO_DIRINFO or ALL_DIRINFO, any authority is matched. + * + * Only ORPorts' addresses are considered. + */ bool router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type) { @@ -281,6 +297,39 @@ router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type) return false; } +/** Return an appropriate usage value describing which authdir port to use + * for a given directory connection purpose. + */ +auth_dirport_usage_t +auth_dirport_usage_for_purpose(int purpose) +{ + switch (purpose) { + case DIR_PURPOSE_FETCH_SERVERDESC: + case DIR_PURPOSE_FETCH_EXTRAINFO: + case DIR_PURPOSE_FETCH_CONSENSUS: + case DIR_PURPOSE_FETCH_CERTIFICATE: + case DIR_PURPOSE_FETCH_MICRODESC: + return AUTH_USAGE_DOWNLOAD; + + case DIR_PURPOSE_UPLOAD_DIR: + return AUTH_USAGE_UPLOAD; + + case DIR_PURPOSE_UPLOAD_VOTE: + case DIR_PURPOSE_UPLOAD_SIGNATURES: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_STATUS_VOTE: + return AUTH_USAGE_VOTING; + + case DIR_PURPOSE_SERVER: + case DIR_PURPOSE_UPLOAD_HSDESC: + case DIR_PURPOSE_FETCH_HSDESC: + case DIR_PURPOSE_HAS_FETCHED_HSDESC: + default: + tor_assert_nonfatal_unreached(); + return AUTH_USAGE_LEGACY; + } +} + /** Create a directory server at <b>address</b>:<b>port</b>, with OR identity * key <b>digest</b> which has DIGEST_LEN bytes. If <b>address</b> is NULL, * add ourself. If <b>is_authority</b>, this is a directory authority. Return @@ -357,6 +406,7 @@ dir_server_new(int is_authority, ent->fake_status.ipv4_dirport = ent->ipv4_dirport; ent->fake_status.ipv4_orport = ent->ipv4_orport; ent->fake_status.ipv6_orport = ent->ipv6_orport; + ent->fake_status.is_authority = !! is_authority; return ent; } @@ -404,10 +454,98 @@ trusted_dir_server_new(const char *nickname, const char *address, ipv6_addrport, digest, v3_auth_digest, type, weight); + + if (ipv4_dirport) { + tor_addr_port_t p; + memset(&p, 0, sizeof(p)); + tor_addr_copy(&p.addr, &ipv4_addr); + p.port = ipv4_dirport; + trusted_dir_server_add_dirport(result, AUTH_USAGE_LEGACY, &p); + } tor_free(hostname); return result; } +/** + * Add @a dirport as an HTTP DirPort contact point for the directory authority + * @a ds, for use when contacting that authority for the given @a usage. + * + * Multiple ports of the same usage are allowed; if present, then only + * the first one of each address family is currently used. + */ +void +trusted_dir_server_add_dirport(dir_server_t *ds, + auth_dirport_usage_t usage, + const tor_addr_port_t *dirport) +{ + tor_assert(ds); + tor_assert(dirport); + + if (BUG(! ds->is_authority)) { + return; + } + + if (ds->auth_dirports == NULL) { + ds->auth_dirports = smartlist_new(); + } + + auth_dirport_t *port = tor_malloc_zero(sizeof(auth_dirport_t)); + port->usage = usage; + tor_addr_port_copy(&port->dirport, dirport); + smartlist_add(ds->auth_dirports, port); +} + +/** + * Helper for trusted_dir_server_get_dirport: only return the exact requested + * usage type. + */ +const tor_addr_port_t * +trusted_dir_server_get_dirport_exact(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family) +{ + tor_assert(ds); + tor_assert_nonfatal(addr_family == AF_INET || addr_family == AF_INET6); + if (ds->auth_dirports == NULL) + return NULL; + + SMARTLIST_FOREACH_BEGIN(ds->auth_dirports, const auth_dirport_t *, port) { + if (port->usage == usage && + tor_addr_family(&port->dirport.addr) == addr_family) { + return &port->dirport; + } + } SMARTLIST_FOREACH_END(port); + + return NULL; +} + +/** + * Return the DirPort of the authority @a ds for with the usage type + * @a usage and address family @a addr_family. If none is found, try + * again with an AUTH_USAGE_LEGACY dirport, if there is one. Return NULL + * if no port can be found. + */ +const tor_addr_port_t * +trusted_dir_server_get_dirport(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family) +{ + const tor_addr_port_t *port; + + while (1) { + port = trusted_dir_server_get_dirport_exact(ds, usage, addr_family); + if (port) + return port; + + // If we tried LEGACY, there is no fallback from this point. + if (usage == AUTH_USAGE_LEGACY) + return NULL; + + // Try again with LEGACY. + usage = AUTH_USAGE_LEGACY; + } +} + /** Return a new dir_server_t for a fallback directory server at * <b>addr</b>:<b>or_port</b>/<b>dir_port</b>, with identity key digest * <b>id_digest</b> */ @@ -447,6 +585,10 @@ dir_server_free_(dir_server_t *ds) if (!ds) return; + if (ds->auth_dirports) { + SMARTLIST_FOREACH(ds->auth_dirports, auth_dirport_t *, p, tor_free(p)); + smartlist_free(ds->auth_dirports); + } tor_free(ds->nickname); tor_free(ds->description); tor_free(ds->address); diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h index f744fecf92..3b4faf07af 100644 --- a/src/feature/nodelist/dirlist.h +++ b/src/feature/nodelist/dirlist.h @@ -11,6 +11,28 @@ #ifndef TOR_DIRLIST_H #define TOR_DIRLIST_H +typedef struct auth_dirport_t auth_dirport_t; +/** + * Different usages for an authority's HTTP directory port. + * + * Historically, only legacy ports existed; proposal 330 added multiple types + * of dirport to better enable authorities to offload work and resist DoS + * attacks. + **/ +typedef enum auth_dirport_usage_t { + /** Flag for an authority's dirport that is intended for misc/legacy + * usage. May be used when no other dirport is available. */ + AUTH_USAGE_LEGACY, + /** Flag for an authority's dirport that is intended for descriptor uploads + * only. */ + AUTH_USAGE_UPLOAD, + /** Flag for an authority's dirport that is intended for voting only */ + AUTH_USAGE_VOTING, + /** Flag for an authority's dirport that is intended for relay downloads + * only. */ + AUTH_USAGE_DOWNLOAD, +} auth_dirport_usage_t; + int get_n_authorities(dirinfo_type_t type); const smartlist_t *router_get_trusted_dir_servers(void); const smartlist_t *router_get_fallback_dir_servers(void); @@ -18,6 +40,8 @@ smartlist_t *router_get_trusted_dir_servers_mutable(void); smartlist_t *router_get_fallback_dir_servers_mutable(void); void mark_all_dirservers_up(smartlist_t *server_list); +auth_dirport_usage_t auth_dirport_usage_for_purpose(int purpose); + dir_server_t *router_get_trusteddirserver_by_digest(const char *d); dir_server_t *router_get_fallback_dirserver_by_digest( const char *digest); @@ -28,6 +52,14 @@ MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest, MOCK_DECL(int, router_digest_is_trusted_dir_type, (const char *digest, dirinfo_type_t type)); +const tor_addr_port_t *trusted_dir_server_get_dirport(const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family); +const tor_addr_port_t *trusted_dir_server_get_dirport_exact( + const dir_server_t *ds, + auth_dirport_usage_t usage, + int addr_family); + bool router_addr_is_trusted_dir_type(const tor_addr_t *addr, dirinfo_type_t type); #define router_addr_is_trusted_dir(d) \ @@ -41,6 +73,9 @@ dir_server_t *trusted_dir_server_new(const char *nickname, const char *address, const tor_addr_port_t *addrport_ipv6, const char *digest, const char *v3_auth_digest, dirinfo_type_t type, double weight); +void trusted_dir_server_add_dirport(dir_server_t *ds, + auth_dirport_usage_t usage, + const tor_addr_port_t *dirport); dir_server_t *fallback_dir_server_new(const tor_addr_t *addr, uint16_t dir_port, uint16_t or_port, const tor_addr_port_t *addrport_ipv6, diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c index 6db40c0b68..95379a7721 100644 --- a/src/feature/nodelist/fmt_routerstatus.c +++ b/src/feature/nodelist/fmt_routerstatus.c @@ -87,7 +87,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, goto done; smartlist_add_asprintf(chunks, - "s%s%s%s%s%s%s%s%s%s%s%s%s\n", + "s%s%s%s%s%s%s%s%s%s%s%s%s%s\n", /* These must stay in alphabetical order. */ rs->is_authority?" Authority":"", rs->is_bad_exit?" BadExit":"", @@ -95,6 +95,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, rs->is_fast?" Fast":"", rs->is_possible_guard?" Guard":"", rs->is_hs_dir?" HSDir":"", + rs->is_middle_only?" MiddleOnly":"", rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", rs->is_staledesc?" StaleDesc":"", diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index af808a6ba7..77e2b547f5 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -45,6 +45,8 @@ #include "core/or/channel.h" #include "core/or/channelpadding.h" #include "core/or/circuitpadding.h" +#include "core/or/congestion_control_common.h" +#include "core/or/congestion_control_flow.h" #include "core/or/circuitmux.h" #include "core/or/circuitmux_ewma.h" #include "core/or/circuitstats.h" @@ -1664,7 +1666,6 @@ notify_before_networkstatus_changes(const networkstatus_t *old_c, dos_consensus_has_changed(new_c); relay_consensus_has_changed(new_c); hs_dos_consensus_has_changed(new_c); - rep_hist_consensus_has_changed(new_c); } /* Called after a new consensus has been put in the global state. It is safe @@ -1701,6 +1702,11 @@ notify_after_networkstatus_changes(void) channelpadding_new_consensus_params(c); circpad_new_consensus_params(c); router_new_consensus_params(c); + congestion_control_new_consensus_params(c); + flow_control_new_consensus_params(c); + + /* Maintenance of our L2 guard list */ + maintain_layer2_guards(); } /** Copy all the ancillary information (like router download status and so on) diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h index b15e7154c4..df67a47ada 100644 --- a/src/feature/nodelist/node_st.h +++ b/src/feature/nodelist/node_st.h @@ -70,6 +70,8 @@ struct node_t { unsigned int is_exit:1; /**< Do we think this is an OK exit? */ unsigned int is_bad_exit:1; /**< Do we think this exit is censored, borked, * or otherwise nasty? */ + /** Is this unsuitable for use as anything besides a middle relay? */ + unsigned int is_middle_only:1; unsigned int is_hs_dir:1; /**< True iff this router is a hidden service * directory according to the authorities. */ diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 121dc8823a..c676e8dfb4 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -2820,6 +2820,7 @@ update_router_have_minimum_dir_info(void) const networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor()); int using_md; + static int be_loud_when_things_work_again = 0; if (!consensus) { if (!networkstatus_get_latest_consensus()) @@ -2875,8 +2876,9 @@ update_router_have_minimum_dir_info(void) if (res && !have_min_dir_info) { control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO"); control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0); - log_info(LD_DIR, - "We now have enough directory information to build circuits."); + tor_log(be_loud_when_things_work_again ? LOG_NOTICE : LOG_INFO, LD_DIR, + "We now have enough directory information to build circuits."); + be_loud_when_things_work_again = 0; } /* If paths have just become unavailable in this update. */ @@ -2885,6 +2887,10 @@ update_router_have_minimum_dir_info(void) tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR, "Our directory information is no longer up-to-date " "enough to build circuits: %s", dir_info_status); + if (!quiet) { + /* remember to do a notice-level log when things come back */ + be_loud_when_things_work_again = 1; + } /* a) make us log when we next complete a circuit, so we know when Tor * is back up and usable, and b) disable some activities that Tor diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index 565d4596d4..c00f7ffb26 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -1617,6 +1617,13 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, "descriptor for router %s", router_describe(router)); } else { + if (router->purpose == ROUTER_PURPOSE_BRIDGE) { + /* Even if we're not going to keep this descriptor, we need to + * let the bridge descriptor fetch subsystem know that we + * succeeded at getting it -- so we can adjust the retry schedule + * to stop trying for a while. */ + learned_bridge_descriptor(router, from_cache, 0); + } log_info(LD_DIR, "Dropping descriptor that we already have for router %s", router_describe(router)); @@ -2012,6 +2019,30 @@ routerlist_remove_old_routers(void) router_rebuild_store(RRS_DONT_REMOVE_OLD,&routerlist->extrainfo_store); } +/* Drop every bridge descriptor in our routerlist. Used by the external + * 'bridgestrap' tool to discard bridge descriptors so that it can then + * do a clean reachability test. */ +void +routerlist_drop_bridge_descriptors(void) +{ + routerinfo_t *router; + int i; + + if (!routerlist) + return; + + for (i = 0; i < smartlist_len(routerlist->routers); ++i) { + router = smartlist_get(routerlist->routers, i); + if (router->purpose == ROUTER_PURPOSE_BRIDGE) { + log_notice(LD_DIR, + "Dropping existing bridge descriptor for %s", + router_describe(router)); + routerlist_remove(routerlist, router, 0, time(NULL)); + i--; + } + } +} + /** We just added a new set of descriptors. Take whatever extra steps * we need. */ void @@ -2023,7 +2054,7 @@ routerlist_descriptors_added(smartlist_t *sl, int from_cache) control_event_descriptors_changed(sl); SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) { if (ri->purpose == ROUTER_PURPOSE_BRIDGE) - learned_bridge_descriptor(ri, from_cache); + learned_bridge_descriptor(ri, from_cache, 1); if (ri->needs_retest_if_added) { ri->needs_retest_if_added = 0; dirserv_single_reachability_test(approx_time(), ri); diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h index 7dc748c94b..7ba305baf6 100644 --- a/src/feature/nodelist/routerlist.h +++ b/src/feature/nodelist/routerlist.h @@ -145,6 +145,7 @@ was_router_added_t router_add_extrainfo_to_routerlist( int from_cache, int from_fetch); void routerlist_descriptors_added(smartlist_t *sl, int from_cache); void routerlist_remove_old_routers(void); +void routerlist_drop_bridge_descriptors(void); int router_load_single_router(const char *s, uint8_t purpose, int cache, const char **msg); int router_load_routers_from_string(const char *s, const char *eos, diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h index 46ff0bdeac..55b76de581 100644 --- a/src/feature/nodelist/routerstatus_st.h +++ b/src/feature/nodelist/routerstatus_st.h @@ -51,6 +51,8 @@ struct routerstatus_t { * choice as an entry guard. */ unsigned int is_bad_exit:1; /**< True iff this node is a bad choice for * an exit node. */ + unsigned int is_middle_only:1; /**< True iff this node is marked as bad + * for anything besides middle positions. */ unsigned int is_hs_dir:1; /**< True iff this router is a v2-or-later hidden * service directory. */ unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index 6a703f2ab3..c6e0439338 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1539,6 +1539,16 @@ evdns_callback(int result, char type, int count, int ttl, void *addresses, tor_addr_make_unspec(&addr); + /* Note down any DNS errors to the statistics module */ + if (result == DNS_ERR_TIMEOUT) { + /* libevent timed out while resolving a name. However, because libevent + * handles retries and timeouts internally, this means that all attempts of + * libevent timed out. If we wanted to get more granular information about + * individual libevent attempts, we would have to implement our own DNS + * timeout/retry logic */ + rep_hist_note_overload(OVERLOAD_GENERAL); + } + /* Keep track of whether IPv6 is working */ if (type == DNS_IPv6_AAAA) { if (result == DNS_ERR_TIMEOUT) { @@ -1642,7 +1652,7 @@ evdns_callback(int result, char type, int count, int ttl, void *addresses, /* The result can be changed within this function thus why we note the result * at the end. */ - rep_hist_note_dns_query(type, result); + rep_hist_note_dns_error(type, result); tor_free(arg_); } @@ -1662,6 +1672,9 @@ launch_one_resolve(const char *address, uint8_t query_type, addr[0] = (char) query_type; memcpy(addr+1, address, addr_len + 1); + /* Note the query for our statistics. */ + rep_hist_note_dns_request(query_type); + switch (query_type) { case DNS_IPv4_A: req = evdns_base_resolve_ipv4(the_evdns_base, diff --git a/src/feature/relay/include.am b/src/feature/relay/include.am index 84bb1ff35e..8a121cef01 100644 --- a/src/feature/relay/include.am +++ b/src/feature/relay/include.am @@ -15,6 +15,7 @@ MODULE_RELAY_SOURCES = \ src/feature/relay/routermode.c \ src/feature/relay/relay_config.c \ src/feature/relay/relay_handshake.c \ + src/feature/relay/relay_metrics.c \ src/feature/relay/relay_periodic.c \ src/feature/relay/relay_sys.c \ src/feature/relay/routerkeys.c \ @@ -30,6 +31,7 @@ noinst_HEADERS += \ src/feature/relay/onion_queue.h \ src/feature/relay/relay_config.h \ src/feature/relay/relay_handshake.h \ + src/feature/relay/relay_metrics.h \ src/feature/relay/relay_periodic.h \ src/feature/relay/relay_sys.h \ src/feature/relay/relay_find_addr.h \ diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c index 85ec0dc74a..c09f4d5b9b 100644 --- a/src/feature/relay/onion_queue.c +++ b/src/feature/relay/onion_queue.c @@ -164,6 +164,7 @@ onion_pending_add(or_circuit_t *circ, create_cell_t *onionskin) #define WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL (60) static ratelim_t last_warned = RATELIM_INIT(WARN_TOO_MANY_CIRC_CREATIONS_INTERVAL); + rep_hist_note_circuit_handshake_dropped(onionskin->handshake_type); if (onionskin->handshake_type == ONION_HANDSHAKE_TYPE_NTOR) { char *m; /* Note this ntor onionskin drop as an overload */ diff --git a/src/feature/relay/relay_metrics.c b/src/feature/relay/relay_metrics.c new file mode 100644 index 0000000000..fc8eb10d1b --- /dev/null +++ b/src/feature/relay/relay_metrics.c @@ -0,0 +1,401 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_metrics.c + * @brief Relay metrics exposed through the MetricsPort + **/ + +#define RELAY_METRICS_ENTRY_PRIVATE + +#include "orconfig.h" + +#include "core/or/or.h" +#include "core/or/relay.h" + +#include "lib/malloc/malloc.h" +#include "lib/container/smartlist.h" +#include "lib/metrics/metrics_store.h" +#include "lib/log/util_bug.h" + +#include "feature/relay/relay_metrics.h" +#include "feature/stats/rephist.h" + +#include <event2/dns.h> + +/** Declarations of each fill function for metrics defined in base_metrics. */ +static void fill_dns_error_values(void); +static void fill_dns_query_values(void); +static void fill_global_bw_limit_values(void); +static void fill_socket_values(void); +static void fill_onionskins_values(void); +static void fill_oom_values(void); +static void fill_tcp_exhaustion_values(void); + +/** The base metrics that is a static array of metrics added to the metrics + * store. + * + * The key member MUST be also the index of the entry in the array. */ +static const relay_metrics_entry_t base_metrics[] = +{ + { + .key = RELAY_METRICS_NUM_OOM_BYTES, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_oom_bytes_total), + .help = "Total number of bytes the OOM has freed by subsystem", + .fill_fn = fill_oom_values, + }, + { + .key = RELAY_METRICS_NUM_ONIONSKINS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_onionskins_total), + .help = "Total number of onionskins handled", + .fill_fn = fill_onionskins_values, + }, + { + .key = RELAY_METRICS_NUM_SOCKETS, + .type = METRICS_TYPE_GAUGE, + .name = METRICS_NAME(relay_load_socket_total), + .help = "Total number of sockets", + .fill_fn = fill_socket_values, + }, + { + .key = RELAY_METRICS_NUM_GLOBAL_RW_LIMIT, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_global_rate_limit_reached_total), + .help = "Total number of global connection bucket limit reached", + .fill_fn = fill_global_bw_limit_values, + }, + { + .key = RELAY_METRICS_NUM_DNS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_exit_dns_query_total), + .help = "Total number of DNS queries done by this relay", + .fill_fn = fill_dns_query_values, + }, + { + .key = RELAY_METRICS_NUM_DNS_ERRORS, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_exit_dns_error_total), + .help = "Total number of DNS errors encountered by this relay", + .fill_fn = fill_dns_error_values, + }, + { + .key = RELAY_METRICS_NUM_TCP_EXHAUSTION, + .type = METRICS_TYPE_COUNTER, + .name = METRICS_NAME(relay_load_tcp_exhaustion_total), + .help = "Total number of times we ran out of TCP ports", + .fill_fn = fill_tcp_exhaustion_values, + }, +}; +static const size_t num_base_metrics = ARRAY_LENGTH(base_metrics); + +/** The only and single store of all the relay metrics. */ +static metrics_store_t *the_store; + +/** Helper function to convert an handshake type into a string. */ +static inline const char * +handshake_type_to_str(const uint16_t type) +{ + switch (type) { + case ONION_HANDSHAKE_TYPE_TAP: + return "tap"; + case ONION_HANDSHAKE_TYPE_FAST: + return "fast"; + case ONION_HANDSHAKE_TYPE_NTOR: + return "ntor"; + default: + // LCOV_EXCL_START + tor_assert_unreached(); + // LCOV_EXCL_STOP + } +} + +/** Fill function for the RELAY_METRICS_NUM_DNS metrics. */ +static void +fill_tcp_exhaustion_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_TCP_EXHAUSTION]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_update(sentry, rep_hist_get_n_tcp_exhaustion()); +} + +/* NOTE: Disable the record type label until libevent is fixed. */ +#if 0 +/** Helper array containing mapping for the name of the different DNS records + * and their corresponding libevent values. */ +static struct dns_type { + const char *name; + uint8_t type; +} dns_types[] = { + { .name = "A", .type = DNS_IPv4_A }, + { .name = "PTR", .type = DNS_PTR }, + { .name = "AAAA", .type = DNS_IPv6_AAAA }, +}; +static const size_t num_dns_types = ARRAY_LENGTH(dns_types); +#endif + +/** Fill function for the RELAY_METRICS_NUM_DNS_ERRORS metrics. */ +static void +fill_dns_error_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_DNS_ERRORS]; + + /* Helper array to map libeven DNS errors to their names and so we can + * iterate over this array to add all metrics. */ + static struct dns_error { + const char *name; + uint8_t key; + } errors[] = { + { .name = "success", .key = DNS_ERR_NONE }, + { .name = "format", .key = DNS_ERR_FORMAT }, + { .name = "serverfailed", .key = DNS_ERR_SERVERFAILED }, + { .name = "notexist", .key = DNS_ERR_NOTEXIST }, + { .name = "notimpl", .key = DNS_ERR_NOTIMPL }, + { .name = "refused", .key = DNS_ERR_REFUSED }, + { .name = "truncated", .key = DNS_ERR_TRUNCATED }, + { .name = "unknown", .key = DNS_ERR_UNKNOWN }, + { .name = "tor_timeout", .key = DNS_ERR_TIMEOUT }, + { .name = "shutdown", .key = DNS_ERR_SHUTDOWN }, + { .name = "cancel", .key = DNS_ERR_CANCEL }, + { .name = "nodata", .key = DNS_ERR_NODATA }, + }; + static const size_t num_errors = ARRAY_LENGTH(errors); + + /* NOTE: Disable the record type label until libevent is fixed. */ +#if 0 + for (size_t i = 0; i < num_dns_types; i++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *record_label = + tor_strdup(metrics_format_label("record", dns_types[i].name)); + + for (size_t j = 0; j < num_errors; j++) { + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, record_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("reason", errors[j].name)); + metrics_store_entry_update(sentry, + rep_hist_get_n_dns_error(dns_types[i].type, errors[j].key)); + } + tor_free(record_label); + } +#endif + + /* Put in the DNS errors, unfortunately not per-type for now. */ + for (size_t j = 0; j < num_errors; j++) { + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("reason", errors[j].name)); + metrics_store_entry_update(sentry, + rep_hist_get_n_dns_error(0, errors[j].key)); + } +} + +/** Fill function for the RELAY_METRICS_NUM_DNS metrics. */ +static void +fill_dns_query_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_DNS]; + + /* NOTE: Disable the record type label until libevent is fixed (#40490). */ +#if 0 + for (size_t i = 0; i < num_dns_types; i++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *record_label = + tor_strdup(metrics_format_label("record", dns_types[i].name)); + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, record_label); + metrics_store_entry_update(sentry, + rep_hist_get_n_dns_request(dns_types[i].type)); + tor_free(record_label); + } +#endif + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_update(sentry, rep_hist_get_n_dns_request(0)); +} + +/** Fill function for the RELAY_METRICS_NUM_GLOBAL_RW_LIMIT metrics. */ +static void +fill_global_bw_limit_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_GLOBAL_RW_LIMIT]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("side", "read")); + metrics_store_entry_update(sentry, rep_hist_get_n_read_limit_reached()); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("side", "write")); + metrics_store_entry_update(sentry, rep_hist_get_n_write_limit_reached()); +} + +/** Fill function for the RELAY_METRICS_NUM_SOCKETS metrics. */ +static void +fill_socket_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_SOCKETS]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("state", "opened")); + metrics_store_entry_update(sentry, get_n_open_sockets()); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_update(sentry, get_max_sockets()); +} + +/** Fill function for the RELAY_METRICS_NUM_ONIONSKINS metrics. */ +static void +fill_onionskins_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_ONIONSKINS]; + + for (uint16_t t = 0; t <= MAX_ONION_HANDSHAKE_TYPE; t++) { + /* Dup the label because metrics_format_label() returns a pointer to a + * string on the stack and we need that label for all metrics. */ + char *type_label = + tor_strdup(metrics_format_label("type", handshake_type_to_str(t))); + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, type_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("action", "processed")); + metrics_store_entry_update(sentry, + rep_hist_get_circuit_n_handshake_assigned(t)); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, type_label); + metrics_store_entry_add_label(sentry, + metrics_format_label("action", "dropped")); + metrics_store_entry_update(sentry, + rep_hist_get_circuit_n_handshake_dropped(t)); + tor_free(type_label); + } +} + +/** Fill function for the RELAY_METRICS_NUM_OOM_BYTES metrics. */ +static void +fill_oom_values(void) +{ + metrics_store_entry_t *sentry; + const relay_metrics_entry_t *rentry = + &base_metrics[RELAY_METRICS_NUM_OOM_BYTES]; + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "cell")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_cell); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "dns")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_dns); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "geoip")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_geoip); + + sentry = metrics_store_add(the_store, rentry->type, rentry->name, + rentry->help); + metrics_store_entry_add_label(sentry, + metrics_format_label("subsys", "hsdir")); + metrics_store_entry_update(sentry, oom_stats_n_bytes_removed_hsdir); +} + +/** Reset the global store and fill it with all the metrics from base_metrics + * and their associated values. + * + * To pull this off, every metrics has a "fill" function that is called and in + * charge of adding the metrics to the store, appropriate labels and finally + * updating the value to report. */ +static void +fill_store(void) +{ + /* Reset the current store, we are about to fill it with all the things. */ + metrics_store_reset(the_store); + + /* Call the fill function for each metrics. */ + for (size_t i = 0; i < num_base_metrics; i++) { + if (BUG(!base_metrics[i].fill_fn)) { + continue; + } + base_metrics[i].fill_fn(); + } +} + +/** Return a list of all the relay metrics stores. This is the + * function attached to the .get_metrics() member of the subsys_t. */ +const smartlist_t * +relay_metrics_get_stores(void) +{ + /* We can't have the caller to free the returned list so keep it static, + * simply update it. */ + static smartlist_t *stores_list = NULL; + + /* We dynamically fill the store with all the metrics upon a request. The + * reason for this is because the exposed metrics of a relay are often + * internal counters in the fast path and thus we fetch the value when a + * metrics port request arrives instead of keeping a local metrics store of + * those values. */ + fill_store(); + + if (!stores_list) { + stores_list = smartlist_new(); + smartlist_add(stores_list, the_store); + } + + return stores_list; +} + +/** Initialize the relay metrics. */ +void +relay_metrics_init(void) +{ + if (BUG(the_store)) { + return; + } + the_store = metrics_store_new(); +} + +/** Free the relay metrics. */ +void +relay_metrics_free(void) +{ + if (!the_store) { + return; + } + /* NULL is set with this call. */ + metrics_store_free(the_store); +} diff --git a/src/feature/relay/relay_metrics.h b/src/feature/relay/relay_metrics.h new file mode 100644 index 0000000000..00dfeaa624 --- /dev/null +++ b/src/feature/relay/relay_metrics.h @@ -0,0 +1,55 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file relay_metrics.h + * @brief Header for feature/relay/relay_metrics.c + **/ + +#ifndef TOR_FEATURE_RELAY_RELAY_METRICS_H +#define TOR_FEATURE_RELAY_RELAY_METRICS_H + +#include "lib/container/smartlist.h" +#include "lib/metrics/metrics_common.h" + +/** Metrics key for each reported metrics. This key is also used as an index in + * the base_metrics array. */ +typedef enum { + /** Number of OOM invocation. */ + RELAY_METRICS_NUM_OOM_BYTES = 0, + /** Number of onionskines handled. */ + RELAY_METRICS_NUM_ONIONSKINS = 1, + /** Number of sockets. */ + RELAY_METRICS_NUM_SOCKETS = 2, + /** Number of global connection rate limit. */ + RELAY_METRICS_NUM_GLOBAL_RW_LIMIT = 3, + /** Number of DNS queries. */ + RELAY_METRICS_NUM_DNS = 4, + /** Number of DNS query errors. */ + RELAY_METRICS_NUM_DNS_ERRORS = 5, + /** Number of TCP exhaustion reached. */ + RELAY_METRICS_NUM_TCP_EXHAUSTION = 6, +} relay_metrics_key_t; + +/** The metadata of a relay metric. */ +typedef struct relay_metrics_entry_t { + /* Metric key used as a static array index. */ + relay_metrics_key_t key; + /* Metric type. */ + metrics_type_t type; + /* Metrics output name. */ + const char *name; + /* Metrics output help comment. */ + const char *help; + /* Update value function. */ + void (*fill_fn)(void); +} relay_metrics_entry_t; + +/* Init. */ +void relay_metrics_init(void); +void relay_metrics_free(void); + +/* Accessors. */ +const smartlist_t *relay_metrics_get_stores(void); + +#endif /* !defined(TOR_FEATURE_RELAY_RELAY_METRICS_H) */ diff --git a/src/feature/relay/relay_periodic.c b/src/feature/relay/relay_periodic.c index ee94590e01..dd9be4e36f 100644 --- a/src/feature/relay/relay_periodic.c +++ b/src/feature/relay/relay_periodic.c @@ -219,7 +219,7 @@ reachability_warnings_callback(time_t now, const or_options_t *options) tor_asprintf(&where4, "%s:%d", address4, me->ipv4_orport); if (!v6_ok) tor_asprintf(&where6, "[%s]:%d", address6, me->ipv6_orport); - const char *opt_and = (!v4_ok && !v6_ok) ? "and" : ""; + const char *opt_and = (!v4_ok && !v6_ok) ? " and " : ""; /* IPv4 reachability test worked but not the IPv6. We will _not_ * publish the descriptor if our IPv6 was configured. We will if it diff --git a/src/feature/relay/relay_sys.c b/src/feature/relay/relay_sys.c index 25fc0bbd32..9c43734b84 100644 --- a/src/feature/relay/relay_sys.c +++ b/src/feature/relay/relay_sys.c @@ -14,6 +14,7 @@ #include "feature/relay/dns.h" #include "feature/relay/ext_orport.h" +#include "feature/relay/relay_metrics.h" #include "feature/relay/onion_queue.h" #include "feature/relay/relay_periodic.h" #include "feature/relay/relay_sys.h" @@ -25,6 +26,7 @@ static int subsys_relay_initialize(void) { + relay_metrics_init(); relay_register_periodic_events(); return 0; } @@ -37,6 +39,7 @@ subsys_relay_shutdown(void) clear_pending_onions(); routerkeys_free_all(); router_free_all(); + relay_metrics_free(); } const struct subsys_fns_t sys_relay = { @@ -46,4 +49,6 @@ const struct subsys_fns_t sys_relay = { .level = RELAY_SUBSYS_LEVEL, .initialize = subsys_relay_initialize, .shutdown = subsys_relay_shutdown, + + .get_metrics = relay_metrics_get_stores, }; diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c index df838aa527..8f6a45dfef 100644 --- a/src/feature/rend/rendmid.c +++ b/src/feature/rend/rendmid.c @@ -131,7 +131,11 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, rend_circ = hs_circuitmap_get_rend_circ_relay_side(request); if (!rend_circ) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + /* Once this was a LOG_PROTOCOL_WARN, but it can happen naturally if a + * client gives up on a rendezvous circuit after sending INTRODUCE1, but + * before the onion service sends the RENDEZVOUS1 cell. + */ + log_fn(LOG_DEBUG, LD_PROTOCOL, "Rejecting RENDEZVOUS1 cell with unrecognized rendezvous cookie %s.", hexid); reason = END_CIRC_REASON_TORPROTOCOL; diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index b4b107c3f7..a0fe8597c1 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -1206,11 +1206,11 @@ format_bridge_stats_controller(time_t now) char * format_client_stats_heartbeat(time_t now) { - const int n_hours = 6; + const int n_seconds = get_options()->HeartbeatPeriod; char *out = NULL; int n_clients = 0; clientmap_entry_t **ent; - unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 ); + unsigned cutoff = (unsigned)( (now-n_seconds)/60 ); if (!start_of_bridge_stats_interval) return NULL; /* Not initialized. */ @@ -1226,8 +1226,7 @@ format_client_stats_heartbeat(time_t now) } tor_asprintf(&out, "Heartbeat: " - "In the last %d hours, I have seen %d unique clients.", - n_hours, + "Since last heartbeat message, I have seen %d unique clients.", n_clients); return out; diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c index 2bfa14d326..5ff4ef1d2e 100644 --- a/src/feature/stats/rephist.c +++ b/src/feature/stats/rephist.c @@ -206,18 +206,33 @@ typedef struct { uint64_t overload_fd_exhausted; } overload_stats_t; +/** Current state of overload stats */ +static overload_stats_t overload_stats; + +/** Counters to count the number of times we've reached an overload for the + * global connection read/write limit. Reported on the MetricsPort. */ +static uint64_t stats_n_read_limit_reached = 0; +static uint64_t stats_n_write_limit_reached = 0; + +/** Total number of times we've reached TCP port exhaustion. */ +static uint64_t stats_n_tcp_exhaustion = 0; + /***** DNS statistics *****/ -/** Represents the statistics of DNS queries seen if it is an Exit. */ +/** Overload DNS statistics. The information in this object is used to assess + * if, due to DNS errors, we should emit a general overload signal or not. + * + * NOTE: This structure is _not_ per DNS query type like the statistics below + * because of a libevent bug + * (https://github.com/libevent/libevent/issues/1219), on error, the type is + * not propagated up back to the user and so we need to keep our own stats for + * the overload signal. */ typedef struct { /** Total number of DNS request seen at an Exit. They might not all end * successfully or might even be lost by tor. This counter is incremented * right before the DNS request is initiated. */ uint64_t stats_n_request; - /** Total number of DNS timeout errors. */ - uint64_t stats_n_error_timeout; - /** When is the next assessment time of the general overload for DNS errors. * Once this time is reached, all stats are reset and this time is set to the * next assessment time. */ @@ -227,121 +242,230 @@ typedef struct { /** Keep track of the DNS requests for the general overload state. */ static overload_dns_stats_t overload_dns_stats; -/* We use a scale here so we can represent percentages with decimal points by - * scaling the value by this factor and so 0.5% becomes a value of 500. - * Default is 1% and thus min and max range is 0 to 100%. */ -#define OVERLOAD_DNS_TIMEOUT_PERCENT_SCALE 1000.0 -#define OVERLOAD_DNS_TIMEOUT_PERCENT_DEFAULT 1000 -#define OVERLOAD_DNS_TIMEOUT_PERCENT_MIN 0 -#define OVERLOAD_DNS_TIMEOUT_PERCENT_MAX 100000 - -/** Consensus parameter: indicate what fraction of DNS timeout errors over the - * total number of DNS requests must be reached before we trigger a general - * overload signal .*/ -static double overload_dns_timeout_fraction = - OVERLOAD_DNS_TIMEOUT_PERCENT_DEFAULT / - OVERLOAD_DNS_TIMEOUT_PERCENT_SCALE / 100.0; - -/* Number of seconds for the assessment period. Default is 10 minutes (600) and - * the min max range is within a 32bit value. */ -#define OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_DEFAULT (10 * 60) -#define OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_MIN 0 -#define OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_MAX INT32_MAX - -/** Consensus parameter: Period, in seconds, over which we count the number of - * DNS requests and timeout errors. After that period, we assess if we trigger - * an overload or not. */ -static int32_t overload_dns_timeout_period_secs = - OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_DEFAULT; +/** Represents the statistics of DNS queries seen if it is an Exit. */ +typedef struct { + /* Total number of DNS errors found in RFC 1035 (from 0 to 5 code). */ + uint64_t stats_n_error_none; /* 0 */ + uint64_t stats_n_error_format; /* 1 */ + uint64_t stats_n_error_serverfailed; /* 2 */ + uint64_t stats_n_error_notexist; /* 3 */ + uint64_t stats_n_error_notimpl; /* 4 */ + uint64_t stats_n_error_refused; /* 5 */ + + /* Total number of DNS errors specific to libevent. */ + uint64_t stats_n_error_truncated; /* 65 */ + uint64_t stats_n_error_unknown; /* 66 */ + uint64_t stats_n_error_tor_timeout; /* 67 */ + uint64_t stats_n_error_shutdown; /* 68 */ + uint64_t stats_n_error_cancel; /* 69 */ + uint64_t stats_n_error_nodata; /* 70 */ + + /* Total number of DNS request seen at an Exit. They might not all end + * successfully or might even be lost by tor. This counter is incremented + * right before the DNS request is initiated. */ + uint64_t stats_n_request; +} dns_stats_t; + +/* This is disabled because of the libevent bug where on error we don't get the + * DNS query type back. Once it is fixed, we can re-enable this. */ +#if 0 +/** DNS statistics store for each DNS record type for which tor supports only + * three at the moment: A, PTR and AAAA. */ +static dns_stats_t dns_A_stats; +static dns_stats_t dns_PTR_stats; +static dns_stats_t dns_AAAA_stats; +#endif -/** Current state of overload stats */ -static overload_stats_t overload_stats; +/** DNS query statistics store. It covers all type of queries. */ +static dns_stats_t dns_all_stats; -/** Return true if this overload happened within the last `n_hours`. */ -static bool -overload_happened_recently(time_t overload_time, int n_hours) +/** Return the point to the DNS statistics store. Ignore the type for now + * because of a libevent problem. */ +static inline dns_stats_t * +get_dns_stats_by_type(const int type) { - /* An overload is relevant if it happened in the last 72 hours */ - if (overload_time > approx_time() - 3600 * n_hours) { - return true; - } - return false; + (void) type; + return &dns_all_stats; } -/** Assess the DNS timeout errors and if we have enough to trigger a general - * overload. */ -static void -overload_general_dns_assessment(void) +#if 0 +/** From a libevent record type, return a pointer to the corresponding DNS + * statistics store. NULL is returned if the type is unhandled. */ +static inline dns_stats_t * +get_dns_stats_by_type(const int type) { - /* Initialize the time. Should be done once. */ - if (overload_dns_stats.next_assessment_time == 0) { - goto reset; + switch (type) { + case DNS_IPv4_A: + return &dns_A_stats; + case DNS_PTR: + return &dns_PTR_stats; + case DNS_IPv6_AAAA: + return &dns_AAAA_stats; + default: + return NULL; } +} +#endif - /* Not the time yet. */ - if (overload_dns_stats.next_assessment_time > approx_time()) { - return; +/** Return the DNS error count for the given libevent DNS type and error code. + * The possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */ +uint64_t +rep_hist_get_n_dns_error(int type, uint8_t error) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return 0; } - reset: - /* Reset counters for the next period. */ - overload_dns_stats.stats_n_error_timeout = 0; - overload_dns_stats.stats_n_request = 0; - overload_dns_stats.next_assessment_time = - approx_time() + overload_dns_timeout_period_secs; + switch (error) { + case DNS_ERR_NONE: + return dns_stats->stats_n_error_none; + case DNS_ERR_FORMAT: + return dns_stats->stats_n_error_format; + case DNS_ERR_SERVERFAILED: + return dns_stats->stats_n_error_serverfailed; + case DNS_ERR_NOTEXIST: + return dns_stats->stats_n_error_notexist; + case DNS_ERR_NOTIMPL: + return dns_stats->stats_n_error_notimpl; + case DNS_ERR_REFUSED: + return dns_stats->stats_n_error_refused; + case DNS_ERR_TRUNCATED: + return dns_stats->stats_n_error_truncated; + case DNS_ERR_UNKNOWN: + return dns_stats->stats_n_error_unknown; + case DNS_ERR_TIMEOUT: + return dns_stats->stats_n_error_tor_timeout; + case DNS_ERR_SHUTDOWN: + return dns_stats->stats_n_error_shutdown; + case DNS_ERR_CANCEL: + return dns_stats->stats_n_error_cancel; + case DNS_ERR_NODATA: + return dns_stats->stats_n_error_nodata; + default: + /* Unhandled code sent back by libevent. */ + return 0; + } } -/** Called just before the consensus will be replaced. Update the consensus - * parameters in case they changed. */ -void -rep_hist_consensus_has_changed(const networkstatus_t *ns) +/** Return the total number of DNS request seen for the given libevent DNS + * record type. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */ +uint64_t +rep_hist_get_n_dns_request(int type) { - overload_dns_timeout_fraction = - networkstatus_get_param(ns, "overload_dns_timeout_scale_percent", - OVERLOAD_DNS_TIMEOUT_PERCENT_DEFAULT, - OVERLOAD_DNS_TIMEOUT_PERCENT_MIN, - OVERLOAD_DNS_TIMEOUT_PERCENT_MAX) / - OVERLOAD_DNS_TIMEOUT_PERCENT_SCALE / 100.0; - - overload_dns_timeout_period_secs = - networkstatus_get_param(ns, "overload_dns_timeout_period_secs", - OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_DEFAULT, - OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_MIN, - OVERLOAD_DNS_TIMEOUT_PERIOD_SECS_MAX); + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return 0; + } + return dns_stats->stats_n_request; } /** Note a DNS error for the given given libevent DNS record type and error * code. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. * - * IMPORTANT: Libevent is _not_ returning the type in case of an error and so - * if error is anything but DNS_ERR_NONE, the type is not usable and set to 0. + * NOTE: Libevent is _not_ returning the type in case of an error and so if + * error is anything but DNS_ERR_NONE, the type is not usable and set to 0. * * See: https://gitlab.torproject.org/tpo/core/tor/-/issues/40490 */ void -rep_hist_note_dns_query(int type, uint8_t error) +rep_hist_note_dns_error(int type, uint8_t error) { - (void) type; + overload_dns_stats.stats_n_request++; - /* Assess if we need to trigger a general overload with regards to the DNS - * errors or not. */ - overload_general_dns_assessment(); + /* Again, the libevent bug (see function comment), for an error that is + * anything but DNS_ERR_NONE, the type is always 0 which means that we don't + * have a DNS stat object for it so this code will do nothing until libevent + * is fixed. */ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + /* Unsupported DNS query type. */ + if (!dns_stats) { + return; + } - /* We only care about timeouts for the moment. */ switch (error) { + case DNS_ERR_NONE: + dns_stats->stats_n_error_none++; + break; + case DNS_ERR_FORMAT: + dns_stats->stats_n_error_format++; + break; + case DNS_ERR_SERVERFAILED: + dns_stats->stats_n_error_serverfailed++; + break; + case DNS_ERR_NOTEXIST: + dns_stats->stats_n_error_notexist++; + break; + case DNS_ERR_NOTIMPL: + dns_stats->stats_n_error_notimpl++; + break; + case DNS_ERR_REFUSED: + dns_stats->stats_n_error_refused++; + break; + case DNS_ERR_TRUNCATED: + dns_stats->stats_n_error_truncated++; + break; + case DNS_ERR_UNKNOWN: + dns_stats->stats_n_error_unknown++; + break; case DNS_ERR_TIMEOUT: - overload_dns_stats.stats_n_error_timeout++; + dns_stats->stats_n_error_tor_timeout++; + break; + case DNS_ERR_SHUTDOWN: + dns_stats->stats_n_error_shutdown++; + break; + case DNS_ERR_CANCEL: + dns_stats->stats_n_error_cancel++; + break; + case DNS_ERR_NODATA: + dns_stats->stats_n_error_nodata++; break; default: + /* Unhandled code sent back by libevent. */ break; } +} - /* Increment total number of requests. */ - overload_dns_stats.stats_n_request++; +/** Note a DNS request for the given given libevent DNS record type. */ +void +rep_hist_note_dns_request(int type) +{ + dns_stats_t *dns_stats = get_dns_stats_by_type(type); + if (BUG(!dns_stats)) { + return; + } + dns_stats->stats_n_request++; +} + +/***** END of DNS statistics *****/ + +/** Return true if this overload happened within the last `n_hours`. */ +static bool +overload_happened_recently(time_t overload_time, int n_hours) +{ + /* An overload is relevant if it happened in the last 72 hours */ + if (overload_time > approx_time() - 3600 * n_hours) { + return true; + } + return false; } /* The current version of the overload stats version */ #define OVERLOAD_STATS_VERSION 1 +/** Return the stats_n_read_limit_reached counter. */ +uint64_t +rep_hist_get_n_read_limit_reached(void) +{ + return stats_n_read_limit_reached; +} + +/** Return the stats_n_write_limit_reached counter. */ +uint64_t +rep_hist_get_n_write_limit_reached(void) +{ + return stats_n_write_limit_reached; +} + /** Returns an allocated string for server descriptor for publising information * on whether we are overloaded or not. */ char * @@ -420,6 +544,7 @@ rep_hist_note_overload(overload_type_t overload) SET_TO_START_OF_HOUR(overload_stats.overload_general_time); break; case OVERLOAD_READ: { + stats_n_read_limit_reached++; SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time); if (approx_time() >= last_read_counted + 60) { /* Count once a minute */ overload_stats.overload_read_count++; @@ -428,6 +553,7 @@ rep_hist_note_overload(overload_type_t overload) break; } case OVERLOAD_WRITE: { + stats_n_write_limit_reached++; SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time); if (approx_time() >= last_write_counted + 60) { /* Count once a minute */ overload_stats.overload_write_count++; @@ -442,6 +568,22 @@ rep_hist_note_overload(overload_type_t overload) } } +/** Note down that we've reached a TCP port exhaustion. This triggers an + * overload general event. */ +void +rep_hist_note_tcp_exhaustion(void) +{ + stats_n_tcp_exhaustion++; + rep_hist_note_overload(OVERLOAD_GENERAL); +} + +/** Return the total number of TCP exhaustion times we've reached. */ +uint64_t +rep_hist_get_n_tcp_exhaustion(void) +{ + return stats_n_tcp_exhaustion; +} + /** Return the or_history_t for the OR with identity digest <b>id</b>, * creating it if necessary. */ static or_history_t * @@ -641,7 +783,7 @@ rep_hist_downrate_old_runs(time_t now) return stability_last_downrated + STABILITY_INTERVAL; /* Okay, we should downrate the data. By how much? */ - while (stability_last_downrated + STABILITY_INTERVAL < now) { + while (stability_last_downrated + STABILITY_INTERVAL <= now) { stability_last_downrated += STABILITY_INTERVAL; alpha *= STABILITY_ALPHA; } @@ -1908,11 +2050,18 @@ rep_hist_note_desc_served(const char * desc) /** Internal statistics to track how many requests of each type of * handshake we've received, and how many we've assigned to cpuworkers. * Useful for seeing trends in cpu load. + * + * They are reset at every heartbeat. * @{ */ STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; /**@}*/ +/** Counters keeping the same stats as above but for the entire duration of the + * process (not reset). */ +static uint64_t stats_n_onionskin_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; +static uint64_t stats_n_onionskin_dropped[MAX_ONION_HANDSHAKE_TYPE+1] = {0}; + /** A new onionskin (using the <b>type</b> handshake) has arrived. */ void rep_hist_note_circuit_handshake_requested(uint16_t type) @@ -1926,8 +2075,20 @@ rep_hist_note_circuit_handshake_requested(uint16_t type) void rep_hist_note_circuit_handshake_assigned(uint16_t type) { - if (type <= MAX_ONION_HANDSHAKE_TYPE) + if (type <= MAX_ONION_HANDSHAKE_TYPE) { onion_handshakes_assigned[type]++; + stats_n_onionskin_assigned[type]++; + } +} + +/** We've just drop an onionskin (using the <b>type</b> handshake) due to being + * overloaded. */ +void +rep_hist_note_circuit_handshake_dropped(uint16_t type) +{ + if (type <= MAX_ONION_HANDSHAKE_TYPE) { + stats_n_onionskin_dropped[type]++; + } } /** Get the circuit handshake value that is requested. */ @@ -1950,6 +2111,26 @@ rep_hist_get_circuit_handshake_assigned, (uint16_t type)) return onion_handshakes_assigned[type]; } +/** Get the total number of circuit handshake value that is assigned. */ +MOCK_IMPL(uint64_t, +rep_hist_get_circuit_n_handshake_assigned, (uint16_t type)) +{ + if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) { + return 0; + } + return stats_n_onionskin_assigned[type]; +} + +/** Get the total number of circuit handshake value that is dropped. */ +MOCK_IMPL(uint64_t, +rep_hist_get_circuit_n_handshake_dropped, (uint16_t type)) +{ + if (BUG(type > MAX_ONION_HANDSHAKE_TYPE)) { + return 0; + } + return stats_n_onionskin_dropped[type]; +} + /** Log our onionskin statistics since the last time we were called. */ void rep_hist_log_circuit_handshake_stats(time_t now) diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h index 749b4996a8..7f414de4c8 100644 --- a/src/feature/stats/rephist.h +++ b/src/feature/stats/rephist.h @@ -58,11 +58,17 @@ time_t rep_hist_desc_stats_write(time_t now); void rep_hist_note_circuit_handshake_requested(uint16_t type); void rep_hist_note_circuit_handshake_assigned(uint16_t type); +void rep_hist_note_circuit_handshake_dropped(uint16_t type); void rep_hist_log_circuit_handshake_stats(time_t now); MOCK_DECL(int, rep_hist_get_circuit_handshake_requested, (uint16_t type)); MOCK_DECL(int, rep_hist_get_circuit_handshake_assigned, (uint16_t type)); +MOCK_DECL(uint64_t, rep_hist_get_circuit_n_handshake_assigned, + (uint16_t type)); +MOCK_DECL(uint64_t, rep_hist_get_circuit_n_handshake_dropped, + (uint16_t type)); + void rep_hist_hs_stats_init(time_t now); void rep_hist_hs_stats_term(void); time_t rep_hist_hs_stats_write(time_t now, bool is_v3); @@ -72,14 +78,16 @@ void rep_hist_seen_new_rp_cell(bool is_v2); char *rep_hist_get_hs_v3_stats_string(void); void rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key); -void rep_hist_note_dns_query(int type, uint8_t error); - void rep_hist_free_all(void); void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here); void rep_hist_log_link_protocol_counts(void); -void rep_hist_consensus_has_changed(const networkstatus_t *ns); + +uint64_t rep_hist_get_n_dns_error(int type, uint8_t error); +uint64_t rep_hist_get_n_dns_request(int type); +void rep_hist_note_dns_request(int type); +void rep_hist_note_dns_error(int type, uint8_t error); extern uint64_t rephist_total_alloc; extern uint32_t rephist_total_num; @@ -162,6 +170,12 @@ void rep_hist_note_overload(overload_type_t overload); char *rep_hist_get_overload_general_line(void); char *rep_hist_get_overload_stats_lines(void); +void rep_hist_note_tcp_exhaustion(void); +uint64_t rep_hist_get_n_tcp_exhaustion(void); + +uint64_t rep_hist_get_n_read_limit_reached(void); +uint64_t rep_hist_get_n_write_limit_reached(void); + #ifdef TOR_UNIT_TESTS struct hs_v2_stats_t; const struct hs_v2_stats_t *rep_hist_get_hs_v2_stats(void); diff --git a/src/include.am b/src/include.am index 0826da7548..36d323e6eb 100644 --- a/src/include.am +++ b/src/include.am @@ -85,7 +85,6 @@ include src/app/main/include.am include src/core/include.am include src/app/include.am -include src/rust/include.am include src/test/include.am include src/tools/include.am include src/win32/include.am diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h index 50bfedffba..991b33d9e7 100644 --- a/src/lib/cc/compat_compiler.h +++ b/src/lib/cc/compat_compiler.h @@ -15,6 +15,15 @@ #include "orconfig.h" #include <inttypes.h> +#if defined(__MINGW32__) || defined(__MINGW64__) +#define MINGW_ANY +#endif + +#ifdef MINGW_ANY +/* We need this for __MINGW_PRINTF_FORMAT, alas. */ +#include <stdio.h> +#endif + #if defined(__has_feature) # if __has_feature(address_sanitizer) /* Some of the fancy glibc strcmp() macros include references to memory that @@ -36,16 +45,30 @@ #error "It seems that you encode characters in something other than ASCII." #endif +/* Use the right magic attribute on mingw, which might be printf, gnu_printf, + * or ms_printf, depending on how we're set up to build. + */ +#ifdef __MINGW_PRINTF_FORMAT +#define PRINTF_FORMAT_ATTR __MINGW_PRINTF_FORMAT +#else +#define PRINTF_FORMAT_ATTR printf +#endif +#ifdef __MINGW_SCANF_FORMAT +#define SCANF_FORMAT_ATTR __MINGW_SCANF_FORMAT +#else +#define SCANF_FORMAT_ATTR scanf +#endif + /* GCC can check printf and scanf types on arbitrary functions. */ #ifdef __GNUC__ #define CHECK_PRINTF(formatIdx, firstArg) \ - __attribute__ ((format(printf, formatIdx, firstArg))) + __attribute__ ((format(PRINTF_FORMAT_ATTR, formatIdx, firstArg))) #else #define CHECK_PRINTF(formatIdx, firstArg) #endif /* defined(__GNUC__) */ #ifdef __GNUC__ #define CHECK_SCANF(formatIdx, firstArg) \ - __attribute__ ((format(scanf, formatIdx, firstArg))) + __attribute__ ((format(SCANF_FORMAT_ATTR, formatIdx, firstArg))) #else #define CHECK_SCANF(formatIdx, firstArg) #endif /* defined(__GNUC__) */ @@ -191,10 +214,6 @@ #define OP_EQ == #define OP_NE != -#if defined(__MINGW32__) || defined(__MINGW64__) -#define MINGW_ANY -#endif - /** Macro: yield a pointer to the field at position <b>off</b> within the * structure <b>st</b>. Example: * <pre> diff --git a/src/lib/metrics/metrics_common.c b/src/lib/metrics/metrics_common.c index 5941a4d892..f3f7e22d88 100644 --- a/src/lib/metrics/metrics_common.c +++ b/src/lib/metrics/metrics_common.c @@ -11,6 +11,7 @@ #include "orconfig.h" #include "lib/log/util_bug.h" +#include "lib/string/printf.h" #include "lib/metrics/metrics_common.h" @@ -27,3 +28,15 @@ metrics_type_to_str(const metrics_type_t type) tor_assert_unreached(); } } + +/** Return a static buffer pointer that contains a formatted label on the form + * of key=value. + * + * Subsequent call to this function invalidates the previous buffer. */ +const char * +metrics_format_label(const char *key, const char *value) +{ + static char buf[128]; + tor_snprintf(buf, sizeof(buf), "%s=\"%s\"", key, value); + return buf; +} diff --git a/src/lib/metrics/metrics_common.h b/src/lib/metrics/metrics_common.h index 59aa9c0e90..3644ad3d50 100644 --- a/src/lib/metrics/metrics_common.h +++ b/src/lib/metrics/metrics_common.h @@ -42,4 +42,7 @@ typedef struct metrics_gauge_t { const char *metrics_type_to_str(const metrics_type_t type); +/* Helpers. */ +const char *metrics_format_label(const char *key, const char *value); + #endif /* !defined(TOR_LIB_METRICS_METRICS_COMMON_H) */ diff --git a/src/lib/metrics/metrics_store.c b/src/lib/metrics/metrics_store.c index 4cab5245f3..33b1780438 100644 --- a/src/lib/metrics/metrics_store.c +++ b/src/lib/metrics/metrics_store.c @@ -34,7 +34,8 @@ struct metrics_store_t { }; /** Function pointer to the format function of a specific driver. */ -typedef void (fmt_driver_fn_t)(const metrics_store_entry_t *, buf_t *); +typedef void (fmt_driver_fn_t)(const metrics_store_entry_t *, buf_t *, + bool no_comment); /** Helper: Free a single entry in a metrics_store_t taking a void pointer * parameter. */ @@ -47,6 +48,8 @@ metrics_store_free_void(void *p) smartlist_free(list); } +#include <stdio.h> + /** Put the given store output in the buffer data and use the format function * given in fmt to get it for each entry. */ static void @@ -57,8 +60,11 @@ get_output(const metrics_store_t *store, buf_t *data, fmt_driver_fn_t fmt) tor_assert(fmt); STRMAP_FOREACH(store->entries, key, const smartlist_t *, entries) { + /* Indicate that we've formatted the coment already for the entries. */ + bool comment_formatted = false; SMARTLIST_FOREACH_BEGIN(entries, const metrics_store_entry_t *, entry) { - fmt(entry, data); + fmt(entry, data, comment_formatted); + comment_formatted = true; } SMARTLIST_FOREACH_END(entry); } STRMAP_FOREACH_END; } @@ -138,3 +144,14 @@ metrics_store_get_output(const metrics_format_t fmt, // LCOV_EXCL_STOP } } + +/** Reset a store as in free its content. */ +void +metrics_store_reset(metrics_store_t *store) +{ + if (store == NULL) { + return; + } + strmap_free(store->entries, metrics_store_free_void); + store->entries = strmap_new(); +} diff --git a/src/lib/metrics/metrics_store.h b/src/lib/metrics/metrics_store.h index 42bc56e8fd..d85f484bd6 100644 --- a/src/lib/metrics/metrics_store.h +++ b/src/lib/metrics/metrics_store.h @@ -28,6 +28,7 @@ metrics_store_t *metrics_store_new(void); metrics_store_entry_t *metrics_store_add(metrics_store_t *store, metrics_type_t type, const char *name, const char *help); +void metrics_store_reset(metrics_store_t *store); /* Accessors. */ smartlist_t *metrics_store_get_all(const metrics_store_t *store, diff --git a/src/lib/metrics/prometheus.c b/src/lib/metrics/prometheus.c index 65241ed6c1..aac23ac92e 100644 --- a/src/lib/metrics/prometheus.c +++ b/src/lib/metrics/prometheus.c @@ -42,14 +42,17 @@ format_labels(smartlist_t *labels) /** Format the given entry in to the buffer data. */ void -prometheus_format_store_entry(const metrics_store_entry_t *entry, buf_t *data) +prometheus_format_store_entry(const metrics_store_entry_t *entry, buf_t *data, + bool no_comment) { tor_assert(entry); tor_assert(data); - buf_add_printf(data, "# HELP %s %s\n", entry->name, entry->help); - buf_add_printf(data, "# TYPE %s %s\n", entry->name, - metrics_type_to_str(entry->type)); + if (!no_comment) { + buf_add_printf(data, "# HELP %s %s\n", entry->name, entry->help); + buf_add_printf(data, "# TYPE %s %s\n", entry->name, + metrics_type_to_str(entry->type)); + } buf_add_printf(data, "%s%s %" PRIi64 "\n", entry->name, format_labels(entry->labels), metrics_store_entry_get_value(entry)); diff --git a/src/lib/metrics/prometheus.h b/src/lib/metrics/prometheus.h index 19770e7911..faa7681daa 100644 --- a/src/lib/metrics/prometheus.h +++ b/src/lib/metrics/prometheus.h @@ -13,6 +13,6 @@ #include "lib/metrics/metrics_store_entry.h" void prometheus_format_store_entry(const metrics_store_entry_t *entry, - buf_t *data); + buf_t *data, bool no_comment); #endif /* !defined(TOR_LIB_METRICS_PROMETHEUS_H) */ diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 26b155bc4c..085eb8c458 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -2005,20 +2005,15 @@ parse_port_range(const char *port, uint16_t *port_min_out, char *endptr = NULL; port_min = (int)tor_parse_long(port, 10, 0, 65535, &ok, &endptr); if (!ok) { - log_warn(LD_GENERAL, - "Malformed port %s on address range; rejecting.", - escaped(port)); - return -1; - } else if (endptr && *endptr == '-') { + goto malformed_port; + } else if (endptr && *endptr != '\0') { + if (*endptr != '-') + goto malformed_port; port = endptr+1; endptr = NULL; port_max = (int)tor_parse_long(port, 10, 1, 65535, &ok, &endptr); - if (!ok) { - log_warn(LD_GENERAL, - "Malformed port %s on address range; rejecting.", - escaped(port)); - return -1; - } + if (!ok) + goto malformed_port; } else { port_max = port_min; } @@ -2037,6 +2032,11 @@ parse_port_range(const char *port, uint16_t *port_min_out, *port_max_out = (uint16_t) port_max; return 0; + malformed_port: + log_warn(LD_GENERAL, + "Malformed port %s on address range; rejecting.", + escaped(port)); + return -1; } /** Given a host-order <b>addr</b>, call tor_inet_ntop() on it diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 02222e5a1c..a15f99ad76 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -58,6 +58,10 @@ #include <linux/futex.h> #include <sys/file.h> +#ifdef ENABLE_FRAGILE_HARDENING +#include <sys/ptrace.h> +#endif + #include <stdarg.h> #include <seccomp.h> #include <signal.h> @@ -148,7 +152,11 @@ static sandbox_cfg_t *filter_dynamic = NULL; static int filter_nopar_gen[] = { SCMP_SYS(access), SCMP_SYS(brk), +#ifdef __NR_clock_gettime64 + SCMP_SYS(clock_gettime64), +#else SCMP_SYS(clock_gettime), +#endif SCMP_SYS(close), SCMP_SYS(clone), SCMP_SYS(dup), @@ -191,6 +199,9 @@ static int filter_nopar_gen[] = { SCMP_SYS(getgid32), #endif SCMP_SYS(getpid), +#ifdef ENABLE_FRAGILE_HARDENING + SCMP_SYS(getppid), +#endif #ifdef __NR_getrlimit SCMP_SYS(getrlimit), #endif @@ -241,6 +252,9 @@ static int filter_nopar_gen[] = { SCMP_SYS(sigreturn), #endif SCMP_SYS(stat), +#if defined(__i386__) && defined(__NR_statx) + SCMP_SYS(statx), +#endif SCMP_SYS(uname), SCMP_SYS(wait4), SCMP_SYS(write), @@ -339,6 +353,7 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return rc; } +#ifdef __NR_time /** * Function responsible for setting up the time syscall for * the seccomp filter sandbox. @@ -347,13 +362,11 @@ static int sb_time(scmp_filter_ctx ctx, sandbox_cfg_t *filter) { (void) filter; -#ifdef __NR_time + return seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(time), SCMP_CMP(0, SCMP_CMP_EQ, 0)); -#else - return 0; -#endif /* defined(__NR_time) */ } +#endif /* defined(__NR_time) */ /** * Function responsible for setting up the accept4 syscall for @@ -532,6 +545,24 @@ sb_open(scmp_filter_ctx ctx, sandbox_cfg_t *filter) int use_openat = libc_uses_openat_for_open(); +#ifdef ENABLE_FRAGILE_HARDENING + /* AddressSanitizer uses the "open" syscall to access information about the + * running process via the filesystem, so that call must be allowed without + * restriction or the sanitizer will be unable to execute normally when the + * process terminates. */ + rc = seccomp_rule_add_0(ctx, SCMP_ACT_ALLOW, SCMP_SYS(open)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add open syscall, received " + "libseccomp error %d", rc); + return rc; + } + + /* If glibc also uses only the "open" syscall to open files on this system + * there is no need to consider any additional rules. */ + if (!use_openat) + return 0; +#endif + // for each dynamic parameter filters for (elem = filter; elem != NULL; elem = elem->next) { smp_param_t *param = elem->param; @@ -575,6 +606,32 @@ sb_chmod(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } +#ifdef __i386__ +static int +sb_chown32(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + sandbox_cfg_t *elem = NULL; + + // for each dynamic parameter filters + for (elem = filter; elem != NULL; elem = elem->next) { + smp_param_t *param = elem->param; + + if (param != NULL && param->prot == 1 && param->syscall + == SCMP_SYS(chown32)) { + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chown32), + SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value)); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add chown32 syscall, received " + "libseccomp error %d", rc); + return rc; + } + } + } + + return 0; +} +#else static int sb_chown(scmp_filter_ctx ctx, sandbox_cfg_t *filter) { @@ -599,6 +656,7 @@ sb_chown(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } +#endif /* defined(__i386__) */ /** * Function responsible for setting up the rename syscall for @@ -687,6 +745,34 @@ sb_opendir(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } +#ifdef ENABLE_FRAGILE_HARDENING +/** + * Function responsible for setting up the ptrace syscall for + * the seccomp filter sandbox. + */ +static int +sb_ptrace(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + pid_t pid = getpid(); + (void) filter; + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(ptrace), + SCMP_CMP(0, SCMP_CMP_EQ, PTRACE_ATTACH), + SCMP_CMP(1, SCMP_CMP_EQ, pid)); + if (rc) + return rc; + + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(ptrace), + SCMP_CMP(0, SCMP_CMP_EQ, PTRACE_GETREGS), + SCMP_CMP(1, SCMP_CMP_EQ, pid)); + if (rc) + return rc; + + return 0; +} +#endif + /** * Function responsible for setting up the socket syscall for * the seccomp filter sandbox. @@ -1009,6 +1095,18 @@ sb_prctl(scmp_filter_ctx ctx, sandbox_cfg_t *filter) int rc = 0; (void) filter; +#ifdef ENABLE_FRAGILE_HARDENING + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(prctl), + SCMP_CMP(0, SCMP_CMP_EQ, PR_GET_DUMPABLE)); + if (rc) + return rc; + + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(prctl), + SCMP_CMP(0, SCMP_CMP_EQ, PR_SET_PTRACER)); + if (rc) + return rc; +#endif + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(prctl), SCMP_CMP(0, SCMP_CMP_EQ, PR_SET_DUMPABLE)); if (rc) @@ -1053,6 +1151,13 @@ sb_rt_sigprocmask(scmp_filter_ctx ctx, sandbox_cfg_t *filter) int rc = 0; (void) filter; +#ifdef ENABLE_FRAGILE_HARDENING + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(rt_sigprocmask), + SCMP_CMP(0, SCMP_CMP_EQ, SIG_BLOCK)); + if (rc) + return rc; +#endif + rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(rt_sigprocmask), SCMP_CMP(0, SCMP_CMP_EQ, SIG_UNBLOCK)); if (rc) @@ -1192,16 +1297,25 @@ sb_kill(scmp_filter_ctx ctx, sandbox_cfg_t *filter) static sandbox_filter_func_t filter_func[] = { sb_rt_sigaction, sb_rt_sigprocmask, +#ifdef __NR_time sb_time, +#endif sb_accept4, #ifdef __NR_mmap2 sb_mmap2, #endif +#ifdef __i386__ + sb_chown32, +#else sb_chown, +#endif sb_chmod, sb_open, sb_openat, sb_opendir, +#ifdef ENABLE_FRAGILE_HARDENING + sb_ptrace, +#endif sb_rename, #ifdef __NR_fcntl64 sb_fcntl64, @@ -1468,6 +1582,12 @@ new_element(int syscall, char *value) return new_element2(syscall, value, NULL); } +#ifdef __i386__ +#define SCMP_chown SCMP_SYS(chown32) +#else +#define SCMP_chown SCMP_SYS(chown) +#endif + #ifdef __NR_stat64 #define SCMP_stat SCMP_SYS(stat64) #else @@ -1518,7 +1638,7 @@ sandbox_cfg_allow_chown_filename(sandbox_cfg_t **cfg, char *file) { sandbox_cfg_t *elem = NULL; - elem = new_element(SCMP_SYS(chown), file); + elem = new_element(SCMP_chown, file); elem->next = *cfg; *cfg = elem; diff --git a/src/lib/string/printf.c b/src/lib/string/printf.c index 62758093a7..bd35b76d1b 100644 --- a/src/lib/string/printf.c +++ b/src/lib/string/printf.c @@ -8,9 +8,9 @@ * \brief Compatibility wrappers around snprintf and its friends **/ +#include "lib/cc/torint.h" #include "lib/string/printf.h" #include "lib/err/torerr.h" -#include "lib/cc/torint.h" #include "lib/malloc/malloc.h" #include <stdlib.h> @@ -45,7 +45,7 @@ tor_vsnprintf(char *str, size_t size, const char *format, va_list args) return -1; /* no place for the NUL */ if (size > SIZE_T_CEILING) return -1; -#ifdef _WIN32 +#if defined(_WIN32) && !defined(HAVE_VSNPRINTF) r = _vsnprintf(str, size, format, args); #else r = vsnprintf(str, size, format, args); diff --git a/src/rust/.cargo/config.in b/src/rust/.cargo/config.in deleted file mode 100644 index 6eddc75459..0000000000 --- a/src/rust/.cargo/config.in +++ /dev/null @@ -1,12 +0,0 @@ -[source] - -@RUST_DL@ [source.crates-io] -@RUST_DL@ registry = 'https://github.com/rust-lang/crates.io-index' -@RUST_DL@ replace-with = 'vendored-sources' - -@RUST_DL@ [source.vendored-sources] -@RUST_DL@ directory = '@TOR_RUST_DEPENDENCIES@' - -[build] -@RUST_WARN@ rustflags = [ "-D", "warnings" ] -@RUST_TARGET_PROP@ diff --git a/src/rust/.rustfmt.toml b/src/rust/.rustfmt.toml deleted file mode 100644 index 4ff839dcf3..0000000000 --- a/src/rust/.rustfmt.toml +++ /dev/null @@ -1,12 +0,0 @@ -max_width = 100 -hard_tabs = false -tab_spaces = 4 -newline_style = "Unix" -#use_small_heuristics = "Default" -reorder_imports = true -reorder_modules = true -remove_nested_parens = true -merge_derives = true -use_try_shorthand = false -use_field_init_shorthand = false -force_explicit_abi = true diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock deleted file mode 100644 index e2f24b0af7..0000000000 --- a/src/rust/Cargo.lock +++ /dev/null @@ -1,122 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -[[package]] -name = "crypto" -version = "0.0.1" -dependencies = [ - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "external 0.0.1", - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)", - "smartlist 0.0.1", - "tor_allocate 0.0.1", - "tor_log 0.1.0", -] - -[[package]] -name = "digest" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "external" -version = "0.0.1" -dependencies = [ - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", - "smartlist 0.0.1", - "tor_allocate 0.0.1", -] - -[[package]] -name = "generic-array" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "libc" -version = "0.2.39" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "protover" -version = "0.0.1" -dependencies = [ - "external 0.0.1", - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", - "smartlist 0.0.1", - "tor_allocate 0.0.1", - "tor_log 0.1.0", - "tor_util 0.0.1", -] - -[[package]] -name = "rand" -version = "0.5.0-pre.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rand_core" -version = "0.2.0-pre.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "smartlist" -version = "0.0.1" -dependencies = [ - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tor_allocate" -version = "0.0.1" -dependencies = [ - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tor_log" -version = "0.1.0" -dependencies = [ - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", - "tor_allocate 0.0.1", -] - -[[package]] -name = "tor_rust" -version = "0.1.0" -dependencies = [ - "protover 0.0.1", - "tor_util 0.0.1", -] - -[[package]] -name = "tor_util" -version = "0.0.1" -dependencies = [ - "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", - "tor_allocate 0.0.1", - "tor_log 0.1.0", -] - -[[package]] -name = "typenum" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[metadata] -"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603" -"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" -"checksum libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)" = "f54263ad99207254cf58b5f701ecb432c717445ea2ee8af387334bdd1a03fdff" -"checksum rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3795e4701d9628a63a84d0289e66279883b40df165fca7caed7b87122447032a" -"checksum rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7255ffbdb188d5be1a69b6f9f3cf187de4207430b9e79ed5b76458a6b20de9a" -"checksum typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a99dc6780ef33c78780b826cf9d2a78840b72cae9474de4bcaf9051e60ebbd" diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml deleted file mode 100644 index de8693ea33..0000000000 --- a/src/rust/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[workspace] -members = [ - "crypto", - "external", - "protover", - "smartlist", - "tor_allocate", - "tor_log", - "tor_rust", - "tor_util", -] - -# Can remove panic="abort" when this issue is fixed: -# https://github.com/rust-lang/rust/issues/52652 -[profile.dev] -panic = "abort" - -[profile.release] -debug = true -panic = "abort" - -[profile.test] -panic = "abort" - -[profile.bench] -panic = "abort" diff --git a/src/rust/build.rs b/src/rust/build.rs deleted file mode 100644 index 5626b35f75..0000000000 --- a/src/rust/build.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! Build script for Rust modules in Tor. -//! -//! We need to use this because some of our Rust tests need to use some -//! of our C modules, which need to link some external libraries. -//! -//! This script works by looking at a "config.rust" file generated by our -//! configure script, and then building a set of options for cargo to pass to -//! the compiler. - -use std::collections::HashMap; -use std::env; -use std::fs::File; -use std::io; -use std::io::prelude::*; -use std::path::PathBuf; - -/// Wrapper around a key-value map. -struct Config(HashMap<String, String>); - -/// Locate a config.rust file generated by autoconf, starting in the OUT_DIR -/// location provided by cargo and recursing up the directory tree. Note that -/// we need to look in the OUT_DIR, since autoconf will place generated files -/// in the build directory. -fn find_cfg() -> io::Result<String> { - let mut path = PathBuf::from(env::var("OUT_DIR").unwrap()); - loop { - path.push("config.rust"); - if path.exists() { - return Ok(path.to_str().unwrap().to_owned()); - } - path.pop(); // remove config.rust - if !path.pop() { - // can't remove last part of directory - return Err(io::Error::new(io::ErrorKind::NotFound, "No config.rust")); - } - } -} - -impl Config { - /// Find the config.rust file and try to parse it. - /// - /// The file format is a series of lines of the form KEY=VAL, with - /// any blank lines and lines starting with # ignored. - fn load() -> io::Result<Config> { - let path = find_cfg()?; - let f = File::open(&path)?; - let reader = io::BufReader::new(f); - let mut map = HashMap::new(); - for line in reader.lines() { - let s = line?; - if s.trim().starts_with("#") || s.trim() == "" { - continue; - } - let idx = match s.find("=") { - None => { - return Err(io::Error::new(io::ErrorKind::InvalidData, "missing =")); - } - Some(x) => x, - }; - let (var, eq_val) = s.split_at(idx); - let val = &eq_val[1..]; - map.insert(var.to_owned(), val.to_owned()); - } - Ok(Config(map)) - } - - /// Return a reference to the value whose key is 'key'. - /// - /// Panics if 'key' is not found in the configuration. - fn get(&self, key: &str) -> &str { - self.0.get(key).unwrap() - } - - /// Add a dependency on a static C library that is part of Tor, by name. - fn component(&self, s: &str) { - println!("cargo:rustc-link-lib=static={}", s); - } - - /// Add a dependency on a native library that is not part of Tor, by name. - fn dependency(&self, s: &str) { - println!("cargo:rustc-link-lib={}", s); - } - - /// Add a link path, relative to Tor's build directory. - fn link_relpath(&self, s: &str) { - let builddir = self.get("BUILDDIR"); - println!("cargo:rustc-link-search=native={}/{}", builddir, s); - } - - /// Add an absolute link path. - fn link_path(&self, s: &str) { - println!("cargo:rustc-link-search=native={}", s); - } - - /// Parse the CFLAGS in s, looking for -l and -L items, and adding - /// rust configuration as appropriate. - fn from_cflags(&self, s: &str) { - let mut next_is_lib = false; - let mut next_is_path = false; - for ent in self.get(s).split_whitespace() { - if next_is_lib { - self.dependency(ent); - next_is_lib = false; - } else if next_is_path { - self.link_path(ent); - next_is_path = false; - } else if ent == "-l" { - next_is_lib = true; - } else if ent == "-L" { - next_is_path = true; - } else if ent.starts_with("-L") { - self.link_path(&ent[2..]); - } else if ent.starts_with("-l") { - self.dependency(&ent[2..]); - } - } - } -} - -pub fn main() { - let cfg = Config::load().unwrap(); - let package = env::var("CARGO_PKG_NAME").unwrap(); - - match package.as_ref() { - "crypto" => { - // Right now, I'm having a separate configuration for each Rust - // package, since I'm hoping we can trim them down. Once we have a - // second Rust package that needs to use this build script, let's - // extract some of this stuff into a module. - // - // This is a ridiculous amount of code to be pulling in just - // to test our crypto library: modularity would be our - // friend here. - cfg.from_cflags("TOR_LDFLAGS_zlib"); - cfg.from_cflags("TOR_LDFLAGS_openssl"); - cfg.from_cflags("TOR_LDFLAGS_libevent"); - - cfg.link_relpath("src/lib"); - cfg.link_relpath("src/ext/keccak-tiny"); - cfg.link_relpath("src/ext/ed25519/ref10"); - cfg.link_relpath("src/ext/ed25519/donna"); - cfg.link_relpath("src/trunnel"); - - // Note that we can't pull in "libtor-testing", or else we - // will have dependencies on all the other rust packages that - // tor uses. We must be careful with factoring and dependencies - // moving forward! - cfg.component("tor-crypt-ops-testing"); - cfg.component("tor-sandbox-testing"); - cfg.component("tor-encoding-testing"); - cfg.component("tor-fs-testing"); - cfg.component("tor-net-testing"); - cfg.component("tor-buf-testing"); - cfg.component("tor-time-testing"); - cfg.component("tor-thread-testing"); - cfg.component("tor-memarea-testing"); - cfg.component("tor-log-testing"); - cfg.component("tor-lock-testing"); - cfg.component("tor-fdio-testing"); - cfg.component("tor-container-testing"); - cfg.component("tor-smartlist-core-testing"); - cfg.component("tor-string-testing"); - cfg.component("tor-malloc"); - cfg.component("tor-wallclock"); - cfg.component("tor-err-testing"); - cfg.component("tor-version-testing"); - cfg.component("tor-intmath-testing"); - cfg.component("tor-ctime-testing"); - cfg.component("curve25519_donna"); - cfg.component("keccak-tiny"); - cfg.component("ed25519_ref10"); - cfg.component("ed25519_donna"); - cfg.component("or-trunnel-testing"); - - cfg.from_cflags("TOR_ZLIB_LIBS"); - cfg.from_cflags("TOR_LIB_MATH"); - cfg.from_cflags("NSS_LIBS"); - cfg.from_cflags("TOR_OPENSSL_LIBS"); - cfg.from_cflags("TOR_LIBEVENT_LIBS"); - cfg.from_cflags("TOR_LIB_WS32"); - cfg.from_cflags("TOR_LIB_GDI"); - cfg.from_cflags("TOR_LIB_USERENV"); - cfg.from_cflags("CURVE25519_LIBS"); - cfg.from_cflags("TOR_LZMA_LIBS"); - cfg.from_cflags("TOR_ZSTD_LIBS"); - cfg.from_cflags("LIBS"); - } - _ => { - panic!("No configuration in build.rs for package {}", package); - } - } -} diff --git a/src/rust/crypto/Cargo.toml b/src/rust/crypto/Cargo.toml deleted file mode 100644 index a7ff7f78d9..0000000000 --- a/src/rust/crypto/Cargo.toml +++ /dev/null @@ -1,37 +0,0 @@ -[package] -authors = ["The Tor Project", - "Isis Lovecruft <isis@torproject.org>"] -name = "crypto" -version = "0.0.1" -publish = false -build = "../build.rs" - -[lib] -name = "crypto" -path = "lib.rs" - -[dependencies] -libc = "=0.2.39" -digest = "=0.7.2" -rand_core = { version = "=0.2.0-pre.0", default-features = false } - -external = { path = "../external" } -smartlist = { path = "../smartlist" } -tor_allocate = { path = "../tor_allocate" } -tor_log = { path = "../tor_log" } - -[dev-dependencies] -rand = { version = "=0.5.0-pre.2", default-features = false } -rand_core = { version = "=0.2.0-pre.0", default-features = false } - -[features] -# If this feature is enabled, test code which calls Tor C code from Rust will -# execute with `cargo test`. Due to numerous linker issues (#25386), this is -# currently disabled by default. -test-c-from-rust = [] - -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/crypto/digests/mod.rs b/src/rust/crypto/digests/mod.rs deleted file mode 100644 index 58343b9ca7..0000000000 --- a/src/rust/crypto/digests/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Hash Digests and eXtendible Output Functions (XOFs) - -pub mod sha2; diff --git a/src/rust/crypto/digests/sha2.rs b/src/rust/crypto/digests/sha2.rs deleted file mode 100644 index 91e8b2b3c9..0000000000 --- a/src/rust/crypto/digests/sha2.rs +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Hash Digests and eXtendible Output Functions (XOFs) - -pub use digest::Digest; - -use digest::generic_array::typenum::U32; -use digest::generic_array::typenum::U64; -use digest::generic_array::GenericArray; -use digest::BlockInput; -use digest::FixedOutput; -use digest::Input; - -use external::crypto_digest::get_256_bit_digest; -use external::crypto_digest::get_512_bit_digest; -use external::crypto_digest::CryptoDigest; -use external::crypto_digest::DigestAlgorithm; - -pub use external::crypto_digest::DIGEST256_LEN; -pub use external::crypto_digest::DIGEST512_LEN; - -/// The block size for both SHA-256 and SHA-512 digests is 512 bits/64 bytes. -/// -/// Unfortunately, we have to use the generic_array crate currently to express -/// this at compile time. Later, in the future, when Rust implements const -/// generics, we'll be able to remove this dependency (actually, it will get -/// removed from the digest crate, which is currently `pub use`ing it). -type BlockSize = U64; - -/// A SHA2-256 digest. -/// -/// # C_RUST_COUPLED -/// -/// * `crypto_digest_dup` -#[derive(Clone)] -pub struct Sha256 { - engine: CryptoDigest, -} - -/// Construct a new, default instance of a `Sha256` hash digest function. -/// -/// # Examples -/// -/// ```rust,no_run -/// use crypto::digests::sha2::{Sha256, Digest}; -/// -/// let mut hasher: Sha256 = Sha256::default(); -/// ``` -/// -/// # Returns -/// -/// A new `Sha256` digest. -impl Default for Sha256 { - fn default() -> Sha256 { - Sha256 { - engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_256)), - } - } -} - -impl BlockInput for Sha256 { - type BlockSize = BlockSize; -} - -/// Input `msg` into the digest. -/// -/// # Examples -/// -/// ```rust,no_run -/// use crypto::digests::sha2::{Sha256, Digest}; -/// -/// let mut hasher: Sha256 = Sha256::default(); -/// -/// hasher.input(b"foo"); -/// hasher.input(b"bar"); -/// ``` -impl Input for Sha256 { - fn process(&mut self, msg: &[u8]) { - self.engine.add_bytes(&msg); - } -} - -/// Retrieve the output hash from everything which has been fed into this -/// `Sha256` digest thus far. -/// -// -// FIXME: Once const generics land in Rust, we should genericise calling -// crypto_digest_get_digest in external::crypto_digest. -impl FixedOutput for Sha256 { - type OutputSize = U32; - - fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> { - let buffer: [u8; DIGEST256_LEN] = get_256_bit_digest(self.engine); - - GenericArray::from(buffer) - } -} - -/// A SHA2-512 digest. -/// -/// # C_RUST_COUPLED -/// -/// * `crypto_digest_dup` -#[derive(Clone)] -pub struct Sha512 { - engine: CryptoDigest, -} - -/// Construct a new, default instance of a `Sha512` hash digest function. -/// -/// # Examples -/// -/// ```rust,no_run -/// use crypto::digests::sha2::{Sha512, Digest}; -/// -/// let mut hasher: Sha512 = Sha512::default(); -/// ``` -/// -/// # Returns -/// -/// A new `Sha512` digest. -impl Default for Sha512 { - fn default() -> Sha512 { - Sha512 { - engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_512)), - } - } -} - -impl BlockInput for Sha512 { - type BlockSize = BlockSize; -} - -/// Input `msg` into the digest. -/// -/// # Examples -/// -/// ```rust,no_run -/// use crypto::digests::sha2::{Sha512, Digest}; -/// -/// let mut hasher: Sha512 = Sha512::default(); -/// -/// hasher.input(b"foo"); -/// hasher.input(b"bar"); -/// ``` -impl Input for Sha512 { - fn process(&mut self, msg: &[u8]) { - self.engine.add_bytes(&msg); - } -} - -/// Retrieve the output hash from everything which has been fed into this -/// `Sha512` digest thus far. -/// -// -// FIXME: Once const generics land in Rust, we should genericise calling -// crypto_digest_get_digest in external::crypto_digest. -impl FixedOutput for Sha512 { - type OutputSize = U64; - - fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> { - let buffer: [u8; DIGEST512_LEN] = get_512_bit_digest(self.engine); - - GenericArray::clone_from_slice(&buffer) - } -} - -#[cfg(test)] -mod test { - #[cfg(feature = "test-c-from-rust")] - use digest::Digest; - - #[cfg(feature = "test-c-from-rust")] - use super::*; - - #[cfg(feature = "test-c-from-rust")] - #[test] - fn sha256_default() { - let _: Sha256 = Sha256::default(); - } - - #[cfg(feature = "test-c-from-rust")] - #[test] - fn sha256_digest() { - let mut h: Sha256 = Sha256::new(); - let mut result: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN]; - let expected = [ - 151, 223, 53, 136, 181, 163, 242, 75, 171, 195, 133, 27, 55, 47, 11, 167, 26, 157, 205, - 222, 212, 59, 20, 185, 208, 105, 97, 191, 193, 112, 125, 157, - ]; - - h.input(b"foo"); - h.input(b"bar"); - h.input(b"baz"); - - result.copy_from_slice(h.fixed_result().as_slice()); - - println!("{:?}", &result[..]); - - assert_eq!(result, expected); - } - - #[cfg(feature = "test-c-from-rust")] - #[test] - fn sha512_default() { - let _: Sha512 = Sha512::default(); - } - - #[cfg(feature = "test-c-from-rust")] - #[test] - fn sha512_digest() { - let mut h: Sha512 = Sha512::new(); - let mut result: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN]; - - let expected = [ - 203, 55, 124, 16, 176, 245, 166, 44, 128, 54, 37, 167, 153, 217, 233, 8, 190, 69, 231, - 103, 245, 209, 71, 212, 116, 73, 7, 203, 5, 89, 122, 164, 237, 211, 41, 160, 175, 20, - 122, 221, 12, 244, 24, 30, 211, 40, 250, 30, 121, 148, 38, 88, 38, 179, 237, 61, 126, - 246, 240, 103, 202, 153, 24, 90, - ]; - - h.input(b"foo"); - h.input(b"bar"); - h.input(b"baz"); - - result.copy_from_slice(h.fixed_result().as_slice()); - - println!("{:?}", &result[..]); - - assert_eq!(&result[..], &expected[..]); - } -} diff --git a/src/rust/crypto/lib.rs b/src/rust/crypto/lib.rs deleted file mode 100644 index 866ea93547..0000000000 --- a/src/rust/crypto/lib.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Common cryptographic functions and utilities. -//! -//! # Hash Digests and eXtendable Output Functions (XOFs) -//! -//! The `digests` module contains submodules for specific hash digests -//! and extendable output functions. -//! -//! ```rust,no_run -//! use crypto::digests::sha2::*; -//! -//! let mut hasher: Sha256 = Sha256::default(); -//! let mut result: [u8; 32] = [0u8; 32]; -//! -//! hasher.input(b"foo"); -//! hasher.input(b"bar"); -//! hasher.input(b"baz"); -//! -//! result.copy_from_slice(hasher.result().as_slice()); -//! -//! assert!(result == [b'X'; DIGEST256_LEN]); -//! ``` - -// XXX: add missing docs -//#![deny(missing_docs)] - -// External crates from cargo or TOR_RUST_DEPENDENCIES. -extern crate digest; -extern crate libc; -extern crate rand_core; - -// External dependencies for tests. -#[cfg(test)] -extern crate rand as rand_crate; - -// Our local crates. -extern crate external; -#[cfg(not(test))] -#[macro_use] -extern crate tor_log; - -pub mod digests; // Unfortunately named "digests" plural to avoid name conflict with the digest crate -pub mod rand; diff --git a/src/rust/crypto/rand/mod.rs b/src/rust/crypto/rand/mod.rs deleted file mode 100644 index da8b3bd8a5..0000000000 --- a/src/rust/crypto/rand/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -// Internal dependencies -pub mod rng; diff --git a/src/rust/crypto/rand/rng.rs b/src/rust/crypto/rand/rng.rs deleted file mode 100644 index 644a5c20b1..0000000000 --- a/src/rust/crypto/rand/rng.rs +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Wrappers for Tor's random number generators to provide implementations of -//! `rand_core` traits. - -// This is the real implementation, in use in production, which calls into our C -// wrappers in /src/common/crypto_rand.c, which call into OpenSSL, system -// libraries, and make syscalls. -#[cfg(not(test))] -mod internal { - use std::u64; - - use rand_core::impls::next_u32_via_fill; - use rand_core::impls::next_u64_via_fill; - use rand_core::CryptoRng; - use rand_core::Error; - use rand_core::RngCore; - - use external::c_tor_crypto_rand; - use external::c_tor_crypto_seed_rng; - use external::c_tor_crypto_strongest_rand; - - use tor_log::LogDomain; - use tor_log::LogSeverity; - - /// Largest strong entropy request permitted. - // - // C_RUST_COUPLED: `MAX_STRONGEST_RAND_SIZE` /src/common/crypto_rand.c - const MAX_STRONGEST_RAND_SIZE: usize = 256; - - /// A wrapper around OpenSSL's RNG. - pub struct TorRng { - // This private, zero-length field forces the struct to be treated the - // same as its opaque C counterpart. - _unused: [u8; 0], - } - - /// Mark `TorRng` as being suitable for cryptographic purposes. - impl CryptoRng for TorRng {} - - impl TorRng { - // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c - #[allow(dead_code)] - pub fn new() -> Self { - if !c_tor_crypto_seed_rng() { - tor_log_msg!( - LogSeverity::Warn, - LogDomain::General, - "TorRng::from_seed()", - "The RNG could not be seeded!" - ); - } - // XXX also log success at info level —isis - TorRng { _unused: [0u8; 0] } - } - } - - impl RngCore for TorRng { - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn next_u32(&mut self) -> u32 { - next_u32_via_fill(self) - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn next_u64(&mut self) -> u64 { - next_u64_via_fill(self) - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn fill_bytes(&mut self, dest: &mut [u8]) { - c_tor_crypto_rand(dest); - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> { - Ok(self.fill_bytes(dest)) - } - } - - /// A CSPRNG which hashes together randomness from OpenSSL's RNG and entropy - /// obtained from the operating system. - pub struct TorStrongestRng { - // This private, zero-length field forces the struct to be treated the - // same as its opaque C counterpart. - _unused: [u8; 0], - } - - /// Mark `TorRng` as being suitable for cryptographic purposes. - impl CryptoRng for TorStrongestRng {} - - impl TorStrongestRng { - // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c - #[allow(dead_code)] - pub fn new() -> Self { - if !c_tor_crypto_seed_rng() { - tor_log_msg!( - LogSeverity::Warn, - LogDomain::General, - "TorStrongestRng::from_seed()", - "The RNG could not be seeded!" - ); - } - // XXX also log success at info level —isis - TorStrongestRng { _unused: [0u8; 0] } - } - } - - impl RngCore for TorStrongestRng { - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn next_u32(&mut self) -> u32 { - next_u32_via_fill(self) - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn next_u64(&mut self) -> u64 { - next_u64_via_fill(self) - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn fill_bytes(&mut self, dest: &mut [u8]) { - debug_assert!(dest.len() <= MAX_STRONGEST_RAND_SIZE); - - c_tor_crypto_strongest_rand(dest); - } - - // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c - fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> { - Ok(self.fill_bytes(dest)) - } - } -} - -// For testing, we expose a pure-Rust implementation. -#[cfg(test)] -mod internal { - // It doesn't matter if we pretend ChaCha is a CSPRNG in tests. - pub use rand_crate::ChaChaRng as TorRng; - pub use rand_crate::ChaChaRng as TorStrongestRng; -} - -// Finally, expose the public functionality of whichever appropriate internal -// module. -pub use self::internal::*; diff --git a/src/rust/external/Cargo.toml b/src/rust/external/Cargo.toml deleted file mode 100644 index 5f443645bb..0000000000 --- a/src/rust/external/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -authors = ["The Tor Project"] -version = "0.0.1" -name = "external" - -[dependencies] -libc = "=0.2.39" -smartlist = { path = "../smartlist" } -tor_allocate = { path = "../tor_allocate" } - -[lib] -name = "external" -path = "lib.rs" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/external/crypto_digest.rs b/src/rust/external/crypto_digest.rs deleted file mode 100644 index 873f75e7a3..0000000000 --- a/src/rust/external/crypto_digest.rs +++ /dev/null @@ -1,454 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Bindings to external digest and XOF functions which live within -//! src/common/crypto_digest.[ch]. -//! -//! We wrap our C implementations in src/common/crypto_digest.[ch] with more -//! Rusty types and interfaces in src/rust/crypto/digest/. - -use std::process::abort; - -use libc::c_char; -use libc::c_int; -use libc::size_t; -use libc::uint8_t; - -use smartlist::Stringlist; - -/// Length of the output of our message digest. -pub const DIGEST_LEN: usize = 20; - -/// Length of the output of our second (improved) message digests. (For now -/// this is just sha256, but it could be any other 256-bit digest.) -pub const DIGEST256_LEN: usize = 32; - -/// Length of the output of our 64-bit optimized message digests (SHA512). -pub const DIGEST512_LEN: usize = 64; - -/// Length of a sha1 message digest when encoded in base32 with trailing = signs -/// removed. -pub const BASE32_DIGEST_LEN: usize = 32; - -/// Length of a sha1 message digest when encoded in base64 with trailing = signs -/// removed. -pub const BASE64_DIGEST_LEN: usize = 27; - -/// Length of a sha256 message digest when encoded in base64 with trailing = -/// signs removed. -pub const BASE64_DIGEST256_LEN: usize = 43; - -/// Length of a sha512 message digest when encoded in base64 with trailing = -/// signs removed. -pub const BASE64_DIGEST512_LEN: usize = 86; - -/// Length of hex encoding of SHA1 digest, not including final NUL. -pub const HEX_DIGEST_LEN: usize = 40; - -/// Length of hex encoding of SHA256 digest, not including final NUL. -pub const HEX_DIGEST256_LEN: usize = 64; - -/// Length of hex encoding of SHA512 digest, not including final NUL. -pub const HEX_DIGEST512_LEN: usize = 128; - -/// Our C code uses an enum to declare the digest algorithm types which we know -/// about. However, because enums are implementation-defined in C, we can -/// neither work with them directly nor translate them into Rust enums. -/// Instead, we represent them as a u8 (under the assumption that we'll never -/// support more than 256 hash functions). -#[allow(non_camel_case_types)] -type digest_algorithm_t = u8; - -const DIGEST_SHA1: digest_algorithm_t = 0; -const DIGEST_SHA256: digest_algorithm_t = 1; -const DIGEST_SHA512: digest_algorithm_t = 2; -const DIGEST_SHA3_256: digest_algorithm_t = 3; -const DIGEST_SHA3_512: digest_algorithm_t = 4; - -/// The number of hash digests we produce for a `common_digests_t`. -/// -/// We can't access these from Rust, because their definitions in C require -/// introspecting the `digest_algorithm_t` typedef, which is an enum, so we have -/// to redefine them here. -const N_COMMON_DIGEST_ALGORITHMS: usize = DIGEST_SHA256 as usize + 1; - -/// A digest function. -#[repr(C)] -#[derive(Debug, Copy, Clone)] -#[allow(non_camel_case_types)] -struct crypto_digest_t { - // This private, zero-length field forces the struct to be treated the same - // as its opaque C counterpart. - _unused: [u8; 0], -} - -/// An eXtendible Output Function (XOF). -#[repr(C)] -#[derive(Debug, Copy, Clone)] -#[allow(non_camel_case_types)] -struct crypto_xof_t { - // This private, zero-length field forces the struct to be treated the same - // as its opaque C counterpart. - _unused: [u8; 0], -} - -/// A set of all the digests we commonly compute, taken on a single -/// string. Any digests that are shorter than 512 bits are right-padded -/// with 0 bits. -/// -/// Note that this representation wastes 44 bytes for the SHA1 case, so -/// don't use it for anything where we need to allocate a whole bunch at -/// once. -#[repr(C)] -#[derive(Debug, Copy, Clone)] -#[allow(non_camel_case_types)] -struct common_digests_t { - pub d: [[c_char; N_COMMON_DIGEST_ALGORITHMS]; DIGEST256_LEN], -} - -/// A `smartlist_t` is just an alias for the `#[repr(C)]` type `Stringlist`, to -/// make it more clear that we're working with a smartlist which is owned by C. -#[allow(non_camel_case_types)] -// BINDGEN_GENERATED: This type isn't actually bindgen generated, but the code -// below it which uses it is. As such, this comes up as "dead code" as well. -#[allow(dead_code)] -type smartlist_t = Stringlist; - -/// All of the external functions from `src/common/crypto_digest.h`. -/// -/// These are kept private because they should be wrapped with Rust to make their usage safer. -// -// BINDGEN_GENERATED: These definitions were generated with bindgen and cleaned -// up manually. As such, there are more bindings than are likely necessary or -// which are in use. -#[allow(dead_code)] -extern "C" { - fn crypto_digest(digest: *mut c_char, m: *const c_char, len: size_t) -> c_int; - fn crypto_digest256( - digest: *mut c_char, - m: *const c_char, - len: size_t, - algorithm: digest_algorithm_t, - ) -> c_int; - fn crypto_digest512( - digest: *mut c_char, - m: *const c_char, - len: size_t, - algorithm: digest_algorithm_t, - ) -> c_int; - fn crypto_common_digests(ds_out: *mut common_digests_t, m: *const c_char, len: size_t) - -> c_int; - fn crypto_digest_smartlist_prefix( - digest_out: *mut c_char, - len_out: size_t, - prepend: *const c_char, - lst: *const smartlist_t, - append: *const c_char, - alg: digest_algorithm_t, - ); - fn crypto_digest_smartlist( - digest_out: *mut c_char, - len_out: size_t, - lst: *const smartlist_t, - append: *const c_char, - alg: digest_algorithm_t, - ); - fn crypto_digest_algorithm_get_name(alg: digest_algorithm_t) -> *const c_char; - fn crypto_digest_algorithm_get_length(alg: digest_algorithm_t) -> size_t; - fn crypto_digest_algorithm_parse_name(name: *const c_char) -> c_int; - fn crypto_digest_new() -> *mut crypto_digest_t; - fn crypto_digest256_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t; - fn crypto_digest512_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t; - fn crypto_digest_free_(digest: *mut crypto_digest_t); - fn crypto_digest_add_bytes(digest: *mut crypto_digest_t, data: *const c_char, len: size_t); - fn crypto_digest_get_digest(digest: *mut crypto_digest_t, out: *mut c_char, out_len: size_t); - fn crypto_digest_dup(digest: *const crypto_digest_t) -> *mut crypto_digest_t; - fn crypto_digest_assign(into: *mut crypto_digest_t, from: *const crypto_digest_t); - fn crypto_hmac_sha256( - hmac_out: *mut c_char, - key: *const c_char, - key_len: size_t, - msg: *const c_char, - msg_len: size_t, - ); - fn crypto_mac_sha3_256( - mac_out: *mut uint8_t, - len_out: size_t, - key: *const uint8_t, - key_len: size_t, - msg: *const uint8_t, - msg_len: size_t, - ); - fn crypto_xof_new() -> *mut crypto_xof_t; - fn crypto_xof_add_bytes(xof: *mut crypto_xof_t, data: *const uint8_t, len: size_t); - fn crypto_xof_squeeze_bytes(xof: *mut crypto_xof_t, out: *mut uint8_t, len: size_t); - fn crypto_xof_free(xof: *mut crypto_xof_t); -} - -/// A wrapper around a `digest_algorithm_t`. -pub enum DigestAlgorithm { - SHA2_256, - SHA2_512, - SHA3_256, - SHA3_512, -} - -impl From<DigestAlgorithm> for digest_algorithm_t { - fn from(digest: DigestAlgorithm) -> digest_algorithm_t { - match digest { - DigestAlgorithm::SHA2_256 => DIGEST_SHA256, - DigestAlgorithm::SHA2_512 => DIGEST_SHA512, - DigestAlgorithm::SHA3_256 => DIGEST_SHA3_256, - DigestAlgorithm::SHA3_512 => DIGEST_SHA3_512, - } - } -} - -/// A wrapper around a mutable pointer to a `crypto_digest_t`. -pub struct CryptoDigest(*mut crypto_digest_t); - -/// Explicitly copy the state of a `CryptoDigest` hash digest context. -/// -/// # C_RUST_COUPLED -/// -/// * `crypto_digest_dup` -impl Clone for CryptoDigest { - fn clone(&self) -> CryptoDigest { - let digest: *mut crypto_digest_t; - - unsafe { - digest = crypto_digest_dup(self.0 as *const crypto_digest_t); - } - - // See the note in the implementation of CryptoDigest for the - // reasoning for `abort()` here. - if digest.is_null() { - abort(); - } - - CryptoDigest(digest) - } -} - -impl CryptoDigest { - /// A wrapper to call one of the C functions `crypto_digest_new`, - /// `crypto_digest256_new`, or `crypto_digest512_new`. - /// - /// # Warnings - /// - /// This function will `abort()` the entire process in an "abnormal" fashion, - /// i.e. not unwinding this or any other thread's stack, running any - /// destructors, or calling any panic/exit hooks) if `tor_malloc()` (called in - /// `crypto_digest256_new()`) is unable to allocate memory. - /// - /// # Returns - /// - /// A new `CryptoDigest`, which is a wrapper around a opaque representation - /// of a `crypto_digest_t`. The underlying `crypto_digest_t` _MUST_ only - /// ever be handled via a raw pointer, and never introspected. - /// - /// # C_RUST_COUPLED - /// - /// * `crypto_digest_new` - /// * `crypto_digest256_new` - /// * `crypto_digest512_new` - /// * `tor_malloc` (called by `crypto_digest256_new`, but we make - /// assumptions about its behaviour and return values here) - pub fn new(algorithm: Option<DigestAlgorithm>) -> CryptoDigest { - let digest: *mut crypto_digest_t; - - if algorithm.is_none() { - unsafe { - digest = crypto_digest_new(); - } - } else { - let algo: digest_algorithm_t = algorithm.unwrap().into(); // can't fail because it's Some - - unsafe { - // XXX This is a pretty awkward API to use from Rust... - digest = match algo { - DIGEST_SHA1 => crypto_digest_new(), - DIGEST_SHA256 => crypto_digest256_new(DIGEST_SHA256), - DIGEST_SHA3_256 => crypto_digest256_new(DIGEST_SHA3_256), - DIGEST_SHA512 => crypto_digest512_new(DIGEST_SHA512), - DIGEST_SHA3_512 => crypto_digest512_new(DIGEST_SHA3_512), - _ => abort(), - } - } - } - - // In our C code, `crypto_digest*_new()` allocates memory with - // `tor_malloc()`. In `tor_malloc()`, if the underlying malloc - // implementation fails to allocate the requested memory and returns a - // NULL pointer, we call `exit(1)`. In the case that this `exit(1)` is - // called within a worker, be that a process or a thread, the inline - // comments within `tor_malloc()` mention "that's ok, since the parent - // will run out of memory soon anyway". However, if it takes long - // enough for the worker to die, and it manages to return a NULL pointer - // to our Rust code, our Rust is now in an irreparably broken state and - // may exhibit undefined behaviour. An even worse scenario, if/when we - // have parent/child processes/threads controlled by Rust, would be that - // the UB contagion in Rust manages to spread to other children before - // the entire process (hopefully terminates). - // - // However, following the assumptions made in `tor_malloc()` that - // calling `exit(1)` in a child is okay because the parent will - // eventually run into the same errors, and also to stymie any UB - // contagion in the meantime, we call abort!() here to terminate the - // entire program immediately. - if digest.is_null() { - abort(); - } - - CryptoDigest(digest) - } - - /// A wrapper to call the C function `crypto_digest_add_bytes`. - /// - /// # Inputs - /// - /// * `bytes`: a byte slice of bytes to be added into this digest. - /// - /// # C_RUST_COUPLED - /// - /// * `crypto_digest_add_bytes` - pub fn add_bytes(&self, bytes: &[u8]) { - unsafe { - crypto_digest_add_bytes( - self.0 as *mut crypto_digest_t, - bytes.as_ptr() as *const c_char, - bytes.len() as size_t, - ) - } - } -} - -impl Drop for CryptoDigest { - fn drop(&mut self) { - unsafe { - crypto_digest_free_(self.0 as *mut crypto_digest_t); - } - } -} - -/// Get the 256-bit digest output of a `crypto_digest_t`. -/// -/// # Inputs -/// -/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA256` or a -/// `DIGEST_SHA3_256`. -/// -/// # Warning -/// -/// Calling this function with a `CryptoDigest` which is neither SHA2-256 or -/// SHA3-256 is a programming error. Since we cannot introspect the opaque -/// struct from Rust, however, there is no way for us to check that the correct -/// one is being passed in. That is up to you, dear programmer. If you mess -/// up, you will get a incorrectly-sized hash digest in return, and it will be -/// your fault. Don't do that. -/// -/// # Returns -/// -/// A 256-bit hash digest, as a `[u8; 32]`. -/// -/// # C_RUST_COUPLED -/// -/// * `crypto_digest_get_digest` -/// * `DIGEST256_LEN` -// -// FIXME: Once const generics land in Rust, we should genericise calling -// crypto_digest_get_digest w.r.t. output array size. -pub fn get_256_bit_digest(digest: CryptoDigest) -> [u8; DIGEST256_LEN] { - let mut buffer: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN]; - - unsafe { - crypto_digest_get_digest( - digest.0, - buffer.as_mut_ptr() as *mut c_char, - DIGEST256_LEN as size_t, - ); - - if buffer.as_ptr().is_null() { - abort(); - } - } - buffer -} - -/// Get the 512-bit digest output of a `crypto_digest_t`. -/// -/// # Inputs -/// -/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA512` or a -/// `DIGEST_SHA3_512`. -/// -/// # Warning -/// -/// Calling this function with a `CryptoDigest` which is neither SHA2-512 or -/// SHA3-512 is a programming error. Since we cannot introspect the opaque -/// struct from Rust, however, there is no way for us to check that the correct -/// one is being passed in. That is up to you, dear programmer. If you mess -/// up, you will get a incorrectly-sized hash digest in return, and it will be -/// your fault. Don't do that. -/// -/// # Returns -/// -/// A 512-bit hash digest, as a `[u8; 64]`. -/// -/// # C_RUST_COUPLED -/// -/// * `crypto_digest_get_digest` -/// * `DIGEST512_LEN` -// -// FIXME: Once const generics land in Rust, we should genericise calling -// crypto_digest_get_digest w.r.t. output array size. -pub fn get_512_bit_digest(digest: CryptoDigest) -> [u8; DIGEST512_LEN] { - let mut buffer: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN]; - - unsafe { - crypto_digest_get_digest( - digest.0, - buffer.as_mut_ptr() as *mut c_char, - DIGEST512_LEN as size_t, - ); - - if buffer.as_ptr().is_null() { - abort(); - } - } - buffer -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_layout_common_digests_t() { - assert_eq!( - ::std::mem::size_of::<common_digests_t>(), - 64usize, - concat!("Size of: ", stringify!(common_digests_t)) - ); - assert_eq!( - ::std::mem::align_of::<common_digests_t>(), - 1usize, - concat!("Alignment of ", stringify!(common_digests_t)) - ); - } - - #[test] - fn test_layout_crypto_digest_t() { - assert_eq!( - ::std::mem::size_of::<crypto_digest_t>(), - 0usize, - concat!("Size of: ", stringify!(crypto_digest_t)) - ); - assert_eq!( - ::std::mem::align_of::<crypto_digest_t>(), - 1usize, - concat!("Alignment of ", stringify!(crypto_digest_t)) - ); - } -} diff --git a/src/rust/external/crypto_rand.rs b/src/rust/external/crypto_rand.rs deleted file mode 100644 index 703382093c..0000000000 --- a/src/rust/external/crypto_rand.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Bindings to external (P)RNG interfaces and utilities in -//! src/common/crypto_rand.[ch]. -//! -//! We wrap our C implementations in src/common/crypto_rand.[ch] here in order -//! to provide wrappers with native Rust types, and then provide more Rusty -//! types and and trait implementations in src/rust/crypto/rand/. - -use std::time::Duration; - -use libc::c_double; -use libc::c_int; -use libc::size_t; -use libc::time_t; -use libc::uint8_t; - -extern "C" { - fn crypto_seed_rng() -> c_int; - fn crypto_rand(out: *mut uint8_t, out_len: size_t); - fn crypto_strongest_rand(out: *mut uint8_t, out_len: size_t); - fn crypto_rand_time_range(min: time_t, max: time_t) -> time_t; - fn crypto_rand_double() -> c_double; -} - -/// Seed OpenSSL's random number generator with bytes from the operating -/// system. -/// -/// # Returns -/// -/// `true` on success; `false` on failure. -pub fn c_tor_crypto_seed_rng() -> bool { - let ret: c_int; - - unsafe { - ret = crypto_seed_rng(); - } - match ret { - 0 => return true, - _ => return false, - } -} - -/// Fill the bytes of `dest` with random data. -pub fn c_tor_crypto_rand(dest: &mut [u8]) { - unsafe { - crypto_rand(dest.as_mut_ptr(), dest.len() as size_t); - } -} - -/// Fill the bytes of `dest` with "strong" random data by hashing -/// together randomness obtained from OpenSSL's RNG and the operating -/// system. -pub fn c_tor_crypto_strongest_rand(dest: &mut [u8]) { - // We'll let the C side panic if the len is larger than - // MAX_STRONGEST_RAND_SIZE, rather than potentially panicking here. A - // paranoid caller should assert on the length of dest *before* calling this - // function. - unsafe { - crypto_strongest_rand(dest.as_mut_ptr(), dest.len() as size_t); - } -} - -/// Get a random time, in seconds since the Unix Epoch. -/// -/// # Returns -/// -/// A `std::time::Duration` of seconds since the Unix Epoch. -pub fn c_tor_crypto_rand_time_range(min: &Duration, max: &Duration) -> Duration { - let ret: time_t; - - unsafe { - ret = crypto_rand_time_range(min.as_secs() as time_t, max.as_secs() as time_t); - } - - Duration::from_secs(ret as u64) -} - -/// Return a pseudorandom 64-bit float, chosen uniformly from the range [0.0, 1.0). -pub fn c_tor_crypto_rand_double() -> f64 { - unsafe { crypto_rand_double() } -} diff --git a/src/rust/external/external.rs b/src/rust/external/external.rs deleted file mode 100644 index 0d324c8820..0000000000 --- a/src/rust/external/external.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -use libc::{c_char, c_int}; -use std::ffi::CString; - -extern "C" { - fn tor_version_as_new_as(platform: *const c_char, cutoff: *const c_char) -> c_int; -} - -/// Wrap calls to tor_version_as_new_as, defined in routerparse.c -pub fn c_tor_version_as_new_as(platform: &str, cutoff: &str) -> bool { - // CHK: These functions should log a warning if an error occurs. This - // can be added when integration with tor's logger is added to rust - let c_platform = match CString::new(platform) { - Ok(n) => n, - Err(_) => return false, - }; - - let c_cutoff = match CString::new(cutoff) { - Ok(n) => n, - Err(_) => return false, - }; - - let result: c_int = unsafe { tor_version_as_new_as(c_platform.as_ptr(), c_cutoff.as_ptr()) }; - - result == 1 -} - -extern "C" { - fn tor_is_using_nss() -> c_int; -} - -/// Return true if Tor was built to use NSS. -pub fn c_tor_is_using_nss() -> bool { - 0 != unsafe { tor_is_using_nss() } -} diff --git a/src/rust/external/lib.rs b/src/rust/external/lib.rs deleted file mode 100644 index 2f50610a4d..0000000000 --- a/src/rust/external/lib.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Copyright (c) 2016-2019, The Tor Project, Inc. */ -//! See LICENSE for licensing information */ - -//! Interface for external calls to tor C ABI -//! -//! The purpose of this module is to provide a clean interface for when Rust -//! modules need to interact with functionality in tor C code rather than each -//! module implementing this functionality repeatedly. - -extern crate libc; -extern crate tor_allocate; -extern crate smartlist; - -pub mod crypto_digest; -mod crypto_rand; -mod external; - -pub use crypto_rand::*; -pub use external::*; diff --git a/src/rust/include.am b/src/rust/include.am deleted file mode 100644 index 5e5b0b3faf..0000000000 --- a/src/rust/include.am +++ /dev/null @@ -1,41 +0,0 @@ -include src/rust/tor_rust/include.am - -EXTRA_DIST +=\ - src/rust/build.rs \ - src/rust/Cargo.toml \ - src/rust/Cargo.lock \ - src/rust/.cargo/config.in \ - src/rust/crypto/Cargo.toml \ - src/rust/crypto/lib.rs \ - src/rust/crypto/digests/mod.rs \ - src/rust/crypto/digests/sha2.rs \ - src/rust/crypto/rand/mod.rs \ - src/rust/crypto/rand/rng.rs \ - src/rust/external/Cargo.toml \ - src/rust/external/crypto_digest.rs \ - src/rust/external/crypto_rand.rs \ - src/rust/external/external.rs \ - src/rust/external/lib.rs \ - src/rust/protover/Cargo.toml \ - src/rust/protover/errors.rs \ - src/rust/protover/protoset.rs \ - src/rust/protover/ffi.rs \ - src/rust/protover/lib.rs \ - src/rust/protover/protover.rs \ - src/rust/protover/tests/protover.rs \ - src/rust/smartlist/Cargo.toml \ - src/rust/smartlist/lib.rs \ - src/rust/smartlist/smartlist.rs \ - src/rust/tor_allocate/Cargo.toml \ - src/rust/tor_allocate/lib.rs \ - src/rust/tor_allocate/tor_allocate.rs \ - src/rust/tor_log/Cargo.toml \ - src/rust/tor_log/lib.rs \ - src/rust/tor_log/tor_log.rs \ - src/rust/tor_rust/Cargo.toml \ - src/rust/tor_rust/include.am \ - src/rust/tor_rust/lib.rs \ - src/rust/tor_util/Cargo.toml \ - src/rust/tor_util/ffi.rs \ - src/rust/tor_util/lib.rs \ - src/rust/tor_util/strings.rs diff --git a/src/rust/protover/Cargo.toml b/src/rust/protover/Cargo.toml deleted file mode 100644 index 84a7c71c1a..0000000000 --- a/src/rust/protover/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -[package] -authors = ["The Tor Project"] -version = "0.0.1" -name = "protover" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] - -[dependencies] -libc = "=0.2.39" - -[dependencies.smartlist] -path = "../smartlist" - -[dependencies.external] -path = "../external" - -[dependencies.tor_util] -path = "../tor_util" - -[dependencies.tor_allocate] -path = "../tor_allocate" - -[dependencies.tor_log] -path = "../tor_log" - -[lib] -name = "protover" -path = "lib.rs" diff --git a/src/rust/protover/errors.rs b/src/rust/protover/errors.rs deleted file mode 100644 index 04397ac4fe..0000000000 --- a/src/rust/protover/errors.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Various errors which may occur during protocol version parsing. - -use std::fmt; -use std::fmt::Display; - -/// All errors which may occur during protover parsing routines. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -#[allow(missing_docs)] // See Display impl for error descriptions -pub enum ProtoverError { - Overlap, - LowGreaterThanHigh, - Unparseable, - ExceedsMax, - ExceedsExpansionLimit, - UnknownProtocol, - ExceedsNameLimit, - InvalidProtocol, -} - -/// Descriptive error messages for `ProtoverError` variants. -impl Display for ProtoverError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - ProtoverError::Overlap => write!( - f, - "Two or more (low, high) protover ranges would overlap once expanded." - ), - ProtoverError::LowGreaterThanHigh => write!( - f, - "The low in a (low, high) protover range was greater than high." - ), - ProtoverError::Unparseable => write!(f, "The protover string was unparseable."), - ProtoverError::ExceedsMax => write!( - f, - "The high in a (low, high) protover range exceeds 63." - ), - ProtoverError::ExceedsExpansionLimit => write!( - f, - "The protover string would exceed the maximum expansion limit." - ), - ProtoverError::UnknownProtocol => write!( - f, - "A protocol in the protover string we attempted to parse is unknown." - ), - ProtoverError::ExceedsNameLimit => { - write!(f, "An unrecognised protocol name was too long.") - } - ProtoverError::InvalidProtocol => { - write!(f, "A protocol name includes invalid characters.") - } - } - } -} diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs deleted file mode 100644 index 2bf8d3a987..0000000000 --- a/src/rust/protover/ffi.rs +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -//! FFI functions, only to be called from C. -//! -//! Equivalent C versions of this api are in `protover.c` - -use libc::{c_char, c_int, uint32_t}; -use std::ffi::CStr; - -use smartlist::*; -use tor_allocate::allocate_and_copy_string; - -use errors::ProtoverError; -use protover::*; - -/// Translate C enums to Rust Proto enums, using the integer value of the C -/// enum to map to its associated Rust enum. -/// -/// C_RUST_COUPLED: protover.h `protocol_type_t` -fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> { - match c_proto { - 0 => Ok(Protocol::Link), - 1 => Ok(Protocol::LinkAuth), - 2 => Ok(Protocol::Relay), - 3 => Ok(Protocol::DirCache), - 4 => Ok(Protocol::HSDir), - 5 => Ok(Protocol::HSIntro), - 6 => Ok(Protocol::HSRend), - 7 => Ok(Protocol::Desc), - 8 => Ok(Protocol::Microdesc), - 9 => Ok(Protocol::Cons), - 10 => Ok(Protocol::Padding), - 11 => Ok(Protocol::FlowCtrl), - _ => Err(ProtoverError::UnknownProtocol), - } -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::all_supported -#[no_mangle] -pub extern "C" fn protover_all_supported( - c_relay_version: *const c_char, - missing_out: *mut *mut c_char, -) -> c_int { - if c_relay_version.is_null() { - return 1; - } - - // Require an unsafe block to read the version from a C string. The pointer - // is checked above to ensure it is not null. - let c_str: &CStr = unsafe { CStr::from_ptr(c_relay_version) }; - - let relay_version = match c_str.to_str() { - Ok(n) => n, - Err(_) => return 1, - }; - - let relay_proto_entry: UnvalidatedProtoEntry = - match UnvalidatedProtoEntry::from_str_any_len(relay_version) { - Ok(n) => n, - Err(_) => return 1, - }; - - if let Some(unsupported) = relay_proto_entry.all_supported() { - if missing_out.is_null() { - return 0; - } - let ptr = allocate_and_copy_string(&unsupported.to_string()); - unsafe { *missing_out = ptr }; - - return 0; - } - - 1 -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::list_supports_protocol -#[no_mangle] -pub extern "C" fn protocol_list_supports_protocol( - c_protocol_list: *const c_char, - c_protocol: uint32_t, - version: uint32_t, -) -> c_int { - if c_protocol_list.is_null() { - return 0; - } - - // Require an unsafe block to read the version from a C string. The pointer - // is checked above to ensure it is not null. - let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) }; - - let protocol_list = match c_str.to_str() { - Ok(n) => n, - Err(_) => return 0, - }; - let proto_entry: UnvalidatedProtoEntry = match protocol_list.parse() { - Ok(n) => n, - Err(_) => return 0, - }; - let protocol: UnknownProtocol = match translate_to_rust(c_protocol) { - Ok(n) => n.into(), - Err(_) => return 0, - }; - if proto_entry.supports_protocol(&protocol, &version) { - 1 - } else { - 0 - } -} - -#[no_mangle] -pub extern "C" fn protover_contains_long_protocol_names_(c_protocol_list: *const c_char) -> c_int { - if c_protocol_list.is_null() { - return 1; - } - - // Require an unsafe block to read the version from a C string. The pointer - // is checked above to ensure it is not null. - let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) }; - - let protocol_list = match c_str.to_str() { - Ok(n) => n, - Err(_) => return 1, - }; - - match protocol_list.parse::<UnvalidatedProtoEntry>() { - Ok(_) => 0, - Err(_) => 1, - } -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::list_supports_protocol_or_later -#[no_mangle] -pub extern "C" fn protocol_list_supports_protocol_or_later( - c_protocol_list: *const c_char, - c_protocol: uint32_t, - version: uint32_t, -) -> c_int { - if c_protocol_list.is_null() { - return 0; - } - - // Require an unsafe block to read the version from a C string. The pointer - // is checked above to ensure it is not null. - let c_str: &CStr = unsafe { CStr::from_ptr(c_protocol_list) }; - - let protocol_list = match c_str.to_str() { - Ok(n) => n, - Err(_) => return 0, - }; - - let protocol = match translate_to_rust(c_protocol) { - Ok(n) => n, - Err(_) => return 0, - }; - - let proto_entry: UnvalidatedProtoEntry = match protocol_list.parse() { - Ok(n) => n, - Err(_) => return 0, - }; - - if proto_entry.supports_protocol_or_later(&protocol.into(), &version) { - return 1; - } - 0 -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::get_supported_protocols -#[no_mangle] -pub extern "C" fn protover_get_supported_protocols() -> *const c_char { - let supported: &'static CStr; - - supported = get_supported_protocols_cstr(); - supported.as_ptr() -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::compute_vote -// -// Why is the threshold a signed integer? —isis -#[no_mangle] -pub extern "C" fn protover_compute_vote(list: *const Stringlist, threshold: c_int) -> *mut c_char { - if list.is_null() { - return allocate_and_copy_string(""); - } - - // Dereference of raw pointer requires an unsafe block. The pointer is - // checked above to ensure it is not null. - let data: Vec<String> = unsafe { (*list).get_list() }; - let hold: usize = threshold as usize; - let mut proto_entries: Vec<UnvalidatedProtoEntry> = Vec::new(); - - for datum in data { - let entry: UnvalidatedProtoEntry = match datum.parse() { - Ok(n) => n, - Err(_) => continue, - }; - proto_entries.push(entry); - } - let vote: UnvalidatedProtoEntry = ProtoverVote::compute(&proto_entries, &hold); - - allocate_and_copy_string(&vote.to_string()) -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::is_supported_here -#[no_mangle] -pub extern "C" fn protover_is_supported_here(c_protocol: uint32_t, version: uint32_t) -> c_int { - let protocol = match translate_to_rust(c_protocol) { - Ok(n) => n, - Err(_) => return 0, - }; - - let is_supported = is_supported_here(&protocol, &version); - - return if is_supported { 1 } else { 0 }; -} - -/// Provide an interface for C to translate arguments and return types for -/// protover::compute_for_old_tor -#[no_mangle] -pub extern "C" fn protover_compute_for_old_tor(version: *const c_char) -> *const c_char { - let supported: &'static CStr; - let empty: &'static CStr; - - empty = cstr!(""); - - if version.is_null() { - return empty.as_ptr(); - } - - // Require an unsafe block to read the version from a C string. The pointer - // is checked above to ensure it is not null. - let c_str: &CStr = unsafe { CStr::from_ptr(version) }; - - let version = match c_str.to_str() { - Ok(n) => n, - Err(_) => return empty.as_ptr(), - }; - - supported = compute_for_old_tor_cstr(&version); - supported.as_ptr() -} diff --git a/src/rust/protover/lib.rs b/src/rust/protover/lib.rs deleted file mode 100644 index 35c4106ae5..0000000000 --- a/src/rust/protover/lib.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Copyright (c) 2016-2019, The Tor Project, Inc. */ -//! See LICENSE for licensing information */ - -//! Versioning information for different pieces of the Tor protocol. -//! -//! The below description is taken from src/rust/protover.c, which is currently -//! enabled by default. We are in the process of experimenting with Rust in -//! tor, and this protover module is implemented to help achieve this goal. -//! -//! Starting in version 0.2.9.3-alpha, Tor places separate version numbers on -//! each of the different components of its protocol. Relays use these numbers -//! to advertise what versions of the protocols they can support, and clients -//! use them to find what they can ask a given relay to do. Authorities vote -//! on the supported protocol versions for each relay, and also vote on the -//! which protocols you should have to support in order to be on the Tor -//! network. All Tor instances use these required/recommended protocol versions -//! to tell what level of support for recent protocols each relay has, and -//! to decide whether they should be running given their current protocols. -//! -//! The main advantage of these protocol versions numbers over using Tor -//! version numbers is that they allow different implementations of the Tor -//! protocols to develop independently, without having to claim compatibility -//! with specific versions of Tor. - -// XXX: add missing docs -//#![deny(missing_docs)] - -extern crate external; -extern crate libc; -extern crate smartlist; -extern crate tor_allocate; -#[macro_use] -extern crate tor_util; - -pub mod errors; -pub mod ffi; -pub mod protoset; -mod protover; - -pub use protover::*; diff --git a/src/rust/protover/protoset.rs b/src/rust/protover/protoset.rs deleted file mode 100644 index 0ab94457c5..0000000000 --- a/src/rust/protover/protoset.rs +++ /dev/null @@ -1,697 +0,0 @@ -// Copyright (c) 2018-2019, The Tor Project, Inc. -// Copyright (c) 2018, isis agora lovecruft -// See LICENSE for licensing information - -//! Sets for lazily storing ordered, non-overlapping ranges of integers. - -use std::cmp; -use std::iter; -use std::slice; -use std::str::FromStr; -use std::u32; - -use errors::ProtoverError; - -/// A single version number. -pub type Version = u32; - -/// A `ProtoSet` stores an ordered `Vec<T>` of `(low, high)` pairs of ranges of -/// non-overlapping protocol versions. -/// -/// # Examples -/// -/// ``` -/// use std::str::FromStr; -/// -/// use protover::errors::ProtoverError; -/// use protover::protoset::ProtoSet; -/// use protover::protoset::Version; -/// -/// # fn do_test() -> Result<ProtoSet, ProtoverError> { -/// let protoset: ProtoSet = ProtoSet::from_str("3-5,8")?; -/// -/// // We could also equivalently call: -/// let protoset: ProtoSet = "3-5,8".parse()?; -/// -/// assert!(protoset.contains(&4)); -/// assert!(!protoset.contains(&7)); -/// -/// let expanded: Vec<Version> = protoset.clone().into(); -/// -/// assert_eq!(&expanded[..], &[3, 4, 5, 8]); -/// -/// let contracted: String = protoset.clone().to_string(); -/// -/// assert_eq!(contracted, "3-5,8".to_string()); -/// # Ok(protoset) -/// # } -/// # fn main() { do_test(); } // wrap the test so we can use the ? operator -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -pub struct ProtoSet { - pub(crate) pairs: Vec<(Version, Version)>, -} - -impl Default for ProtoSet { - fn default() -> Self { - let pairs: Vec<(Version, Version)> = Vec::new(); - - ProtoSet { pairs } - } -} - -impl<'a> ProtoSet { - /// Create a new `ProtoSet` from a slice of `(low, high)` pairs. - /// - /// # Inputs - /// - /// We do not assume the input pairs are deduplicated or ordered. - pub fn from_slice(low_high_pairs: &'a [(Version, Version)]) -> Result<Self, ProtoverError> { - let mut pairs: Vec<(Version, Version)> = Vec::with_capacity(low_high_pairs.len()); - - for &(low, high) in low_high_pairs { - pairs.push((low, high)); - } - // Sort the pairs without reallocation and remove all duplicate pairs. - pairs.sort_unstable(); - pairs.dedup(); - - ProtoSet { pairs }.is_ok() - } -} - -/// Expand this `ProtoSet` to a `Vec` of all its `Version`s. -/// -/// # Examples -/// -/// ``` -/// use std::str::FromStr; -/// use protover::protoset::ProtoSet; -/// use protover::protoset::Version; -/// # use protover::errors::ProtoverError; -/// -/// # fn do_test() -> Result<Vec<Version>, ProtoverError> { -/// let protoset: ProtoSet = ProtoSet::from_str("3-5,21")?; -/// let versions: Vec<Version> = protoset.into(); -/// -/// assert_eq!(&versions[..], &[3, 4, 5, 21]); -/// # -/// # Ok(versions) -/// # } -/// # fn main() { do_test(); } // wrap the test so we can use the ? operator -/// ``` -impl Into<Vec<Version>> for ProtoSet { - fn into(self) -> Vec<Version> { - let mut versions: Vec<Version> = Vec::new(); - - for &(low, high) in self.iter() { - versions.extend(low..high + 1); - } - versions - } -} - -impl ProtoSet { - /// Get an iterator over the `(low, high)` `pairs` in this `ProtoSet`. - pub fn iter(&self) -> slice::Iter<(Version, Version)> { - self.pairs.iter() - } - - /// Expand this `ProtoSet` into a `Vec` of all its `Version`s. - /// - /// # Examples - /// - /// ``` - /// # use protover::errors::ProtoverError; - /// use protover::protoset::ProtoSet; - /// - /// # fn do_test() -> Result<bool, ProtoverError> { - /// let protoset: ProtoSet = "3-5,9".parse()?; - /// - /// assert_eq!(protoset.expand(), vec![3, 4, 5, 9]); - /// - /// let protoset: ProtoSet = "1,3,5-7".parse()?; - /// - /// assert_eq!(protoset.expand(), vec![1, 3, 5, 6, 7]); - /// # - /// # Ok(true) - /// # } - /// # fn main() { do_test(); } // wrap the test so we can use the ? operator - /// ``` - pub fn expand(self) -> Vec<Version> { - self.into() - } - - pub fn len(&self) -> usize { - let mut length: usize = 0; - - for &(low, high) in self.iter() { - length += (high as usize - low as usize) + 1; - } - - length - } - - /// Check that this `ProtoSet` is well-formed. - /// - /// This is automatically called in `ProtoSet::from_str()`. - /// - /// # Errors - /// - /// * `ProtoverError::LowGreaterThanHigh`: if its `pairs` were not - /// well-formed, i.e. a `low` in a `(low, high)` was higher than the - /// previous `high`, - /// * `ProtoverError::Overlap`: if one or more of the `pairs` are - /// overlapping, - /// * `ProtoverError::ExceedsMax`: if the number of versions when expanded - /// would exceed `MAX_PROTOCOLS_TO_EXPAND`, and - /// - /// # Returns - /// - /// A `Result` whose `Ok` is this `Protoset`, and whose `Err` is one of the - /// errors enumerated in the Errors section above. - fn is_ok(self) -> Result<ProtoSet, ProtoverError> { - let mut last_high: Version = 0; - - for &(low, high) in self.iter() { - if low == u32::MAX || high == u32::MAX { - return Err(ProtoverError::ExceedsMax); - } - if low <= last_high { - return Err(ProtoverError::Overlap); - } else if low > high { - return Err(ProtoverError::LowGreaterThanHigh); - } - last_high = high; - } - - Ok(self) - } - - /// Determine if this `ProtoSet` contains no `Version`s. - /// - /// # Returns - /// - /// * `true` if this `ProtoSet`'s length is zero, and - /// * `false` otherwise. - /// - /// # Examples - /// - /// ``` - /// use protover::protoset::ProtoSet; - /// - /// let protoset: ProtoSet = ProtoSet::default(); - /// - /// assert!(protoset.is_empty()); - /// ``` - pub fn is_empty(&self) -> bool { - self.pairs.len() == 0 - } - - /// Determine if `version` is included within this `ProtoSet`. - /// - /// # Inputs - /// - /// * `version`: a `Version`. - /// - /// # Returns - /// - /// `true` if the `version` is contained within this set; `false` otherwise. - /// - /// # Examples - /// - /// ``` - /// # use protover::errors::ProtoverError; - /// use protover::protoset::ProtoSet; - /// - /// # fn do_test() -> Result<ProtoSet, ProtoverError> { - /// let protoset: ProtoSet = ProtoSet::from_slice(&[(0, 5), (7, 9), (13, 14)])?; - /// - /// assert!(protoset.contains(&5)); - /// assert!(!protoset.contains(&10)); - /// # - /// # Ok(protoset) - /// # } - /// # fn main() { do_test(); } // wrap the test so we can use the ? operator - /// ``` - pub fn contains(&self, version: &Version) -> bool { - for &(low, high) in self.iter() { - if low <= *version && *version <= high { - return true; - } - } - false - } - - /// Returns all the `Version`s in `self` which are not also in the `other` - /// `ProtoSet`. - /// - /// # Examples - /// - /// ``` - /// # use protover::errors::ProtoverError; - /// use protover::protoset::ProtoSet; - /// - /// # fn do_test() -> Result<bool, ProtoverError> { - /// let protoset: ProtoSet = "1,3-6,10-12,15-16".parse()?; - /// let other: ProtoSet = "2,5-7,9-11,14-20".parse()?; - /// - /// let subset: ProtoSet = protoset.and_not_in(&other); - /// - /// assert_eq!(subset.expand(), vec![1, 3, 4, 12]); - /// # - /// # Ok(true) - /// # } - /// # fn main() { do_test(); } // wrap the test so we can use the ? operator - /// ``` - pub fn and_not_in(&self, other: &Self) -> Self { - if self.is_empty() || other.is_empty() { - return self.clone(); - } - - let pairs = self.iter().flat_map(|&(lo, hi)| { - let the_end = (hi + 1, hi + 1); // special case to mark the end of the range. - let excluded_ranges = other - .iter() - .cloned() // have to be owned tuples, to match iter::once(the_end). - .skip_while(move|&(_, hi2)| hi2 < lo) // skip the non-overlapping ranges. - .take_while(move|&(lo2, _)| lo2 <= hi) // take all the overlapping ones. - .chain(iter::once(the_end)); - - let mut nextlo = lo; - excluded_ranges.filter_map(move |(excluded_lo, excluded_hi)| { - let pair = if nextlo < excluded_lo { - Some((nextlo, excluded_lo - 1)) - } else { - None - }; - nextlo = cmp::min(excluded_hi, u32::MAX - 1) + 1; - pair - }) - }); - - let pairs = pairs.collect(); - ProtoSet::is_ok(ProtoSet { pairs }).expect("should be already sorted") - } -} - -/// Largest allowed protocol version. -/// C_RUST_COUPLED: protover.c `MAX_PROTOCOL_VERSION` -const MAX_PROTOCOL_VERSION: Version = 63; - -impl FromStr for ProtoSet { - type Err = ProtoverError; - - /// Parse the unique version numbers supported by a subprotocol from a string. - /// - /// # Inputs - /// - /// * `version_string`, a string comprised of "[0-9,-]" - /// - /// # Returns - /// - /// A `Result` whose `Ok` value is a `ProtoSet` holding all of the unique - /// version numbers. - /// - /// The returned `Result`'s `Err` value is an `ProtoverError` appropriate to - /// the error. - /// - /// # Errors - /// - /// This function will error if: - /// - /// * the `version_string` is an equals (`"="`) sign, - /// * the expansion of a version range produces an error (see - /// `expand_version_range`), - /// * any single version number is not parseable as an `u32` in radix 10, or - /// * there are greater than 2^16 version numbers to expand. - /// - /// # Examples - /// - /// ``` - /// use std::str::FromStr; - /// - /// use protover::errors::ProtoverError; - /// use protover::protoset::ProtoSet; - /// - /// # fn do_test() -> Result<ProtoSet, ProtoverError> { - /// let protoset: ProtoSet = ProtoSet::from_str("2-5,8")?; - /// - /// assert!(protoset.contains(&5)); - /// assert!(!protoset.contains(&10)); - /// - /// // We can also equivalently call `ProtoSet::from_str` by doing (all - /// // implementations of `FromStr` can be called this way, this one isn't - /// // special): - /// let protoset: ProtoSet = "4-6,12".parse()?; - /// - /// // Calling it (either way) can take really large ranges (up to `u32::MAX`): - /// let protoset: ProtoSet = "1-70000".parse()?; - /// let protoset: ProtoSet = "1-4294967296".parse()?; - /// - /// // There are lots of ways to get an `Err` from this function. Here are - /// // a few: - /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("=")); - /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-")); - /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("not_an_int")); - /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("3-")); - /// assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1-,4")); - /// - /// // An empty string is, however, legal, and results in an - /// // empty `ProtoSet`: - /// assert_eq!(Ok(ProtoSet::default()), ProtoSet::from_str("")); - /// # - /// # Ok(protoset) - /// # } - /// # fn main() { do_test(); } // wrap the test so we can use the ? operator - /// ``` - fn from_str(version_string: &str) -> Result<Self, Self::Err> { - // If we were passed in an empty string, then return an empty ProtoSet. - if version_string.is_empty() { - return Ok(Self::default()); - } - - let mut pairs: Vec<(Version, Version)> = Vec::new(); - let pieces: ::std::str::Split<char> = version_string.split(','); - - for p in pieces { - let (lo,hi) = if p.contains('-') { - let mut pair = p.splitn(2, '-'); - - let low = pair.next().ok_or(ProtoverError::Unparseable)?; - let high = pair.next().ok_or(ProtoverError::Unparseable)?; - - let lo: Version = low.parse().or(Err(ProtoverError::Unparseable))?; - let hi: Version = high.parse().or(Err(ProtoverError::Unparseable))?; - - (lo,hi) - } else { - let v: u32 = p.parse().or(Err(ProtoverError::Unparseable))?; - - (v, v) - }; - - if lo > MAX_PROTOCOL_VERSION || hi > MAX_PROTOCOL_VERSION { - return Err(ProtoverError::ExceedsMax); - } - pairs.push((lo, hi)); - } - - ProtoSet::from_slice(&pairs[..]) - } -} - -impl ToString for ProtoSet { - /// Contracts a `ProtoSet` of versions into a string. - /// - /// # Returns - /// - /// A `String` representation of this `ProtoSet` in ascending order. - fn to_string(&self) -> String { - let mut final_output: Vec<String> = Vec::new(); - - for &(lo, hi) in self.iter() { - if lo != hi { - debug_assert!(lo < hi); - final_output.push(format!("{}-{}", lo, hi)); - } else { - final_output.push(format!("{}", lo)); - } - } - final_output.join(",") - } -} - -/// Checks to see if there is a continuous range of integers, starting at the -/// first in the list. Returns the last integer in the range if a range exists. -/// -/// # Inputs -/// -/// `list`, an ordered vector of `u32` integers of "[0-9,-]" representing the -/// supported versions for a single protocol. -/// -/// # Returns -/// -/// A `bool` indicating whether the list contains a range, starting at the first -/// in the list, a`Version` of the last integer in the range, and a `usize` of -/// the index of that version. -/// -/// For example, if given vec![1, 2, 3, 5], find_range will return true, -/// as there is a continuous range, and 3, which is the last number in the -/// continuous range, and 2 which is the index of 3. -fn find_range(list: &Vec<Version>) -> (bool, Version, usize) { - if list.len() == 0 { - return (false, 0, 0); - } - - let mut index: usize = 0; - let mut iterable = list.iter().peekable(); - let mut range_end = match iterable.next() { - Some(n) => *n, - None => return (false, 0, 0), - }; - - let mut has_range = false; - - while iterable.peek().is_some() { - let n = *iterable.next().unwrap(); - if n != range_end + 1 { - break; - } - - has_range = true; - range_end = n; - index += 1; - } - - (has_range, range_end, index) -} - -impl From<Vec<Version>> for ProtoSet { - fn from(mut v: Vec<Version>) -> ProtoSet { - let mut version_pairs: Vec<(Version, Version)> = Vec::new(); - - v.sort_unstable(); - v.dedup(); - - 'vector: while !v.is_empty() { - let (has_range, end, index): (bool, Version, usize) = find_range(&v); - - if has_range { - let first: Version = match v.first() { - Some(x) => *x, - None => continue, - }; - let last: Version = match v.get(index) { - Some(x) => *x, - None => continue, - }; - debug_assert!(last == end, format!("last = {}, end = {}", last, end)); - - version_pairs.push((first, last)); - v = v.split_off(index + 1); - - if v.len() == 0 { - break 'vector; - } - } else { - let last: Version = match v.get(index) { - Some(x) => *x, - None => continue, - }; - version_pairs.push((last, last)); - v.remove(index); - } - } - ProtoSet::from_slice(&version_pairs[..]).unwrap_or(ProtoSet::default()) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_find_range() { - assert_eq!((false, 0, 0), find_range(&vec![])); - assert_eq!((false, 1, 0), find_range(&vec![1])); - assert_eq!((true, 2, 1), find_range(&vec![1, 2])); - assert_eq!((true, 3, 2), find_range(&vec![1, 2, 3])); - assert_eq!((true, 3, 2), find_range(&vec![1, 2, 3, 5])); - } - - macro_rules! assert_contains_each { - ($protoset:expr, $versions:expr) => { - for version in $versions { - assert!($protoset.contains(version)); - } - }; - } - - macro_rules! test_protoset_contains_versions { - ($list:expr, $str:expr) => { - let versions: &[Version] = $list; - let protoset: Result<ProtoSet, ProtoverError> = ProtoSet::from_str($str); - - assert!(protoset.is_ok()); - let p = protoset.unwrap(); - assert_contains_each!(p, versions); - }; - } - - #[test] - fn test_versions_from_str() { - test_protoset_contains_versions!(&[], ""); - test_protoset_contains_versions!(&[1], "1"); - test_protoset_contains_versions!(&[1, 2], "1,2"); - test_protoset_contains_versions!(&[1, 2, 3], "1-3"); - test_protoset_contains_versions!(&[1, 2, 5], "1-2,5"); - test_protoset_contains_versions!(&[1, 3, 4, 5], "1,3-5"); - test_protoset_contains_versions!(&[42, 55, 56, 57, 58], "42,55-58"); - } - - #[test] - fn test_versions_from_str_ab() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("a,b")); - } - - #[test] - fn test_versions_from_str_negative_1() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-1")); - } - - #[test] - fn test_versions_from_str_commas() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str(",")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,,2")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,2,")); - } - - #[test] - fn test_versions_from_str_hyphens() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("--1")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("-1-2")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1--2")); - } - - #[test] - fn test_versions_from_str_triple() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1-2-3")); - } - - #[test] - fn test_versions_from_str_1exclam() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,!")); - } - - #[test] - fn test_versions_from_str_percent_equal() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("%=")); - } - - #[test] - fn test_versions_from_str_whitespace() { - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,2\n")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1\r,2")); - assert_eq!(Err(ProtoverError::Unparseable), ProtoSet::from_str("1,\t2")); - } - - #[test] - fn test_versions_from_str_overlap() { - assert_eq!(Err(ProtoverError::Overlap), ProtoSet::from_str("1-3,2-4")); - } - - #[test] - fn test_versions_from_slice_overlap() { - assert_eq!( - Err(ProtoverError::Overlap), - ProtoSet::from_slice(&[(1, 3), (2, 4)]) - ); - } - - #[test] - fn test_versions_from_str_max() { - assert_eq!( - Err(ProtoverError::ExceedsMax), - ProtoSet::from_str("4294967295") - ); - } - - #[test] - fn test_versions_from_slice_max() { - assert_eq!( - Err(ProtoverError::ExceedsMax), - ProtoSet::from_slice(&[(4294967295, 4294967295)]) - ); - } - - #[test] - fn test_protoset_contains() { - let protoset: ProtoSet = ProtoSet::from_slice(&[(1, 5), (7, 9), (13, 14)]).unwrap(); - - for x in 1..6 { - assert!(protoset.contains(&x), format!("should contain {}", x)); - } - for x in 7..10 { - assert!(protoset.contains(&x), format!("should contain {}", x)); - } - for x in 13..15 { - assert!(protoset.contains(&x), format!("should contain {}", x)); - } - - for x in [6, 10, 11, 12, 15, 42, 43, 44, 45, 1234584].iter() { - assert!(!protoset.contains(&x), format!("should not contain {}", x)); - } - } - - #[test] - fn test_protoset_contains_1_3() { - let protoset: ProtoSet = ProtoSet::from_slice(&[(1, 3)]).unwrap(); - - for x in 1..4 { - assert!(protoset.contains(&x), format!("should contain {}", x)); - } - } - - macro_rules! assert_protoset_from_vec_contains_all { - ($($x:expr),*) => ( - let vec: Vec<Version> = vec!($($x),*); - let protoset: ProtoSet = vec.clone().into(); - - for x in vec.iter() { - assert!(protoset.contains(&x)); - } - ) - } - - #[test] - fn test_protoset_from_vec_123() { - assert_protoset_from_vec_contains_all!(1, 2, 3); - } - - #[test] - fn test_protoset_from_vec_1_315() { - assert_protoset_from_vec_contains_all!(1, 2, 3, 15); - } - - #[test] - fn test_protoset_from_vec_unordered() { - let v: Vec<Version> = vec![2, 3, 8, 4, 3, 9, 7, 2]; - let ps: ProtoSet = v.into(); - - assert_eq!(ps.to_string(), "2-4,7-9"); - } - - #[test] - fn test_protoset_into_vec() { - let ps: ProtoSet = "1-13,42".parse().unwrap(); - let v: Vec<Version> = ps.into(); - - assert!(v.contains(&7)); - assert!(v.contains(&42)); - } -} - -#[cfg(all(test, feature = "bench"))] -mod bench { - use super::*; -} diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs deleted file mode 100644 index da87509ffa..0000000000 --- a/src/rust/protover/protover.rs +++ /dev/null @@ -1,984 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -use std::collections::hash_map; -use std::collections::HashMap; -use std::ffi::CStr; -use std::fmt; -use std::str; -use std::str::FromStr; -use std::string::String; - -use external::c_tor_version_as_new_as; - -use errors::ProtoverError; -use protoset::ProtoSet; -use protoset::Version; - -/// The first version of Tor that included "proto" entries in its descriptors. -/// Authorities should use this to decide whether to guess proto lines. -/// -/// C_RUST_COUPLED: -/// protover.h `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS` -const FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS: &'static str = "0.2.9.3-alpha"; - -/// The maximum number of subprotocol version numbers we will attempt to expand -/// before concluding that someone is trying to DoS us -/// -/// C_RUST_COUPLED: protover.c `MAX_PROTOCOLS_TO_EXPAND` -const MAX_PROTOCOLS_TO_EXPAND: usize = 1 << 16; - -/// The maximum size an `UnknownProtocol`'s name may be. -pub(crate) const MAX_PROTOCOL_NAME_LENGTH: usize = 100; - -/// Known subprotocols in Tor. Indicates which subprotocol a relay supports. -/// -/// C_RUST_COUPLED: protover.h `protocol_type_t` -#[derive(Clone, Hash, Eq, PartialEq, Debug)] -pub enum Protocol { - Cons, - Desc, - DirCache, - HSDir, - HSIntro, - HSRend, - Link, - LinkAuth, - Microdesc, - Relay, - Padding, - FlowCtrl, -} - -impl fmt::Display for Protocol { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", self) - } -} - -/// Translates a string representation of a protocol into a Proto type. -/// Error if the string is an unrecognized protocol name. -/// -/// C_RUST_COUPLED: protover.c `PROTOCOL_NAMES` -impl FromStr for Protocol { - type Err = ProtoverError; - - fn from_str(s: &str) -> Result<Self, Self::Err> { - match s { - "Cons" => Ok(Protocol::Cons), - "Desc" => Ok(Protocol::Desc), - "DirCache" => Ok(Protocol::DirCache), - "HSDir" => Ok(Protocol::HSDir), - "HSIntro" => Ok(Protocol::HSIntro), - "HSRend" => Ok(Protocol::HSRend), - "Link" => Ok(Protocol::Link), - "LinkAuth" => Ok(Protocol::LinkAuth), - "Microdesc" => Ok(Protocol::Microdesc), - "Relay" => Ok(Protocol::Relay), - "Padding" => Ok(Protocol::Padding), - "FlowCtrl" => Ok(Protocol::FlowCtrl), - _ => Err(ProtoverError::UnknownProtocol), - } - } -} - -/// A protocol string which is not one of the `Protocols` we currently know -/// about. -#[derive(Clone, Debug, Hash, Eq, PartialEq)] -pub struct UnknownProtocol(String); - -impl fmt::Display for UnknownProtocol { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -fn is_valid_proto(s: &str) -> bool { - s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') -} - -impl FromStr for UnknownProtocol { - type Err = ProtoverError; - - fn from_str(s: &str) -> Result<Self, Self::Err> { - if !is_valid_proto(s) { - Err(ProtoverError::InvalidProtocol) - } else if s.len() <= MAX_PROTOCOL_NAME_LENGTH { - Ok(UnknownProtocol(s.to_string())) - } else { - Err(ProtoverError::ExceedsNameLimit) - } - } -} - -impl UnknownProtocol { - /// Create an `UnknownProtocol`, ignoring whether or not it - /// exceeds MAX_PROTOCOL_NAME_LENGTH. - fn from_str_any_len(s: &str) -> Result<Self, ProtoverError> { - if !is_valid_proto(s) { - return Err(ProtoverError::InvalidProtocol); - } - Ok(UnknownProtocol(s.to_string())) - } -} - -impl From<Protocol> for UnknownProtocol { - fn from(p: Protocol) -> UnknownProtocol { - UnknownProtocol(p.to_string()) - } -} - -#[cfg(feature = "test_linking_hack")] -fn have_linkauth_v1() -> bool { - true -} - -#[cfg(not(feature = "test_linking_hack"))] -fn have_linkauth_v1() -> bool { - use external::c_tor_is_using_nss; - !c_tor_is_using_nss() -} - -/// Get a CStr representation of current supported protocols, for -/// passing to C, or for converting to a `&str` for Rust. -/// -/// # Returns -/// -/// An `&'static CStr` whose value is the existing protocols supported by tor. -/// Returned data is in the format as follows: -/// -/// "HSDir=1-1 LinkAuth=1" -/// -/// # Note -/// -/// Rust code can use the `&'static CStr` as a normal `&'a str` by -/// calling `protover::get_supported_protocols`. -/// -// C_RUST_COUPLED: protover.c `protover_get_supported_protocols` -pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { - if !have_linkauth_v1() { - cstr!( - "Cons=1-2 \ - Desc=1-2 \ - DirCache=2 \ - FlowCtrl=1 \ - HSDir=1-2 \ - HSIntro=3-5 \ - HSRend=1-2 \ - Link=1-5 \ - LinkAuth=3 \ - Microdesc=1-2 \ - Padding=2 \ - Relay=1-3" - ) - } else { - cstr!( - "Cons=1-2 \ - Desc=1-2 \ - DirCache=2 \ - FlowCtrl=1 \ - HSDir=1-2 \ - HSIntro=3-5 \ - HSRend=1-2 \ - Link=1-5 \ - LinkAuth=1,3 \ - Microdesc=1-2 \ - Padding=2 \ - Relay=1-3" - ) - } -} - -/// A map of protocol names to the versions of them which are supported. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ProtoEntry(HashMap<Protocol, ProtoSet>); - -impl Default for ProtoEntry { - fn default() -> ProtoEntry { - ProtoEntry(HashMap::new()) - } -} - -impl ProtoEntry { - /// Get an iterator over the `Protocol`s and their `ProtoSet`s in this `ProtoEntry`. - pub fn iter(&self) -> hash_map::Iter<Protocol, ProtoSet> { - self.0.iter() - } - - /// Translate the supported tor versions from a string into a - /// ProtoEntry, which is useful when looking up a specific - /// subprotocol. - pub fn supported() -> Result<Self, ProtoverError> { - let supported_cstr: &'static CStr = get_supported_protocols_cstr(); - let supported: &str = supported_cstr.to_str().unwrap_or(""); - - supported.parse() - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn get(&self, protocol: &Protocol) -> Option<&ProtoSet> { - self.0.get(protocol) - } - - pub fn insert(&mut self, key: Protocol, value: ProtoSet) { - self.0.insert(key, value); - } - - pub fn remove(&mut self, key: &Protocol) -> Option<ProtoSet> { - self.0.remove(key) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -impl FromStr for ProtoEntry { - type Err = ProtoverError; - - /// Parse a string of subprotocol types and their version numbers. - /// - /// # Inputs - /// - /// * A `protocol_entry` string, comprised of a keywords, an "=" sign, and - /// one or more version numbers, each separated by a space. For example, - /// `"Cons=3-4 HSDir=1"`. - /// - /// # Returns - /// - /// A `Result` whose `Ok` value is a `ProtoEntry`. - /// Otherwise, the `Err` value of this `Result` is a `ProtoverError`. - fn from_str(protocol_entry: &str) -> Result<ProtoEntry, ProtoverError> { - let mut proto_entry: ProtoEntry = ProtoEntry::default(); - - if protocol_entry.is_empty() { - return Ok(proto_entry); - } - - let entries = protocol_entry.split(' '); - - for entry in entries { - let mut parts = entry.splitn(2, '='); - - let proto = match parts.next() { - Some(n) => n, - None => return Err(ProtoverError::Unparseable), - }; - - let vers = match parts.next() { - Some(n) => n, - None => return Err(ProtoverError::Unparseable), - }; - let versions: ProtoSet = vers.parse()?; - let proto_name: Protocol = proto.parse()?; - - proto_entry.insert(proto_name, versions); - - if proto_entry.len() > MAX_PROTOCOLS_TO_EXPAND { - return Err(ProtoverError::ExceedsMax); - } - } - Ok(proto_entry) - } -} - -/// Generate an implementation of `ToString` for either a `ProtoEntry` or an -/// `UnvalidatedProtoEntry`. -macro_rules! impl_to_string_for_proto_entry { - ($t:ty) => { - impl ToString for $t { - fn to_string(&self) -> String { - let mut parts: Vec<String> = Vec::new(); - - for (protocol, versions) in self.iter() { - parts.push(format!("{}={}", protocol.to_string(), versions.to_string())); - } - parts.sort_unstable(); - parts.join(" ") - } - } - }; -} - -impl_to_string_for_proto_entry!(ProtoEntry); -impl_to_string_for_proto_entry!(UnvalidatedProtoEntry); - -/// A `ProtoEntry`, but whose `Protocols` can be any `UnknownProtocol`, not just -/// the supported ones enumerated in `Protocols`. The protocol versions are -/// validated, however. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct UnvalidatedProtoEntry(HashMap<UnknownProtocol, ProtoSet>); - -impl Default for UnvalidatedProtoEntry { - fn default() -> UnvalidatedProtoEntry { - UnvalidatedProtoEntry(HashMap::new()) - } -} - -impl UnvalidatedProtoEntry { - /// Get an iterator over the `Protocol`s and their `ProtoSet`s in this `ProtoEntry`. - pub fn iter(&self) -> hash_map::Iter<UnknownProtocol, ProtoSet> { - self.0.iter() - } - - pub fn get(&self, protocol: &UnknownProtocol) -> Option<&ProtoSet> { - self.0.get(protocol) - } - - pub fn insert(&mut self, key: UnknownProtocol, value: ProtoSet) { - self.0.insert(key, value); - } - - pub fn remove(&mut self, key: &UnknownProtocol) -> Option<ProtoSet> { - self.0.remove(key) - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn len(&self) -> usize { - let mut total: usize = 0; - - for (_, versions) in self.iter() { - total += versions.len(); - } - total - } - - /// Determine if we support every protocol a client supports, and if not, - /// determine which protocols we do not have support for. - /// - /// # Returns - /// - /// Optionally, return parameters which the client supports but which we do not. - /// - /// # Examples - /// ``` - /// use protover::UnvalidatedProtoEntry; - /// - /// let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap(); - /// let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - /// assert_eq!(true, unsupported.is_none()); - /// - /// let protocols: UnvalidatedProtoEntry = "Link=1-2 Wombat=9".parse().unwrap(); - /// let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - /// assert_eq!(true, unsupported.is_some()); - /// assert_eq!("Wombat=9", &unsupported.unwrap().to_string()); - /// ``` - pub fn all_supported(&self) -> Option<UnvalidatedProtoEntry> { - let mut unsupported: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default(); - let supported: ProtoEntry = match ProtoEntry::supported() { - Ok(x) => x, - Err(_) => return None, - }; - - for (protocol, versions) in self.iter() { - let is_supported: Result<Protocol, ProtoverError> = protocol.0.parse(); - let supported_protocol: Protocol; - - // If the protocol wasn't even in the enum, then we definitely don't - // know about it and don't support any of its versions. - if is_supported.is_err() { - if !versions.is_empty() { - unsupported.insert(protocol.clone(), versions.clone()); - } - continue; - } else { - supported_protocol = is_supported.unwrap(); - } - - let maybe_supported_versions: Option<&ProtoSet> = supported.get(&supported_protocol); - let supported_versions: &ProtoSet; - - // If the protocol wasn't in the map, then we don't know about it - // and don't support any of its versions. Add its versions to the - // map (if it has versions). - if maybe_supported_versions.is_none() { - if !versions.is_empty() { - unsupported.insert(protocol.clone(), versions.clone()); - } - continue; - } else { - supported_versions = maybe_supported_versions.unwrap(); - } - let unsupported_versions = versions.and_not_in(supported_versions); - - if !unsupported_versions.is_empty() { - unsupported.insert(protocol.clone(), unsupported_versions); - } - } - - if unsupported.is_empty() { - return None; - } - Some(unsupported) - } - - /// Determine if we have support for some protocol and version. - /// - /// # Inputs - /// - /// * `proto`, an `UnknownProtocol` to test support for - /// * `vers`, a `Version` which we will go on to determine whether the - /// specified protocol supports. - /// - /// # Return - /// - /// Returns `true` iff this `UnvalidatedProtoEntry` includes support for the - /// indicated protocol and version, and `false` otherwise. - /// - /// # Examples - /// - /// ``` - /// # use std::str::FromStr; - /// use protover::*; - /// # use protover::errors::ProtoverError; - /// - /// # fn do_test () -> Result<UnvalidatedProtoEntry, ProtoverError> { - /// let proto: UnvalidatedProtoEntry = "Link=3-4 Cons=1 Doggo=3-5".parse()?; - /// assert_eq!(true, proto.supports_protocol(&Protocol::Cons.into(), &1)); - /// assert_eq!(false, proto.supports_protocol(&Protocol::Cons.into(), &5)); - /// assert_eq!(true, proto.supports_protocol(&UnknownProtocol::from_str("Doggo")?, &4)); - /// # Ok(proto) - /// # } fn main () { do_test(); } - /// ``` - pub fn supports_protocol(&self, proto: &UnknownProtocol, vers: &Version) -> bool { - let supported_versions: &ProtoSet = match self.get(proto) { - Some(n) => n, - None => return false, - }; - supported_versions.contains(&vers) - } - - /// As `UnvalidatedProtoEntry::supports_protocol()`, but also returns `true` - /// if any later version of the protocol is supported. - /// - /// # Examples - /// ``` - /// use protover::*; - /// # use protover::errors::ProtoverError; - /// - /// # fn do_test () -> Result<UnvalidatedProtoEntry, ProtoverError> { - /// let proto: UnvalidatedProtoEntry = "Link=3-4 Cons=5".parse()?; - /// - /// assert_eq!(true, proto.supports_protocol_or_later(&Protocol::Cons.into(), &5)); - /// assert_eq!(true, proto.supports_protocol_or_later(&Protocol::Cons.into(), &4)); - /// assert_eq!(false, proto.supports_protocol_or_later(&Protocol::Cons.into(), &6)); - /// # Ok(proto) - /// # } fn main () { do_test(); } - /// ``` - pub fn supports_protocol_or_later(&self, proto: &UnknownProtocol, vers: &Version) -> bool { - let supported_versions: &ProtoSet = match self.get(&proto) { - Some(n) => n, - None => return false, - }; - supported_versions.iter().any(|v| v.1 >= *vers) - } - - /// Split a string containing (potentially) several protocols and their - /// versions into a `Vec` of tuples of string in `(protocol, versions)` - /// form. - /// - /// # Inputs - /// - /// A &str in the form `"Link=3-4 Cons=5"`. - /// - /// # Returns - /// - /// A `Result` whose `Ok` variant is a `Vec<(&str, &str)>` of `(protocol, - /// versions)`, or whose `Err` variant is a `ProtoverError`. - /// - /// # Errors - /// - /// This will error with a `ProtoverError::Unparseable` if any of the - /// following are true: - /// - /// * If a protocol name is an empty string, e.g. `"Cons=1,3 =3-5"`. - /// * If an entry has no equals sign, e.g. `"Cons=1,3 Desc"`. - /// * If there is leading or trailing whitespace, e.g. `" Cons=1,3 Link=3"`. - /// * If there is any other extra whitespice, e.g. `"Cons=1,3 Link=3"`. - fn parse_protocol_and_version_str<'a>( - protocol_string: &'a str, - ) -> Result<Vec<(&'a str, &'a str)>, ProtoverError> { - let mut protovers: Vec<(&str, &str)> = Vec::new(); - - if protocol_string.is_empty() { - return Ok(protovers); - } - - for subproto in protocol_string.split(' ') { - let mut parts = subproto.splitn(2, '='); - - let name = match parts.next() { - Some("") => return Err(ProtoverError::Unparseable), - Some(n) => n, - None => return Err(ProtoverError::Unparseable), - }; - let vers = match parts.next() { - Some(n) => n, - None => return Err(ProtoverError::Unparseable), - }; - protovers.push((name, vers)); - } - Ok(protovers) - } -} - -impl FromStr for UnvalidatedProtoEntry { - type Err = ProtoverError; - - /// Parses a protocol list without validating the protocol names. - /// - /// # Inputs - /// - /// * `protocol_string`, a string comprised of keys and values, both which are - /// strings. The keys are the protocol names while values are a string - /// representation of the supported versions. - /// - /// The input is _not_ expected to be a subset of the Protocol types - /// - /// # Returns - /// - /// A `Result` whose `Ok` value is an `UnvalidatedProtoEntry`. - /// - /// The returned `Result`'s `Err` value is an `ProtoverError`. - /// - /// # Errors - /// - /// This function will error if: - /// - /// * The protocol string does not follow the "protocol_name=version_list" - /// expected format, or - /// * If the version string is malformed. See `impl FromStr for ProtoSet`. - fn from_str(protocol_string: &str) -> Result<UnvalidatedProtoEntry, ProtoverError> { - let mut parsed: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default(); - let parts: Vec<(&str, &str)> = - UnvalidatedProtoEntry::parse_protocol_and_version_str(protocol_string)?; - - for &(name, vers) in parts.iter() { - let versions = ProtoSet::from_str(vers)?; - let protocol = UnknownProtocol::from_str(name)?; - - parsed.insert(protocol, versions); - } - Ok(parsed) - } -} - -impl UnvalidatedProtoEntry { - /// Create an `UnknownProtocol`, ignoring whether or not it - /// exceeds MAX_PROTOCOL_NAME_LENGTH. - pub(crate) fn from_str_any_len( - protocol_string: &str, - ) -> Result<UnvalidatedProtoEntry, ProtoverError> { - let mut parsed: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default(); - let parts: Vec<(&str, &str)> = - UnvalidatedProtoEntry::parse_protocol_and_version_str(protocol_string)?; - - for &(name, vers) in parts.iter() { - let versions = ProtoSet::from_str(vers)?; - let protocol = UnknownProtocol::from_str_any_len(name)?; - - parsed.insert(protocol, versions); - } - Ok(parsed) - } -} - -/// Pretend a `ProtoEntry` is actually an `UnvalidatedProtoEntry`. -impl From<ProtoEntry> for UnvalidatedProtoEntry { - fn from(proto_entry: ProtoEntry) -> UnvalidatedProtoEntry { - let mut unvalidated: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default(); - - for (protocol, versions) in proto_entry.iter() { - unvalidated.insert(UnknownProtocol::from(protocol.clone()), versions.clone()); - } - unvalidated - } -} - -/// A mapping of protocols to a count of how many times each of their `Version`s -/// were voted for or supported. -/// -/// # Warning -/// -/// The "protocols" are *not* guaranteed to be known/supported `Protocol`s, in -/// order to allow new subprotocols to be introduced even if Directory -/// Authorities don't yet know of them. -pub struct ProtoverVote(HashMap<UnknownProtocol, HashMap<Version, usize>>); - -impl Default for ProtoverVote { - fn default() -> ProtoverVote { - ProtoverVote(HashMap::new()) - } -} - -impl IntoIterator for ProtoverVote { - type Item = (UnknownProtocol, HashMap<Version, usize>); - type IntoIter = hash_map::IntoIter<UnknownProtocol, HashMap<Version, usize>>; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -impl ProtoverVote { - pub fn entry( - &mut self, - key: UnknownProtocol, - ) -> hash_map::Entry<UnknownProtocol, HashMap<Version, usize>> { - self.0.entry(key) - } - - /// Protocol voting implementation. - /// - /// Given a slice of `UnvalidatedProtoEntry`s and a vote `threshold`, return - /// a new `UnvalidatedProtoEntry` encoding all of the protocols that are - /// listed by at least `threshold` of the inputs. - /// - /// # Examples - /// - /// ``` - /// use protover::ProtoverVote; - /// use protover::UnvalidatedProtoEntry; - /// - /// let protos: &[UnvalidatedProtoEntry] = &["Link=3-4".parse().unwrap(), - /// "Link=3".parse().unwrap()]; - /// let vote = ProtoverVote::compute(protos, &2); - /// assert_eq!("Link=3", vote.to_string()); - /// ``` - // C_RUST_COUPLED: protover.c protover_compute_vote - pub fn compute( - proto_entries: &[UnvalidatedProtoEntry], - threshold: &usize, - ) -> UnvalidatedProtoEntry { - let mut all_count: ProtoverVote = ProtoverVote::default(); - let mut final_output: UnvalidatedProtoEntry = UnvalidatedProtoEntry::default(); - - if proto_entries.is_empty() { - return final_output; - } - - // parse and collect all of the protos and their versions and collect them - for vote in proto_entries { - // C_RUST_DIFFERS: This doesn't actually differ, bu this check on - // the total is here to make it match. Because the C version calls - // expand_protocol_list() which checks if there would be too many - // subprotocols *or* individual version numbers, i.e. more than - // MAX_PROTOCOLS_TO_EXPAND, and does this *per vote*, we need to - // match it's behaviour and ensure we're not allowing more than it - // would. - if vote.len() > MAX_PROTOCOLS_TO_EXPAND { - continue; - } - - for (protocol, versions) in vote.iter() { - let supported_vers: &mut HashMap<Version, usize> = - all_count.entry(protocol.clone()).or_insert(HashMap::new()); - - for version in versions.clone().expand() { - let counter: &mut usize = supported_vers.entry(version).or_insert(0); - *counter += 1; - } - } - } - - for (protocol, mut versions) in all_count { - // Go through and remove versions that are less than the threshold - versions.retain(|_, count| *count as usize >= *threshold); - - if versions.len() > 0 { - let voted_versions: Vec<Version> = versions.keys().cloned().collect(); - let voted_protoset: ProtoSet = ProtoSet::from(voted_versions); - - final_output.insert(protocol, voted_protoset); - } - } - final_output - } -} - -/// Returns a boolean indicating whether the given protocol and version is -/// supported in any of the existing Tor protocols -/// -/// # Examples -/// ``` -/// use protover::is_supported_here; -/// use protover::Protocol; -/// -/// let is_supported = is_supported_here(&Protocol::Link, &10); -/// assert_eq!(false, is_supported); -/// -/// let is_supported = is_supported_here(&Protocol::Link, &1); -/// assert_eq!(true, is_supported); -/// ``` -pub fn is_supported_here(proto: &Protocol, vers: &Version) -> bool { - let currently_supported: ProtoEntry = match ProtoEntry::supported() { - Ok(result) => result, - Err(_) => return false, - }; - let supported_versions = match currently_supported.get(proto) { - Some(n) => n, - None => return false, - }; - supported_versions.contains(vers) -} - -/// Since older versions of Tor cannot infer their own subprotocols, -/// determine which subprotocols are supported by older Tor versions. -/// -/// # Inputs -/// -/// * `version`, a string comprised of "[0-9a-z.-]" -/// -/// # Returns -/// -/// A `&'static CStr` encoding a list of protocol names and supported -/// versions. The string takes the following format: -/// -/// "HSDir=1-1 LinkAuth=1" -/// -/// This function returns the protocols that are supported by the version input, -/// only for tor versions older than `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS` -/// (but not older than 0.2.4.19). For newer tors (or older than 0.2.4.19), it -/// returns an empty string. -/// -/// # Note -/// -/// This function is meant to be called for/within FFI code. If you'd -/// like to use this code in Rust, please see `compute_for_old_tor()`. -// -// C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor` -pub(crate) fn compute_for_old_tor_cstr(version: &str) -> &'static CStr { - let empty: &'static CStr = cstr!(""); - - if c_tor_version_as_new_as(version, FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS) { - return empty; - } - if c_tor_version_as_new_as(version, "0.2.9.1-alpha") { - return cstr!( - "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 \ - Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2" - ); - } - if c_tor_version_as_new_as(version, "0.2.7.5") { - return cstr!( - "Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ - Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2" - ); - } - if c_tor_version_as_new_as(version, "0.2.4.19") { - return cstr!( - "Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ - Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2" - ); - } - empty -} - -/// Since older versions of Tor cannot infer their own subprotocols, -/// determine which subprotocols are supported by older Tor versions. -/// -/// # Inputs -/// -/// * `version`, a string comprised of "[0-9a-z.-]" -/// -/// # Returns -/// -/// A `Result` whose `Ok` value is an `&'static str` encoding a list of protocol -/// names and supported versions. The string takes the following format: -/// -/// "HSDir=1-1 LinkAuth=1" -/// -/// This function returns the protocols that are supported by the version input, -/// only for tor versions older than `FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS`. -/// (but not older than 0.2.4.19). For newer tors (or older than 0.2.4.19), its -/// `Ok` `Result` contains an empty string. -/// -/// Otherwise, its `Err` contains a `ProtoverError::Unparseable` if the -/// `version` string was invalid utf-8. -/// -/// # Note -/// -/// This function is meant to be called for/within non-FFI Rust code. -// -// C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor` -pub fn compute_for_old_tor(version: &str) -> Result<&'static str, ProtoverError> { - // .to_str() fails with a Utf8Error if it couldn't validate the - // utf-8, so convert that here into an Unparseable ProtoverError. - compute_for_old_tor_cstr(version) - .to_str() - .or(Err(ProtoverError::Unparseable)) -} - -#[cfg(test)] -mod test { - use std::str::FromStr; - use std::string::ToString; - - use super::*; - - macro_rules! parse_proto { - ($e:expr) => {{ - let proto: Result<UnknownProtocol, _> = $e.parse(); - let proto2 = UnknownProtocol::from_str_any_len($e); - assert_eq!(proto, proto2); - proto - }}; - } - - #[test] - fn test_protocol_from_str() { - assert!(parse_proto!("Cons").is_ok()); - assert!(parse_proto!("123").is_ok()); - assert!(parse_proto!("1-2-3").is_ok()); - - let err = Err(ProtoverError::InvalidProtocol); - assert_eq!(err, parse_proto!("a_b_c")); - assert_eq!(err, parse_proto!("a b")); - assert_eq!(err, parse_proto!("a,")); - assert_eq!(err, parse_proto!("b.")); - assert_eq!(err, parse_proto!("é")); - } - - macro_rules! assert_protoentry_is_parseable { - ($e:expr) => { - let protoentry: Result<ProtoEntry, ProtoverError> = $e.parse(); - - assert!(protoentry.is_ok(), format!("{:?}", protoentry.err())); - }; - } - - macro_rules! assert_protoentry_is_unparseable { - ($e:expr) => { - let protoentry: Result<ProtoEntry, ProtoverError> = $e.parse(); - - assert!(protoentry.is_err()); - }; - } - - #[test] - fn test_protoentry_from_str_multiple_protocols_multiple_versions() { - assert_protoentry_is_parseable!("Cons=3-4 Link=1,3-5"); - } - - #[test] - fn test_protoentry_from_str_empty() { - assert_protoentry_is_parseable!(""); - assert!(UnvalidatedProtoEntry::from_str("").is_ok()); - } - - #[test] - fn test_protoentry_from_str_single_protocol_single_version() { - assert_protoentry_is_parseable!("HSDir=1"); - } - - #[test] - fn test_protoentry_from_str_unknown_protocol() { - assert_protoentry_is_unparseable!("Ducks=5-7,8"); - } - - #[test] - fn test_protoentry_from_str_allowed_number_of_versions() { - assert_protoentry_is_parseable!("Desc=1-63"); - } - - #[test] - fn test_protoentry_from_str_too_many_versions() { - assert_protoentry_is_unparseable!("Desc=1-64"); - } - - #[test] - fn test_protoentry_all_supported_single_protocol_single_version() { - let protocol: UnvalidatedProtoEntry = "Cons=1".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); - } - - #[test] - fn test_protoentry_all_supported_multiple_protocol_multiple_versions() { - let protocols: UnvalidatedProtoEntry = "Link=3-4 Desc=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); - } - - #[test] - fn test_protoentry_all_supported_three_values() { - let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); - } - - #[test] - fn test_protoentry_all_supported_unknown_protocol() { - let protocols: UnvalidatedProtoEntry = "Wombat=9".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("Wombat=9", &unsupported.unwrap().to_string()); - } - - #[test] - fn test_protoentry_all_supported_unsupported_high_version() { - let protocols: UnvalidatedProtoEntry = "HSDir=12-60".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("HSDir=12-60", &unsupported.unwrap().to_string()); - } - - #[test] - fn test_protoentry_all_supported_unsupported_low_version() { - let protocols: UnvalidatedProtoEntry = "HSIntro=2-3".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("HSIntro=2", &unsupported.unwrap().to_string()); - } - - #[test] - fn test_contract_protocol_list() { - let mut versions = ""; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1-2"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1,3"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1-4"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1,3,5-7"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - - versions = "1-3,50"; - assert_eq!( - String::from(versions), - ProtoSet::from_str(&versions).unwrap().to_string() - ); - } -} diff --git a/src/rust/protover/tests/protover.rs b/src/rust/protover/tests/protover.rs deleted file mode 100644 index a6305ac39a..0000000000 --- a/src/rust/protover/tests/protover.rs +++ /dev/null @@ -1,365 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -extern crate protover; - -use protover::errors::ProtoverError; -use protover::ProtoEntry; -use protover::ProtoverVote; -use protover::UnvalidatedProtoEntry; - -#[test] -fn parse_protocol_with_single_proto_and_single_version() { - let _: ProtoEntry = "Cons=1".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_single_protocol_and_multiple_versions() { - let _: ProtoEntry = "Cons=1-2".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_different_single_protocol_and_single_version() { - let _: ProtoEntry = "HSDir=1".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_single_protocol_and_supported_version() { - let _: ProtoEntry = "Desc=2".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_two_protocols_and_single_version() { - let _: ProtoEntry = "Cons=1 HSDir=1".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_single_protocol_and_two_sequential_versions() { - let _: ProtoEntry = "Desc=1-2".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_single_protocol_and_protocol_range() { - let _: ProtoEntry = "Link=1-4".parse().unwrap(); -} - -#[test] -fn parse_protocol_with_single_protocol_and_protocol_set() { - let _: ProtoEntry = "Link=3-4 Desc=2".parse().unwrap(); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_protocol_set() { - let protocols: UnvalidatedProtoEntry = "Link=3-4 Desc=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_two_values() { - let protocols: UnvalidatedProtoEntry = "Microdesc=1-2 Relay=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_one_value() { - let protocols: UnvalidatedProtoEntry = "Microdesc=1-2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_three_values() { - let protocols: UnvalidatedProtoEntry = "LinkAuth=1 Microdesc=1-2 Relay=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_unsupported_protocol() { - let protocols: UnvalidatedProtoEntry = "Wombat=9".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("Wombat=9", &unsupported.unwrap().to_string()); -} - -#[test] -fn protocol_all_supported_with_unsupported_versions() { - let protocols: UnvalidatedProtoEntry = "Link=3-63".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("Link=6-63", &unsupported.unwrap().to_string()); -} - -#[test] -fn protocol_all_supported_with_unsupported_low_version() { - let protocols: UnvalidatedProtoEntry = "HSIntro=2-3".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("HSIntro=2", &unsupported.unwrap().to_string()); -} - -#[test] -fn protocol_all_supported_with_unsupported_high_version() { - let protocols: UnvalidatedProtoEntry = "Cons=1-2,60".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("Cons=60", &unsupported.unwrap().to_string()); -} - -#[test] -fn protocol_all_supported_with_mix_of_supported_and_unsupproted() { - let protocols: UnvalidatedProtoEntry = "Link=3-4 Wombat=9".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_some()); - assert_eq!("Wombat=9", &unsupported.unwrap().to_string()); -} - -#[test] -fn protover_string_supports_protocol_returns_true_for_single_supported() { - let protocols: UnvalidatedProtoEntry = "Link=3-4 Cons=1".parse().unwrap(); - let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &1); - assert_eq!(true, is_supported); -} - -#[test] -fn protover_string_supports_protocol_returns_false_for_single_unsupported() { - let protocols: UnvalidatedProtoEntry = "Link=3-4 Cons=1".parse().unwrap(); - let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &2); - assert_eq!(false, is_supported); -} - -#[test] -fn protover_string_supports_protocol_returns_false_for_unsupported() { - let protocols: UnvalidatedProtoEntry = "Link=3-4".parse().unwrap(); - let is_supported = protocols.supports_protocol(&protover::Protocol::Cons.into(), &2); - assert_eq!(false, is_supported); -} - -#[test] -#[should_panic] -fn parse_protocol_with_unexpected_characters() { - let _: UnvalidatedProtoEntry = "Cons=*-%".parse().unwrap(); -} - -#[test] -fn protover_compute_vote_returns_empty_for_empty_string() { - let protocols: &[UnvalidatedProtoEntry] = &["".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_single_protocol_for_matching() { - let protocols: &[UnvalidatedProtoEntry] = &["Cons=1".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("Cons=1", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_two_protocols_for_two_matching() { - let protocols: &[UnvalidatedProtoEntry] = &["Link=1 Cons=1".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("Cons=1 Link=1", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_one_protocol_when_one_out_of_two_matches() { - let protocols: &[UnvalidatedProtoEntry] = - &["Cons=1 Link=2".parse().unwrap(), "Cons=1".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &2); - assert_eq!("Cons=1", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_protocols_that_it_doesnt_currently_support() { - let protocols: &[UnvalidatedProtoEntry] = - &["Foo=1 Cons=2".parse().unwrap(), "Bar=1".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("Bar=1 Cons=2 Foo=1", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_matching_for_mix() { - let protocols: &[UnvalidatedProtoEntry] = &["Link=1-10,50 Cons=1,3-7,8".parse().unwrap()]; - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("Cons=1,3-8 Link=1-10,50", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_matching_for_longer_mix() { - let protocols: &[UnvalidatedProtoEntry] = &[ - "Desc=1-10,50 Cons=1,3-7,8".parse().unwrap(), - "Link=12-45,8 Cons=2-6,8 Desc=9".parse().unwrap(), - ]; - - let listed = ProtoverVote::compute(protocols, &1); - assert_eq!("Cons=1-8 Desc=1-10,50 Link=8,12-45", listed.to_string()); -} - -#[test] -fn protover_compute_vote_returns_matching_for_longer_mix_with_threshold_two() { - let protocols: &[UnvalidatedProtoEntry] = &[ - "Desc=1-10,50 Cons=1,3-7,8".parse().unwrap(), - "Link=8,12-45 Cons=2-6,8 Desc=9".parse().unwrap(), - ]; - - let listed = ProtoverVote::compute(protocols, &2); - assert_eq!("Cons=3-6,8 Desc=9", listed.to_string()); -} - -#[test] -fn protover_compute_vote_handles_duplicated_versions() { - let protocols: &[UnvalidatedProtoEntry] = - &["Cons=1".parse().unwrap(), "Cons=1".parse().unwrap()]; - assert_eq!("Cons=1", ProtoverVote::compute(protocols, &2).to_string()); - - let protocols: &[UnvalidatedProtoEntry] = - &["Cons=1-2".parse().unwrap(), "Cons=1-2".parse().unwrap()]; - assert_eq!("Cons=1-2", ProtoverVote::compute(protocols, &2).to_string()); -} - -#[test] -fn protover_compute_vote_handles_invalid_proto_entries() { - let protocols: &[UnvalidatedProtoEntry] = &[ - "Cons=1".parse().unwrap(), - "Cons=1".parse().unwrap(), - "Dinosaur=1".parse().unwrap(), - ]; - assert_eq!("Cons=1", ProtoverVote::compute(protocols, &2).to_string()); -} - -#[test] -fn parse_protocol_with_single_protocol_and_two_nonsequential_versions() { - let _: ProtoEntry = "Desc=1,2".parse().unwrap(); -} - -#[test] -fn protover_is_supported_here_returns_true_for_supported_protocol() { - assert_eq!( - true, - protover::is_supported_here(&protover::Protocol::Cons, &1) - ); -} - -#[test] -fn protover_is_supported_here_returns_false_for_unsupported_protocol() { - assert_eq!( - false, - protover::is_supported_here(&protover::Protocol::Cons, &5) - ); -} - -#[test] -fn protocol_all_supported_with_single_proto_and_single_version() { - let protocol: UnvalidatedProtoEntry = "Cons=1".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_multiple_versions() { - let protocol: UnvalidatedProtoEntry = "Cons=1-2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_different_single_protocol_and_single_version() { - let protocol: UnvalidatedProtoEntry = "HSDir=1".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_supported_version() { - let protocol: UnvalidatedProtoEntry = "Desc=2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_two_protocols_and_single_version() { - let protocols: UnvalidatedProtoEntry = "Cons=1 HSDir=1".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocols.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_two_nonsequential_versions() { - let protocol: UnvalidatedProtoEntry = "Desc=1,2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_two_sequential_versions() { - let protocol: UnvalidatedProtoEntry = "Desc=1-2".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protocol_all_supported_with_single_protocol_and_protocol_range() { - let protocol: UnvalidatedProtoEntry = "Link=1-4".parse().unwrap(); - let unsupported: Option<UnvalidatedProtoEntry> = protocol.all_supported(); - assert_eq!(true, unsupported.is_none()); -} - -#[test] -fn protover_all_supported_should_exclude_versions_we_actually_do_support() { - let proto: UnvalidatedProtoEntry = "Link=3-63".parse().unwrap(); - let result: String = proto.all_supported().unwrap().to_string(); - - assert_eq!(result, "Link=6-63".to_string()); -} - -#[test] -fn protover_all_supported_should_exclude_versions_we_actually_do_support_complex1() { - let proto: UnvalidatedProtoEntry = "Link=1-3,30-63".parse().unwrap(); - let result: String = proto.all_supported().unwrap().to_string(); - - assert_eq!(result, "Link=30-63".to_string()); -} - -#[test] -fn protover_all_supported_should_exclude_versions_we_actually_do_support_complex2() { - let proto: UnvalidatedProtoEntry = "Link=1-3,5-12".parse().unwrap(); - let result: String = proto.all_supported().unwrap().to_string(); - - assert_eq!(result, "Link=6-12".to_string()); -} - -#[test] -fn protover_all_supported_should_exclude_some_versions_and_entire_protocols() { - let proto: UnvalidatedProtoEntry = "Link=1-3,5-12 Quokka=50-51".parse().unwrap(); - let result: String = proto.all_supported().unwrap().to_string(); - - assert_eq!(result, "Link=6-12 Quokka=50-51".to_string()); -} - -#[test] -// C_RUST_DIFFERS: The C will return true (e.g. saying "yes, that's supported") -// but set the msg to NULL (??? seems maybe potentially bad). The Rust will -// simply return a None. -fn protover_all_supported_should_return_empty_string_for_weird_thing() { - let proto: UnvalidatedProtoEntry = "Fribble=".parse().unwrap(); - let result: Option<UnvalidatedProtoEntry> = proto.all_supported(); - - assert!(result.is_none()); -} - -#[test] -fn protover_unvalidatedprotoentry_should_err_entirely_unparseable_things() { - let proto: Result<UnvalidatedProtoEntry, ProtoverError> = "Fribble".parse(); - - assert_eq!(Err(ProtoverError::Unparseable), proto); -} - -#[test] -fn protover_all_supported_over_maximum_limit() { - let proto: Result<UnvalidatedProtoEntry, ProtoverError> = "Sleen=1-4294967295".parse(); - - assert_eq!(Err(ProtoverError::ExceedsMax), proto); -} diff --git a/src/rust/smartlist/Cargo.toml b/src/rust/smartlist/Cargo.toml deleted file mode 100644 index a5afe7bf74..0000000000 --- a/src/rust/smartlist/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -authors = ["The Tor Project"] -version = "0.0.1" -name = "smartlist" - -[dependencies] -libc = "0.2.39" - -[lib] -name = "smartlist" -path = "lib.rs" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/smartlist/lib.rs b/src/rust/smartlist/lib.rs deleted file mode 100644 index 23301f88c3..0000000000 --- a/src/rust/smartlist/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -extern crate libc; - -mod smartlist; - -pub use smartlist::*; - -// When testing we may be compiled with sanitizers which are incompatible with -// Rust's default allocator, jemalloc (unsure why at this time). Most crates -// link to `tor_allocate` which switches by default to a non-jemalloc allocator, -// but we don't already depend on `tor_allocate` so make sure that while testing -// we don't use jemalloc. (but rather malloc/free) -#[global_allocator] -#[cfg(test)] -static A: std::alloc::System = std::alloc::System; diff --git a/src/rust/smartlist/smartlist.rs b/src/rust/smartlist/smartlist.rs deleted file mode 100644 index d8f8083dff..0000000000 --- a/src/rust/smartlist/smartlist.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -use libc::{c_char, c_int}; -use std::ffi::CStr; -use std::slice; - -/// Smartlists are a type used in C code in tor to define a collection of a -/// generic type, which has a capacity and a number used. Each Smartlist -/// defines how to extract the list of values from the underlying C structure -/// -/// Implementations are required to have a C representation, as this module -/// serves purely to translate smartlists as defined in tor to vectors in Rust. -pub trait Smartlist<T> { - fn get_list(&self) -> Vec<T>; -} - -#[repr(C)] -pub struct Stringlist { - pub list: *const *const c_char, - pub num_used: c_int, - pub capacity: c_int, -} - -impl Smartlist<String> for Stringlist { - fn get_list(&self) -> Vec<String> { - let empty: Vec<String> = Vec::new(); - let mut rust_list: Vec<String> = Vec::new(); - - if self.list.is_null() || self.num_used == 0 { - return empty; - } - - // unsafe, as we need to extract the smartlist list into a vector of - // pointers, and then transform each element into a Rust string. - let elems: &[*const c_char] = - unsafe { slice::from_raw_parts(self.list, self.num_used as usize) }; - - for elem in elems.iter() { - if elem.is_null() { - continue; - } - - // unsafe, as we need to create a cstring from the referenced - // element - let c_string = unsafe { CStr::from_ptr(*elem) }; - - let r_string = match c_string.to_str() { - Ok(n) => n, - Err(_) => return empty, - }; - - rust_list.push(String::from(r_string)); - } - - rust_list - } -} - -// TODO: CHK: this module maybe should be tested from a test in C with a -// smartlist as defined in tor. -#[cfg(test)] -mod test { - #[test] - fn test_get_list_of_strings() { - extern crate libc; - - use libc::c_char; - use std::ffi::CString; - - use super::Smartlist; - use super::Stringlist; - - { - // test to verify that null pointers are gracefully handled - use std::ptr; - - let sl = Stringlist { - list: ptr::null(), - num_used: 0, - capacity: 0, - }; - - let data = sl.get_list(); - assert_eq!(0, data.len()); - } - - { - let args = vec![String::from("a"), String::from("b")]; - - // for each string, transform it into a CString - let c_strings: Vec<_> = args - .iter() - .map(|arg| CString::new(arg.as_str()).unwrap()) - .collect(); - - // then, collect a pointer for each CString - let p_args: Vec<_> = c_strings.iter().map(|arg| arg.as_ptr()).collect(); - - let p: *const *const c_char = p_args.as_ptr(); - - // This is the representation that we expect when receiving a - // smartlist at the Rust/C FFI layer. - let sl = Stringlist { - list: p, - num_used: 2, - capacity: 2, - }; - - let data = sl.get_list(); - assert_eq!("a", &data[0]); - assert_eq!("b", &data[1]); - } - } -} diff --git a/src/rust/tor_allocate/Cargo.toml b/src/rust/tor_allocate/Cargo.toml deleted file mode 100644 index 06ac605f17..0000000000 --- a/src/rust/tor_allocate/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -authors = ["The Tor Project"] -version = "0.0.1" -name = "tor_allocate" - -[dependencies] -libc = "=0.2.39" - -[lib] -name = "tor_allocate" -path = "lib.rs" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/tor_allocate/lib.rs b/src/rust/tor_allocate/lib.rs deleted file mode 100644 index fff8a08006..0000000000 --- a/src/rust/tor_allocate/lib.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -//! Allocation helper functions that allow data to be allocated in Rust -//! using tor's specified allocator. In doing so, this can be later freed -//! from C. -//! -//! This is currently a temporary solution, we will later use tor's allocator -//! by default for any allocation that occurs in Rust. However, as this will -//! stabalize in 2018, we can use this as a temporary measure. - -extern crate libc; - -use std::alloc::System; - -mod tor_allocate; -pub use tor_allocate::*; - -#[global_allocator] -static A: System = System; diff --git a/src/rust/tor_allocate/tor_allocate.rs b/src/rust/tor_allocate/tor_allocate.rs deleted file mode 100644 index 7b35e2451f..0000000000 --- a/src/rust/tor_allocate/tor_allocate.rs +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ -// No-op defined purely for testing at the module level -use libc::c_char; - -use libc::c_void; -#[cfg(not(feature = "testing"))] -use std::{mem, ptr, slice}; - -// Define a no-op implementation for testing Rust modules without linking to C -#[cfg(feature = "testing")] -pub fn allocate_and_copy_string(s: &str) -> *mut c_char { - use std::ffi::CString; - CString::new(s).unwrap().into_raw() -} - -// Defined only for tests, used for testing purposes, so that we don't need -// to link to tor C files. Uses the system allocator -#[cfg(test)] -unsafe extern "C" fn tor_malloc_(size: usize) -> *mut c_void { - use libc::malloc; - malloc(size) -} - -#[cfg(all(not(test), not(feature = "testing")))] -extern "C" { - fn tor_malloc_(size: usize) -> *mut c_void; -} - -/// Allocate memory using tor_malloc_ and copy an existing string into the -/// allocated buffer, returning a pointer that can later be called in C. -/// -/// # Inputs -/// -/// * `src`, a reference to a String. -/// -/// # Returns -/// -/// A `*mut c_char` that should be freed by tor_free in C -/// -#[cfg(not(feature = "testing"))] -pub fn allocate_and_copy_string(src: &str) -> *mut c_char { - let bytes: &[u8] = src.as_bytes(); - - let size = mem::size_of_val::<[u8]>(bytes); - let size_one_byte = mem::size_of::<u8>(); - - // handle integer overflow when adding one to the calculated length - let size_with_null_byte = match size.checked_add(size_one_byte) { - Some(n) => n, - None => return ptr::null_mut(), - }; - - let dest = unsafe { tor_malloc_(size_with_null_byte) as *mut u8 }; - - if dest.is_null() { - return ptr::null_mut(); - } - - unsafe { ptr::copy_nonoverlapping(bytes.as_ptr(), dest, size) }; - - // set the last byte as null, using the ability to index into a slice - // rather than doing pointer arithmetic - let slice = unsafe { slice::from_raw_parts_mut(dest, size_with_null_byte) }; - slice[size] = 0; // add a null terminator - - dest as *mut c_char -} - -#[cfg(test)] -mod test { - - #[test] - fn test_allocate_and_copy_string_with_empty() { - use libc::{c_void, free}; - use std::ffi::CStr; - - use tor_allocate::allocate_and_copy_string; - - let allocated_empty = allocate_and_copy_string(""); - - let allocated_empty_rust = unsafe { CStr::from_ptr(allocated_empty).to_str().unwrap() }; - - assert_eq!("", allocated_empty_rust); - - unsafe { free(allocated_empty as *mut c_void) }; - } - - #[test] - fn test_allocate_and_copy_string_with_not_empty_string() { - use libc::{c_void, free}; - use std::ffi::CStr; - - use tor_allocate::allocate_and_copy_string; - - let allocated_empty = allocate_and_copy_string("foo bar biz"); - - let allocated_empty_rust = unsafe { CStr::from_ptr(allocated_empty).to_str().unwrap() }; - - assert_eq!("foo bar biz", allocated_empty_rust); - - unsafe { free(allocated_empty as *mut c_void) }; - } -} diff --git a/src/rust/tor_log/Cargo.toml b/src/rust/tor_log/Cargo.toml deleted file mode 100644 index 14d9ae803a..0000000000 --- a/src/rust/tor_log/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "tor_log" -version = "0.1.0" -authors = ["The Tor Project"] - -[lib] -name = "tor_log" -path = "lib.rs" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] - -[dependencies] -libc = "0.2.39" - -[dependencies.tor_allocate] -path = "../tor_allocate" diff --git a/src/rust/tor_log/lib.rs b/src/rust/tor_log/lib.rs deleted file mode 100644 index 4aa658e35b..0000000000 --- a/src/rust/tor_log/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Copyright (c) 2016-2019, The Tor Project, Inc. */ -//! See LICENSE for licensing information */ - -//! Logging wrapper for Rust to utilize Tor's logger, found at -//! src/common/log.c and src/common/torlog.h -//! -//! Exposes different interfaces depending on whether we are running in test -//! or non-test mode. When testing, we use a no-op implementation, -//! otherwise we link directly to C. - -extern crate libc; -extern crate tor_allocate; - -mod tor_log; - -pub use tor_log::*; diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs deleted file mode 100644 index 391cb32ab3..0000000000 --- a/src/rust/tor_log/tor_log.rs +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -// Note that these functions are untested due to the fact that there are no -// return variables to test and they are calling into a C API. - -/// The related domain which the logging message is relevant. For example, -/// log messages relevant to networking would use LogDomain::LdNet, whereas -/// general messages can use LdGeneral. -#[derive(Eq, PartialEq)] -pub enum LogDomain { - Net, - General, -} - -/// The severity level at which to log messages. -#[derive(Eq, PartialEq)] -pub enum LogSeverity { - Notice, - Warn, -} - -/// Main entry point for Rust modules to log messages. -/// -/// # Inputs -/// -/// * A `severity` of type LogSeverity, which defines the level of severity the -/// message will be logged. -/// * A `domain` of type LogDomain, which defines the domain the log message -/// will be associated with. -/// * A `function` of type &str, which defines the name of the function where -/// the message is being logged. There is a current RFC for a macro that -/// defines function names. When it is, we should use it. See -/// https://github.com/rust-lang/rfcs/pull/1719 -/// * A `message` of type &str, which is the log message itself. -#[macro_export] -macro_rules! tor_log_msg { - ($severity: path, - $domain: path, - $function: expr, - $($message:tt)*) => - { - { - let msg = format!($($message)*); - $crate::tor_log_msg_impl($severity, $domain, $function, msg) - } - }; -} - -#[inline] -pub fn tor_log_msg_impl(severity: LogSeverity, domain: LogDomain, function: &str, message: String) { - use std::ffi::CString; - - /// Default function name to log in case of errors when converting - /// a function name to a CString - const ERR_LOG_FUNCTION: &str = "tor_log_msg"; - - /// Default message to log in case of errors when converting a log - /// message to a CString - const ERR_LOG_MSG: &str = "Unable to log message from Rust \ - module due to error when converting to CString"; - - let func = match CString::new(function) { - Ok(n) => n, - Err(_) => CString::new(ERR_LOG_FUNCTION).unwrap(), - }; - - let msg = match CString::new(message) { - Ok(n) => n, - Err(_) => CString::new(ERR_LOG_MSG).unwrap(), - }; - - // Bind to a local variable to preserve ownership. This is essential so - // that ownership is guaranteed until these local variables go out of scope - let func_ptr = func.as_ptr(); - let msg_ptr = msg.as_ptr(); - - let c_severity = unsafe { log::translate_severity(severity) }; - let c_domain = unsafe { log::translate_domain(domain) }; - - unsafe { log::tor_log_string(c_severity, c_domain, func_ptr, msg_ptr) } -} - -/// This implementation is used when compiling for actual use, as opposed to -/// testing. -#[cfg(not(test))] -pub mod log { - use super::LogDomain; - use super::LogSeverity; - use libc::{c_char, c_int}; - - /// Severity log types. These mirror definitions in src/lib/log/log.h - /// C_RUST_COUPLED: src/lib/log/log.c, log domain types - extern "C" { - static LOG_WARN_: c_int; - static LOG_NOTICE_: c_int; - } - - /// Domain log types. These mirror definitions in src/lib/log/log.h - /// C_RUST_COUPLED: src/lib/log/log.c, log severity types - extern "C" { - static LD_NET_: u64; - static LD_GENERAL_: u64; - } - - /// Translate Rust definitions of log domain levels to C. This exposes a 1:1 - /// mapping between types. - #[inline] - pub unsafe fn translate_domain(domain: LogDomain) -> u64 { - match domain { - LogDomain::Net => LD_NET_, - LogDomain::General => LD_GENERAL_, - } - } - - /// Translate Rust definitions of log severity levels to C. This exposes a - /// 1:1 mapping between types. - #[inline] - pub unsafe fn translate_severity(severity: LogSeverity) -> c_int { - match severity { - LogSeverity::Warn => LOG_WARN_, - LogSeverity::Notice => LOG_NOTICE_, - } - } - - /// The main entry point into Tor's logger. When in non-test mode, this - /// will link directly with `tor_log_string` in torlog.c - extern "C" { - pub fn tor_log_string( - severity: c_int, - domain: u64, - function: *const c_char, - string: *const c_char, - ); - } -} - -/// This module exposes no-op functionality for testing other Rust modules -/// without linking to C. -#[cfg(test)] -pub mod log { - use super::LogDomain; - use super::LogSeverity; - use libc::{c_char, c_int}; - - pub static mut LAST_LOGGED_FUNCTION: *mut String = 0 as *mut String; - pub static mut LAST_LOGGED_MESSAGE: *mut String = 0 as *mut String; - - pub unsafe fn tor_log_string( - _severity: c_int, - _domain: u32, - function: *const c_char, - message: *const c_char, - ) { - use std::ffi::CStr; - - let f = CStr::from_ptr(function); - let fct = match f.to_str() { - Ok(n) => n, - Err(_) => "", - }; - LAST_LOGGED_FUNCTION = Box::into_raw(Box::new(String::from(fct))); - - let m = CStr::from_ptr(message); - let msg = match m.to_str() { - Ok(n) => n, - Err(_) => "", - }; - LAST_LOGGED_MESSAGE = Box::into_raw(Box::new(String::from(msg))); - } - - pub unsafe fn translate_domain(_domain: LogDomain) -> u32 { - 1 - } - - pub unsafe fn translate_severity(_severity: LogSeverity) -> c_int { - 1 - } -} - -#[cfg(test)] -mod test { - use tor_log::log::{LAST_LOGGED_FUNCTION, LAST_LOGGED_MESSAGE}; - use tor_log::*; - - #[test] - fn test_get_log_message() { - { - fn test_macro() { - tor_log_msg!( - LogSeverity::Warn, - LogDomain::Net, - "test_macro", - "test log message {}", - "a", - ); - } - - test_macro(); - - let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; - assert_eq!("test_macro", *function); - - let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; - assert_eq!("test log message a", *message); - } - - // test multiple inputs into the log message - { - fn test_macro() { - tor_log_msg!( - LogSeverity::Warn, - LogDomain::Net, - "next_test_macro", - "test log message {} {} {} {} {}", - 1, - 2, - 3, - 4, - 5 - ); - } - - test_macro(); - - let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; - assert_eq!("next_test_macro", *function); - - let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; - assert_eq!("test log message 1 2 3 4 5", *message); - } - - // test how a long log message will be formatted - { - fn test_macro() { - tor_log_msg!( - LogSeverity::Warn, - LogDomain::Net, - "test_macro", - "{}", - "All the world's a stage, and all the men and women \ - merely players: they have their exits and their \ - entrances; and one man in his time plays many parts, his \ - acts being seven ages." - ); - } - - test_macro(); - - let expected_string = "All the world's a \ - stage, and all the men \ - and women merely players: \ - they have their exits and \ - their entrances; and one man \ - in his time plays many parts, \ - his acts being seven ages."; - - let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; - assert_eq!("test_macro", *function); - - let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; - assert_eq!(expected_string, *message); - } - } -} diff --git a/src/rust/tor_rust/Cargo.toml b/src/rust/tor_rust/Cargo.toml deleted file mode 100644 index 35c629882e..0000000000 --- a/src/rust/tor_rust/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -authors = ["The Tor Project"] -name = "tor_rust" -version = "0.1.0" - -[lib] -name = "tor_rust" -path = "lib.rs" -crate_type = ["staticlib"] - -[dependencies.tor_util] -path = "../tor_util" - -[dependencies.protover] -path = "../protover" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/tor_rust/include.am b/src/rust/tor_rust/include.am deleted file mode 100644 index ce673abbee..0000000000 --- a/src/rust/tor_rust/include.am +++ /dev/null @@ -1,28 +0,0 @@ -EXTRA_DIST +=\ - src/rust/tor_rust/Cargo.toml \ - src/rust/tor_rust/lib.rs - -EXTRA_CARGO_OPTIONS= - -@TOR_RUST_LIB_PATH@: FORCE - ( cd "$(abs_top_builddir)/src/rust" ; \ - CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \ - $(CARGO) build --release $(EXTRA_CARGO_OPTIONS) \ - $(CARGO_ONLINE) \ - --manifest-path "$(abs_top_srcdir)/src/rust/tor_rust/Cargo.toml" ) - -distclean-rust: - ( cd "$(abs_top_builddir)/src/rust" ; \ - CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \ - $(CARGO) clean $(EXTRA_CARGO_OPTIONS) \ - $(CARGO_ONLINE) \ - --manifest-path "$(abs_top_srcdir)/src/rust/tor_rust/Cargo.toml" ) - rm -rf "$(abs_top_builddir)/src/rust/registry" - -if USE_RUST -build-rust: @TOR_RUST_LIB_PATH@ -else -build-rust: -endif - -FORCE: diff --git a/src/rust/tor_rust/lib.rs b/src/rust/tor_rust/lib.rs deleted file mode 100644 index 18519f8497..0000000000 --- a/src/rust/tor_rust/lib.rs +++ /dev/null @@ -1,5 +0,0 @@ -extern crate protover; -extern crate tor_util; - -pub use protover::*; -pub use tor_util::*; diff --git a/src/rust/tor_util/Cargo.toml b/src/rust/tor_util/Cargo.toml deleted file mode 100644 index 9ffaeda8a6..0000000000 --- a/src/rust/tor_util/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -authors = ["The Tor Project"] -name = "tor_util" -version = "0.0.1" - -[lib] -name = "tor_util" -path = "lib.rs" - -[dependencies.tor_allocate] -path = "../tor_allocate" - -[dependencies.tor_log] -path = "../tor_log" - -[dependencies] -libc = "=0.2.39" - -[features] -# We have to define a feature here because doctests don't get cfg(test), -# and we need to disable some C dependencies when running the doctests -# because of the various linker issues. See -# https://github.com/rust-lang/rust/issues/45599 -test_linking_hack = [] diff --git a/src/rust/tor_util/ffi.rs b/src/rust/tor_util/ffi.rs deleted file mode 100644 index b71b2bd093..0000000000 --- a/src/rust/tor_util/ffi.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -//! FFI functions to announce Rust support during tor startup, only to be -//! called from C. -//! - -use tor_log::{LogDomain, LogSeverity}; - -/// Returns a short string to announce Rust support during startup. -/// -/// # Examples -/// ```c -/// char *rust_str = rust_welcome_string(); -/// printf("%s", rust_str); -/// tor_free(rust_str); -/// ``` -#[no_mangle] -pub extern "C" fn rust_log_welcome_string() { - tor_log_msg!( - LogSeverity::Notice, - LogDomain::General, - "rust_log_welcome_string", - "Tor is running with Rust integration. Please report \ - any bugs you encounter." - ); -} diff --git a/src/rust/tor_util/lib.rs b/src/rust/tor_util/lib.rs deleted file mode 100644 index 8886767ede..0000000000 --- a/src/rust/tor_util/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -//! Small module to announce Rust support during startup for demonstration -//! purposes. - -extern crate libc; -extern crate tor_allocate; - -#[macro_use] -extern crate tor_log; - -pub mod ffi; -pub mod strings; diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs deleted file mode 100644 index ede42c6ea8..0000000000 --- a/src/rust/tor_util/strings.rs +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) 2016-2019, The Tor Project, Inc. */ -// See LICENSE for licensing information */ - -//! Utilities for working with static strings. - -/// Create a `CStr` from a literal byte slice, appending a NUL byte to it first. -/// -/// # Warning -/// -/// The literal byte slice which is taken as an argument *MUST NOT* have any NUL -/// bytes (`b"\0"`) in it, anywhere, or else an empty string will be returned -/// (`CStr::from_bytes_with_nul_unchecked(b"\0")`) so as to avoid `panic!()`ing. -/// -/// # Examples -/// -/// ``` -/// #[macro_use] -/// extern crate tor_util; -/// -/// use std::ffi::CStr; -/// -/// # fn do_test() -> Result<&'static CStr, &'static str> { -/// let message: &'static str = "This is a test of the tsunami warning system."; -/// let tuesday: &'static CStr; -/// let original: &str; -/// -/// tuesday = cstr!("This is a test of the tsunami warning system."); -/// original = tuesday.to_str().or(Err("Couldn't unwrap CStr!"))?; -/// -/// assert!(original == message); -/// # -/// # Ok(tuesday) -/// # } -/// # fn main() { -/// # do_test(); // so that we can use the ? operator in the test -/// # } -/// ``` -/// It is also possible to pass several string literals to this macro. They -/// will be concatenated together in the order of the arguments, unmodified, -/// before finally being suffixed with a NUL byte: -/// -/// ``` -/// #[macro_use] -/// extern crate tor_util; -/// # -/// # use std::ffi::CStr; -/// # -/// # fn do_test() -> Result<&'static CStr, &'static str> { -/// -/// let quux: &'static CStr = cstr!("foo", "bar", "baz"); -/// let orig: &'static str = quux.to_str().or(Err("Couldn't unwrap CStr!"))?; -/// -/// assert!(orig == "foobarbaz"); -/// # Ok(quux) -/// # } -/// # fn main() { -/// # do_test(); // so that we can use the ? operator in the test -/// # } -/// ``` -/// This is useful for passing static strings to C from Rust FFI code. To do so -/// so, use the `.as_ptr()` method on the resulting `&'static CStr` to convert -/// it to the Rust equivalent of a C `const char*`: -/// -/// ``` -/// #[macro_use] -/// extern crate tor_util; -/// -/// use std::ffi::CStr; -/// use std::os::raw::c_char; -/// -/// pub extern "C" fn give_static_borrowed_string_to_c() -> *const c_char { -/// let hello: &'static CStr = cstr!("Hello, language my parents wrote."); -/// -/// hello.as_ptr() -/// } -/// # fn main() { -/// # let greetings = give_static_borrowed_string_to_c(); -/// # } -/// ``` -/// Note that the C code this static borrowed string is passed to *MUST NOT* -/// attempt to free the memory for the string. -/// -/// # Note -/// -/// An unfortunate limitation of the rustc compiler (as of 1.25.0-nightly), is -/// that the first example above compiles, but if we were to change the -/// assignment of `tuesday` as follows, it will fail to compile, because Rust -/// macros are expanded at parse time, and at parse time there is no symbol -/// table available. -/// -/// ```ignore -/// tuesday = cstr!(message); -/// ``` -/// with the error message `error: expected a literal`. -/// -/// # Returns -/// -/// If the string literals passed as arguments contain no NUL bytes anywhere, -/// then an `&'static CStr` containing the (concatenated) bytes of the string -/// literal(s) passed as arguments, with a NUL byte appended, is returned. -/// Otherwise, an `&'static CStr` containing a single NUL byte is returned (an -/// "empty" string in C). -#[macro_export] -macro_rules! cstr { - ($($bytes:expr),*) => ( - ::std::ffi::CStr::from_bytes_with_nul( - concat!($($bytes),*, "\0").as_bytes() - ).unwrap_or_default() - ) -} - -#[cfg(test)] -mod test { - use std::ffi::CStr; - - #[test] - fn cstr_macro() { - let _: &'static CStr = cstr!("boo"); - } - - #[test] - fn cstr_macro_multi_input() { - let quux: &'static CStr = cstr!("foo", "bar", "baz"); - - assert!(quux.to_str().unwrap() == "foobarbaz"); - } - - #[test] - fn cstr_macro_bad_input() { - let waving: &'static CStr = cstr!("waving not drowning o/"); - let drowning: &'static CStr = cstr!("\0 drowning not waving"); - - assert!(waving.to_str().unwrap() == "waving not drowning o/"); - assert!(drowning.to_str().unwrap() == "") - } -} diff --git a/src/test/fuzz/fuzz_address.c b/src/test/fuzz/fuzz_address.c new file mode 100644 index 0000000000..6dccd65e9d --- /dev/null +++ b/src/test/fuzz/fuzz_address.c @@ -0,0 +1,26 @@ +#include "lib/net/address.h" +#include "lib/malloc/malloc.h" + +#include "test/fuzz/fuzzing.h" + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +int +fuzz_main(const uint8_t *data, size_t sz) +{ + tor_addr_t addr; + char *fuzzing_data = tor_memdup_nulterm(data, sz); + tor_addr_parse(&addr, fuzzing_data); + tor_free(fuzzing_data); + return 0; +} diff --git a/src/test/fuzz/fuzz_addressPTR.c b/src/test/fuzz/fuzz_addressPTR.c new file mode 100644 index 0000000000..b503d53666 --- /dev/null +++ b/src/test/fuzz/fuzz_addressPTR.c @@ -0,0 +1,32 @@ +#include "lib/net/address.h" +#include "lib/net/socket.h" +#include "lib/cc/ctassert.h" +#include "lib/container/smartlist.h" +#include "lib/ctime/di_ops.h" +#include "lib/log/log.h" +#include "lib/log/escape.h" +#include "lib/malloc/malloc.h" +#include "lib/net/address.h" +#include "test/fuzz/fuzzing.h" + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +int +fuzz_main(const uint8_t *data, size_t sz) +{ + tor_addr_t addr_result; + char *fuzzing_data = tor_memdup_nulterm(data, sz); + tor_addr_parse_PTR_name(&addr_result, fuzzing_data, AF_UNSPEC, 1); + tor_free(fuzzing_data); + return 0; +} diff --git a/src/test/fuzz/fuzz_hsdescv3_inner.c b/src/test/fuzz/fuzz_hsdescv3_inner.c new file mode 100644 index 0000000000..5aa719f5c3 --- /dev/null +++ b/src/test/fuzz/fuzz_hsdescv3_inner.c @@ -0,0 +1,119 @@ +/* Copyright (c) 2017-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define HS_DESCRIPTOR_PRIVATE + +#include "core/or/or.h" +#include "trunnel/ed25519_cert.h" /* Trunnel interface. */ +#include "lib/crypt_ops/crypto_ed25519.h" +#include "feature/hs/hs_descriptor.h" +#include "feature/dirparse/unparseable.h" + +#include "test/fuzz/fuzzing.h" + +static void +mock_dump_desc__nodump(const char *desc, const char *type) +{ + (void)desc; + (void)type; +} + +static int +mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert, + const size_t crosscert_len, + const crypto_pk_t *rsa_id_key, + const ed25519_public_key_t *master_key, + const time_t reject_if_expired_before) +{ + (void) crosscert; + (void) crosscert_len; + (void) rsa_id_key; + (void) master_key; + (void) reject_if_expired_before; + return 0; +} + +static size_t +mock_decrypt_desc_layer(const hs_descriptor_t *desc, + const uint8_t *descriptor_cookie, + bool is_superencrypted_layer, + char **decrypted_out) +{ + (void)is_superencrypted_layer; + (void)desc; + (void)descriptor_cookie; + const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN; + const uint8_t *encrypted_blob = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob + : desc->superencrypted_data.encrypted_blob; + size_t encrypted_blob_size = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob_size + : desc->superencrypted_data.encrypted_blob_size; + + if (encrypted_blob_size < overhead) + return 0; + *decrypted_out = tor_memdup_nulterm( + encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN, + encrypted_blob_size - overhead); + size_t result = strlen(*decrypted_out); + if (result) { + return result; + } else { + tor_free(*decrypted_out); + return 0; + } +} + +static const uint8_t *decrypted_data = NULL; +static size_t decrypted_len = 0; +static size_t +mock_desc_decrypt_encrypted(const hs_descriptor_t *desc, + const curve25519_secret_key_t *client_auth_sk, + char **decrypted_out) +{ + (void)desc; + (void)client_auth_sk; + *decrypted_out = (char*)tor_memdup_nulterm(decrypted_data, decrypted_len); + return decrypted_len; +} + +int +fuzz_init(void) +{ + disable_signature_checking(); + MOCK(dump_desc, mock_dump_desc__nodump); + MOCK(rsa_ed25519_crosscert_check, mock_rsa_ed25519_crosscert_check); + MOCK(decrypt_desc_layer, mock_decrypt_desc_layer); + MOCK(desc_decrypt_encrypted, mock_desc_decrypt_encrypted); + ed25519_init(); + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +int +fuzz_main(const uint8_t *data, size_t sz) +{ + decrypted_data = data; + decrypted_len = sz; + + hs_descriptor_t *desc = tor_malloc_zero(sizeof(hs_descriptor_t)); + hs_desc_encrypted_data_t *output = tor_malloc_zero(sizeof(*output)); + curve25519_secret_key_t *client_auth_sk = NULL; + hs_desc_decode_status_t status; + + status = desc_decode_encrypted_v3(desc, client_auth_sk, output); + if (status == HS_DESC_DECODE_OK) { + log_debug(LD_GENERAL, "Decoding okay"); + } else { + log_debug(LD_GENERAL, "Decoding failed"); + } + + hs_descriptor_free(desc); + hs_desc_encrypted_data_free(output); + return 0; +} diff --git a/src/test/fuzz/fuzz_hsdescv3_middle.c b/src/test/fuzz/fuzz_hsdescv3_middle.c new file mode 100644 index 0000000000..66a9d52cf3 --- /dev/null +++ b/src/test/fuzz/fuzz_hsdescv3_middle.c @@ -0,0 +1,116 @@ +/* Copyright (c) 2017-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#define HS_DESCRIPTOR_PRIVATE + +#include "core/or/or.h" +#include "trunnel/ed25519_cert.h" /* Trunnel interface. */ +#include "lib/crypt_ops/crypto_ed25519.h" +#include "feature/hs/hs_descriptor.h" +#include "feature/dirparse/unparseable.h" + +#include "test/fuzz/fuzzing.h" + +static void +mock_dump_desc__nodump(const char *desc, const char *type) +{ + (void)desc; + (void)type; +} + +static int +mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert, + const size_t crosscert_len, + const crypto_pk_t *rsa_id_key, + const ed25519_public_key_t *master_key, + const time_t reject_if_expired_before) +{ + (void) crosscert; + (void) crosscert_len; + (void) rsa_id_key; + (void) master_key; + (void) reject_if_expired_before; + return 0; +} + +static size_t +mock_decrypt_desc_layer(const hs_descriptor_t *desc, + const uint8_t *descriptor_cookie, + bool is_superencrypted_layer, + char **decrypted_out) +{ + (void)is_superencrypted_layer; + (void)desc; + (void)descriptor_cookie; + const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN; + const uint8_t *encrypted_blob = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob + : desc->superencrypted_data.encrypted_blob; + size_t encrypted_blob_size = (is_superencrypted_layer) + ? desc->plaintext_data.superencrypted_blob_size + : desc->superencrypted_data.encrypted_blob_size; + + if (encrypted_blob_size < overhead) + return 0; + *decrypted_out = tor_memdup_nulterm( + encrypted_blob + HS_DESC_ENCRYPTED_SALT_LEN, + encrypted_blob_size - overhead); + size_t result = strlen(*decrypted_out); + if (result) { + return result; + } else { + tor_free(*decrypted_out); + return 0; + } +} + +static const uint8_t *decrypted_data = NULL; +static size_t decrypted_len = 0; +static size_t +mock_desc_decrypt_superencrypted(const hs_descriptor_t *desc, + char **decrypted_out) +{ + (void)desc; + *decrypted_out = (char*)tor_memdup_nulterm(decrypted_data, decrypted_len); + return decrypted_len; +} + +int +fuzz_init(void) +{ + disable_signature_checking(); + MOCK(dump_desc, mock_dump_desc__nodump); + MOCK(rsa_ed25519_crosscert_check, mock_rsa_ed25519_crosscert_check); + MOCK(decrypt_desc_layer, mock_decrypt_desc_layer); + MOCK(desc_decrypt_superencrypted, mock_desc_decrypt_superencrypted); + ed25519_init(); + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +int +fuzz_main(const uint8_t *data, size_t sz) +{ + decrypted_data = data; + decrypted_len = sz; + + hs_descriptor_t *desc = tor_malloc_zero(sizeof(hs_descriptor_t)); + hs_desc_superencrypted_data_t *output = tor_malloc_zero(sizeof(*output)); + hs_desc_decode_status_t status; + + status = desc_decode_superencrypted_v3(desc, output); + if (status == HS_DESC_DECODE_OK) { + log_debug(LD_GENERAL, "Decoding okay"); + } else { + log_debug(LD_GENERAL, "Decoding failed"); + } + + hs_descriptor_free(desc); + hs_desc_superencrypted_data_free(output); + return 0; +} diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 9bdced9e6f..9fece7d004 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -8,7 +8,6 @@ FUZZING_LDFLAG = \ @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@ FUZZING_LIBS = \ src/test/libtor-testing.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ @@ -23,17 +22,36 @@ oss-fuzz-prereqs: \ noinst_HEADERS += \ src/test/fuzz/fuzzing.h -LIBFUZZER = -lFuzzer LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) -LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -fsanitize=fuzzer +LIBFUZZER_LIBS = $(FUZZING_LIBS) -lstdc++ LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) # ===== AFL fuzzers if UNITTESTS_ENABLED +src_test_fuzz_fuzz_address_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_address.c +src_test_fuzz_fuzz_address_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_address_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_address_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_address_LDADD = $(FUZZING_LIBS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_fuzz_addressPTR_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_addressPTR.c +src_test_fuzz_fuzz_addressPTR_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_addressPTR_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_addressPTR_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_addressPTR_LDADD = $(FUZZING_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_fuzz_consensus_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_consensus.c @@ -94,6 +112,28 @@ src_test_fuzz_fuzz_hsdescv3_LDADD = $(FUZZING_LIBS) endif if UNITTESTS_ENABLED +src_test_fuzz_fuzz_hsdescv3_inner_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_hsdescv3_inner.c +src_test_fuzz_fuzz_hsdescv3_inner_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_hsdescv3_inner_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_hsdescv3_inner_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_hsdescv3_inner_LDADD = $(FUZZING_LIBS) +endif + + +if UNITTESTS_ENABLED +src_test_fuzz_fuzz_hsdescv3_middle_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_hsdescv3_middle.c +src_test_fuzz_fuzz_hsdescv3_middle_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_hsdescv3_middle_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_hsdescv3_middle_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_hsdescv3_middle_LDADD = $(FUZZING_LIBS) +endif + + +if UNITTESTS_ENABLED src_test_fuzz_fuzz_http_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_http.c @@ -155,12 +195,16 @@ endif if UNITTESTS_ENABLED FUZZERS = \ + src/test/fuzz/fuzz-address \ + src/test/fuzz/fuzz-addressPTR \ src/test/fuzz/fuzz-consensus \ src/test/fuzz/fuzz-descriptor \ src/test/fuzz/fuzz-diff \ src/test/fuzz/fuzz-diff-apply \ src/test/fuzz/fuzz-extrainfo \ src/test/fuzz/fuzz-hsdescv3 \ + src/test/fuzz/fuzz-hsdescv3-inner \ + src/test/fuzz/fuzz-hsdescv3-middle \ src/test/fuzz/fuzz-http \ src/test/fuzz/fuzz-http-connect \ src/test/fuzz/fuzz-microdesc \ @@ -173,6 +217,24 @@ endif if LIBFUZZER_ENABLED if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_address_SOURCES = \ + $(src_test_fuzz_fuzz_address_SOURCES) +src_test_fuzz_lf_fuzz_address_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_address_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_address_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_address_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_addressPTR_SOURCES = \ + $(src_test_fuzz_fuzz_addressPTR_SOURCES) +src_test_fuzz_lf_fuzz_addressPTR_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_addressPTR_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_addressPTR_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_addressPTR_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_consensus_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_lf_fuzz_consensus_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -227,6 +289,25 @@ src_test_fuzz_lf_fuzz_hsdescv3_LDADD = $(LIBFUZZER_LIBS) endif if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_hsdescv3_inner_SOURCES = \ + $(src_test_fuzz_fuzz_hsdescv3_inner_SOURCES) +src_test_fuzz_lf_fuzz_hsdescv3_inner_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_hsdescv3_inner_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_hsdescv3_inner_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_hsdescv3_inner_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_hsdescv3_middle_SOURCES = \ + $(src_test_fuzz_fuzz_hsdescv3_middle_SOURCES) +src_test_fuzz_lf_fuzz_hsdescv3_middle_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_hsdescv3_middle_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_hsdescv3_middle_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_hsdescv3_middle_LDADD = $(LIBFUZZER_LIBS) +endif + + +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_http_SOURCES = \ $(src_test_fuzz_fuzz_http_SOURCES) src_test_fuzz_lf_fuzz_http_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -281,12 +362,16 @@ src_test_fuzz_lf_fuzz_vrs_LDADD = $(LIBFUZZER_LIBS) endif LIBFUZZER_FUZZERS = \ + src/test/fuzz/lf-fuzz-address \ + src/test/fuzz/lf-fuzz-addressPTR \ src/test/fuzz/lf-fuzz-consensus \ src/test/fuzz/lf-fuzz-descriptor \ src/test/fuzz/lf-fuzz-diff \ src/test/fuzz/lf-fuzz-diff-apply \ src/test/fuzz/lf-fuzz-extrainfo \ src/test/fuzz/lf-fuzz-hsdescv3 \ + src/test/fuzz/lf-fuzz-hsdescv3-inner \ + src/test/fuzz/lf-fuzz-hsdescv3-middle \ src/test/fuzz/lf-fuzz-http \ src/test/fuzz/lf-fuzz-http-connect \ src/test/fuzz/lf-fuzz-microdesc \ @@ -302,6 +387,20 @@ endif if OSS_FUZZ_ENABLED if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_address_a_SOURCES = \ + $(src_test_fuzz_fuzz_address_SOURCES) +src_test_fuzz_liboss_fuzz_address_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_address_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_addressPTR_a_SOURCES = \ + $(src_test_fuzz_fuzz_addressPTR_SOURCES) +src_test_fuzz_liboss_fuzz_addressPTR_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_addressPTR_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_consensus_a_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_liboss_fuzz_consensus_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -344,6 +443,20 @@ src_test_fuzz_liboss_fuzz_hsdescv3_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) endif if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_hsdescv3_inner_a_SOURCES = \ + $(src_test_fuzz_fuzz_hsdescv3_inner_SOURCES) +src_test_fuzz_liboss_fuzz_hsdescv3_inner_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_hsdescv3_inner_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_hsdescv3_middle_a_SOURCES = \ + $(src_test_fuzz_fuzz_hsdescv3_middle_SOURCES) +src_test_fuzz_liboss_fuzz_hsdescv3_middle_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_hsdescv3_middle_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_http_a_SOURCES = \ $(src_test_fuzz_fuzz_http_SOURCES) src_test_fuzz_liboss_fuzz_http_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -386,12 +499,16 @@ src_test_fuzz_liboss_fuzz_vrs_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) endif OSS_FUZZ_FUZZERS = \ + src/test/fuzz/liboss-fuzz-address.a \ + src/test/fuzz/liboss-fuzz-addressPTR.a \ src/test/fuzz/liboss-fuzz-consensus.a \ src/test/fuzz/liboss-fuzz-descriptor.a \ src/test/fuzz/liboss-fuzz-diff.a \ src/test/fuzz/liboss-fuzz-diff-apply.a \ src/test/fuzz/liboss-fuzz-extrainfo.a \ src/test/fuzz/liboss-fuzz-hsdescv3.a \ + src/test/fuzz/liboss-fuzz-hsdescv3-inner.a \ + src/test/fuzz/liboss-fuzz-hsdescv3-middle.a \ src/test/fuzz/liboss-fuzz-http.a \ src/test/fuzz/liboss-fuzz-http-connect.a \ src/test/fuzz/liboss-fuzz-microdesc.a \ diff --git a/src/test/include.am b/src/test/include.am index d5dcebfaea..2765cf27d0 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -8,11 +8,7 @@ TESTS_ENVIRONMENT = \ export abs_top_builddir="$(abs_top_builddir)"; \ export builddir="$(builddir)"; \ export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; \ - export CARGO="$(CARGO)"; \ - export EXTRA_CARGO_OPTIONS="$(EXTRA_CARGO_OPTIONS)"; \ - export CARGO_ONLINE="$(CARGO_ONLINE)"; \ - export CCLD="$(CCLD)"; \ - export RUSTFLAGS="-C linker=`echo '$(CC)' | cut -d' ' -f 1` $(RUST_LINKER_OPTIONS)"; + export CCLD="$(CCLD)"; TESTSCRIPTS = \ src/test/fuzz_static_testcases.sh \ @@ -35,11 +31,6 @@ TESTSCRIPTS = \ src/test/unittest_part7.sh \ src/test/unittest_part8.sh -if USE_RUST -TESTSCRIPTS += \ - src/test/test_rust.sh -endif - if USEPYTHON TESTSCRIPTS += \ src/test/test_ntor.sh \ @@ -170,6 +161,7 @@ src_test_test_SOURCES += \ src/test/test_crypto_rng.c \ src/test/test_data.c \ src/test/test_dir.c \ + src/test/test_dirauth_ports.c \ src/test/test_dirvote.c \ src/test/test_dir_common.c \ src/test/test_dir_handle_get.c \ @@ -203,6 +195,7 @@ src_test_test_SOURCES += \ src/test/test_namemap.c \ src/test/test_netinfo.c \ src/test/test_nodelist.c \ + src/test/test_ntor_v3.c \ src/test/test_oom.c \ src/test/test_oos.c \ src/test/test_options.c \ @@ -229,6 +222,7 @@ src_test_test_SOURCES += \ src/test/test_routerkeys.c \ src/test/test_routerlist.c \ src/test/test_routerset.c \ + src/test/test_sandbox.c \ src/test/test_scheduler.c \ src/test/test_sendme.c \ src/test/test_shared_random.c \ @@ -303,7 +297,6 @@ src_test_test_switch_id_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_test_test_switch_id_LDFLAGS = @TOR_LDFLAGS_zlib@ src_test_test_switch_id_LDADD = \ $(TOR_UTIL_TESTING_LIBS) \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_USERENV@ \ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@ @@ -311,7 +304,6 @@ src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ @TOR_LDFLAGS_libevent@ src_test_test_LDADD = \ src/test/libtor-testing.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ @@ -340,7 +332,6 @@ src_test_bench_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ @TOR_LDFLAGS_libevent@ src_test_bench_LDADD = \ libtor.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ @@ -350,7 +341,6 @@ src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \ @TOR_LDFLAGS_libevent@ src_test_test_workqueue_LDADD = \ src/test/libtor-testing.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ @@ -362,7 +352,6 @@ src_test_test_timers_LDADD = \ src/lib/libtor-evloop-testing.a \ $(TOR_CRYPTO_TESTING_LIBS) \ $(TOR_UTIL_TESTING_LIBS) \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ \ @@ -398,7 +387,6 @@ src_test_test_ntor_cl_SOURCES = src/test/test_ntor_cl.c src_test_test_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) src_test_test_ntor_cl_LDADD = \ libtor.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ $(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @CURVE25519_LIBS@ @TOR_LZMA_LIBS@ @TOR_TRACE_LIBS@ @@ -421,7 +409,6 @@ noinst_PROGRAMS += src/test/test-bt-cl src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c src_test_test_bt_cl_LDADD = \ $(TOR_UTIL_TESTING_LIBS) \ - $(rust_ldadd) \ @TOR_LIB_MATH@ \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ @TOR_TRACE_LIBS@ @@ -443,13 +430,11 @@ EXTRA_DIST += \ src/test/test_include.py \ src/test/zero_length_keys.sh \ scripts/maint/run_check_subsystem_order.sh \ - src/test/rust_supp.txt \ src/test/test_keygen.sh \ src/test/test_key_expiration.sh \ src/test/test_zero_length_keys.sh \ src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh \ src/test/test-network.sh \ - src/test/test_rust.sh \ src/test/test_switch_id.sh \ src/test/test_workqueue_cancel.sh \ src/test/test_workqueue_efd.sh \ @@ -467,6 +452,3 @@ EXTRA_DIST += \ src/test/unittest_part6.sh \ src/test/unittest_part7.sh \ src/test/unittest_part8.sh - -test-rust: - $(TESTS_ENVIRONMENT) "$(abs_top_srcdir)/src/test/test_rust.sh" diff --git a/src/test/ntor_v3_ref.py b/src/test/ntor_v3_ref.py new file mode 100755 index 0000000000..28bc077105 --- /dev/null +++ b/src/test/ntor_v3_ref.py @@ -0,0 +1,308 @@ +#!/usr/bin/python + +import binascii +import hashlib +import os +import struct + +import donna25519 +from Crypto.Cipher import AES +from Crypto.Util import Counter + +# Define basic wrappers. + +DIGEST_LEN = 32 +ENC_KEY_LEN = 32 +PUB_KEY_LEN = 32 +SEC_KEY_LEN = 32 +IDENTITY_LEN = 32 + +def sha3_256(s): + d = hashlib.sha3_256(s).digest() + assert len(d) == DIGEST_LEN + return d + +def shake_256(s): + # Note: In reality, you wouldn't want to generate more bytes than needed. + MAX_KEY_BYTES = 1024 + return hashlib.shake_256(s).digest(MAX_KEY_BYTES) + +def curve25519(pk, sk): + assert len(pk) == PUB_KEY_LEN + assert len(sk) == SEC_KEY_LEN + private = donna25519.PrivateKey.load(sk) + public = donna25519.PublicKey(pk) + return private.do_exchange(public) + +def keygen(): + private = donna25519.PrivateKey() + public = private.get_public() + return (private.private, public.public) + +def aes256_ctr(k, s): + assert len(k) == ENC_KEY_LEN + cipher = AES.new(k, AES.MODE_CTR, counter=Counter.new(128, initial_value=0)) + return cipher.encrypt(s) + +# Byte-oriented helper. We use this for decoding keystreams and messages. + +class ByteSeq: + def __init__(self, data): + self.data = data + + def take(self, n): + assert n <= len(self.data) + result = self.data[:n] + self.data = self.data[n:] + return result + + def exhausted(self): + return len(self.data) == 0 + + def remaining(self): + return len(self.data) + +# Low-level functions + +MAC_KEY_LEN = 32 +MAC_LEN = DIGEST_LEN + +hash_func = sha3_256 + +def encapsulate(s): + """encapsulate `s` with a length prefix. + + We use this whenever we need to avoid message ambiguities in + cryptographic inputs. + """ + assert len(s) <= 0xffffffff + header = b"\0\0\0\0" + struct.pack("!L", len(s)) + assert len(header) == 8 + return header + s + +def h(s, tweak): + return hash_func(encapsulate(tweak) + s) + +def mac(s, key, tweak): + return hash_func(encapsulate(tweak) + encapsulate(key) + s) + +def kdf(s, tweak): + data = shake_256(encapsulate(tweak) + s) + return ByteSeq(data) + +def enc(s, k): + return aes256_ctr(k, s) + +# Tweaked wrappers + +PROTOID = b"ntor3-curve25519-sha3_256-1" +T_KDF_PHASE1 = PROTOID + b":kdf_phase1" +T_MAC_PHASE1 = PROTOID + b":msg_mac" +T_KDF_FINAL = PROTOID + b":kdf_final" +T_KEY_SEED = PROTOID + b":key_seed" +T_VERIFY = PROTOID + b":verify" +T_AUTH = PROTOID + b":auth_final" + +def kdf_phase1(s): + return kdf(s, T_KDF_PHASE1) + +def kdf_final(s): + return kdf(s, T_KDF_FINAL) + +def mac_phase1(s, key): + return mac(s, key, T_MAC_PHASE1) + +def h_key_seed(s): + return h(s, T_KEY_SEED) + +def h_verify(s): + return h(s, T_VERIFY) + +def h_auth(s): + return h(s, T_AUTH) + +# Handshake. + +def client_phase1(msg, verification, B, ID): + assert len(B) == PUB_KEY_LEN + assert len(ID) == IDENTITY_LEN + + (x,X) = keygen() + p(["x", "X"], locals()) + p(["msg", "verification"], locals()) + Bx = curve25519(B, x) + secret_input_phase1 = Bx + ID + X + B + PROTOID + encapsulate(verification) + + phase1_keys = kdf_phase1(secret_input_phase1) + enc_key = phase1_keys.take(ENC_KEY_LEN) + mac_key = phase1_keys.take(MAC_KEY_LEN) + p(["enc_key", "mac_key"], locals()) + + msg_0 = ID + B + X + enc(msg, enc_key) + mac = mac_phase1(msg_0, mac_key) + p(["mac"], locals()) + + client_handshake = msg_0 + mac + state = dict(x=x, X=X, B=B, ID=ID, Bx=Bx, mac=mac, verification=verification) + + p(["client_handshake"], locals()) + + return (client_handshake, state) + +# server. + +class Reject(Exception): + pass + +def server_part1(cmsg, verification, b, B, ID): + assert len(B) == PUB_KEY_LEN + assert len(ID) == IDENTITY_LEN + assert len(b) == SEC_KEY_LEN + + if len(cmsg) < (IDENTITY_LEN + PUB_KEY_LEN * 2 + MAC_LEN): + raise Reject() + + mac_covered_portion = cmsg[0:-MAC_LEN] + cmsg = ByteSeq(cmsg) + cmsg_id = cmsg.take(IDENTITY_LEN) + cmsg_B = cmsg.take(PUB_KEY_LEN) + cmsg_X = cmsg.take(PUB_KEY_LEN) + cmsg_msg = cmsg.take(cmsg.remaining() - MAC_LEN) + cmsg_mac = cmsg.take(MAC_LEN) + + assert cmsg.exhausted() + + # XXXX for real purposes, you would use constant-time checks here + if cmsg_id != ID or cmsg_B != B: + raise Reject() + + Xb = curve25519(cmsg_X, b) + secret_input_phase1 = Xb + ID + cmsg_X + B + PROTOID + encapsulate(verification) + + phase1_keys = kdf_phase1(secret_input_phase1) + enc_key = phase1_keys.take(ENC_KEY_LEN) + mac_key = phase1_keys.take(MAC_KEY_LEN) + + mac_received = mac_phase1(mac_covered_portion, mac_key) + if mac_received != cmsg_mac: + raise Reject() + + client_msg = enc(cmsg_msg, enc_key) + state = dict( + b=b, + B=B, + X=cmsg_X, + mac_received=mac_received, + Xb=Xb, + ID=ID, + verification=verification) + + return (client_msg, state) + +def server_part2(state, server_msg): + X = state['X'] + Xb = state['Xb'] + B = state['B'] + b = state['b'] + ID = state['ID'] + mac_received = state['mac_received'] + verification = state['verification'] + + p(["server_msg"], locals()) + + (y,Y) = keygen() + p(["y", "Y"], locals()) + Xy = curve25519(X, y) + + secret_input = Xy + Xb + ID + B + X + Y + PROTOID + encapsulate(verification) + key_seed = h_key_seed(secret_input) + verify = h_verify(secret_input) + p(["key_seed", "verify"], locals()) + + keys = kdf_final(key_seed) + server_enc_key = keys.take(ENC_KEY_LEN) + p(["server_enc_key"], locals()) + + smsg_msg = enc(server_msg, server_enc_key) + + auth_input = verify + ID + B + Y + X + mac_received + encapsulate(smsg_msg) + PROTOID + b"Server" + + auth = h_auth(auth_input) + server_handshake = Y + auth + smsg_msg + p(["auth", "server_handshake"], locals()) + + return (server_handshake, keys) + +def client_phase2(state, smsg): + x = state['x'] + X = state['X'] + B = state['B'] + ID = state['ID'] + Bx = state['Bx'] + mac_sent = state['mac'] + verification = state['verification'] + + if len(smsg) < PUB_KEY_LEN + DIGEST_LEN: + raise Reject() + + smsg = ByteSeq(smsg) + Y = smsg.take(PUB_KEY_LEN) + auth_received = smsg.take(DIGEST_LEN) + server_msg = smsg.take(smsg.remaining()) + + Yx = curve25519(Y,x) + + secret_input = Yx + Bx + ID + B + X + Y + PROTOID + encapsulate(verification) + key_seed = h_key_seed(secret_input) + verify = h_verify(secret_input) + + auth_input = verify + ID + B + Y + X + mac_sent + encapsulate(server_msg) + PROTOID + b"Server" + + auth = h_auth(auth_input) + if auth != auth_received: + raise Reject() + + keys = kdf_final(key_seed) + enc_key = keys.take(ENC_KEY_LEN) + + server_msg_decrypted = enc(server_msg, enc_key) + + return (keys, server_msg_decrypted) + +def p(varnames, localvars): + for v in varnames: + label = v + val = localvars[label] + print('{} = "{}"'.format(label, binascii.b2a_hex(val).decode("ascii"))) + +def test(): + (b,B) = keygen() + ID = os.urandom(IDENTITY_LEN) + + p(["b", "B", "ID"], locals()) + + print("# ============") + (c_handshake, c_state) = client_phase1(b"hello world", b"xyzzy", B, ID) + + print("# ============") + + (c_msg_got, s_state) = server_part1(c_handshake, b"xyzzy", b, B, ID) + + #print(repr(c_msg_got)) + + (s_handshake, s_keys) = server_part2(s_state, b"Hola Mundo") + + print("# ============") + + (c_keys, s_msg_got) = client_phase2(c_state, s_handshake) + + #print(repr(s_msg_got)) + + c_keys_256 = c_keys.take(256) + p(["c_keys_256"], locals()) + + assert (c_keys_256 == s_keys.take(256)) + + +if __name__ == '__main__': + test() diff --git a/src/test/test.c b/src/test/test.c index 40c053a660..c38d78da30 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1,5 +1,4 @@ /* Copyright (c) 2001-2004, Roger Dingledine. -->a * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2021, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -53,6 +52,7 @@ #include "core/crypto/onion_fast.h" #include "core/crypto/onion_tap.h" #include "core/or/policies.h" +#include "lib/sandbox/sandbox.h" #include "app/config/statefile.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "feature/nodelist/networkstatus.h" @@ -673,6 +673,7 @@ struct testgroup_t testgroups[] = { { "crypto/pem/", pem_tests }, { "crypto/rng/", crypto_rng_tests }, { "dir/", dir_tests }, + { "dir/auth/ports/", dirauth_port_tests }, { "dir/auth/process_descs/", process_descs_tests }, { "dir/md/", microdesc_tests }, { "dirauth/dirvote/", dirvote_tests}, @@ -707,6 +708,7 @@ struct testgroup_t testgroups[] = { { "netinfo/", netinfo_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, + { "onion-handshake/ntor-v3/", ntor_v3_tests }, { "oos/", oos_tests }, { "options/", options_tests }, { "options/act/", options_act_tests }, @@ -731,6 +733,9 @@ struct testgroup_t testgroups[] = { { "routerkeys/", routerkeys_tests }, { "routerlist/", routerlist_tests }, { "routerset/" , routerset_tests }, +#ifdef USE_LIBSECCOMP + { "sandbox/" , sandbox_tests }, +#endif { "scheduler/", scheduler_tests }, { "sendme/", sendme_tests }, { "shared-random/", sr_tests }, diff --git a/src/test/test.h b/src/test/test.h index f88bc98498..e17bce427c 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -120,6 +120,7 @@ extern struct testcase_t crypto_ope_tests[]; extern struct testcase_t crypto_openssl_tests[]; extern struct testcase_t crypto_rng_tests[]; extern struct testcase_t crypto_tests[]; +extern struct testcase_t dirauth_port_tests[]; extern struct testcase_t dir_handle_get_tests[]; extern struct testcase_t dir_tests[]; extern struct testcase_t dirvote_tests[]; @@ -154,6 +155,7 @@ extern struct testcase_t microdesc_tests[]; extern struct testcase_t namemap_tests[]; extern struct testcase_t netinfo_tests[]; extern struct testcase_t nodelist_tests[]; +extern struct testcase_t ntor_v3_tests[]; extern struct testcase_t oom_tests[]; extern struct testcase_t oos_tests[]; extern struct testcase_t options_tests[]; @@ -182,6 +184,7 @@ extern struct testcase_t router_tests[]; extern struct testcase_t routerkeys_tests[]; extern struct testcase_t routerlist_tests[]; extern struct testcase_t routerset_tests[]; +extern struct testcase_t sandbox_tests[]; extern struct testcase_t scheduler_tests[]; extern struct testcase_t sendme_tests[]; extern struct testcase_t socks_tests[]; diff --git a/src/test/test_address.c b/src/test/test_address.c index 9c1415419c..015ca0807c 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -1326,6 +1326,42 @@ test_address_dirserv_router_addr_private(void *opt_dir_allow_private) UNMOCK(get_options); } +static void +test_address_parse_port_range(void *arg) +{ + int ret; + uint16_t min_out = 0; + uint16_t max_out = 0; + + (void) arg; + + /* Invalid. */ + ret = parse_port_range("0x00", &min_out, &max_out); + tt_int_op(ret, OP_EQ, -1); + ret = parse_port_range("0x01", &min_out, &max_out); + tt_int_op(ret, OP_EQ, -1); + ret = parse_port_range("1817161", &min_out, &max_out); + tt_int_op(ret, OP_EQ, -1); + ret = parse_port_range("65536", &min_out, &max_out); + tt_int_op(ret, OP_EQ, -1); + ret = parse_port_range("1-65536", &min_out, &max_out); + tt_int_op(ret, OP_EQ, -1); + + /* Valid. */ + ret = parse_port_range("65535", &min_out, &max_out); + tt_int_op(ret, OP_EQ, 0); + tt_int_op(min_out, OP_EQ, 65535); + tt_int_op(max_out, OP_EQ, 65535); + + ret = parse_port_range("1-65535", &min_out, &max_out); + tt_int_op(ret, OP_EQ, 0); + tt_int_op(min_out, OP_EQ, 1); + tt_int_op(max_out, OP_EQ, 65535); + + done: + ; +} + #define ADDRESS_TEST(name, flags) \ { #name, test_address_ ## name, flags, NULL, NULL } #define ADDRESS_TEST_STR_ARG(name, flags, str_arg) \ @@ -1364,5 +1400,6 @@ struct testcase_t address_tests[] = { ADDRESS_TEST(tor_node_in_same_network_family, 0), ADDRESS_TEST(dirserv_router_addr_private, 0), ADDRESS_TEST_STR_ARG(dirserv_router_addr_private, 0, "allow_private"), + ADDRESS_TEST(parse_port_range, 0), END_OF_TESTCASES }; diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index 5219c86097..ca7fee2c53 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -20,6 +20,7 @@ #include "lib/tls/tortls.h" #include "core/or/or_connection_st.h" +#include "core/or/congestion_control_common.h" /* Test suite stuff */ #include "test/test.h" @@ -155,7 +156,7 @@ test_channeltls_num_bytes_queued(void *arg) * - 2 cells. */ n = ch->num_cells_writeable(ch); - tt_int_op(n, OP_EQ, CEIL_DIV(OR_CONN_HIGHWATER, 512) - 2); + tt_int_op(n, OP_EQ, CEIL_DIV(or_conn_highwatermark(), 512) - 2); UNMOCK(buf_datalen); tlschan_buf_datalen_mock_target = NULL; tlschan_buf_datalen_mock_size = 0; diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 873391a84f..0a5c3530bd 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -113,7 +113,7 @@ test_new_route_len_safe_exit(void *arg) /* hidden service connecting to introduction point */ r = new_route_len(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, &dummy_ei, &dummy_nodes); - tt_int_op(DEFAULT_ROUTE_LEN, OP_EQ, r); + tt_int_op(DEFAULT_ROUTE_LEN+1, OP_EQ, r); /* router testing its own reachability */ r = new_route_len(CIRCUIT_PURPOSE_TESTING, &dummy_ei, &dummy_nodes); diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c index 86baf54f40..6ced3f4111 100644 --- a/src/test/test_circuitpadding.c +++ b/src/test/test_circuitpadding.c @@ -1367,7 +1367,7 @@ test_circuitpadding_wronghop(void *arg) tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); - tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_relay_cells, OP_EQ, 2); tt_int_op(n_client_cells, OP_EQ, 2); /* 6. Sending negotiated command to relay does nothing */ @@ -1396,11 +1396,9 @@ test_circuitpadding_wronghop(void *arg) /* verify no padding was negotiated */ tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); - tt_int_op(n_relay_cells, OP_EQ, 3); - tt_int_op(n_client_cells, OP_EQ, 2); /* verify no echo was sent */ - tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_relay_cells, OP_EQ, 2); tt_int_op(n_client_cells, OP_EQ, 2); /* Finish circuit */ diff --git a/src/test/test_connection.c b/src/test/test_connection.c index 9c726c07f8..87940f71e6 100644 --- a/src/test/test_connection.c +++ b/src/test/test_connection.c @@ -992,12 +992,12 @@ test_conn_describe(void *arg) #define STR(x) #x /* where arg is an expression (constant, variable, compound expression) */ -#define CONNECTION_TESTCASE_ARG(name, fork, setup, arg) \ - { #name "_" STR(x), \ +#define CONNECTION_TESTCASE_ARG(name, extra, fork, setup, arg) \ + { STR(name)"/"extra, \ test_conn_##name, \ - fork, \ - &setup, \ - (void *)arg } + (fork), \ + &(setup), \ + (void *)(arg) } #endif /* !defined(COCCI) */ static const unsigned int PROXY_CONNECT_ARG = PROXY_CONNECT; @@ -1007,14 +1007,14 @@ struct testcase_t connection_tests[] = { CONNECTION_TESTCASE(get_basic, TT_FORK, test_conn_get_basic_st), CONNECTION_TESTCASE(get_rsrc, TT_FORK, test_conn_get_rsrc_st), - CONNECTION_TESTCASE_ARG(download_status, TT_FORK, + CONNECTION_TESTCASE_ARG(download_status, "microdesc", TT_FORK, test_conn_download_status_st, "microdesc"), - CONNECTION_TESTCASE_ARG(download_status, TT_FORK, + CONNECTION_TESTCASE_ARG(download_status, "ns", TT_FORK, test_conn_download_status_st, "ns"), - CONNECTION_TESTCASE_ARG(https_proxy_connect, TT_FORK, + CONNECTION_TESTCASE_ARG(https_proxy_connect, "https", TT_FORK, test_conn_proxy_connect_st, &PROXY_CONNECT_ARG), - CONNECTION_TESTCASE_ARG(haproxy_proxy_connect, TT_FORK, + CONNECTION_TESTCASE_ARG(haproxy_proxy_connect, "haproxy", TT_FORK, test_conn_proxy_connect_st, &PROXY_HAPROXY_ARG), //CONNECTION_TESTCASE(func_suffix, TT_FORK, setup_func_pair), diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 0d2d6800ba..186e09f236 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -6652,13 +6652,7 @@ test_dir_find_dl_min_delay(void* data) dls.schedule = DL_SCHED_BRIDGE; /* client */ - mock_options->ClientOnly = 1; - mock_options->UseBridges = 1; - if (num_bridges_usable(0) > 0) { - tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge); - } else { - tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge_bootstrap); - } + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge_bootstrap); done: UNMOCK(networkstatus_consensus_is_bootstrapping); diff --git a/src/test/test_dirauth_ports.c b/src/test/test_dirauth_ports.c new file mode 100644 index 0000000000..5dc0b0b631 --- /dev/null +++ b/src/test/test_dirauth_ports.c @@ -0,0 +1,152 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CONFIG_PRIVATE + +#include "core/or/or.h" +#include "feature/dirclient/dir_server_st.h" +#include "feature/nodelist/dirlist.h" +#include "app/config/config.h" +#include "test/test.h" +#include "test/log_test_helpers.h" + +static void +test_dirauth_port_parsing(void *arg) +{ + (void)arg; + + // This one is okay. + int rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.39:9131/ " + "download=http://128.31.0.39:9131 " + "vote=http://128.31.0.39:9131/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,0); + + // These have bad syntax. + setup_capture_of_logs(LOG_WARN); + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "uploadx=http://128.31.0.39:9131/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,0); + expect_log_msg_containing("Unrecognized flag"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=https://128.31.0.39:9131/ " // https is not recognized + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unsupported URL scheme"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.39:9131/tor " // suffix is not supported + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unsupported URL prefix"); + mock_clean_saved_logs(); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.256:9131/ " // "256" is not ipv4. + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unable to parse address"); + + rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://xyz.example.com/ " // hostnames not supported. + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 1); + tt_int_op(rv,OP_EQ,-1); + expect_log_msg_containing("Unable to parse address"); + + done: + teardown_capture_of_logs(); +} + +static void +test_dirauth_port_lookup(void *arg) +{ + (void)arg; + + clear_dir_servers(); + + int rv = parse_dir_authority_line( + "moria1 orport=9101 " + "v3ident=D586D18309DED4CD6D57C18FDB97EFA96D330566 " + "upload=http://128.31.0.40:9132/ " + "download=http://128.31.0.41:9133 " + "vote=http://128.31.0.42:9134/ " + "128.31.0.39:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 0); + tt_int_op(rv,OP_EQ,0); + + rv = parse_dir_authority_line( + "morgoth orport=9101 " + "v3ident=D586D18309DED4CDFFFFFFFFDB97EFA96D330566 " + "upload=http://128.31.0.43:9140/ " + "128.31.0.44:9131 9695 DFC3 5FFE B861 329B 9F1A B04C 4639 7020 CE31", + NO_DIRINFO, 0); + tt_int_op(rv,OP_EQ,0); + + const smartlist_t *servers = router_get_trusted_dir_servers(); + tt_assert(servers); + tt_int_op(smartlist_len(servers), OP_EQ, 2); + const dir_server_t *moria = smartlist_get(servers, 0); + const dir_server_t *morgoth = smartlist_get(servers, 1); + tt_str_op(moria->nickname, OP_EQ, "moria1"); + tt_str_op(morgoth->nickname, OP_EQ, "morgoth"); + + const tor_addr_port_t *dirport; + + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_UPLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9132); + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_DOWNLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9133); + dirport = trusted_dir_server_get_dirport(moria, + AUTH_USAGE_VOTING, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9134); + + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_UPLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9140); + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_DOWNLOAD, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9131); // fallback + dirport = trusted_dir_server_get_dirport(morgoth, + AUTH_USAGE_VOTING, AF_INET); + tt_int_op(dirport->port, OP_EQ, 9131); // fallback + + done: + ; +} + +#define T(name) \ + { #name, test_dirauth_port_ ## name, TT_FORK, NULL, NULL } + +struct testcase_t dirauth_port_tests[] = { + T(parsing), + T(lookup), + END_OF_TESTCASES +}; diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index c94b5d6a23..118b66dfa7 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -92,6 +92,12 @@ bfn_mock_node_get_by_id(const char *id) return NULL; } +static int +mock_router_have_minimum_dir_info(void) +{ + return 1; +} + /* Helper function to free a test node. */ static void test_node_free(node_t *n) @@ -3087,6 +3093,38 @@ test_entry_guard_vanguard_path_selection(void *arg) circuit_free_(circ); } +static void +test_entry_guard_layer2_guards(void *arg) +{ + (void) arg; + MOCK(router_have_minimum_dir_info, mock_router_have_minimum_dir_info); + + /* First check the enable/disable switch */ + get_options_mutable()->VanguardsLiteEnabled = 0; + tt_int_op(vanguards_lite_is_enabled(), OP_EQ, 0); + + get_options_mutable()->VanguardsLiteEnabled = 1; + tt_int_op(vanguards_lite_is_enabled(), OP_EQ, 1); + + get_options_mutable()->VanguardsLiteEnabled = -1; + tt_int_op(vanguards_lite_is_enabled(), OP_EQ, 1); + + /* OK now let's move to actual testing */ + + /* Remove restrictions to route around Big Fake Network restrictions */ + get_options_mutable()->EnforceDistinctSubnets = 0; + + /* Create the L2 guardset */ + maintain_layer2_guards(); + + const routerset_t *l2_guards = get_layer2_guards(); + tt_assert(l2_guards); + tt_int_op(routerset_len(l2_guards), OP_EQ, 4); + + done: + UNMOCK(router_have_minimum_dir_info); +} + static const struct testcase_setup_t big_fake_network = { big_fake_network_setup, big_fake_network_cleanup }; @@ -3152,6 +3190,8 @@ struct testcase_t entrynodes_tests[] = { BFN_TEST(manage_primary), BFN_TEST(correct_cascading_order), + BFN_TEST(layer2_guards), + EN_TEST_FORK(guard_preferred), BFN_TEST(select_for_circuit_no_confirmed), diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index 7cb6a36f8e..347a5b7174 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -808,13 +808,11 @@ test_parse_extended_hostname(void *arg) tt_assert(parse_extended_hostname(address4, &type)); tt_int_op(type, OP_EQ, NORMAL_HOSTNAME); - tt_assert(parse_extended_hostname(address5, &type)); - tt_int_op(type, OP_EQ, ONION_V2_HOSTNAME); - tt_str_op(address5, OP_EQ, "abcdefghijklmnop"); + tt_assert(!parse_extended_hostname(address5, &type)); + tt_int_op(type, OP_EQ, BAD_HOSTNAME); - tt_assert(parse_extended_hostname(address6, &type)); - tt_int_op(type, OP_EQ, ONION_V2_HOSTNAME); - tt_str_op(address6, OP_EQ, "abcdefghijklmnop"); + tt_assert(!parse_extended_hostname(address6, &type)); + tt_int_op(type, OP_EQ, BAD_HOSTNAME); tt_assert(!parse_extended_hostname(address7, &type)); tt_int_op(type, OP_EQ, BAD_HOSTNAME); diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c index b036c5eada..c32803b380 100644 --- a/src/test/test_hs_control.c +++ b/src/test/test_hs_control.c @@ -798,7 +798,7 @@ test_hs_control_add_onion_helper_add_service(void *arg) hs_service_ht *global_map; hs_port_config_t *portcfg; smartlist_t *portcfgs; - char *address_out_good, *address_out_bad; + char *address_out_good = NULL, *address_out_bad = NULL; hs_service_t *service_good = NULL; hs_service_t *service_bad = NULL; diff --git a/src/test/test_hs_ob.c b/src/test/test_hs_ob.c index 3485655c2e..2f69bf31e0 100644 --- a/src/test/test_hs_ob.c +++ b/src/test/test_hs_ob.c @@ -174,6 +174,7 @@ test_get_subcredentials(void *arg) hs_subcredential_t *subcreds = NULL; (void) arg; + memset(&config, 0, sizeof(config)); MOCK(networkstatus_get_live_consensus, mock_networkstatus_get_live_consensus); diff --git a/src/test/test_ntor_v3.c b/src/test/test_ntor_v3.c new file mode 100644 index 0000000000..096ac6668f --- /dev/null +++ b/src/test/test_ntor_v3.c @@ -0,0 +1,172 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define ONION_NTOR_V3_PRIVATE +#include "core/or/or.h" +#include "test/test.h" +#include "lib/crypt_ops/crypto_curve25519.h" +#include "lib/crypt_ops/crypto_ed25519.h" +#include "core/crypto/onion_ntor_v3.h" + +#define unhex(arry, s) \ + { tt_int_op(sizeof(arry), OP_EQ, \ + base16_decode((char*)arry, sizeof(arry), s, strlen(s))); \ + } + +static void +test_ntor3_testvecs(void *arg) +{ + (void)arg; + char *mem_op_hex_tmp = NULL; // temp val to make test_memeq_hex work. + + ntor3_server_handshake_state_t *relay_state = NULL; + uint8_t *onion_skin = NULL; + size_t onion_skin_len; + ntor3_handshake_state_t *client_state = NULL; + uint8_t *cm = NULL, *sm = NULL; + size_t cm_len, sm_len; + di_digest256_map_t *private_keys = NULL; + uint8_t *server_handshake = NULL; + size_t server_handshake_len; + + // Test vectors from python implementation, confirmed with rust + // implementation. + curve25519_keypair_t relay_keypair_b; + curve25519_keypair_t client_keypair_x; + curve25519_keypair_t relay_keypair_y; + ed25519_public_key_t relay_id; + + unhex(relay_keypair_b.seckey.secret_key, + "4051daa5921cfa2a1c27b08451324919538e79e788a81b38cbed097a5dff454a"); + unhex(relay_keypair_b.pubkey.public_key, + "f8307a2bc1870b00b828bb74dbb8fd88e632a6375ab3bcd1ae706aaa8b6cdd1d"); + unhex(relay_id.pubkey, + "9fad2af287ef942632833d21f946c6260c33fae6172b60006e86e4a6911753a2"); + unhex(client_keypair_x.seckey.secret_key, + "b825a3719147bcbe5fb1d0b0fcb9c09e51948048e2e3283d2ab7b45b5ef38b49"); + unhex(client_keypair_x.pubkey.public_key, + "252fe9ae91264c91d4ecb8501f79d0387e34ad8ca0f7c995184f7d11d5da4f46"); + unhex(relay_keypair_y.seckey.secret_key, + "4865a5b7689dafd978f529291c7171bc159be076b92186405d13220b80e2a053"); + unhex(relay_keypair_y.pubkey.public_key, + "4bf4814326fdab45ad5184f5518bd7fae25dc59374062698201a50a22954246d"); + + uint8_t client_message[11]; + uint8_t verification[5]; + unhex(client_message, "68656c6c6f20776f726c64"); + unhex(verification, "78797a7a79"); + + // ========= Client handshake 1. + + onion_skin_ntor3_create_nokeygen( + &client_keypair_x, + &relay_id, + &relay_keypair_b.pubkey, + verification, + sizeof(verification), + client_message, + sizeof(client_message), + &client_state, + &onion_skin, + &onion_skin_len); + + const char expect_client_handshake[] = "9fad2af287ef942632833d21f946c6260c" + "33fae6172b60006e86e4a6911753a2f8307a2bc1870b00b828bb74dbb8fd88e632a6375" + "ab3bcd1ae706aaa8b6cdd1d252fe9ae91264c91d4ecb8501f79d0387e34ad8ca0f7c995" + "184f7d11d5da4f463bebd9151fd3b47c180abc9e044d53565f04d82bbb3bebed3d06cea" + "65db8be9c72b68cd461942088502f67"; + + tt_int_op(onion_skin_len, OP_EQ, strlen(expect_client_handshake)/2); + test_memeq_hex(onion_skin, expect_client_handshake); + + // ========= Relay handshake. + + dimap_add_entry(&private_keys, + relay_keypair_b.pubkey.public_key, + &relay_keypair_b); + + int r = onion_skin_ntor3_server_handshake_part1( + private_keys, + &client_keypair_x, + &relay_id, + onion_skin, + onion_skin_len, + verification, + sizeof(verification), + &cm, + &cm_len, + &relay_state); + tt_int_op(r, OP_EQ, 0); + tt_int_op(cm_len, OP_EQ, sizeof(client_message)); + tt_mem_op(cm, OP_EQ, client_message, cm_len); + + uint8_t server_message[10]; + unhex(server_message, "486f6c61204d756e646f"); + + uint8_t server_keys[256]; + onion_skin_ntor3_server_handshake_part2_nokeygen( + &relay_keypair_y, + relay_state, + verification, + sizeof(verification), + server_message, + sizeof(server_message), + &server_handshake, + &server_handshake_len, + server_keys, + sizeof(server_keys)); + + const char expect_server_handshake[] = "4bf4814326fdab45ad5184f5518bd7fae25" + "dc59374062698201a50a22954246d2fc5f8773ca824542bc6cf6f57c7c29bbf4e5476461" + "ab130c5b18ab0a91276651202c3e1e87c0d32054c"; + tt_int_op(server_handshake_len, OP_EQ, strlen(expect_server_handshake)/2); + test_memeq_hex(server_handshake, expect_server_handshake); + + uint8_t expect_keys[256]; + unhex(expect_keys, "9c19b631fd94ed86a817e01f6c80b0743a43f5faebd39cfaa8b00f" + "a8bcc65c3bfeaa403d91acbd68a821bf6ee8504602b094a254392a07737d5662768" + "c7a9fb1b2814bb34780eaee6e867c773e28c212ead563e98a1cd5d5b4576f5ee61c" + "59bde025ff2851bb19b721421694f263818e3531e43a9e4e3e2c661e2ad547d8984" + "caa28ebecd3e4525452299be26b9185a20a90ce1eac20a91f2832d731b54502b097" + "49b5a2a2949292f8cfcbeffb790c7790ed935a9d251e7e336148ea83b063a5618fc" + "ff674a44581585fd22077ca0e52c59a24347a38d1a1ceebddbf238541f226b8f88d" + "0fb9c07a1bcd2ea764bbbb5dacdaf5312a14c0b9e4f06309b0333b4a"); + tt_mem_op(server_keys, OP_EQ, expect_keys, 256); + + // ===== Client handshake 2 + + uint8_t client_keys[256]; + r = onion_ntor3_client_handshake( + client_state, + server_handshake, + server_handshake_len, + verification, + sizeof(verification), + client_keys, + sizeof(client_keys), + &sm, + &sm_len); + + tt_int_op(r, OP_EQ, 0); + tt_int_op(sm_len, OP_EQ, sizeof(server_message)); + tt_mem_op(sm, OP_EQ, server_message, sizeof(server_message)); + tt_mem_op(client_keys, OP_EQ, server_keys, 256); + + done: + tor_free(onion_skin); + tor_free(server_handshake); + tor_free(mem_op_hex_tmp); + ntor3_handshake_state_free(client_state); + ntor3_server_handshake_state_free(relay_state); + tor_free(cm); + tor_free(sm); + dimap_free(private_keys, NULL); +} + +struct testcase_t ntor_v3_tests[] = { + { "testvecs", test_ntor3_testvecs, 0, NULL, NULL, }, + END_OF_TESTCASES, +}; diff --git a/src/test/test_process_descs.c b/src/test/test_process_descs.c index fa2657f6c2..5503fc69ee 100644 --- a/src/test/test_process_descs.c +++ b/src/test/test_process_descs.c @@ -39,12 +39,20 @@ test_process_descs_versions(void *arg) { "Tor 0.4.1.1-alpha", true }, { "Tor 0.4.1.4-rc", true }, { "Tor 0.4.1.5", true }, + { "Tor 0.4.2.1-alpha", true }, + { "Tor 0.4.2.4-rc", true }, + { "Tor 0.4.2.5", true }, + { "Tor 0.4.3.0-alpha-dev", true }, + { "Tor 0.4.3.8", true }, + { "Tor 0.4.4.9", true }, + { "Tor 0.4.5.5-rc", true }, // new enough to be supported { "Tor 0.3.5.7", false }, { "Tor 0.3.5.8", false }, - { "Tor 0.4.2.1-alpha", false }, - { "Tor 0.4.2.4-rc", false }, - { "Tor 0.4.3.0-alpha-dev", false }, + { "Tor 0.4.5.6", false }, + { "Tor 0.4.6.0-alpha-dev", false }, + { "Tor 0.4.6.5", false }, + { "Tor 0.4.7.0-alpha-dev", false }, // Very far in the future { "Tor 100.100.1.5", false }, }; diff --git a/src/test/test_protover.c b/src/test/test_protover.c index 4b25a86e26..8cc3bcf0e5 100644 --- a/src/test/test_protover.c +++ b/src/test/test_protover.c @@ -23,13 +23,6 @@ static void test_protover_parse(void *arg) { (void) arg; -#ifdef HAVE_RUST - /** This test is disabled on rust builds, because it only exists to test - * internal C functions. */ - tt_skip(); - done: - ; -#else /* !defined(HAVE_RUST) */ char *re_encoded = NULL; const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16"; @@ -64,18 +57,12 @@ test_protover_parse(void *arg) SMARTLIST_FOREACH(elts, proto_entry_t *, ent, proto_entry_free(ent)); smartlist_free(elts); tor_free(re_encoded); -#endif /* defined(HAVE_RUST) */ } static void test_protover_parse_fail(void *arg) { (void)arg; -#ifdef HAVE_RUST - /** This test is disabled on rust builds, because it only exists to test - * internal C functions. */ - tt_skip(); -#else smartlist_t *elts; /* random junk */ @@ -108,7 +95,6 @@ test_protover_parse_fail(void *arg) "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); tt_ptr_op(elts, OP_EQ, NULL); -#endif /* defined(HAVE_RUST) */ done: ; } @@ -265,7 +251,7 @@ test_protover_all_supported(void *arg) #endif /* !defined(ALL_BUGS_ARE_FATAL) */ /* Protocol name too long */ -#if !defined(HAVE_RUST) && !defined(ALL_BUGS_ARE_FATAL) +#if !defined(ALL_BUGS_ARE_FATAL) tor_capture_bugs_(1); tt_assert(protover_all_supported( "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @@ -273,7 +259,7 @@ test_protover_all_supported(void *arg) "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaa=1-65536", &msg)); tor_end_capture_bugs_(); -#endif /* !defined(HAVE_RUST) && !defined(ALL_BUGS_ARE_FATAL) */ +#endif /* !defined(ALL_BUGS_ARE_FATAL) */ done: tor_end_capture_bugs_(); diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 27e74d5ebf..07c5032933 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -233,6 +233,10 @@ test_pt_protocol(void *arg) handle_proxy_line(line, mp); tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); + strlcpy(line,"CMETHOD-ERROR fakename not supported",sizeof(line)); + handle_proxy_line(line, mp); + tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); + strlcpy(line,"CMETHODS DONE",sizeof(line)); handle_proxy_line(line, mp); tt_assert(mp->conf_state == PT_PROTO_CONFIGURED); diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh deleted file mode 100755 index 804d2ada36..0000000000 --- a/src/test/test_rust.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -# Test all Rust crates - -set -e - -export LSAN_OPTIONS=suppressions=${abs_top_srcdir:-../../..}/src/test/rust_supp.txt - -# When testing Cargo we pass a number of very specific linker flags down -# through Cargo. We do not, however, want these flags to affect things like -# build scripts, only the tests that we're compiling. To ensure this happens -# we unconditionally pass `--target` into Cargo, ensuring that `RUSTFLAGS` in -# the environment won't make their way into build scripts. -rustc_host=$(rustc -vV | grep host | sed 's/host: //') - -for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do - if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then - # shellcheck disable=SC2086 - cd "${abs_top_builddir:-../../..}/src/rust" && \ - CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \ - "${CARGO:-cargo}" test "${CARGO_ONLINE-'--frozen'}" \ - --features "test_linking_hack" \ - --target "$rustc_host" \ - ${EXTRA_CARGO_OPTIONS} \ - --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1 - fi -done - -exit $exitcode diff --git a/src/test/test_sandbox.c b/src/test/test_sandbox.c new file mode 100644 index 0000000000..7ec08a3546 --- /dev/null +++ b/src/test/test_sandbox.c @@ -0,0 +1,349 @@ +/* Copyright (c) 2021, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef _LARGEFILE64_SOURCE +/** + * Temporarily required for O_LARGEFILE flag. Needs to be removed + * with the libevent fix. + */ +#define _LARGEFILE64_SOURCE +#endif /* !defined(_LARGEFILE64_SOURCE) */ + +#include "orconfig.h" + +#include "lib/sandbox/sandbox.h" + +#ifdef USE_LIBSECCOMP + +#include <dirent.h> +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#include "core/or/or.h" + +#include "test/test.h" +#include "test/log_test_helpers.h" + +typedef struct { + sandbox_cfg_t *cfg; + + char *file_ops_allowed; + char *file_ops_blocked; + + char *file_rename_target_allowed; + + char *dir_ops_allowed; + char *dir_ops_blocked; +} sandbox_data_t; + +/* All tests are skipped when coverage support is enabled (see further below) + * as the sandbox interferes with the use of gcov. Prevent a compiler warning + * by omitting these definitions in that case. */ +#ifndef ENABLE_COVERAGE +static void * +setup_sandbox(const struct testcase_t *testcase) +{ + sandbox_data_t *data = tor_malloc_zero(sizeof(*data)); + + (void)testcase; + + /* Establish common file and directory names within the test suite's + * temporary directory. */ + data->file_ops_allowed = tor_strdup(get_fname("file_ops_allowed")); + data->file_ops_blocked = tor_strdup(get_fname("file_ops_blocked")); + + data->file_rename_target_allowed = + tor_strdup(get_fname("file_rename_target_allowed")); + + data->dir_ops_allowed = tor_strdup(get_fname("dir_ops_allowed")); + data->dir_ops_blocked = tor_strdup(get_fname("dir_ops_blocked")); + + /* Create the corresponding filesystem objects. */ + creat(data->file_ops_allowed, S_IRWXU); + creat(data->file_ops_blocked, S_IRWXU); + mkdir(data->dir_ops_allowed, S_IRWXU); + mkdir(data->dir_ops_blocked, S_IRWXU); + + /* Create the sandbox configuration. */ + data->cfg = sandbox_cfg_new(); + + sandbox_cfg_allow_open_filename(&data->cfg, + tor_strdup(data->file_ops_allowed)); + sandbox_cfg_allow_open_filename(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + + sandbox_cfg_allow_chmod_filename(&data->cfg, + tor_strdup(data->file_ops_allowed)); + sandbox_cfg_allow_chmod_filename(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + sandbox_cfg_allow_chown_filename(&data->cfg, + tor_strdup(data->file_ops_allowed)); + sandbox_cfg_allow_chown_filename(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + + sandbox_cfg_allow_rename(&data->cfg, tor_strdup(data->file_ops_allowed), + tor_strdup(data->file_rename_target_allowed)); + + sandbox_cfg_allow_openat_filename(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + + sandbox_cfg_allow_opendir_dirname(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + + sandbox_cfg_allow_stat_filename(&data->cfg, + tor_strdup(data->file_ops_allowed)); + sandbox_cfg_allow_stat_filename(&data->cfg, + tor_strdup(data->dir_ops_allowed)); + + /* Activate the sandbox, which will remain in effect until the process + * terminates. */ + sandbox_init(data->cfg); + + return data; +} + +static int +cleanup_sandbox(const struct testcase_t *testcase, void *data_) +{ + sandbox_data_t *data = data_; + + (void)testcase; + + tor_free(data->dir_ops_blocked); + tor_free(data->dir_ops_allowed); + tor_free(data->file_rename_target_allowed); + tor_free(data->file_ops_blocked); + tor_free(data->file_ops_allowed); + + tor_free(data); + + return 1; +} + +static const struct testcase_setup_t sandboxed_testcase_setup = { + .setup_fn = setup_sandbox, + .cleanup_fn = cleanup_sandbox +}; +#endif /* !defined(ENABLE_COVERAGE) */ + +static void +test_sandbox_is_active(void *ignored) +{ + (void)ignored; + + tt_assert(!sandbox_is_active()); + + sandbox_init(sandbox_cfg_new()); + tt_assert(sandbox_is_active()); + + done: + (void)0; +} + +static void +test_sandbox_open_filename(void *arg) +{ + sandbox_data_t *data = arg; + int fd, errsv; + + fd = open(sandbox_intern_string(data->file_ops_allowed), O_RDONLY); + if (fd == -1) + tt_abort_perror("open"); + close(fd); + + /* It might be nice to use sandbox_intern_string() in the line below as well + * (and likewise in the test cases that follow) but this would require + * capturing the warning message it logs, and the mechanism for doing so + * relies on system calls that are normally blocked by the sandbox and may + * vary across architectures. */ + fd = open(data->file_ops_blocked, O_RDONLY); + errsv = errno; + tt_int_op(fd, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + if (fd >= 0) + close(fd); +} + +static void +test_sandbox_chmod_filename(void *arg) +{ + sandbox_data_t *data = arg; + int rc, errsv; + + if (chmod(sandbox_intern_string(data->file_ops_allowed), + S_IRUSR | S_IWUSR) != 0) + tt_abort_perror("chmod"); + + rc = chmod(data->file_ops_blocked, S_IRUSR | S_IWUSR); + errsv = errno; + tt_int_op(rc, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + (void)0; +} + +static void +test_sandbox_chown_filename(void *arg) +{ + sandbox_data_t *data = arg; + int rc, errsv; + + if (chown(sandbox_intern_string(data->file_ops_allowed), -1, -1) != 0) + tt_abort_perror("chown"); + + rc = chown(data->file_ops_blocked, -1, -1); + errsv = errno; + tt_int_op(rc, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + (void)0; +} + +static void +test_sandbox_rename_filename(void *arg) +{ + sandbox_data_t *data = arg; + const char *fname_old = sandbox_intern_string(data->file_ops_allowed), + *fname_new = sandbox_intern_string(data->file_rename_target_allowed); + int rc, errsv; + + if (rename(fname_old, fname_new) != 0) + tt_abort_perror("rename"); + + rc = rename(fname_new, fname_old); + errsv = errno; + tt_int_op(rc, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + (void)0; +} + +static void +test_sandbox_openat_filename(void *arg) +{ + sandbox_data_t *data = arg; + int flags = O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_DIRECTORY | O_CLOEXEC; + int fd, errsv; + + fd = openat(AT_FDCWD, sandbox_intern_string(data->dir_ops_allowed), flags); + if (fd < 0) + tt_abort_perror("openat"); + close(fd); + + fd = openat(AT_FDCWD, data->dir_ops_blocked, flags); + errsv = errno; + tt_int_op(fd, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + if (fd >= 0) + close(fd); +} + +static void +test_sandbox_opendir_dirname(void *arg) +{ + sandbox_data_t *data = arg; + DIR *dir; + int errsv; + + dir = opendir(sandbox_intern_string(data->dir_ops_allowed)); + if (dir == NULL) + tt_abort_perror("opendir"); + closedir(dir); + + dir = opendir(data->dir_ops_blocked); + errsv = errno; + tt_ptr_op(dir, OP_EQ, NULL); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + if (dir) + closedir(dir); +} + +static void +test_sandbox_stat_filename(void *arg) +{ + sandbox_data_t *data = arg; + struct stat st; + + if (stat(sandbox_intern_string(data->file_ops_allowed), &st) != 0) + tt_abort_perror("stat"); + + int rc = stat(data->file_ops_blocked, &st); + int errsv = errno; + tt_int_op(rc, OP_EQ, -1); + tt_int_op(errsv, OP_EQ, EPERM); + + done: + (void)0; +} + +#define SANDBOX_TEST_SKIPPED(name) \ + { #name, test_sandbox_ ## name, TT_SKIP, NULL, NULL } + +/* Skip all tests when coverage support is enabled, as the sandbox interferes + * with gcov and prevents it from producing any results. */ +#ifdef ENABLE_COVERAGE +#define SANDBOX_TEST(name, flags) SANDBOX_TEST_SKIPPED(name) +#define SANDBOX_TEST_IN_SANDBOX(name) SANDBOX_TEST_SKIPPED(name) +#else +#define SANDBOX_TEST(name, flags) \ + { #name, test_sandbox_ ## name, flags, NULL, NULL } +#define SANDBOX_TEST_IN_SANDBOX(name) \ + { #name, test_sandbox_ ## name, TT_FORK, &sandboxed_testcase_setup, NULL } +#endif /* defined(ENABLE_COVERAGE) */ + +struct testcase_t sandbox_tests[] = { + SANDBOX_TEST(is_active, TT_FORK), + +/* When Tor is built with fragile compiler-hardening the sandbox is unable to + * filter requests to open files or directories (on systems where glibc uses + * the "open" system call to provide this functionality), as doing so would + * interfere with the address sanitizer as it retrieves information about the + * running process via the filesystem. Skip these tests in that case as the + * corresponding functions are likely to have no effect and this will cause the + * tests to fail. */ +#ifdef ENABLE_FRAGILE_HARDENING + SANDBOX_TEST_SKIPPED(open_filename), + SANDBOX_TEST_SKIPPED(opendir_dirname), +#else + SANDBOX_TEST_IN_SANDBOX(open_filename), + SANDBOX_TEST_IN_SANDBOX(opendir_dirname), +#endif /* defined(ENABLE_FRAGILE_HARDENING) */ + + SANDBOX_TEST_IN_SANDBOX(openat_filename), + SANDBOX_TEST_IN_SANDBOX(chmod_filename), + SANDBOX_TEST_IN_SANDBOX(chown_filename), + SANDBOX_TEST_IN_SANDBOX(rename_filename), + +/* Currently the sandbox is unable to filter stat() calls on systems where + * glibc implements this function using either of the legacy "stat" or "stat64" + * system calls, or (in glibc version 2.33 and later) either of the newer + * "newfstatat" or "statx" syscalls. + * + * Skip testing sandbox_cfg_allow_stat_filename() if it seems the likely the + * function will have no effect and the test will therefore not succeed. */ +#if !defined(__NR_stat) && !defined(__NR_stat64) && !defined(__NR_newfstatat) \ + && !(defined(__i386__) && defined(__NR_statx)) + SANDBOX_TEST_IN_SANDBOX(stat_filename), +#else + SANDBOX_TEST_SKIPPED(stat_filename), +#endif + END_OF_TESTCASES +}; + +#endif /* defined(USE_SECCOMP) */ diff --git a/src/test/test_stats.c b/src/test/test_stats.c index af035ae54c..c7e0c10845 100644 --- a/src/test/test_stats.c +++ b/src/test/test_stats.c @@ -51,7 +51,7 @@ #include "feature/stats/bw_array_st.h" #include "feature/relay/router.h" -#include "event2/dns.h" +#include <event2/dns.h> /** Run unit tests for some stats code. */ static void @@ -721,7 +721,7 @@ test_overload_stats(void *arg) stats_str = rep_hist_get_overload_stats_lines(); tt_assert(!stats_str); - /* Note a overload */ + /* Note an overload */ rep_hist_note_overload(OVERLOAD_GENERAL); /* Move the time forward one hour */ @@ -742,7 +742,7 @@ test_overload_stats(void *arg) /* Now the time should be 2002-01-07 00:00:00 */ - /* Note a overload */ + /* Note an overload */ rep_hist_note_overload(OVERLOAD_GENERAL); stats_str = rep_hist_get_overload_general_line(); diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c index 4c5f3a3270..457b0fa796 100644 --- a/src/test/test_voting_flags.c +++ b/src/test/test_voting_flags.c @@ -62,7 +62,8 @@ check_result(flag_vote_test_cfg_t *c) bool result = false; routerstatus_t rs; memset(&rs, 0, sizeof(rs)); - dirauth_set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0); + dirauth_set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, + 0, 0); tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on); tt_str_op(rs.nickname, OP_EQ, c->expected.nickname); diff --git a/src/tools/include.am b/src/tools/include.am index 6daa27f6de..86cd0acbfd 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -10,7 +10,6 @@ src_tools_tor_resolve_LDADD = \ src/trunnel/libor-trunnel.a \ $(TOR_UTIL_LIBS) \ $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\ - $(rust_ldadd) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_USERENV@ if COVERAGE_ENABLED @@ -34,7 +33,6 @@ src_tools_tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) src_tools_tor_gencert_LDADD = \ $(TOR_CRYPTO_LIBS) \ $(TOR_UTIL_LIBS) \ - $(rust_ldadd) \ @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ endif diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 09ff8df4ab..ad52fdaa22 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -253,7 +253,7 @@ build_socks_resolve_request(uint8_t **out, } static void -onion_warning(const char *hostname) +onion_hs_warning(const char *hostname) { log_warn(LD_NET, "%s is a hidden service; those don't have IP addresses. " @@ -264,6 +264,15 @@ onion_warning(const char *hostname) hostname); } +static void +onion_exit_warning(const char *hostname) +{ + log_warn(LD_NET, + "%s is a link pointing to an exit node; however, .exit domains" + "have been long defunct and are not valid anymore.", + hostname); +} + /** Given a <b>len</b>-byte SOCKS4a response in <b>response</b>, set * *<b>addr_out</b> to the address it contains (in host order). * Return 0 on success, -1 on error. @@ -306,9 +315,15 @@ parse_socks4a_resolve_response(const char *hostname, if (status != 90) { log_warn(LD_NET,"Got status response '%d': socks request failed.", status); if (!strcasecmpend(hostname, ".onion")) { - onion_warning(hostname); + onion_hs_warning(hostname); result = -1; goto cleanup; } + + if (!strcasecmpend(hostname, ".exit")) { + onion_exit_warning(hostname); + result = -1; goto cleanup; + } + result = -1; goto cleanup; } @@ -493,7 +508,11 @@ do_resolve(const char *hostname, (unsigned)reply_field, socks5_reason_to_string(reply_field)); if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) { - onion_warning(hostname); + onion_hs_warning(hostname); + } + + if (reply_field == 4 && !strcasecmpend(hostname, ".exit")) { + onion_exit_warning(hostname); } socks5_server_reply_free(reply); diff --git a/src/trunnel/flow_control_cells.c b/src/trunnel/flow_control_cells.c new file mode 100644 index 0000000000..df44756d6b --- /dev/null +++ b/src/trunnel/flow_control_cells.c @@ -0,0 +1,382 @@ +/* flow_control_cells.c -- generated by Trunnel v1.5.3. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include <stdlib.h> +#include "trunnel-impl.h" + +#include "flow_control_cells.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int flowcontrolcells_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || flowcontrolcells_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +xoff_cell_t * +xoff_cell_new(void) +{ + xoff_cell_t *val = trunnel_calloc(1, sizeof(xoff_cell_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +xoff_cell_clear(xoff_cell_t *obj) +{ + (void) obj; +} + +void +xoff_cell_free(xoff_cell_t *obj) +{ + if (obj == NULL) + return; + xoff_cell_clear(obj); + trunnel_memwipe(obj, sizeof(xoff_cell_t)); + trunnel_free_(obj); +} + +uint8_t +xoff_cell_get_version(const xoff_cell_t *inp) +{ + return inp->version; +} +int +xoff_cell_set_version(xoff_cell_t *inp, uint8_t val) +{ + if (! ((val == 0))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +const char * +xoff_cell_check(const xoff_cell_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0)) + return "Integer out of bounds"; + return NULL; +} + +ssize_t +xoff_cell_encoded_len(const xoff_cell_t *obj) +{ + ssize_t result = 0; + + if (NULL != xoff_cell_check(obj)) + return -1; + + + /* Length of u8 version IN [0] */ + result += 1; + return result; +} +int +xoff_cell_clear_errors(xoff_cell_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +xoff_cell_encode(uint8_t *output, const size_t avail, const xoff_cell_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = xoff_cell_encoded_len(obj); +#endif + + if (NULL != (msg = xoff_cell_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As xoff_cell_parse(), but do not allocate the output object. + */ +static ssize_t +xoff_cell_parse_into(xoff_cell_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0)) + goto fail; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +xoff_cell_parse(xoff_cell_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = xoff_cell_new(); + if (NULL == *output) + return -1; + result = xoff_cell_parse_into(*output, input, len_in); + if (result < 0) { + xoff_cell_free(*output); + *output = NULL; + } + return result; +} +xon_cell_t * +xon_cell_new(void) +{ + xon_cell_t *val = trunnel_calloc(1, sizeof(xon_cell_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +xon_cell_clear(xon_cell_t *obj) +{ + (void) obj; +} + +void +xon_cell_free(xon_cell_t *obj) +{ + if (obj == NULL) + return; + xon_cell_clear(obj); + trunnel_memwipe(obj, sizeof(xon_cell_t)); + trunnel_free_(obj); +} + +uint8_t +xon_cell_get_version(const xon_cell_t *inp) +{ + return inp->version; +} +int +xon_cell_set_version(xon_cell_t *inp, uint8_t val) +{ + if (! ((val == 0))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +uint32_t +xon_cell_get_kbps_ewma(const xon_cell_t *inp) +{ + return inp->kbps_ewma; +} +int +xon_cell_set_kbps_ewma(xon_cell_t *inp, uint32_t val) +{ + inp->kbps_ewma = val; + return 0; +} +const char * +xon_cell_check(const xon_cell_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0)) + return "Integer out of bounds"; + return NULL; +} + +ssize_t +xon_cell_encoded_len(const xon_cell_t *obj) +{ + ssize_t result = 0; + + if (NULL != xon_cell_check(obj)) + return -1; + + + /* Length of u8 version IN [0] */ + result += 1; + + /* Length of u32 kbps_ewma */ + result += 4; + return result; +} +int +xon_cell_clear_errors(xon_cell_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +xon_cell_encode(uint8_t *output, const size_t avail, const xon_cell_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = xon_cell_encoded_len(obj); +#endif + + if (NULL != (msg = xon_cell_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + /* Encode u32 kbps_ewma */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->kbps_ewma)); + written += 4; ptr += 4; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As xon_cell_parse(), but do not allocate the output object. + */ +static ssize_t +xon_cell_parse_into(xon_cell_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0)) + goto fail; + + /* Parse u32 kbps_ewma */ + CHECK_REMAINING(4, truncated); + obj->kbps_ewma = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +xon_cell_parse(xon_cell_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = xon_cell_new(); + if (NULL == *output) + return -1; + result = xon_cell_parse_into(*output, input, len_in); + if (result < 0) { + xon_cell_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/flow_control_cells.h b/src/trunnel/flow_control_cells.h new file mode 100644 index 0000000000..b8108b9a24 --- /dev/null +++ b/src/trunnel/flow_control_cells.h @@ -0,0 +1,120 @@ +/* flow_control_cells.h -- generated by Trunnel v1.5.3. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_FLOW_CONTROL_CELLS_H +#define TRUNNEL_FLOW_CONTROL_CELLS_H + +#include <stdint.h> +#include "trunnel.h" + +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_XOFF_CELL) +struct xoff_cell_st { + uint8_t version; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct xoff_cell_st xoff_cell_t; +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_XON_CELL) +struct xon_cell_st { + uint8_t version; + uint32_t kbps_ewma; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct xon_cell_st xon_cell_t; +/** Return a newly allocated xoff_cell with all elements set to zero. + */ +xoff_cell_t *xoff_cell_new(void); +/** Release all storage held by the xoff_cell in 'victim'. (Do nothing + * if 'victim' is NULL.) + */ +void xoff_cell_free(xoff_cell_t *victim); +/** Try to parse a xoff_cell from the buffer in 'input', using up to + * 'len_in' bytes from the input buffer. On success, return the number + * of bytes consumed and set *output to the newly allocated + * xoff_cell_t. On failure, return -2 if the input appears truncated, + * and -1 if the input is otherwise invalid. + */ +ssize_t xoff_cell_parse(xoff_cell_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * xoff_cell in 'obj'. On failure, return a negative value. Note that + * this value may be an overestimate, and can even be an underestimate + * for certain unencodeable objects. + */ +ssize_t xoff_cell_encoded_len(const xoff_cell_t *obj); +/** Try to encode the xoff_cell from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t xoff_cell_encode(uint8_t *output, size_t avail, const xoff_cell_t *input); +/** Check whether the internal state of the xoff_cell in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *xoff_cell_check(const xoff_cell_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int xoff_cell_clear_errors(xoff_cell_t *obj); +/** Return the value of the version field of the xoff_cell_t in 'inp' + */ +uint8_t xoff_cell_get_version(const xoff_cell_t *inp); +/** Set the value of the version field of the xoff_cell_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int xoff_cell_set_version(xoff_cell_t *inp, uint8_t val); +/** Return a newly allocated xon_cell with all elements set to zero. + */ +xon_cell_t *xon_cell_new(void); +/** Release all storage held by the xon_cell in 'victim'. (Do nothing + * if 'victim' is NULL.) + */ +void xon_cell_free(xon_cell_t *victim); +/** Try to parse a xon_cell from the buffer in 'input', using up to + * 'len_in' bytes from the input buffer. On success, return the number + * of bytes consumed and set *output to the newly allocated + * xon_cell_t. On failure, return -2 if the input appears truncated, + * and -1 if the input is otherwise invalid. + */ +ssize_t xon_cell_parse(xon_cell_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * xon_cell in 'obj'. On failure, return a negative value. Note that + * this value may be an overestimate, and can even be an underestimate + * for certain unencodeable objects. + */ +ssize_t xon_cell_encoded_len(const xon_cell_t *obj); +/** Try to encode the xon_cell from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t xon_cell_encode(uint8_t *output, size_t avail, const xon_cell_t *input); +/** Check whether the internal state of the xon_cell in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *xon_cell_check(const xon_cell_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int xon_cell_clear_errors(xon_cell_t *obj); +/** Return the value of the version field of the xon_cell_t in 'inp' + */ +uint8_t xon_cell_get_version(const xon_cell_t *inp); +/** Set the value of the version field of the xon_cell_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int xon_cell_set_version(xon_cell_t *inp, uint8_t val); +/** Return the value of the kbps_ewma field of the xon_cell_t in 'inp' + */ +uint32_t xon_cell_get_kbps_ewma(const xon_cell_t *inp); +/** Set the value of the kbps_ewma field of the xon_cell_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int xon_cell_set_kbps_ewma(xon_cell_t *inp, uint32_t val); + + +#endif diff --git a/src/trunnel/flow_control_cells.trunnel b/src/trunnel/flow_control_cells.trunnel new file mode 100644 index 0000000000..9d07b568a9 --- /dev/null +++ b/src/trunnel/flow_control_cells.trunnel @@ -0,0 +1,20 @@ +/* This file contains the xon and xoff cell definitions, for flow control. */ + +/* xoff cell definition. Tells the other endpoint to stop sending, because + * we have too much data queued for this stream. */ +struct xoff_cell { + /* Version field. */ + u8 version IN [0x00]; +} + +/* xon cell declaration. Tells the other endpoint to resume sending and/or + * update its sending rate on this stream based on advisory information. */ +struct xon_cell { + /* Version field. */ + u8 version IN [0x00]; + + /* Advisory field: The ewma rate of socket drain we have seen so far + * on this stream, in kilobytes/sec (1000 bytes/sec). May be zero, + * which means no rate advice. */ + u32 kbps_ewma; +} diff --git a/src/trunnel/include.am b/src/trunnel/include.am index 6c3a5ff06b..00a96536f1 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -12,6 +12,7 @@ TRUNNELINPUTS = \ src/trunnel/pwbox.trunnel \ src/trunnel/channelpadding_negotiation.trunnel \ src/trunnel/sendme_cell.trunnel \ + src/trunnel/flow_control_cells.trunnel \ src/trunnel/socks5.trunnel \ src/trunnel/circpad_negotiation.trunnel @@ -26,6 +27,7 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ src/trunnel/sendme_cell.c \ + src/trunnel/flow_control_cells.c \ src/trunnel/socks5.c \ src/trunnel/netinfo.c \ src/trunnel/circpad_negotiation.c @@ -43,6 +45,7 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ src/trunnel/sendme_cell.h \ + src/trunnel/flow_control_cells.h \ src/trunnel/socks5.h \ src/trunnel/netinfo.h \ src/trunnel/circpad_negotiation.h diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 141a603e2e..036314d39a 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -217,7 +217,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.4.6.9-dev" +#define VERSION "0.4.7.3-alpha-dev" #define HAVE_STRUCT_SOCKADDR_IN6 #define HAVE_STRUCT_IN6_ADDR |