diff options
Diffstat (limited to 'src')
354 files changed, 26810 insertions, 4519 deletions
diff --git a/src/app/config/config.c b/src/app/config/config.c index 0b1b758d96..1676e9349a 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -64,6 +64,7 @@ #include "app/config/confparse.h" #include "app/config/statefile.h" #include "app/main/main.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" @@ -112,9 +113,9 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" -#include "lib/log/git_revision.h" #include "lib/net/resolve.h" #include "lib/sandbox/sandbox.h" +#include "lib/version/torversion.h" #ifdef ENABLE_NSS #include "lib/crypt_ops/crypto_nss_mgt.h" @@ -144,7 +145,7 @@ #include "lib/process/pidfile.h" #include "lib/process/restrict.h" #include "lib/process/setuid.h" -#include "lib/process/subprocess.h" +#include "lib/process/process.h" #include "lib/net/gethostname.h" #include "lib/thread/numcpus.h" @@ -342,6 +343,7 @@ static config_var_t option_vars_[] = { V(ClientOnly, BOOL, "0"), V(ClientPreferIPv6ORPort, AUTOBOOL, "auto"), V(ClientPreferIPv6DirPort, AUTOBOOL, "auto"), + V(ClientAutoIPv6ORPort, BOOL, "0"), V(ClientRejectInternalAddresses, BOOL, "1"), V(ClientTransportPlugin, LINELIST, NULL), V(ClientUseIPv6, BOOL, "0"), @@ -391,6 +393,10 @@ static config_var_t option_vars_[] = { OBSOLETE("DynamicDHGroups"), VPORT(DNSPort), OBSOLETE("DNSListenAddress"), + V(DormantClientTimeout, INTERVAL, "24 hours"), + V(DormantTimeoutDisabledByIdleStreams, BOOL, "1"), + V(DormantOnFirstStartup, BOOL, "0"), + V(DormantCanceledByStartup, BOOL, "0"), /* DoS circuit creation options. */ V(DoSCircuitCreationEnabled, AUTOBOOL, "auto"), V(DoSCircuitCreationMinConnections, UINT, "0"), @@ -416,6 +422,10 @@ static config_var_t option_vars_[] = { V(ExcludeExitNodes, ROUTERSET, NULL), OBSOLETE("ExcludeSingleHopRelays"), V(ExitNodes, ROUTERSET, NULL), + /* Researchers need a way to tell their clients to use specific + * middles that they also control, to allow safe live-network + * experimentation with new padding machines. */ + V(MiddleNodes, ROUTERSET, NULL), V(ExitPolicy, LINELIST, NULL), V(ExitPolicyRejectPrivate, BOOL, "1"), V(ExitPolicyRejectLocalInterfaces, BOOL, "0"), @@ -975,42 +985,6 @@ set_options(or_options_t *new_val, char **msg) return 0; } -/** The version of this Tor process, as parsed. */ -static char *the_tor_version = NULL; -/** A shorter version of this Tor process's version, for export in our router - * descriptor. (Does not include the git version, if any.) */ -static char *the_short_tor_version = NULL; - -/** Return the current Tor version. */ -const char * -get_version(void) -{ - if (the_tor_version == NULL) { - if (strlen(tor_git_revision)) { - tor_asprintf(&the_tor_version, "%s (git-%s)", get_short_version(), - tor_git_revision); - } else { - the_tor_version = tor_strdup(get_short_version()); - } - } - return the_tor_version; -} - -/** Return the current Tor version, without any git tag. */ -const char * -get_short_version(void) -{ - - if (the_short_tor_version == NULL) { -#ifdef TOR_BUILD_TAG - tor_asprintf(&the_short_tor_version, "%s (%s)", VERSION, TOR_BUILD_TAG); -#else - the_short_tor_version = tor_strdup(VERSION); -#endif - } - return the_short_tor_version; -} - /** Release additional memory allocated in options */ STATIC void @@ -1070,9 +1044,6 @@ config_free_all(void) tor_free(torrc_defaults_fname); tor_free(global_dirfrontpagecontents); - tor_free(the_short_tor_version); - tor_free(the_tor_version); - cleanup_protocol_warning_severity_level(); have_parsed_cmdline = 0; @@ -1443,10 +1414,10 @@ options_act_reversible(const or_options_t *old_options, char **msg) * processes. */ if (running_tor && options->RunAsDaemon) { if (! start_daemon_has_been_called()) - crypto_prefork(); + subsystems_prefork(); /* No need to roll back, since you can't change the value. */ if (start_daemon()) - crypto_postfork(); + subsystems_postfork(); } #ifdef HAVE_SYSTEMD @@ -1735,6 +1706,7 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out) int routerset_usage = routerset_needs_geoip(options->EntryNodes) || routerset_needs_geoip(options->ExitNodes) || + routerset_needs_geoip(options->MiddleNodes) || routerset_needs_geoip(options->ExcludeExitNodes) || routerset_needs_geoip(options->ExcludeNodes) || routerset_needs_geoip(options->HSLayer2Nodes) || @@ -2042,9 +2014,6 @@ options_act(const or_options_t *old_options) finish_daemon(options->DataDirectory); } - /* See whether we need to enable/disable our once-a-second timer. */ - reschedule_per_second_timer(); - /* We want to reinit keys as needed before we do much of anything else: keys are important, and other things can depend on them. */ if (transition_affects_workers || @@ -2177,6 +2146,7 @@ options_act(const or_options_t *old_options) options->HSLayer2Nodes) || !routerset_equal(old_options->HSLayer3Nodes, options->HSLayer3Nodes) || + !routerset_equal(old_options->MiddleNodes, options->MiddleNodes) || options->StrictNodes != old_options->StrictNodes) { log_info(LD_CIRC, "Changed to using entry guards or bridges, or changed " @@ -3441,6 +3411,8 @@ options_validate(or_options_t *old_options, or_options_t *options, if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo))) REJECT("ContactInfo config option must be UTF-8."); + check_network_configuration(server_mode(options)); + /* Special case on first boot if no Log options are given. */ if (!options->Logs && !options->RunAsDaemon && !from_setconf) { if (quiet_level == 0) @@ -3592,7 +3564,8 @@ options_validate(or_options_t *old_options, or_options_t *options, "(Bridge/V3)AuthoritativeDir is set."); /* If we have a v3bandwidthsfile and it's broken, complain on startup */ if (options->V3BandwidthsFile && !old_options) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL); + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL, + NULL); } /* same for guardfraction file */ if (options->GuardfractionFile && !old_options) { @@ -3897,6 +3870,10 @@ options_validate(or_options_t *old_options, or_options_t *options, "default."); } + if (options->DormantClientTimeout < 10*60 && !options->TestingTorNetwork) { + REJECT("DormantClientTimeout is too low. It must be at least 10 minutes."); + } + if (options->PathBiasNoticeRate > 1.0) { tor_asprintf(msg, "PathBiasNoticeRate is too high. " @@ -3948,7 +3925,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } if (options->HeartbeatPeriod && - options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD) { + options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD && + !options->TestingTorNetwork) { log_warn(LD_CONFIG, "HeartbeatPeriod option is too short; " "raising to %d seconds.", MIN_HEARTBEAT_PERIOD); options->HeartbeatPeriod = MIN_HEARTBEAT_PERIOD; diff --git a/src/app/config/config.h b/src/app/config/config.h index 301faf7067..46db02f944 100644 --- a/src/app/config/config.h +++ b/src/app/config/config.h @@ -41,8 +41,6 @@ const char *escaped_safe_str_client(const char *address); const char *escaped_safe_str(const char *address); void init_protocol_warning_severity_level(void); int get_protocol_warning_severity_level(void); -const char *get_version(void); -const char *get_short_version(void); /** An error from options_trial_assign() or options_init_from_string(). */ typedef enum setopt_err_t { diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h index 74d2fefa16..bd707fd193 100644 --- a/src/app/config/or_options_st.h +++ b/src/app/config/or_options_st.h @@ -72,6 +72,9 @@ struct or_options_t { routerset_t *ExitNodes; /**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as exits. */ + routerset_t *MiddleNodes; /**< Structure containing nicknames, digests, + * country codes and IP address patterns of ORs to + * consider as middles. */ routerset_t *EntryNodes;/**< Structure containing nicknames, digests, * country codes and IP address patterns of ORs to * consider as entry points. */ @@ -666,6 +669,9 @@ struct or_options_t { * accessing this value directly. */ int ClientPreferIPv6DirPort; + /** If true, prefer an IPv4 or IPv6 OR port at random. */ + int ClientAutoIPv6ORPort; + /** The length of time that we think a consensus should be fresh. */ int V3AuthVotingInterval; /** The length of time we think it will take to distribute votes. */ @@ -1072,6 +1078,25 @@ struct or_options_t { /** Autobool: Do we refuse single hop client rendezvous? */ int DoSRefuseSingleHopClientRendezvous; + + /** Interval: how long without activity does it take for a client + * to become dormant? + **/ + int DormantClientTimeout; + + /** Boolean: true if having an idle stream is sufficient to prevent a client + * from becoming dormant. + **/ + int DormantTimeoutDisabledByIdleStreams; + + /** Boolean: true if Tor should be dormant the first time it starts with + * a datadirectory; false otherwise. */ + int DormantOnFirstStartup; + /** + * Boolean: true if Tor should treat every startup event as cancelling + * a possible previous dormant state. + **/ + int DormantCanceledByStartup; }; #endif diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h index 5f8214d146..cdb9b38287 100644 --- a/src/app/config/or_state_st.h +++ b/src/app/config/or_state_st.h @@ -87,6 +87,13 @@ struct or_state_t { /** When did we last rotate our onion key? "0" for 'no idea'. */ time_t LastRotatedOnionKey; + + /** Number of minutes since the last user-initiated request (as defined by + * the dormant net-status system.) Set to zero if we are dormant. */ + int MinutesSinceUserActivity; + /** True if we were dormant when we last wrote the file; false if we + * weren't. "auto" on initial startup. */ + int Dormant; }; #endif diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c index 89039a05b5..9681f6f8b3 100644 --- a/src/app/config/statefile.c +++ b/src/app/config/statefile.c @@ -34,6 +34,7 @@ #include "app/config/config.h" #include "app/config/confparse.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "feature/client/entrynodes.h" @@ -45,6 +46,7 @@ #include "app/config/statefile.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" +#include "lib/version/torversion.h" #include "app/config/or_state_st.h" @@ -131,6 +133,9 @@ static config_var_t state_vars_[] = { VAR("CircuitBuildTimeBin", LINELIST_S, BuildtimeHistogram, NULL), VAR("BuildtimeHistogram", LINELIST_V, BuildtimeHistogram, NULL), + V(MinutesSinceUserActivity, UINT, NULL), + V(Dormant, AUTOBOOL, "auto"), + END_OF_CONFIG_VARS }; @@ -308,6 +313,8 @@ or_state_set(or_state_t *new_state) get_circuit_build_times_mutable(),global_state) < 0) { ret = -1; } + netstatus_load_from_state(global_state, time(NULL)); + return ret; } @@ -499,6 +506,8 @@ or_state_save(time_t now) entry_guards_update_state(global_state); rep_hist_update_state(global_state); circuit_build_times_update_state(get_circuit_build_times(), global_state); + netstatus_flush_to_state(global_state, now); + if (accounting_is_enabled(get_options())) accounting_run_housekeeping(now); diff --git a/src/app/main/main.c b/src/app/main/main.c index 67f2181cd5..4b60763f75 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -15,12 +15,14 @@ #include "app/config/statefile.h" #include "app/main/main.h" #include "app/main/ntmain.h" +#include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" #include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/channelpadding.h" +#include "core/or/circuitpadding.h" #include "core/or/channeltls.h" #include "core/or/circuitlist.h" #include "core/or/circuitmux_ewma.h" @@ -33,6 +35,7 @@ #include "core/or/relay.h" #include "core/or/scheduler.h" #include "core/or/status.h" +#include "core/or/versions.h" #include "feature/api/tor_api.h" #include "feature/api/tor_api_internal.h" #include "feature/client/addressmap.h" @@ -65,11 +68,11 @@ #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" -#include "lib/err/backtrace.h" #include "lib/geoip/geoip.h" +#include "lib/net/resolve.h" #include "lib/process/waitpid.h" @@ -83,6 +86,7 @@ #include "lib/encoding/confline.h" #include "lib/evloop/timers.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" #include <event2/event.h> @@ -301,6 +305,19 @@ process_signal(int sig) log_heartbeat(time(NULL)); control_event_signal(sig); break; + case SIGACTIVE: + /* "SIGACTIVE" counts as ersatz user activity. */ + note_user_activity(approx_time()); + control_event_signal(sig); + break; + case SIGDORMANT: + /* "SIGDORMANT" means to ignore past user activity */ + log_notice(LD_GENERAL, "Going dormant because of controller request."); + reset_user_activity(0); + set_network_participation(false); + schedule_rescan_periodic_events(); + control_event_signal(sig); + break; } } @@ -426,18 +443,6 @@ dumpstats(int severity) rend_service_dump_stats(severity); } -/** Called by exit() as we shut down the process. - */ -static void -exit_function(void) -{ - /* NOTE: If we ever daemonize, this gets called immediately. That's - * okay for now, because we only use this on Windows. */ -#ifdef _WIN32 - WSACleanup(); -#endif -} - #ifdef _WIN32 #define UNIX_ONLY 0 #else @@ -482,6 +487,8 @@ static struct { { SIGNEWNYM, 0, NULL }, { SIGCLEARDNSCACHE, 0, NULL }, { SIGHEARTBEAT, 0, NULL }, + { SIGACTIVE, 0, NULL }, + { SIGDORMANT, 0, NULL }, { -1, -1, NULL } }; @@ -546,18 +553,13 @@ tor_init(int argc, char *argv[]) tor_snprintf(progname, sizeof(progname), "Tor %s", get_version()); log_set_application_name(progname); - /* Set up the crypto nice and early */ - if (crypto_early_init() < 0) { - log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!"); - return -1; - } - /* Initialize the history structures. */ rep_hist_init(); /* Initialize the service cache. */ rend_cache_init(); addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ + /* Initialize the HS subsystem. */ hs_init(); @@ -632,12 +634,6 @@ tor_init(int argc, char *argv[]) rust_log_welcome_string(); #endif /* defined(HAVE_RUST) */ - if (network_init()<0) { - log_err(LD_BUG,"Error initializing network; exiting."); - return -1; - } - atexit(exit_function); - int init_rv = options_init_from_torrc(argc,argv); if (init_rv < 0) { log_err(LD_CONFIG,"Reading config failed--see warnings above."); @@ -651,9 +647,13 @@ tor_init(int argc, char *argv[]) /* The options are now initialised */ const or_options_t *options = get_options(); - /* Initialize channelpadding parameters to defaults until we get - * a consensus */ + /* Initialize channelpadding and circpad parameters to defaults + * until we get a consensus */ channelpadding_new_consensus_params(NULL); + circpad_new_consensus_params(NULL); + + /* Initialize circuit padding to defaults+torrc until we get a consensus */ + circpad_machines_init(); /* Initialize predicted ports list after loading options */ predicted_ports_init(); @@ -772,6 +772,7 @@ tor_free_all(int postfork) dns_free_all(); clear_pending_onions(); circuit_free_all(); + circpad_machines_free(); entry_guards_free_all(); pt_free_all(); channel_tls_free_all(); @@ -784,7 +785,6 @@ tor_free_all(int postfork) routerparse_free_all(); ext_orport_free_all(); control_free_all(); - tor_free_getaddrinfo_cache(); protover_free_all(); bridges_free_all(); consdiffmgr_free_all(); @@ -792,6 +792,7 @@ tor_free_all(int postfork) dos_free_all(); circuitmux_ewma_free_all(); accounting_free_all(); + protover_summary_cache_free_all(); if (!postfork) { config_free_all(); @@ -801,7 +802,6 @@ tor_free_all(int postfork) policies_free_all(); } if (!postfork) { - tor_tls_free_all(); #ifndef _WIN32 tor_getpwnam(NULL); #endif @@ -814,12 +814,12 @@ tor_free_all(int postfork) release_lockfile(); } tor_libevent_free_all(); + + subsystems_shutdown(); + /* Stuff in util.c and address.c*/ if (!postfork) { - escaped(NULL); esc_router_info(NULL); - clean_up_backtrace_handler(); - logs_free_all(); /* free log strings. do this last so logs keep working. */ } } @@ -878,7 +878,6 @@ tor_cleanup(void) later, if it makes shutdown unacceptably slow. But for now, leave it here: it's helped us catch bugs in the past. */ - crypto_global_cleanup(); } /** Read/create keys as needed, and echo our fingerprint to stdout. */ @@ -1274,7 +1273,6 @@ int run_tor_main_loop(void) { handle_signals(); - monotime_init(); timers_initialize(); initialize_mainloop_events(); @@ -1386,54 +1384,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) { int result = 0; -#ifdef _WIN32 -#ifndef HeapEnableTerminationOnCorruption -#define HeapEnableTerminationOnCorruption 1 -#endif - /* On heap corruption, just give up; don't try to play along. */ - HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); - - /* SetProcessDEPPolicy is only supported on 32-bit Windows. - * (On 64-bit Windows it always fails, and some compilers don't like the - * PSETDEP cast.) - * 32-bit Windows defines _WIN32. - * 64-bit Windows defines _WIN32 and _WIN64. */ -#ifndef _WIN64 - /* Call SetProcessDEPPolicy to permanently enable DEP. - The function will not resolve on earlier versions of Windows, - and failure is not dangerous. */ - HMODULE hMod = GetModuleHandleA("Kernel32.dll"); - if (hMod) { - typedef BOOL (WINAPI *PSETDEP)(DWORD); - PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, - "SetProcessDEPPolicy"); - if (setdeppolicy) { - /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ - setdeppolicy(3); - } - } -#endif /* !defined(_WIN64) */ -#endif /* defined(_WIN32) */ - - { - int bt_err = configure_backtrace_handler(get_version()); - if (bt_err < 0) { - log_warn(LD_BUG, "Unable to install backtrace handler: %s", - strerror(-bt_err)); - } - } - #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_); #endif - init_protocol_warning_severity_level(); + subsystems_init(); - update_approx_time(time(NULL)); - tor_threads_init(); - tor_compress_init(); - init_logging(0); - monotime_init(); + init_protocol_warning_severity_level(); int argc = tor_cfg->argc + tor_cfg->argc_owned; char **argv = tor_calloc(argc, sizeof(char*)); @@ -1469,6 +1426,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) tor_free_all(0); return -1; } + tor_make_getaddrinfo_cache_active(); // registering libevent rng #ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c new file mode 100644 index 0000000000..e0ca3ce4df --- /dev/null +++ b/src/app/main/subsysmgr.c @@ -0,0 +1,202 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "app/main/subsysmgr.h" +#include "lib/err/torerr.h" + +#include "lib/log/log.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/** + * True iff we have checked tor_subsystems for consistency. + **/ +static bool subsystem_array_validated = false; + +/** + * True if a given subsystem is initialized. Expand this array if there + * are more than this number of subsystems. (We'd rather not + * dynamically allocate in this module.) + **/ +static bool sys_initialized[128]; + +/** + * Exit with a raw assertion if the subsystems list is inconsistent; + * initialize the subsystem_initialized array. + **/ +static void +check_and_setup(void) +{ + if (subsystem_array_validated) + return; + + raw_assert(ARRAY_LENGTH(sys_initialized) >= n_tor_subsystems); + memset(sys_initialized, 0, sizeof(sys_initialized)); + + int last_level = MIN_SUBSYS_LEVEL; + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (sys->level < MIN_SUBSYS_LEVEL || sys->level > MAX_SUBSYS_LEVEL) { + fprintf(stderr, "BUG: Subsystem %s (at %u) has an invalid level %d. " + "It is supposed to be between %d and %d (inclusive).\n", + sys->name, i, sys->level, MIN_SUBSYS_LEVEL, MAX_SUBSYS_LEVEL); + raw_assert_unreached_msg("There is a bug in subsystem_list.c"); + } + if (sys->level < last_level) { + fprintf(stderr, "BUG: Subsystem %s (at #%u) is in the wrong position. " + "Its level is %d; but the previous subsystem's level was %d.\n", + sys->name, i, sys->level, last_level); + raw_assert_unreached_msg("There is a bug in subsystem_list.c"); + } + last_level = sys->level; + } + + subsystem_array_validated = true; +} + +/** + * Initialize all the subsystems; exit on failure. + **/ +int +subsystems_init(void) +{ + return subsystems_init_upto(MAX_SUBSYS_LEVEL); +} + +/** + * Initialize all the subsystems whose level is less than or equal to + * <b>target_level</b>; exit on failure. + **/ +int +subsystems_init_upto(int target_level) +{ + check_and_setup(); + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level > target_level) + break; + if (sys_initialized[i]) + continue; + int r = 0; + if (sys->initialize) { + // Note that the logging subsystem is designed so that it does no harm + // to log a message in an uninitialized state. These messages will be + // discarded for now, however. + log_debug(LD_GENERAL, "Initializing %s", sys->name); + r = sys->initialize(); + } + if (r < 0) { + fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n", + sys->name, i); + raw_assert_unreached_msg("A subsystem couldn't be initialized."); + } + sys_initialized[i] = true; + } + + return 0; +} + +/** + * Shut down all the subsystems. + **/ +void +subsystems_shutdown(void) +{ + subsystems_shutdown_downto(MIN_SUBSYS_LEVEL - 1); +} + +/** + * Shut down all the subsystems whose level is above <b>target_level</b>. + **/ +void +subsystems_shutdown_downto(int target_level) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (sys->level <= target_level) + break; + if (! sys_initialized[i]) + continue; + if (sys->shutdown) { + log_debug(LD_GENERAL, "Shutting down %s", sys->name); + sys->shutdown(); + } + sys_initialized[i] = false; + } +} + +/** + * Run pre-fork code on all subsystems that declare any + **/ +void +subsystems_prefork(void) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->prefork) { + log_debug(LD_GENERAL, "Pre-fork: %s", sys->name); + sys->prefork(); + } + } +} + +/** + * Run post-fork code on all subsystems that declare any + **/ +void +subsystems_postfork(void) +{ + check_and_setup(); + + for (unsigned i = 0; i < n_tor_subsystems; ++i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->postfork) { + log_debug(LD_GENERAL, "Post-fork: %s", sys->name); + sys->postfork(); + } + } +} + +/** + * Run thread-cleanup code on all subsystems that declare any + **/ +void +subsystems_thread_cleanup(void) +{ + check_and_setup(); + + for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) { + const subsys_fns_t *sys = tor_subsystems[i]; + if (!sys->supported) + continue; + if (! sys_initialized[i]) + continue; + if (sys->thread_cleanup) { + log_debug(LD_GENERAL, "Thread cleanup: %s", sys->name); + sys->thread_cleanup(); + } + } +} diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h new file mode 100644 index 0000000000..a5e62f71d9 --- /dev/null +++ b/src/app/main/subsysmgr.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYSMGR_T +#define TOR_SUBSYSMGR_T + +#include "lib/subsys/subsys.h" + +extern const struct subsys_fns_t *tor_subsystems[]; +extern const unsigned n_tor_subsystems; + +int subsystems_init(void); +int subsystems_init_upto(int level); + +void subsystems_shutdown(void); +void subsystems_shutdown_downto(int level); + +void subsystems_prefork(void); +void subsystems_postfork(void); +void subsystems_thread_cleanup(void); + +#endif diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c new file mode 100644 index 0000000000..3834176182 --- /dev/null +++ b/src/app/main/subsystem_list.c @@ -0,0 +1,49 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "app/main/subsysmgr.h" +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" + +#include "core/or/ocirc_event_sys.h" +#include "core/or/orconn_event_sys.h" +#include "feature/control/btrack_sys.h" +#include "lib/compress/compress_sys.h" +#include "lib/crypt_ops/crypto_sys.h" +#include "lib/err/torerr_sys.h" +#include "lib/log/log_sys.h" +#include "lib/net/network_sys.h" +#include "lib/process/winprocess_sys.h" +#include "lib/thread/thread_sys.h" +#include "lib/time/time_sys.h" +#include "lib/tls/tortls_sys.h" +#include "lib/wallclock/wallclock_sys.h" +#include "lib/process/process_sys.h" + +#include <stddef.h> + +/** + * Global list of the subsystems in Tor, in the order of their initialization. + **/ +const subsys_fns_t *tor_subsystems[] = { + &sys_winprocess, /* -100 */ + &sys_torerr, /* -100 */ + &sys_wallclock, /* -99 */ + &sys_threads, /* -95 */ + &sys_logging, /* -90 */ + &sys_time, /* -90 */ + &sys_network, /* -90 */ + &sys_compress, /* -70 */ + &sys_crypto, /* -60 */ + &sys_tortls, /* -50 */ + &sys_process, /* -35 */ + + &sys_orconn_event, /* -33 */ + &sys_ocirc_event, /* -32 */ + &sys_btrack, /* -30 */ +}; + +const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems); diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py index 3a454a3fc1..706a8b03cc 100644 --- a/src/config/mmdb-convert.py +++ b/src/config/mmdb-convert.py @@ -77,7 +77,7 @@ def to_int32(s): def to_int28(s): "Parse a pair of big-endian 28-bit integers from bytestring s." - a, b = unpack("!LL", s + b'\x00') + a, b = struct.unpack("!LL", s + b'\x00') return (((a & 0xf0) << 20) + (a >> 8)), ((a & 0x0f) << 24) + (b >> 8) class Tree(object): diff --git a/src/core/include.am b/src/core/include.am index 1b8ef2ac58..ae47c75e09 100644 --- a/src/core/include.am +++ b/src/core/include.am @@ -11,6 +11,8 @@ LIBTOR_APP_A_SOURCES = \ src/app/config/confparse.c \ src/app/config/statefile.c \ src/app/main/main.c \ + src/app/main/subsystem_list.c \ + src/app/main/subsysmgr.c \ src/core/crypto/hs_ntor.c \ src/core/crypto/onion_crypto.c \ src/core/crypto/onion_fast.c \ @@ -30,6 +32,7 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/circuitlist.c \ src/core/or/circuitmux.c \ src/core/or/circuitmux_ewma.c \ + src/core/or/circuitpadding.c \ src/core/or/circuitstats.c \ src/core/or/circuituse.c \ src/core/or/command.c \ @@ -37,6 +40,8 @@ LIBTOR_APP_A_SOURCES = \ src/core/or/connection_or.c \ src/core/or/dos.c \ src/core/or/onion.c \ + src/core/or/ocirc_event.c \ + src/core/or/orconn_event.c \ src/core/or/policies.c \ src/core/or/protover.c \ src/core/or/protover_rust.c \ @@ -59,7 +64,13 @@ LIBTOR_APP_A_SOURCES = \ src/feature/client/dnsserv.c \ src/feature/client/entrynodes.c \ src/feature/client/transports.c \ + src/feature/control/btrack.c \ + src/feature/control/btrack_circuit.c \ + src/feature/control/btrack_orconn.c \ + src/feature/control/btrack_orconn_cevent.c \ + src/feature/control/btrack_orconn_maps.c \ src/feature/control/control.c \ + src/feature/control/control_bootstrap.c \ src/feature/control/fmt_serverstatus.c \ src/feature/control/getinfo_geoip.c \ src/feature/dirauth/keypin.c \ @@ -106,6 +117,7 @@ LIBTOR_APP_A_SOURCES = \ src/feature/nodelist/microdesc.c \ src/feature/nodelist/networkstatus.c \ src/feature/nodelist/nickname.c \ + src/feature/nodelist/nodefamily.c \ src/feature/nodelist/nodelist.c \ src/feature/nodelist/node_select.c \ src/feature/nodelist/routerinfo.c \ @@ -191,6 +203,7 @@ noinst_HEADERS += \ src/app/config/statefile.h \ src/app/main/main.h \ src/app/main/ntmain.h \ + src/app/main/subsysmgr.h \ src/core/crypto/hs_ntor.h \ src/core/crypto/onion_crypto.h \ src/core/crypto/onion_fast.h \ @@ -215,6 +228,7 @@ noinst_HEADERS += \ src/core/or/circuitmux.h \ src/core/or/circuitmux_ewma.h \ src/core/or/circuitstats.h \ + src/core/or/circuitpadding.h \ src/core/or/circuituse.h \ src/core/or/command.h \ src/core/or/connection_edge.h \ @@ -233,10 +247,14 @@ noinst_HEADERS += \ src/core/or/listener_connection_st.h \ src/core/or/onion.h \ src/core/or/or.h \ + src/core/or/orconn_event.h \ + src/core/or/orconn_event_sys.h \ src/core/or/or_circuit_st.h \ src/core/or/or_connection_st.h \ src/core/or/or_handshake_certs_st.h \ src/core/or/or_handshake_state_st.h \ + src/core/or/ocirc_event.h \ + src/core/or/ocirc_event_sys.h \ src/core/or/origin_circuit_st.h \ src/core/or/policies.h \ src/core/or/port_cfg_st.h \ @@ -263,6 +281,11 @@ noinst_HEADERS += \ src/feature/client/dnsserv.h \ src/feature/client/entrynodes.h \ src/feature/client/transports.h \ + src/feature/control/btrack_circuit.h \ + src/feature/control/btrack_orconn.h \ + src/feature/control/btrack_orconn_cevent.h \ + src/feature/control/btrack_orconn_maps.h \ + src/feature/control/btrack_sys.h \ src/feature/control/control.h \ src/feature/control/control_connection_st.h \ src/feature/control/fmt_serverstatus.h \ @@ -340,6 +363,8 @@ noinst_HEADERS += \ src/feature/nodelist/networkstatus_voter_info_st.h \ src/feature/nodelist/nickname.h \ src/feature/nodelist/node_st.h \ + src/feature/nodelist/nodefamily.h \ + src/feature/nodelist/nodefamily_st.h \ src/feature/nodelist/nodelist.h \ src/feature/nodelist/node_select.h \ src/feature/nodelist/routerinfo.h \ diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c index 3595bba85c..21d7bd6d0d 100644 --- a/src/core/mainloop/connection.c +++ b/src/core/mainloop/connection.c @@ -57,7 +57,7 @@ #define CONNECTION_PRIVATE #include "core/or/or.h" #include "feature/client/bridges.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/err/backtrace.h" @@ -1466,6 +1466,20 @@ connection_listener_new(const struct sockaddr *listensockaddr, tor_socket_strerror(tor_socket_errno(s))); goto err; } + +#ifndef __APPLE__ + /* This code was introduced to help debug #28229. */ + int value; + socklen_t len = sizeof(value); + + if (!getsockopt(s, SOL_SOCKET, SO_ACCEPTCONN, &value, &len)) { + if (value == 0) { + log_err(LD_NET, "Could not listen on %s - " + "getsockopt(.,SO_ACCEPTCONN,.) yields 0.", address); + goto err; + } + } +#endif /* __APPLE__ */ #endif /* defined(HAVE_SYS_UN_H) */ } else { log_err(LD_BUG, "Got unexpected address family %d.", @@ -1867,7 +1881,7 @@ connection_init_accepted_conn(connection_t *conn, /* Initiate Extended ORPort authentication. */ return connection_ext_or_start_auth(TO_OR_CONN(conn)); case CONN_TYPE_OR: - control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); + connection_or_event_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0); rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1); if (rv < 0) { connection_or_close_for_error(TO_OR_CONN(conn), 0); @@ -1880,6 +1894,9 @@ connection_init_accepted_conn(connection_t *conn, TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch(); TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type; + /* Any incoming connection on an entry port counts as user activity. */ + note_user_activity(approx_time()); + switch (TO_CONN(listener)->type) { case CONN_TYPE_AP_LISTENER: conn->state = AP_CONN_STATE_SOCKS_WAIT; @@ -2079,6 +2096,11 @@ connection_connect_log_client_use_ip_version(const connection_t *conn) return; } + if (fascist_firewall_use_ipv6(options)) { + log_info(LD_NET, "Our outgoing connection is using IPv%d.", + tor_addr_family(&real_addr) == AF_INET6 ? 6 : 4); + } + /* Check if we couldn't satisfy an address family preference */ if ((!pref_ipv6 && tor_addr_family(&real_addr) == AF_INET6) || (pref_ipv6 && tor_addr_family(&real_addr) == AF_INET)) { @@ -2901,6 +2923,10 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) retval = -1; #ifdef ENABLE_LISTENER_REBIND + if (smartlist_len(replacements)) + log_debug(LD_NET, "%d replacements - starting rebinding loop.", + smartlist_len(replacements)); + SMARTLIST_FOREACH_BEGIN(replacements, listener_replacement_t *, r) { int addr_in_use = 0; int skip = 0; @@ -2912,8 +2938,11 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol) connection_listener_new_for_port(r->new_port, &skip, &addr_in_use); connection_t *old_conn = r->old_conn; - if (skip) + if (skip) { + log_debug(LD_NET, "Skipping creating new listener for %s:%d", + old_conn->address, old_conn->port); continue; + } connection_close_immediate(old_conn); connection_mark_for_close(old_conn); @@ -4331,6 +4360,23 @@ connection_write_to_buf_impl_,(const char *string, size_t len, connection_write_to_buf_commit(conn, written); } +/** + * Write a <b>string</b> (of size <b>len</b> to directory connection + * <b>dir_conn</b>. Apply compression if connection is configured to use + * it and finalize it if <b>done</b> is true. + */ +void +connection_dir_buf_add(const char *string, size_t len, + dir_connection_t *dir_conn, int done) +{ + if (dir_conn->compress_state != NULL) { + connection_buf_add_compress(string, len, dir_conn, done); + return; + } + + connection_buf_add(string, len, TO_CONN(dir_conn)); +} + void connection_buf_add_compress(const char *string, size_t len, dir_connection_t *conn, int done) @@ -4445,6 +4491,16 @@ connection_get_by_type_state(int type, int state) CONN_GET_TEMPLATE(conn, conn->type == type && conn->state == state); } +/** + * Return a connection of type <b>type</b> that is not an internally linked + * connection, and is not marked for close. + **/ +MOCK_IMPL(connection_t *, +connection_get_by_type_nonlinked,(int type)) +{ + CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked); +} + /** Return a connection of type <b>type</b> that has rendquery equal * to <b>rendquery</b>, and that is not marked for close. If state * is non-zero, conn must be of that state too. @@ -5341,17 +5397,20 @@ assert_connection_ok(connection_t *conn, time_t now) } /** Fills <b>addr</b> and <b>port</b> with the details of the global - * proxy server we are using. - * <b>conn</b> contains the connection we are using the proxy for. + * proxy server we are using. Store a 1 to the int pointed to by + * <b>is_put_out</b> if the connection is using a pluggable + * transport; store 0 otherwise. <b>conn</b> contains the connection + * we are using the proxy for. * * Return 0 on success, -1 on failure. */ int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, - const connection_t *conn) + int *is_pt_out, const connection_t *conn) { const or_options_t *options = get_options(); + *is_pt_out = 0; /* Client Transport Plugins can use another proxy, but that should be hidden * from the rest of tor (as the plugin is responsible for dealing with the * proxy), check it first, then check the rest of the proxy types to allow @@ -5367,6 +5426,7 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, tor_addr_copy(addr, &transport->addr); *port = transport->port; *proxy_type = transport->socks_version; + *is_pt_out = 1; return 0; } @@ -5403,11 +5463,13 @@ log_failed_proxy_connection(connection_t *conn) { tor_addr_t proxy_addr; uint16_t proxy_port; - int proxy_type; + int proxy_type, is_pt; - if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, conn) != 0) + if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt, + conn) != 0) return; /* if we have no proxy set up, leave this function. */ + (void)is_pt; log_warn(LD_NET, "The connection to the %s proxy server at %s just failed. " "Make sure that the proxy server is up and running.", diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h index 8ecdd6f06f..c93f1ef8e8 100644 --- a/src/core/mainloop/connection.h +++ b/src/core/mainloop/connection.h @@ -187,7 +187,7 @@ int connection_proxy_connect(connection_t *conn, int type); int connection_read_proxy_handshake(connection_t *conn); void log_failed_proxy_connection(connection_t *conn); int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type, - const connection_t *conn); + int *is_pt_out, const connection_t *conn); int retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol); @@ -226,6 +226,8 @@ MOCK_DECL(void, connection_write_to_buf_impl_, /* DOCDOC connection_write_to_buf */ static void connection_buf_add(const char *string, size_t len, connection_t *conn); +void connection_dir_buf_add(const char *string, size_t len, + dir_connection_t *dir_conn, int done); static inline void connection_buf_add(const char *string, size_t len, connection_t *conn) { @@ -240,6 +242,7 @@ size_t connection_get_outbuf_len(connection_t *conn); connection_t *connection_get_by_global_id(uint64_t id); connection_t *connection_get_by_type(int type); +MOCK_DECL(connection_t *,connection_get_by_type_nonlinked,(int type)); MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type, const tor_addr_t *addr, uint16_t port, int purpose)); diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index f0aa37e8da..a42a8acc80 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -95,7 +95,7 @@ #include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/err/backtrace.h" #include "lib/tls/buffers_tls.h" @@ -200,12 +200,10 @@ static int can_complete_circuits = 0; #define LAZY_DESCRIPTOR_RETRY_INTERVAL (60) static int conn_close_if_marked(int i); -static int run_main_loop_until_done(void); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); static void conn_read_callback(evutil_socket_t fd, short event, void *_conn); static void conn_write_callback(evutil_socket_t fd, short event, void *_conn); -static void second_elapsed_callback(periodic_timer_t *timer, void *args); static void shutdown_did_not_work_callback(evutil_socket_t fd, short event, void *arg) ATTR_NORETURN; @@ -1379,6 +1377,7 @@ CALLBACK(heartbeat); CALLBACK(hs_service); CALLBACK(launch_descriptor_fetches); CALLBACK(launch_reachability_tests); +CALLBACK(prune_old_routers); CALLBACK(reachability_warnings); CALLBACK(record_bridge_stats); CALLBACK(rend_cache_failure_clean); @@ -1391,78 +1390,93 @@ CALLBACK(save_stability); CALLBACK(save_state); CALLBACK(write_bridge_ns); CALLBACK(write_stats_file); +CALLBACK(control_per_second_events); +CALLBACK(second_elapsed); #undef CALLBACK /* Now we declare an array of periodic_event_item_t for each periodic event */ -#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f) +#define CALLBACK(name, r, f) \ + PERIODIC_EVENT(name, PERIODIC_EVENT_ROLE_ ## r, f) +#define FL(name) (PERIODIC_EVENT_FLAG_ ## name) STATIC periodic_event_item_t periodic_events[] = { - /* Everyone needs to run those. */ - CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0), - CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0), + + /* Everyone needs to run these. They need to have very long timeouts for + * that to be safe. */ + CALLBACK(add_entropy, ALL, 0), + CALLBACK(heartbeat, ALL, 0), + CALLBACK(reset_padding_counts, ALL, 0), + + /* This is a legacy catch-all callback that runs once per second if + * we are online and active. */ + CALLBACK(second_elapsed, NET_PARTICIPANT, + FL(RUN_ON_DISABLE)), + + /* XXXX Do we have a reason to do this on a callback? Does it do any good at + * all? For now, if we're dormant, we can let our listeners decay. */ + CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)), + + /* We need to do these if we're participating in the Tor network. */ + CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0), + CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0), + CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)), + CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0), + CALLBACK(check_network_participation, NET_PARTICIPANT, 0), + + /* We need to do these if we're participating in the Tor network, and + * immediately before we stop. */ + CALLBACK(clean_caches, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(save_state, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(write_stats_file, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), + CALLBACK(prune_old_routers, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), /* Routers (bridge and relay) only. */ - CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0), - CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(check_descriptor, ROUTER, FL(NEED_NET)), + CALLBACK(check_ed_keys, ROUTER, 0), + CALLBACK(check_for_reachability_bw, ROUTER, FL(NEED_NET)), + CALLBACK(check_onion_keys_expiry_time, ROUTER, 0), + CALLBACK(expire_old_ciruits_serverside, ROUTER, FL(NEED_NET)), + CALLBACK(reachability_warnings, ROUTER, FL(NEED_NET)), + CALLBACK(retry_dns, ROUTER, 0), + CALLBACK(rotate_onion_key, ROUTER, 0), /* Authorities (bridge and directory) only. */ - CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), - CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), + CALLBACK(downrate_stability, AUTHORITIES, 0), + CALLBACK(launch_reachability_tests, AUTHORITIES, FL(NEED_NET)), + CALLBACK(save_stability, AUTHORITIES, 0), /* Directory authority only. */ - CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0), - CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_authority_cert, DIRAUTH, 0), + CALLBACK(dirvote, DIRAUTH, FL(NEED_NET)), /* Relay only. */ - CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY, - PERIODIC_EVENT_FLAG_NEED_NET), - CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY, - PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_canonical_channels, RELAY, FL(NEED_NET)), + CALLBACK(check_dns_honesty, RELAY, FL(NEED_NET)), /* Hidden Service service only. */ - CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE, - PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), // XXXX break this down more /* Bridge only. */ - CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0), + CALLBACK(record_bridge_stats, BRIDGE, 0), /* Client only. */ - CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0), + /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */ + CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(RUN_ON_DISABLE)), /* Bridge Authority only. */ - CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0), + CALLBACK(write_bridge_ns, BRIDGEAUTH, 0), /* Directory server only. */ - CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0), + CALLBACK(clean_consdiffmgr, DIRSERVER, 0), + + /* Controller with per-second events only. */ + CALLBACK(control_per_second_events, CONTROLEV, 0), END_OF_PERIODIC_EVENTS }; #undef CALLBACK +#undef FL /* These are pointers to members of periodic_events[] that are used to * implement particular callbacks. We keep them separate here so that we @@ -1474,6 +1488,7 @@ static periodic_event_item_t *fetch_networkstatus_event=NULL; static periodic_event_item_t *launch_descriptor_fetches_event=NULL; static periodic_event_item_t *check_dns_honesty_event=NULL; static periodic_event_item_t *save_state_event=NULL; +static periodic_event_item_t *prune_old_routers_event=NULL; /** Reset all the periodic events so we'll do all our actions again as if we * just started up. @@ -1510,7 +1525,7 @@ get_my_roles(const or_options_t *options) { tor_assert(options); - int roles = 0; + int roles = PERIODIC_EVENT_ROLE_ALL; int is_bridge = options->BridgeRelay; int is_relay = server_mode(options); int is_dirauth = authdir_mode_v3(options); @@ -1518,6 +1533,8 @@ get_my_roles(const or_options_t *options) int is_hidden_service = !!hs_service_get_num_services() || !!rend_num_services(); int is_dirserver = dir_server_mode(options); + int sending_control_events = control_any_per_second_event_enabled(); + /* We also consider tor to have the role of a client if the ControlPort is * set because a lot of things can be done over the control port which * requires tor to have basic functionnalities. */ @@ -1525,6 +1542,9 @@ get_my_roles(const or_options_t *options) options->ControlPort_set || options->OwningControllerFD != UINT64_MAX; + int is_net_participant = is_participating_on_network() || + is_relay || is_hidden_service; + if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY; @@ -1532,6 +1552,8 @@ get_my_roles(const or_options_t *options) if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH; if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE; if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER; + if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT; + if (sending_control_events) roles |= PERIODIC_EVENT_ROLE_CONTROLEV; return roles; } @@ -1576,6 +1598,7 @@ initialize_periodic_events(void) STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END NAMED_CALLBACK(check_descriptor); + NAMED_CALLBACK(prune_old_routers); NAMED_CALLBACK(dirvote); NAMED_CALLBACK(fetch_networkstatus); NAMED_CALLBACK(launch_descriptor_fetches); @@ -1599,6 +1622,30 @@ teardown_periodic_events(void) periodic_events_initialized = 0; } +static mainloop_event_t *rescan_periodic_events_ev = NULL; + +/** Callback: rescan the periodic event list. */ +static void +rescan_periodic_events_cb(mainloop_event_t *event, void *arg) +{ + (void)event; + (void)arg; + rescan_periodic_events(get_options()); +} + +/** + * Schedule an event that will rescan which periodic events should run. + **/ +MOCK_IMPL(void, +schedule_rescan_periodic_events,(void)) +{ + if (!rescan_periodic_events_ev) { + rescan_periodic_events_ev = + mainloop_event_new(rescan_periodic_events_cb, NULL); + } + mainloop_event_activate(rescan_periodic_events_ev); +} + /** Do a pass at all our periodic events, disable those we don't need anymore * and enable those we need now using the given options. */ void @@ -1633,7 +1680,11 @@ rescan_periodic_events(const or_options_t *options) periodic_event_enable(item); } else { log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); - periodic_event_disable(item); + if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) { + periodic_event_schedule_and_disable(item); + } else { + periodic_event_disable(item); + } } } } @@ -1708,6 +1759,30 @@ mainloop_schedule_postloop_cleanup(void) mainloop_event_activate(postloop_cleanup_ev); } +/** Event to run 'scheduled_shutdown_cb' */ +static mainloop_event_t *scheduled_shutdown_ev=NULL; + +/** Callback: run a scheduled shutdown */ +static void +scheduled_shutdown_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + log_notice(LD_GENERAL, "Clean shutdown finished. Exiting."); + tor_shutdown_event_loop_and_exit(0); +} + +/** Schedule the mainloop to exit after <b>delay_sec</b> seconds. */ +void +mainloop_schedule_shutdown(int delay_sec) +{ + const struct timeval delay_tv = { delay_sec, 0 }; + if (! scheduled_shutdown_ev) { + scheduled_shutdown_ev = mainloop_event_new(scheduled_shutdown_cb, NULL); + } + mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv); +} + #define LONGEST_TIMER_PERIOD (30 * 86400) /** Helper: Return the number of seconds between <b>now</b> and <b>next</b>, * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */ @@ -1732,16 +1807,16 @@ safe_timer_diff(time_t now, time_t next) } /** Perform regular maintenance tasks. This function gets run once per - * second by second_elapsed_callback(). + * second. */ -static void -run_scheduled_events(time_t now) +static int +second_elapsed_callback(time_t now, const or_options_t *options) { - const or_options_t *options = get_options(); - - /* 0. See if we've been asked to shut down and our timeout has - * expired; or if our bandwidth limits are exhausted and we - * should hibernate; or if it's time to wake up from hibernation. + /* 0. See if our bandwidth limits are exhausted and we should hibernate + * + * Note: we have redundant mechanisms to handle the case where it's + * time to wake up from hibernation; or where we have a scheduled + * shutdown and it's time to run it, but this will also handle those. */ consider_hibernation(now); @@ -1751,10 +1826,13 @@ run_scheduled_events(time_t now) if (options->UseBridges && !net_is_disabled()) { /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches() * -- the latter is only for fetching consensus-derived directory info. */ + // TODO: client + // Also, schedule this rather than probing 1x / sec fetch_bridge_descriptors(options, now); } if (accounting_is_enabled(options)) { + // TODO: refactor or rewrite? accounting_run_housekeeping(now); } @@ -1765,6 +1843,7 @@ run_scheduled_events(time_t now) */ /* (If our circuit build timeout can ever become lower than a second (which * it can't, currently), we should do this more often.) */ + // TODO: All expire stuff can become NET_PARTICIPANT, RUN_ON_DISABLE circuit_expire_building(); circuit_expire_waiting_for_better_guard(); @@ -1798,9 +1877,8 @@ run_scheduled_events(time_t now) run_connection_housekeeping(i, now); } - /* 11b. check pending unconfigured managed proxies */ - if (!net_is_disabled() && pt_proxies_configuration_pending()) - pt_configure_remaining_proxies(); + /* Run again in a second. */ + return 1; } /* Periodic callback: rotate the onion keys after the period defined by the @@ -1947,6 +2025,55 @@ add_entropy_callback(time_t now, const or_options_t *options) return ENTROPY_INTERVAL; } +/** Periodic callback: if there has been no network usage in a while, + * enter a dormant state. */ +STATIC int +check_network_participation_callback(time_t now, const or_options_t *options) +{ + /* If we're a server, we can't become dormant. */ + if (server_mode(options)) { + goto found_activity; + } + + /* If we're running an onion service, we can't become dormant. */ + /* XXXX this would be nice to change, so that we can be dormant with a + * service. */ + if (hs_service_get_num_services() || rend_num_services()) { + goto found_activity; + } + + /* If we have any currently open entry streams other than "linked" + * connections used for directory requests, those count as user activity. + */ + if (options->DormantTimeoutDisabledByIdleStreams) { + if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) { + goto found_activity; + } + } + + /* XXXX Make this configurable? */ +/** How often do we check whether we have had network activity? */ +#define CHECK_PARTICIPATION_INTERVAL (5*60) + + /* Become dormant if there has been no user activity in a long time. + * (The funny checks below are in order to prevent overflow.) */ + time_t time_since_last_activity = 0; + if (get_last_user_activity_time() < now) + time_since_last_activity = now - get_last_user_activity_time(); + if (time_since_last_activity >= options->DormantClientTimeout) { + log_notice(LD_GENERAL, "No user activity in a long time: becoming" + " dormant."); + set_network_participation(false); + rescan_periodic_events(options); + } + + return CHECK_PARTICIPATION_INTERVAL; + + found_activity: + note_user_activity(now); + return CHECK_PARTICIPATION_INTERVAL; +} + /** * Periodic callback: if we're an authority, make sure we test * the routers on the network for reachability. @@ -2053,11 +2180,9 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options) (void)options; /* Check whether our networkstatus has expired. */ networkstatus_t *ns = networkstatus_get_latest_consensus(); - /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in - * networkstatus_get_reasonably_live_consensus(), but that value is way - * way too high. Arma: is the bridge issue there resolved yet? -NM */ -#define NS_EXPIRY_SLOP (24*60*60) - if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) && + /* Use reasonably live consensuses until they are no longer reasonably live. + */ + if (ns && !networkstatus_consensus_reasonably_live(ns, now) && router_have_minimum_dir_info()) { router_dir_info_changed(); } @@ -2240,6 +2365,27 @@ retry_dns_callback(time_t now, const or_options_t *options) return RETRY_DNS_INTERVAL; } +/** + * Periodic callback: prune routerlist of old information about Tor network. + */ +static int +prune_old_routers_callback(time_t now, const or_options_t *options) +{ +#define ROUTERLIST_PRUNING_INTERVAL (60*60) // 1 hour. + (void)now; + (void)options; + + if (!net_is_disabled()) { + /* If any networkstatus documents are no longer recent, we need to + * update all the descriptors' running status. */ + /* Remove dead routers. */ + log_debug(LD_GENERAL, "Pruning routerlist..."); + routerlist_remove_old_routers(); + } + + return ROUTERLIST_PRUNING_INTERVAL; +} + /** Periodic callback: consider rebuilding or and re-uploading our descriptor * (if we've passed our internal checks). */ static int @@ -2259,12 +2405,6 @@ check_descriptor_callback(time_t now, const or_options_t *options) check_descriptor_ipaddress_changed(now); mark_my_descriptor_dirty_if_too_old(now); consider_publishable_server(0); - /* If any networkstatus documents are no longer recent, we need to - * update all the descriptors' running status. */ - /* Remove dead routers. */ - /* XXXX This doesn't belong here, but it was here in the pre- - * XXXX refactoring code. */ - routerlist_remove_old_routers(); } return CHECK_DESCRIPTOR_INTERVAL; @@ -2522,36 +2662,19 @@ hs_service_callback(time_t now, const or_options_t *options) return 1; } -/** Timer: used to invoke second_elapsed_callback() once per second. */ -static periodic_timer_t *second_timer = NULL; - -/** - * Enable or disable the per-second timer as appropriate, creating it if - * necessary. +/* + * Periodic callback: Send once-per-second events to the controller(s). + * This is called every second. */ -void -reschedule_per_second_timer(void) +static int +control_per_second_events_callback(time_t now, const or_options_t *options) { - struct timeval one_second; - one_second.tv_sec = 1; - one_second.tv_usec = 0; - - if (! second_timer) { - second_timer = periodic_timer_new(tor_libevent_get_base(), - &one_second, - second_elapsed_callback, - NULL); - tor_assert(second_timer); - } + (void) options; + (void) now; - const bool run_per_second_events = - control_any_per_second_event_enabled() || ! net_is_completely_disabled(); + control_per_second_events(); - if (run_per_second_events) { - periodic_timer_launch(second_timer, &one_second); - } else { - periodic_timer_disable(second_timer); - } + return 1; } /** Last time that update_current_time was called. */ @@ -2581,6 +2704,17 @@ update_current_time(time_t now) memcpy(&last_updated, ¤t_second_last_changed, sizeof(last_updated)); monotime_coarse_get(¤t_second_last_changed); + /** How much clock jumping means that we should adjust our idea of when + * to go dormant? */ +#define NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE 20 + + /* Don't go dormant early or late just because we jumped in time. */ + if (ABS(seconds_elapsed) >= NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE) { + if (is_participating_on_network()) { + netstatus_note_clock_jumped(seconds_elapsed); + } + } + /** How much clock jumping do we tolerate? */ #define NUM_JUMPED_SECONDS_BEFORE_WARN 100 @@ -2590,6 +2724,7 @@ update_current_time(time_t now) if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) { // moving back in time is always a bad sign. circuit_note_clock_jumped(seconds_elapsed, false); + } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { /* Compare the monotonic clock to the result of time(). */ const int32_t monotime_msec_passed = @@ -2619,31 +2754,6 @@ update_current_time(time_t now) current_second = now; } -/** Libevent callback: invoked once every second. */ -static void -second_elapsed_callback(periodic_timer_t *timer, void *arg) -{ - /* XXXX This could be sensibly refactored into multiple callbacks, and we - * could use Libevent's timers for this rather than checking the current - * time against a bunch of timeouts every second. */ - time_t now; - (void)timer; - (void)arg; - - now = time(NULL); - - /* We don't need to do this once-per-second any more: time-updating is - * only in this callback _because it is a callback_. It should be fine - * to disable this callback, and the time will still get updated. - */ - update_current_time(now); - - /* Maybe some controller events are ready to fire */ - control_per_second_events(); - - run_scheduled_events(now); -} - #ifdef HAVE_SYSTEMD_209 static periodic_timer_t *systemd_watchdog_timer = NULL; @@ -2734,9 +2844,6 @@ do_main_loop(void) initialize_periodic_events(); initialize_mainloop_events(); - /* set up once-a-second callback. */ - reschedule_per_second_timer(); - #ifdef HAVE_SYSTEMD_209 uint64_t watchdog_delay; /* set up systemd watchdog notification. */ @@ -2758,10 +2865,6 @@ do_main_loop(void) } } #endif /* defined(HAVE_SYSTEMD_209) */ - - main_loop_should_exit = 0; - main_loop_exit_value = 0; - #ifdef ENABLE_RESTART_DEBUGGING { static int first_time = 1; @@ -2887,10 +2990,14 @@ run_main_loop_once(void) * * Shadow won't invoke this function, so don't fill it up with things. */ -static int +STATIC int run_main_loop_until_done(void) { int loop_result = 1; + + main_loop_should_exit = 0; + main_loop_exit_value = 0; + do { loop_result = run_main_loop_once(); } while (loop_result == 1); @@ -2921,7 +3028,6 @@ tor_mainloop_free_all(void) smartlist_free(connection_array); smartlist_free(closeable_connection_lst); smartlist_free(active_linked_connection_lst); - periodic_timer_free(second_timer); teardown_periodic_events(); tor_event_free(shutdown_did_not_work_event); tor_event_free(initialize_periodic_events_event); @@ -2929,6 +3035,8 @@ tor_mainloop_free_all(void) mainloop_event_free(schedule_active_linked_connections_event); mainloop_event_free(postloop_cleanup_ev); mainloop_event_free(handle_deferred_signewnym_ev); + mainloop_event_free(scheduled_shutdown_ev); + mainloop_event_free(rescan_periodic_events_ev); #ifdef HAVE_SYSTEMD_209 periodic_timer_free(systemd_watchdog_timer); diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index c5669fc4e0..6ed93fa900 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -65,6 +65,7 @@ void reschedule_or_state_save(void); void reschedule_dirvote(const or_options_t *options); void mainloop_schedule_postloop_cleanup(void); void rescan_periodic_events(const or_options_t *options); +MOCK_DECL(void, schedule_rescan_periodic_events,(void)); void update_current_time(time_t now); @@ -81,11 +82,12 @@ uint64_t get_main_loop_error_count(void); uint64_t get_main_loop_idle_count(void); void periodic_events_on_new_options(const or_options_t *options); -void reschedule_per_second_timer(void); void do_signewnym(time_t); time_t get_last_signewnym_time(void); +void mainloop_schedule_shutdown(int delay_sec); + void tor_init_connection_lists(void); void initialize_mainloop_events(void); void tor_mainloop_free_all(void); @@ -98,10 +100,14 @@ extern struct token_bucket_rw_t global_bucket; extern struct token_bucket_rw_t global_relayed_bucket; #ifdef MAINLOOP_PRIVATE +STATIC int run_main_loop_until_done(void); STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); STATIC int get_my_roles(const or_options_t *); +STATIC int check_network_participation_callback(time_t now, + const or_options_t *options); + #ifdef TOR_UNIT_TESTS extern smartlist_t *connection_array; diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c index 1444ca5db2..4924888598 100644 --- a/src/core/mainloop/netstatus.c +++ b/src/core/mainloop/netstatus.c @@ -6,9 +6,12 @@ #include "core/or/or.h" #include "core/mainloop/netstatus.h" +#include "core/mainloop/mainloop.h" #include "app/config/config.h" #include "feature/hibernate/hibernate.h" +#include "app/config/or_state_st.h" + /** Return true iff our network is in some sense disabled or shutting down: * either we're hibernating, entering hibernation, or the network is turned * off with DisableNetwork. */ @@ -26,3 +29,136 @@ net_is_completely_disabled(void) { return get_options()->DisableNetwork || we_are_fully_hibernating(); } + +/** + * The time at which we've last seen "user activity" -- that is, any activity + * that should keep us as a participant on the network. + * + * This is not actually the true time. We will adjust this forward if + * our clock jumps, or if Tor is shut down for a while, so that the time + * since our last activity remains as it was before the jump or shutdown. + */ +static time_t last_user_activity_seen = 0; + +/** + * True iff we are currently a "network participant" -- that is, we + * are building circuits, fetching directory information, and so on. + **/ +static bool participating_on_network = false; + +/** + * Record the fact that we have seen "user activity" at the time now. Move + * "last activity seen" time forwards, but never backwards. + * + * If we were previously not participating on the network, set our + * participation status to true, and launch periodic events as appropriate. + **/ +void +note_user_activity(time_t now) +{ + last_user_activity_seen = MAX(now, last_user_activity_seen); + + if (! participating_on_network) { + log_notice(LD_GENERAL, "Tor is no longer dormant."); + set_network_participation(true); + schedule_rescan_periodic_events(); + } +} + +/** + * Change the time at which "user activitiy" was last seen to <b>now</b>. + * + * Unlike note_user_actity, this function sets the time without checking + * whether it is in the past, and without causing any rescan of periodic events + * or change in participation status. + */ +void +reset_user_activity(time_t now) +{ + last_user_activity_seen = now; +} + +/** + * Return the most recent time at which we recorded "user activity". + **/ +time_t +get_last_user_activity_time(void) +{ + return last_user_activity_seen; +} + +/** + * Set the field that remembers whether we are currently participating on the + * network. Does not schedule or un-schedule periodic events. + **/ +void +set_network_participation(bool participation) +{ + participating_on_network = participation; +} + +/** + * Return true iff we are currently participating on the network. + **/ +bool +is_participating_on_network(void) +{ + return participating_on_network; +} + +/** + * Update 'state' with the last time at which we were active on the network. + **/ +void +netstatus_flush_to_state(or_state_t *state, time_t now) +{ + state->Dormant = ! participating_on_network; + if (participating_on_network) { + time_t sec_since_activity = MAX(0, now - last_user_activity_seen); + state->MinutesSinceUserActivity = (int)(sec_since_activity / 60); + } else { + state->MinutesSinceUserActivity = 0; + } +} + +/** + * Update our current view of network participation from an or_state_t object. + **/ +void +netstatus_load_from_state(const or_state_t *state, time_t now) +{ + time_t last_activity; + if (state->Dormant == -1) { // Initial setup. + if (get_options()->DormantOnFirstStartup) { + last_activity = 0; + participating_on_network = false; + } else { + // Start up as active, treat activity as happening now. + last_activity = now; + participating_on_network = true; + } + } else if (state->Dormant) { + last_activity = 0; + participating_on_network = false; + } else { + last_activity = now - 60 * state->MinutesSinceUserActivity; + participating_on_network = true; + } + if (get_options()->DormantCanceledByStartup) { + last_activity = now; + participating_on_network = true; + } + reset_user_activity(last_activity); +} + +/** + * Adjust the time at which the user was last active by <b>seconds_diff</b> + * in response to a clock jump. + */ +void +netstatus_note_clock_jumped(time_t seconds_diff) +{ + time_t last_active = get_last_user_activity_time(); + if (last_active) + reset_user_activity(last_active + seconds_diff); +} diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h index ae8547b30d..aba631e2fb 100644 --- a/src/core/mainloop/netstatus.h +++ b/src/core/mainloop/netstatus.h @@ -10,4 +10,15 @@ int net_is_disabled(void); int net_is_completely_disabled(void); +void note_user_activity(time_t now); +void reset_user_activity(time_t now); +time_t get_last_user_activity_time(void); + +void set_network_participation(bool participation); +bool is_participating_on_network(void); + +void netstatus_flush_to_state(or_state_t *state, time_t now); +void netstatus_load_from_state(const or_state_t *state, time_t now); +void netstatus_note_clock_jumped(time_t seconds_diff); + #endif diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c index 2651bbbc89..5c01213d17 100644 --- a/src/core/mainloop/periodic.c +++ b/src/core/mainloop/periodic.c @@ -45,10 +45,6 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) periodic_event_item_t *event = data; tor_assert(ev == event->ev); - if (BUG(!periodic_event_is_enabled(event))) { - return; - } - time_t now = time(NULL); update_current_time(now); const or_options_t *options = get_options(); @@ -57,7 +53,7 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data) int next_interval = 0; if (!periodic_event_is_enabled(event)) { - /* The event got disabled from inside its callback; no need to + /* The event got disabled from inside its callback, or before: no need to * reschedule. */ return; } @@ -177,3 +173,19 @@ periodic_event_disable(periodic_event_item_t *event) mainloop_event_cancel(event->ev); event->enabled = 0; } + +/** + * Disable an event, then schedule it to run once. + * Do nothing if the event was already disabled. + */ +void +periodic_event_schedule_and_disable(periodic_event_item_t *event) +{ + tor_assert(event); + if (!periodic_event_is_enabled(event)) + return; + + periodic_event_disable(event); + + mainloop_event_activate(event->ev); +} diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h index e49fafd174..344fc9ad25 100644 --- a/src/core/mainloop/periodic.h +++ b/src/core/mainloop/periodic.h @@ -15,6 +15,10 @@ #define PERIODIC_EVENT_ROLE_BRIDGEAUTH (1U << 4) #define PERIODIC_EVENT_ROLE_HS_SERVICE (1U << 5) #define PERIODIC_EVENT_ROLE_DIRSERVER (1U << 6) +#define PERIODIC_EVENT_ROLE_CONTROLEV (1U << 7) + +#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 8) +#define PERIODIC_EVENT_ROLE_ALL (1U << 9) /* Helper macro to make it a bit less annoying to defined groups of roles that * are often used. */ @@ -25,10 +29,6 @@ /* Authorities that is both bridge and directory. */ #define PERIODIC_EVENT_ROLE_AUTHORITIES \ (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH) -/* All roles. */ -#define PERIODIC_EVENT_ROLE_ALL \ - (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \ - PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER) /* * Event flags which can change the behavior of an event. @@ -39,6 +39,11 @@ * the net_is_disabled() check. */ #define PERIODIC_EVENT_FLAG_NEED_NET (1U << 0) +/* Indicate that if the event is enabled, it needs to be run once before + * it becomes disabled. + */ +#define PERIODIC_EVENT_FLAG_RUN_ON_DISABLE (1U << 1) + /** Callback function for a periodic event to take action. The return value * influences the next time the function will get called. Return * PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled @@ -83,6 +88,6 @@ void periodic_event_destroy(periodic_event_item_t *event); void periodic_event_reschedule(periodic_event_item_t *event); void periodic_event_enable(periodic_event_item_t *event); void periodic_event_disable(periodic_event_item_t *event); +void periodic_event_schedule_and_disable(periodic_event_item_t *event); #endif /* !defined(TOR_PERIODIC_H) */ - diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c index 4db283d20e..d508c91988 100644 --- a/src/core/or/channeltls.c +++ b/src/core/or/channeltls.c @@ -59,6 +59,7 @@ #include "feature/nodelist/torcert.h" #include "feature/nodelist/networkstatus.h" #include "trunnel/channelpadding_negotiation.h" +#include "trunnel/netinfo.h" #include "core/or/channelpadding.h" #include "core/or/cell_st.h" @@ -949,7 +950,6 @@ channel_tls_listener_describe_transport_method(channel_listener_t *chan_l) void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, or_connection_t *conn, - uint8_t old_state, uint8_t state) { channel_t *base_chan; @@ -958,8 +958,6 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, tor_assert(conn); tor_assert(conn->chan == chan); tor_assert(chan->conn == conn); - /* Shut the compiler up without triggering -Wtautological-compare */ - (void)old_state; base_chan = TLS_CHAN_TO_BASE(chan); @@ -1645,6 +1643,35 @@ channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan) } /** + * Convert <b>netinfo_addr</b> into corresponding <b>tor_addr</b>. + * Return 0 on success; on failure, return -1 and log a warning. + */ +static int +tor_addr_from_netinfo_addr(tor_addr_t *tor_addr, + const netinfo_addr_t *netinfo_addr) { + tor_assert(tor_addr); + tor_assert(netinfo_addr); + + uint8_t type = netinfo_addr_get_addr_type(netinfo_addr); + uint8_t len = netinfo_addr_get_len(netinfo_addr); + + if (type == NETINFO_ADDR_TYPE_IPV4 && len == 4) { + uint32_t ipv4 = netinfo_addr_get_addr_ipv4(netinfo_addr); + tor_addr_from_ipv4h(tor_addr, ipv4); + } else if (type == NETINFO_ADDR_TYPE_IPV6 && len == 16) { + const uint8_t *ipv6_bytes = netinfo_addr_getconstarray_addr_ipv6( + netinfo_addr); + tor_addr_from_ipv6_bytes(tor_addr, (const char *)ipv6_bytes); + } else { + log_fn(LOG_PROTOCOL_WARN, LD_OR, "Cannot read address from NETINFO " + "- wrong type/length."); + return -1; + } + + return 0; +} + +/** * Helper: compute the absolute value of a time_t. * * (we need this because labs() doesn't always work for time_t, since @@ -1668,8 +1695,6 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) time_t timestamp; uint8_t my_addr_type; uint8_t my_addr_len; - const uint8_t *my_addr_ptr; - const uint8_t *cp, *end; uint8_t n_other_addrs; time_t now = time(NULL); const routerinfo_t *me = router_get_my_routerinfo(); @@ -1740,38 +1765,48 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) } /* Decode the cell. */ - timestamp = ntohl(get_uint32(cell->payload)); - const time_t sent_versions_at = - chan->conn->handshake_state->sent_versions_at; - if (now > sent_versions_at && (now - sent_versions_at) < 180) { - /* If we have gotten the NETINFO cell reasonably soon after having - * sent our VERSIONS cell, maybe we can learn skew information from it. */ - apparent_skew = now - timestamp; + netinfo_cell_t *netinfo_cell = NULL; + + ssize_t parsed = netinfo_cell_parse(&netinfo_cell, cell->payload, + CELL_PAYLOAD_SIZE); + + if (parsed < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Failed to parse NETINFO cell - closing connection."); + connection_or_close_for_error(chan->conn, 0); + return; } - my_addr_type = (uint8_t) cell->payload[4]; - my_addr_len = (uint8_t) cell->payload[5]; - my_addr_ptr = (uint8_t*) cell->payload + 6; - end = cell->payload + CELL_PAYLOAD_SIZE; - cp = cell->payload + 6 + my_addr_len; + timestamp = netinfo_cell_get_timestamp(netinfo_cell); + const netinfo_addr_t *my_addr = + netinfo_cell_getconst_other_addr(netinfo_cell); + + my_addr_type = netinfo_addr_get_addr_type(my_addr); + my_addr_len = netinfo_addr_get_len(my_addr); + + if ((now - chan->conn->handshake_state->sent_versions_at) < 180) { + apparent_skew = now - timestamp; + } /* We used to check: * if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) { * * This is actually never going to happen, since my_addr_len is at most 255, * and CELL_PAYLOAD_LEN - 6 is 503. So we know that cp is < end. */ - if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) { - tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr)); + if (tor_addr_from_netinfo_addr(&my_apparent_addr, my_addr) == -1) { + connection_or_close_for_error(chan->conn, 0); + netinfo_cell_free(netinfo_cell); + return; + } + if (my_addr_type == NETINFO_ADDR_TYPE_IPV4 && my_addr_len == 4) { if (!get_options()->BridgeRelay && me && - get_uint32(my_addr_ptr) == htonl(me->addr)) { + tor_addr_eq_ipv4h(&my_apparent_addr, me->addr)) { TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1; } - - } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) { - tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr); - + } else if (my_addr_type == NETINFO_ADDR_TYPE_IPV6 && + my_addr_len == 16) { if (!get_options()->BridgeRelay && me && !tor_addr_is_null(&me->ipv6_addr) && tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) { @@ -1779,18 +1814,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) } } - n_other_addrs = (uint8_t) *cp++; - while (n_other_addrs && cp < end-2) { + n_other_addrs = netinfo_cell_get_n_my_addrs(netinfo_cell); + for (uint8_t i = 0; i < n_other_addrs; i++) { /* Consider all the other addresses; if any matches, this connection is * "canonical." */ + + const netinfo_addr_t *netinfo_addr = + netinfo_cell_getconst_my_addrs(netinfo_cell, i); + tor_addr_t addr; - const uint8_t *next = - decode_address_from_payload(&addr, cp, (int)(end-cp)); - if (next == NULL) { + + if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) { log_fn(LOG_PROTOCOL_WARN, LD_OR, - "Bad address in netinfo cell; closing connection."); - connection_or_close_for_error(chan->conn, 0); - return; + "Bad address in netinfo cell; Skipping."); + continue; } /* A relay can connect from anywhere and be canonical, so * long as it tells you from where it came. This may sound a bit @@ -1803,10 +1840,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan) connection_or_set_canonical(chan->conn, 1); break; } - cp = next; - --n_other_addrs; } + netinfo_cell_free(netinfo_cell); + if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer && channel_is_canonical(TLS_CHAN_TO_BASE(chan))) { const char *descr = diff --git a/src/core/or/channeltls.h b/src/core/or/channeltls.h index a2648ff537..634a2a00e9 100644 --- a/src/core/or/channeltls.h +++ b/src/core/or/channeltls.h @@ -49,7 +49,6 @@ channel_tls_t * channel_tls_from_base(channel_t *chan); void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn); void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan, or_connection_t *conn, - uint8_t old_state, uint8_t state); void channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn); diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h index d4339ff50d..af343f082e 100644 --- a/src/core/or/circuit_st.h +++ b/src/core/or/circuit_st.h @@ -12,6 +12,11 @@ #include "core/or/cell_queue_st.h" struct hs_token_t; +struct circpad_machine_spec_t; +struct circpad_machine_state_t; + +/** Number of padding state machines on a circuit. */ +#define CIRCPAD_MAX_MACHINES (2) /** "magic" value for an origin_circuit_t */ #define ORIGIN_CIRCUIT_MAGIC 0x35315243u @@ -177,6 +182,27 @@ struct circuit_t { /** Hashtable node: used to look up the circuit by its HS token using the HS circuitmap. */ HT_ENTRY(circuit_t) hs_circuitmap_node; + + /** Adaptive Padding state machines: these are immutable. The state machines + * that come from the consensus are saved to a global structure, to avoid + * per-circuit allocations. This merely points to the global copy in + * origin_padding_machines or relay_padding_machines that should never + * change or get deallocated. + * + * Each element of this array corresponds to a different padding machine, + * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ + const struct circpad_machine_spec_t *padding_machine[CIRCPAD_MAX_MACHINES]; + + /** Adaptive Padding machine info for above machines. This is the + * per-circuit mutable information, such as the current state and + * histogram token counts. Some of it is optional (aka NULL). + * If a machine is being shut down, these indexes can be NULL + * without the corresponding padding_machine being NULL, while we + * wait for the other end to respond to our shutdown request. + * + * Each element of this array corresponds to a different padding machine, + * and we can have up to CIRCPAD_MAX_MACHINES such machines. */ + struct circpad_machine_state_t *padding_info[CIRCPAD_MAX_MACHINES]; }; #endif diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c index 21369fb538..3ec1e01f11 100644 --- a/src/core/or/circuitbuild.c +++ b/src/core/or/circuitbuild.c @@ -26,6 +26,7 @@ **/ #define CIRCUITBUILD_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -42,10 +43,12 @@ #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "core/or/command.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" #include "core/or/onion.h" +#include "core/or/ocirc_event.h" #include "core/or/policies.h" #include "core/or/relay.h" #include "feature/client/bridges.h" @@ -492,7 +495,7 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags) return NULL; } - control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0); + circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); if ((err_reason = circuit_handle_first_hop(circ)) < 0) { circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason); @@ -508,6 +511,28 @@ origin_circuit_get_guard_state(origin_circuit_t *circ) return circ->guard_state; } +/** + * Helper function to publish a channel association message + * + * circuit_handle_first_hop() calls this to notify subscribers about a + * channel launch event, which associates a circuit with a channel. + * This doesn't always correspond to an assignment of the circuit's + * n_chan field, because that seems to be only for fully-open + * channels. + **/ +static void +circuit_chan_publish(const origin_circuit_t *circ, const channel_t *chan) +{ + ocirc_event_msg_t msg; + + msg.type = OCIRC_MSGTYPE_CHAN; + msg.u.chan.gid = circ->global_identifier; + msg.u.chan.chan = chan->global_identifier; + msg.u.chan.onehop = circ->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); +} + /** Start establishing the first hop of our circuit. Figure out what * OR we should connect to, and if necessary start the connection to * it. If we're already connected, then send the 'create' cell. @@ -559,8 +584,6 @@ circuit_handle_first_hop(origin_circuit_t *circ) circ->base_.n_hop = extend_info_dup(firsthop->extend_info); if (should_launch) { - if (circ->build_state->onehop_tunnel) - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); n_chan = channel_connect_for_circuit( &firsthop->extend_info->addr, firsthop->extend_info->port, @@ -570,6 +593,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) log_info(LD_CIRC,"connect to firsthop failed. Closing."); return -END_CIRC_REASON_CONNECTFAILED; } + circuit_chan_publish(circ, n_chan); } log_debug(LD_CIRC,"connecting in progress (or finished). Good."); @@ -581,6 +605,7 @@ circuit_handle_first_hop(origin_circuit_t *circ) } else { /* it's already open. use it. */ tor_assert(!circ->base_.n_hop); circ->base_.n_chan = n_chan; + circuit_chan_publish(circ, n_chan); log_debug(LD_CIRC,"Conn open. Delivering first onion skin."); if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) { log_info(LD_CIRC,"circuit_send_next_onion_skin failed."); @@ -926,12 +951,15 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath); circuit_build_times_handle_completed_hop(circ); + circpad_machine_event_circ_added_hop(circ); + if (hop) { /* Case two: we're on a hop after the first. */ return circuit_send_intermediate_onion_skin(circ, hop); } /* Case three: the circuit is finished. Do housekeeping tasks on it. */ + circpad_machine_event_circ_built(circ); return circuit_build_no_more_hops(circ); } @@ -1416,7 +1444,7 @@ circuit_finish_handshake(origin_circuit_t *circ, hop->state = CPATH_STATE_OPEN; log_info(LD_CIRC,"Finished building circuit hop:"); circuit_log_path(LOG_INFO,LD_CIRC,circ); - control_event_circuit_status(circ, CIRC_EVENT_EXTENDED, 0); + circuit_event_status(circ, CIRC_EVENT_EXTENDED, 0); return 0; } @@ -1657,22 +1685,25 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei) STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes) { - int num_acceptable_routers; int routelen; tor_assert(nodes); routelen = route_len_for_purpose(purpose, exit_ei); - num_acceptable_routers = count_acceptable_nodes(nodes); + int num_acceptable_direct = count_acceptable_nodes(nodes, 1); + int num_acceptable_indirect = count_acceptable_nodes(nodes, 0); - log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).", - routelen, num_acceptable_routers, smartlist_len(nodes)); + log_debug(LD_CIRC,"Chosen route length %d (%d direct and %d indirect " + "routers suitable).", routelen, num_acceptable_direct, + num_acceptable_indirect); - if (num_acceptable_routers < routelen) { + if (num_acceptable_direct < 1 || num_acceptable_indirect < routelen - 1) { log_info(LD_CIRC, - "Not enough acceptable routers (%d/%d). Discarding this circuit.", - num_acceptable_routers, routelen); + "Not enough acceptable routers (%d/%d direct and %d/%d " + "indirect routers suitable). Discarding this circuit.", + num_acceptable_direct, routelen, + num_acceptable_indirect, routelen); return -1; } @@ -2314,7 +2345,7 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei) * particular router. See bug #25885.) */ MOCK_IMPL(STATIC int, -count_acceptable_nodes, (smartlist_t *nodes)) +count_acceptable_nodes, (smartlist_t *nodes, int direct)) { int num=0; @@ -2328,7 +2359,7 @@ count_acceptable_nodes, (smartlist_t *nodes)) if (! node->is_valid) // log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i); continue; - if (! node_has_any_descriptor(node)) + if (! node_has_preferred_descriptor(node, direct)) continue; /* The node has a descriptor, so we can just check the ntor key directly */ if (!node_has_curve25519_onion_key(node)) @@ -2579,7 +2610,24 @@ choose_good_middle_server(uint8_t purpose, return choice; } - choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); + if (options->MiddleNodes) { + smartlist_t *sl = smartlist_new(); + routerset_get_all_nodes(sl, options->MiddleNodes, + options->ExcludeNodes, 1); + + smartlist_subtract(sl, excluded); + + choice = node_sl_choose_by_bandwidth(sl, WEIGHT_FOR_MID); + smartlist_free(sl); + if (choice) { + log_fn(LOG_INFO, LD_CIRC, "Chose fixed middle node: %s", + hex_str(choice->identity, DIGEST_LEN)); + } else { + log_fn(LOG_NOTICE, LD_CIRC, "Restricted middle not available"); + } + } else { + choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); + } smartlist_free(excluded); return choice; } diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h index 969f5b5cc3..b19bc41235 100644 --- a/src/core/or/circuitbuild.h +++ b/src/core/or/circuitbuild.h @@ -84,7 +84,8 @@ void circuit_upgrade_circuits_from_guard_wait(void); STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes); -MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes)); +MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes, + int direct)); STATIC int onion_extend_cpath(origin_circuit_t *circ); diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c index 145004c71d..6b5f30e418 100644 --- a/src/core/or/circuitlist.c +++ b/src/core/or/circuitlist.c @@ -51,6 +51,7 @@ * logic, which was originally circuit-focused. **/ #define CIRCUITLIST_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "lib/cc/torint.h" /* TOR_PRIuSZ */ #include "core/or/or.h" @@ -61,6 +62,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuituse.h" #include "core/or/circuitstats.h" +#include "core/or/circuitpadding.h" #include "core/mainloop/connection.h" #include "app/config/config.h" #include "core/or/connection_edge.h" @@ -94,7 +96,10 @@ #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" + +#define OCIRC_EVENT_PRIVATE +#include "core/or/ocirc_event.h" #include "ht.h" @@ -481,6 +486,56 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id, } } +/** + * Helper function to publish a message about events on an origin circuit + * + * Publishes a message to subscribers of origin circuit events, and + * sends the control event. + **/ +int +circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, + int reason_code) +{ + ocirc_event_msg_t msg; + + tor_assert(circ); + + msg.type = OCIRC_MSGTYPE_CEVENT; + msg.u.cevent.gid = circ->global_identifier; + msg.u.cevent.evtype = tp; + msg.u.cevent.reason = reason_code; + msg.u.cevent.onehop = circ->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); + return control_event_circuit_status(circ, tp, reason_code); +} + +/** + * Helper function to publish a state change message + * + * circuit_set_state() calls this to notify subscribers about a change + * of the state of an origin circuit. + **/ +static void +circuit_state_publish(const circuit_t *circ) +{ + ocirc_event_msg_t msg; + const origin_circuit_t *ocirc; + + if (!CIRCUIT_IS_ORIGIN(circ)) + return; + ocirc = CONST_TO_ORIGIN_CIRCUIT(circ); + /* Only inbound OR circuits can be in this state, not origin circuits. */ + tor_assert(circ->state != CIRCUIT_STATE_ONIONSKIN_PENDING); + + msg.type = OCIRC_MSGTYPE_STATE; + msg.u.state.gid = ocirc->global_identifier; + msg.u.state.state = circ->state; + msg.u.state.onehop = ocirc->build_state->onehop_tunnel; + + ocirc_event_publish(&msg); +} + /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing * it from lists as appropriate. */ void @@ -510,6 +565,7 @@ circuit_set_state(circuit_t *circ, uint8_t state) if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN) tor_assert(!circ->n_chan_create_cell); circ->state = state; + circuit_state_publish(circ); } /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for @@ -1176,6 +1232,9 @@ circuit_free_(circuit_t *circ) CIRCUIT_IS_ORIGIN(circ) ? TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0); + /* Free any circuit padding structures */ + circpad_circuit_free_all_machineinfos(circ); + if (should_free) { memwipe(mem, 0xAA, memlen); /* poison memory */ tor_free(mem); @@ -2270,7 +2329,7 @@ circuit_about_to_free(circuit_t *circ) smartlist_remove(circuits_pending_other_guards, circ); } if (CIRCUIT_IS_ORIGIN(circ)) { - control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ), + circuit_event_status(TO_ORIGIN_CIRCUIT(circ), (circ->state == CIRCUIT_STATE_OPEN || circ->state == CIRCUIT_STATE_GUARD_WAIT) ? CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED, diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h index b87c6a3667..f34f4ed6b7 100644 --- a/src/core/or/circuitlist.h +++ b/src/core/or/circuitlist.h @@ -14,6 +14,7 @@ #include "lib/testsupport/testsupport.h" #include "feature/hs/hs_ident.h" +#include "core/or/ocirc_event.h" /** Circuit state: I'm the origin, still haven't done all my handshakes. */ #define CIRCUIT_STATE_BUILDING 0 @@ -184,6 +185,8 @@ void channel_mark_circid_unusable(channel_t *chan, circid_t id); void channel_mark_circid_usable(channel_t *chan, circid_t id); time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id, channel_t *chan); +int circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp, + int reason_code); void circuit_set_state(circuit_t *circ, uint8_t state); void circuit_close_all_marked(void); int32_t circuit_initial_package_window(void); diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c new file mode 100644 index 0000000000..aa38b0ffc3 --- /dev/null +++ b/src/core/or/circuitpadding.c @@ -0,0 +1,2589 @@ +/* Copyright (c) 2017 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitpadding.c + * \brief Circuit-level padding implementation + * + * \details + * + * This file implements Tor proposal 254 "Padding Negotiation" which is heavily + * inspired by the paper "Toward an Efficient Website Fingerprinting Defense" + * by M. Juarez, M. Imani, M. Perry, C. Diaz, M. Wright. + * + * In particular the code in this file describes mechanisms for clients to + * negotiate various types of circuit-level padding from relays. + * + * Each padding type is described by a state machine (circpad_machine_spec_t), + * which is also referred as a "padding machine" in this file. Currently, + * these state machines are hardcoded in the source code (e.g. see + * circpad_circ_client_machine_init()), but in the future we will be able to + * serialize them in the torrc or the consensus. + * + * As specified by prop#254, clients can negotiate padding with relays by using + * PADDING_NEGOTIATE cells. After successful padding negotiation, padding + * machines are assigned to the circuit in their mutable form as a + * circpad_machine_state_t. + * + * Each state of a padding state machine can be either: + * - A histogram that specifies inter-arrival padding delays. + * - Or a parametrized probability distribution that specifies inter-arrival + * delays (see circpad_distribution_type_t). + * + * Padding machines start from the START state and finish with the END + * state. They can transition between states using the events in + * circpad_event_t. + * + * When a padding machine reaches the END state, it gets wiped from the circuit + * so that other padding machines can take over if needed (see + * circpad_machine_spec_transitioned_to_end()). + **/ + +#define CIRCUITPADDING_PRIVATE + +#include <math.h> +#include "lib/math/fp.h" +#include "lib/math/prob_distr.h" +#include "core/or/or.h" +#include "core/or/circuitpadding.h" +#include "core/or/circuitlist.h" +#include "core/or/circuituse.h" +#include "core/or/relay.h" +#include "feature/stats/rephist.h" +#include "feature/nodelist/networkstatus.h" + +#include "core/or/channel.h" + +#include "lib/time/compat_time.h" +#include "lib/defs/time.h" +#include "lib/crypt_ops/crypto_rand.h" + +#include "core/or/crypt_path_st.h" +#include "core/or/circuit_st.h" +#include "core/or/origin_circuit_st.h" +#include "core/or/or_circuit_st.h" +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "core/or/cell_st.h" +#include "core/or/extend_info_st.h" +#include "core/crypto/relay_crypto.h" +#include "feature/nodelist/nodelist.h" + +#include "app/config/config.h" + +static inline circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t + circ_purpose); +static inline circpad_circuit_state_t circpad_circuit_state( + origin_circuit_t *circ); +static void circpad_setup_machine_on_circ(circuit_t *on_circ, + const circpad_machine_spec_t *machine); +static double circpad_distribution_sample(circpad_distribution_t dist); + +/** Cached consensus params */ +static uint8_t circpad_global_max_padding_percent; +static uint16_t circpad_global_allowed_cells; +static uint16_t circpad_max_circ_queued_cells; + +/** Global cell counts, for rate limiting */ +static uint64_t circpad_global_padding_sent; +static uint64_t circpad_global_nonpadding_sent; + +/** This is the list of circpad_machine_spec_t's parsed from consensus and + * torrc that have origin_side == 1 (ie: are for client side). + * + * The machines in this smartlist are considered immutable and they are used + * as-is by circuits so they should not change or get deallocated in Tor's + * runtime and as long as circuits are alive. */ +STATIC smartlist_t *origin_padding_machines = NULL; + +/** This is the list of circpad_machine_spec_t's parsed from consensus and + * torrc that have origin_side == 0 (ie: are for relay side). + * + * The machines in this smartlist are considered immutable and they are used + * as-is by circuits so they should not change or get deallocated in Tor's + * runtime and as long as circuits are alive. */ +STATIC smartlist_t *relay_padding_machines = NULL; + +/** Loop over the current padding state machines using <b>loop_var</b> as the + * loop variable. */ +#define FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var) \ + STMT_BEGIN \ + for (int loop_var = 0; loop_var < CIRCPAD_MAX_MACHINES; loop_var++) { +#define FOR_EACH_CIRCUIT_MACHINE_END } STMT_END ; + +/** Loop over the current active padding state machines using <b>loop_var</b> + * as the loop variable. If a machine is not active, skip it. */ +#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(loop_var, circ) \ + FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var) \ + if (!(circ)->padding_info[loop_var]) \ + continue; +#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END } STMT_END ; + +/** + * Return a human-readable description for a circuit padding state. + */ +static const char * +circpad_state_to_string(circpad_statenum_t state) +{ + const char *descr; + + switch (state) { + case CIRCPAD_STATE_START: + descr = "START"; + break; + case CIRCPAD_STATE_BURST: + descr = "BURST"; + break; + case CIRCPAD_STATE_GAP: + descr = "GAP"; + break; + case CIRCPAD_STATE_END: + descr = "END"; + break; + default: + descr = "CUSTOM"; // XXX: Just return # in static char buf? + } + + return descr; +} + +/** + * Free the machineinfo at an index + */ +static void +circpad_circuit_machineinfo_free_idx(circuit_t *circ, int idx) +{ + if (circ->padding_info[idx]) { + tor_free(circ->padding_info[idx]->histogram); + timer_free(circ->padding_info[idx]->padding_timer); + tor_free(circ->padding_info[idx]); + } +} + +/** + * Free all the machineinfos in <b>circ</b> that match <b>machine_num</b>. + * + * Returns true if any machineinfos with that number were freed. + * False otherwise. */ +static int +free_circ_machineinfos_with_machine_num(circuit_t *circ, int machine_num) +{ + int found = 0; + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + if (circ->padding_machine[i] && + circ->padding_machine[i]->machine_num == machine_num) { + circpad_circuit_machineinfo_free_idx(circ, i); + circ->padding_machine[i] = NULL; + found = 1; + } + } FOR_EACH_CIRCUIT_MACHINE_END; + + return found; +} + +/** + * Free all padding machines and mutable info associated with circuit + */ +void +circpad_circuit_free_all_machineinfos(circuit_t *circ) +{ + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + circpad_circuit_machineinfo_free_idx(circ, i); + } FOR_EACH_CIRCUIT_MACHINE_END; +} + +/** + * Allocate a new mutable machineinfo structure. + */ +STATIC circpad_machine_state_t * +circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index) +{ + circpad_machine_state_t *mi = + tor_malloc_zero(sizeof(circpad_machine_state_t)); + mi->machine_index = machine_index; + mi->on_circ = on_circ; + + return mi; +} + +/** + * Return the circpad_state_t for the current state based on the + * mutable info. + * + * This function returns NULL when the machine is in the end state or in an + * invalid state. + */ +STATIC const circpad_state_t * +circpad_machine_current_state(const circpad_machine_state_t *mi) +{ + const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); + + if (mi->current_state == CIRCPAD_STATE_END) { + return NULL; + } else if (BUG(mi->current_state >= machine->num_states)) { + log_fn(LOG_WARN,LD_CIRC, + "Invalid circuit padding state %d", + mi->current_state); + + return NULL; + } + + return &machine->states[mi->current_state]; +} + +/** + * Calculate the lower bound of a histogram bin. The upper bound + * is obtained by calling this function with bin+1, and subtracting 1. + * + * The 0th bin has a special value -- it only represents start_usec. + * This is so we can specify a probability on 0-delay values. + * + * After bin 0, bins are exponentially spaced, so that each subsequent + * bin is twice as large as the previous. This is done so that higher + * time resolution is given to lower time values. + * + * The infinity bin is a the last bin in the array (histogram_len-1). + * It has a usec value of CIRCPAD_DELAY_INFINITE (UINT32_MAX). + */ +STATIC circpad_delay_t +circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, + circpad_hist_index_t bin) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + circpad_delay_t start_usec; + + /* Our state should have been checked to be non-null by the caller + * (circpad_machine_remove_token()) */ + if (BUG(state == NULL)) { + return CIRCPAD_DELAY_INFINITE; + } + + if (state->use_rtt_estimate) + start_usec = mi->rtt_estimate_usec+state->start_usec; + else + start_usec = state->start_usec; + + if (bin >= CIRCPAD_INFINITY_BIN(state)) + return CIRCPAD_DELAY_INFINITE; + + if (bin == 0) + return start_usec; + + if (bin == 1) + return start_usec+1; + + /* The bin widths double every index, so that we can have more resolution + * for lower time values in the histogram. */ + const circpad_time_t bin_width_exponent = + 1 << (CIRCPAD_INFINITY_BIN(state) - bin); + return (circpad_delay_t)MIN(start_usec + + state->range_usec/bin_width_exponent, + CIRCPAD_DELAY_INFINITE); +} + +/** Return the midpoint of the histogram bin <b>bin_index</b>. */ +static circpad_delay_t +circpad_get_histogram_bin_midpoint(const circpad_machine_state_t *mi, + int bin_index) +{ + circpad_delay_t left_bound = circpad_histogram_bin_to_usec(mi, bin_index); + circpad_delay_t right_bound = + circpad_histogram_bin_to_usec(mi, bin_index+1)-1; + + return left_bound + (right_bound - left_bound)/2; +} + +/** + * Return the bin that contains the usec argument. + * "Contains" is defined as us in [lower, upper). + * + * This function will never return the infinity bin (histogram_len-1), + * in order to simplify the rest of the code. + * + * This means that technically the last bin (histogram_len-2) + * has range [start_usec+range_usec, CIRCPAD_DELAY_INFINITE]. + */ +STATIC circpad_hist_index_t +circpad_histogram_usec_to_bin(const circpad_machine_state_t *mi, + circpad_delay_t usec) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + circpad_delay_t start_usec; + int32_t bin; /* Larger than return type to properly clamp overflow */ + + /* Our state should have been checked to be non-null by the caller + * (circpad_machine_remove_token()) */ + if (BUG(state == NULL)) { + return 0; + } + + if (state->use_rtt_estimate) + start_usec = mi->rtt_estimate_usec+state->start_usec; + else + start_usec = state->start_usec; + + /* The first bin (#0) has zero width and starts (and ends) at start_usec. */ + if (usec <= start_usec) + return 0; + + if (usec == start_usec+1) + return 1; + + const circpad_time_t histogram_range_usec = state->range_usec; + /* We need to find the bin corresponding to our position in the range. + * Since bins are exponentially spaced in powers of two, we need to + * take the log2 of our position in histogram_range_usec. However, + * since tor_log2() returns the floor(log2(u64)), we have to adjust + * it to behave like ceil(log2(u64)). This is verified in our tests + * to properly invert the operation done in + * circpad_histogram_bin_to_usec(). */ + bin = CIRCPAD_INFINITY_BIN(state) - + tor_log2(2*histogram_range_usec/(usec-start_usec+1)); + + /* Clamp the return value to account for timevals before the start + * of bin 0, or after the last bin. Don't return the infinity bin + * index. */ + bin = MIN(MAX(bin, 1), CIRCPAD_INFINITY_BIN(state)-1); + return bin; +} + +/** + * This function frees any token bins allocated from a previous state + * + * Called after a state transition, or if the bins are empty. + */ +STATIC void +circpad_machine_setup_tokens(circpad_machine_state_t *mi) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + + /* If this state doesn't exist, or doesn't have token removal, + * free any previous state's histogram, and bail */ + if (!state || state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) { + if (mi->histogram) { + tor_free(mi->histogram); + mi->histogram = NULL; + mi->histogram_len = 0; + } + return; + } + + /* Try to avoid re-mallocing if we don't really need to */ + if (!mi->histogram || (mi->histogram + && mi->histogram_len != state->histogram_len)) { + tor_free(mi->histogram); // null ok + mi->histogram = tor_malloc_zero(sizeof(circpad_hist_token_t) + *state->histogram_len); + } + mi->histogram_len = state->histogram_len; + + memcpy(mi->histogram, state->histogram, + sizeof(circpad_hist_token_t)*state->histogram_len); +} + +/** + * Choose a length for this state (in cells), if specified. + */ +static void +circpad_choose_state_length(circpad_machine_state_t *mi) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + double length; + + if (!state || state->length_dist.type == CIRCPAD_DIST_NONE) { + mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE; + return; + } + + length = circpad_distribution_sample(state->length_dist); + length = MAX(0, length); + length += state->start_length; + length = MIN(length, state->max_length); + + mi->state_length = clamp_double_to_int64(length); +} + +/** + * Sample a value from our iat_dist, and clamp it safely + * to circpad_delay_t. + */ +static circpad_delay_t +circpad_distribution_sample_iat_delay(const circpad_state_t *state, + circpad_delay_t start_usec) +{ + double val = circpad_distribution_sample(state->iat_dist); + /* These comparisons are safe, because the output is in the range + * [0, 2**32), and double has a precision of 53 bits. */ + val = MAX(0, val); + val = MIN(val, state->range_usec); + + /* This addition is exact: val is at most 2**32-1, start_usec + * is at most 2**32-1, and doubles have a precision of 53 bits. */ + val += start_usec; + + /* Clamp the distribution at infinite delay val */ + return (circpad_delay_t)MIN(tor_llround(val), CIRCPAD_DELAY_INFINITE); +} + +/** + * Sample an expected time-until-next-packet delay from the histogram. + * + * The bin is chosen with probability proportional to the number + * of tokens in each bin, and then a time value is chosen uniformly from + * that bin's [start,end) time range. + */ +STATIC circpad_delay_t +circpad_machine_sample_delay(circpad_machine_state_t *mi) +{ + const circpad_state_t *state = circpad_machine_current_state(mi); + const circpad_hist_token_t *histogram = NULL; + circpad_hist_index_t curr_bin = 0; + circpad_delay_t bin_start, bin_end; + circpad_delay_t start_usec; + /* These three must all be larger than circpad_hist_token_t, because + * we sum several circpad_hist_token_t values across the histogram */ + uint64_t curr_weight = 0; + uint64_t histogram_total_tokens = 0; + uint64_t bin_choice; + + tor_assert(state); + + if (state->use_rtt_estimate) + start_usec = mi->rtt_estimate_usec+state->start_usec; + else + start_usec = state->start_usec; + + if (state->iat_dist.type != CIRCPAD_DIST_NONE) { + /* Sample from a fixed IAT distribution and return */ + return circpad_distribution_sample_iat_delay(state, start_usec); + } else if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) { + /* We have a mutable histogram. Do basic sanity check and apply: */ + if (BUG(!mi->histogram) || + BUG(mi->histogram_len != state->histogram_len)) { + return CIRCPAD_DELAY_INFINITE; + } + + histogram = mi->histogram; + for (circpad_hist_index_t b = 0; b < state->histogram_len; b++) + histogram_total_tokens += histogram[b]; + } else { + /* We have a histogram, but it's immutable */ + histogram = state->histogram; + histogram_total_tokens = state->histogram_total_tokens; + } + + bin_choice = crypto_rand_uint64(histogram_total_tokens); + + /* Skip all the initial zero bins */ + while (!histogram[curr_bin]) { + curr_bin++; + } + curr_weight = histogram[curr_bin]; + + // TODO: This is not constant-time. Pretty sure we don't + // really need it to be, though. + while (curr_weight < bin_choice) { + curr_bin++; + /* It should be impossible to run past the end of the histogram */ + if (BUG(curr_bin >= state->histogram_len)) { + return CIRCPAD_DELAY_INFINITE; + } + curr_weight += histogram[curr_bin]; + } + + /* Do some basic checking of the current bin we are in */ + if (BUG(curr_bin >= state->histogram_len) || + BUG(histogram[curr_bin] == 0)) { + return CIRCPAD_DELAY_INFINITE; + } + + // Store this index to remove the token upon callback. + if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) { + mi->chosen_bin = curr_bin; + } + + if (curr_bin >= CIRCPAD_INFINITY_BIN(state)) { + if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE && + mi->histogram[curr_bin] > 0) { + mi->histogram[curr_bin]--; + } + + // Infinity: Don't send a padding packet. Wait for a real packet + // and then see if our bins are empty or what else we should do. + return CIRCPAD_DELAY_INFINITE; + } + + tor_assert(curr_bin < CIRCPAD_INFINITY_BIN(state)); + + bin_start = circpad_histogram_bin_to_usec(mi, curr_bin); + /* We don't need to reduct 1 from the upper bound because the random range + * function below samples from [bin_start, bin_end) */ + bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1); + + /* Truncate the high bin in case it's the infinity bin: + * Don't actually schedule an "infinite"-1 delay */ + bin_end = MIN(bin_end, start_usec+state->range_usec); + + // Sample uniformly between histogram[i] to histogram[i+1]-1, + // but no need to sample if they are the same timeval (aka bin 0 or bin 1). + if (bin_end <= bin_start+1) + return bin_start; + else + return (circpad_delay_t)crypto_rand_uint64_range(bin_start, bin_end); +} + +/** + * Sample a value from the specified probability distribution. + * + * This performs inverse transform sampling + * (https://en.wikipedia.org/wiki/Inverse_transform_sampling). + * + * XXX: These formulas were taken verbatim. Need a floating wizard + * to check them for catastropic cancellation and other issues (teor?). + * Also: is 32bits of double from [0.0,1.0) enough? + */ +static double +circpad_distribution_sample(circpad_distribution_t dist) +{ + log_fn(LOG_DEBUG,LD_CIRC, "Sampling delay with distribution %d", + dist.type); + + switch (dist.type) { + case CIRCPAD_DIST_NONE: + { + /* We should not get in here like this */ + tor_assert_nonfatal_unreached(); + return 0; + } + case CIRCPAD_DIST_UNIFORM: + { + // param2 is upper bound, param1 is lower + const struct uniform my_uniform = { + .base = UNIFORM(my_uniform), + .a = dist.param1, + .b = dist.param2, + }; + return dist_sample(&my_uniform.base); + } + case CIRCPAD_DIST_LOGISTIC: + { + /* param1 is Mu, param2 is sigma. */ + const struct logistic my_logistic = { + .base = LOGISTIC(my_logistic), + .mu = dist.param1, + .sigma = dist.param2, + }; + return dist_sample(&my_logistic.base); + } + case CIRCPAD_DIST_LOG_LOGISTIC: + { + /* param1 is Alpha, param2 is 1.0/Beta */ + const struct log_logistic my_log_logistic = { + .base = LOG_LOGISTIC(my_log_logistic), + .alpha = dist.param1, + .beta = dist.param2, + }; + return dist_sample(&my_log_logistic.base); + } + case CIRCPAD_DIST_GEOMETRIC: + { + /* param1 is 'p' (success probability) */ + const struct geometric my_geometric = { + .base = GEOMETRIC(my_geometric), + .p = dist.param1, + }; + return dist_sample(&my_geometric.base); + } + case CIRCPAD_DIST_WEIBULL: + { + /* param1 is k, param2 is Lambda */ + const struct weibull my_weibull = { + .base = WEIBULL(my_weibull), + .k = dist.param1, + .lambda = dist.param2, + }; + return dist_sample(&my_weibull.base); + } + case CIRCPAD_DIST_PARETO: + { + /* param1 is sigma, param2 is xi, no more params for mu so we use 0 */ + const struct genpareto my_genpareto = { + .base = GENPARETO(my_genpareto), + .mu = 0, + .sigma = dist.param1, + .xi = dist.param2, + }; + return dist_sample(&my_genpareto.base); + } + } + + tor_assert_nonfatal_unreached(); + return 0; +} + +/** + * Find the index of the first bin whose upper bound is + * greater than the target, and that has tokens remaining. + */ +static circpad_hist_index_t +circpad_machine_first_higher_index(const circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, + target_bin_usec); + + /* Don't remove from the infinity bin */ + for (; bin < CIRCPAD_INFINITY_BIN(mi); bin++) { + if (mi->histogram[bin] && + circpad_histogram_bin_to_usec(mi, bin+1) > target_bin_usec) { + return bin; + } + } + + return mi->histogram_len; +} + +/** + * Find the index of the first bin whose lower bound is lower or equal to + * <b>target_bin_usec</b>, and that still has tokens remaining. + */ +static circpad_hist_index_t +circpad_machine_first_lower_index(const circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, + target_bin_usec); + + for (; bin >= 0; bin--) { + if (mi->histogram[bin] && + circpad_histogram_bin_to_usec(mi, bin) <= target_bin_usec) { + return bin; + } + } + + return -1; +} + +/** + * Remove a token from the first non-empty bin whose upper bound is + * greater than the target. + */ +STATIC void +circpad_machine_remove_higher_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec) +{ + /* We need to remove the token from the first bin + * whose upper bound is greater than the target, and that + * has tokens remaining. */ + circpad_hist_index_t bin = circpad_machine_first_higher_index(mi, + target_bin_usec); + + if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) { + if (!BUG(mi->histogram[bin] == 0)) { + mi->histogram[bin]--; + } + } +} + +/** + * Remove a token from the first non-empty bin whose upper bound is + * lower than the target. + */ +STATIC void +circpad_machine_remove_lower_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_machine_first_lower_index(mi, + target_bin_usec); + + if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) { + if (!BUG(mi->histogram[bin] == 0)) { + mi->histogram[bin]--; + } + } +} + +/* Helper macro: Ensure that the bin has tokens available, and BUG out of the + * function if it's not the case. */ +#define ENSURE_BIN_CAPACITY(bin_index) \ + if (BUG(mi->histogram[bin_index] == 0)) { \ + return; \ + } + +/** + * Remove a token from the closest non-empty bin to the target. + * + * If use_usec is true, measure "closest" in terms of the next closest bin + * midpoint. + * + * If it is false, use bin index distance only. + */ +STATIC void +circpad_machine_remove_closest_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec, + bool use_usec) +{ + circpad_hist_index_t lower, higher, current; + circpad_hist_index_t bin_to_remove = -1; + + lower = circpad_machine_first_lower_index(mi, target_bin_usec); + higher = circpad_machine_first_higher_index(mi, target_bin_usec); + current = circpad_histogram_usec_to_bin(mi, target_bin_usec); + + /* Sanity check the results */ + if (BUG(lower > current) || BUG(higher < current)) { + return; + } + + /* Take care of edge cases first */ + if (higher == mi->histogram_len && lower == -1) { + /* All bins are empty */ + return; + } else if (higher == mi->histogram_len) { + /* All higher bins are empty */ + ENSURE_BIN_CAPACITY(lower); + mi->histogram[lower]--; + return; + } else if (lower == -1) { + /* All lower bins are empty */ + ENSURE_BIN_CAPACITY(higher); + mi->histogram[higher]--; + return; + } + + /* Now handle the intermediate cases */ + if (use_usec) { + /* Find the closest bin midpoint to the target */ + circpad_delay_t lower_usec = circpad_get_histogram_bin_midpoint(mi, lower); + circpad_delay_t higher_usec = + circpad_get_histogram_bin_midpoint(mi, higher); + + if (target_bin_usec < lower_usec) { + // Lower bin is closer + ENSURE_BIN_CAPACITY(lower); + bin_to_remove = lower; + } else if (target_bin_usec > higher_usec) { + // Higher bin is closer + ENSURE_BIN_CAPACITY(higher); + bin_to_remove = higher; + } else if (target_bin_usec-lower_usec > higher_usec-target_bin_usec) { + // Higher bin is closer + ENSURE_BIN_CAPACITY(higher); + bin_to_remove = higher; + } else { + // Lower bin is closer + ENSURE_BIN_CAPACITY(lower); + bin_to_remove = lower; + } + mi->histogram[bin_to_remove]--; + log_debug(LD_GENERAL, "Removing token from bin %d", bin_to_remove); + return; + } else { + if (current - lower > higher - current) { + // Higher bin is closer + ENSURE_BIN_CAPACITY(higher); + mi->histogram[higher]--; + return; + } else { + // Lower bin is closer + ENSURE_BIN_CAPACITY(lower); + mi->histogram[lower]--; + return; + } + } +} + +#undef ENSURE_BIN_CAPACITY + +/** + * Remove a token from the exact bin corresponding to the target. + * + * If it is empty, do nothing. + */ +static void +circpad_machine_remove_exact(circpad_machine_state_t *mi, + circpad_delay_t target_bin_usec) +{ + circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi, + target_bin_usec); + + if (mi->histogram[bin] > 0) + mi->histogram[bin]--; +} + +/** + * Check our state's cell limit count and tokens. + * + * Returns 1 if either limits are hit and we decide to change states, + * otherwise returns 0. + */ +static circpad_decision_t +check_machine_token_supply(circpad_machine_state_t *mi) +{ + uint32_t histogram_total_tokens = 0; + + /* Check if bins empty. This requires summing up the current mutable + * machineinfo histogram token total and checking if it is zero. + * Machineinfo does not keep a running token count. We're assuming the + * extra space is not worth this short loop iteration. + * + * We also do not count infinity bin in histogram totals. + */ + if (mi->histogram_len && mi->histogram) { + for (circpad_hist_index_t b = 0; b < CIRCPAD_INFINITY_BIN(mi); b++) + histogram_total_tokens += mi->histogram[b]; + + /* If we change state, we're done */ + if (histogram_total_tokens == 0) { + if (circpad_internal_event_bins_empty(mi) == CIRCPAD_STATE_CHANGED) + return CIRCPAD_STATE_CHANGED; + } + } + + if (mi->state_length == 0) { + return circpad_internal_event_state_length_up(mi); + } + + return CIRCPAD_STATE_UNCHANGED; +} + +/** + * Remove a token from the bin corresponding to the delta since + * last packet. If that bin is empty, choose a token based on + * the specified removal strategy in the state machine. + * + * This function also updates and checks rate limit and state + * limit counters. + * + * Returns 1 if we transition states, 0 otherwise. + */ +STATIC circpad_decision_t +circpad_machine_remove_token(circpad_machine_state_t *mi) +{ + const circpad_state_t *state = NULL; + circpad_time_t current_time; + circpad_delay_t target_bin_usec; + + /* Update non-padding counts for rate limiting: We scale at UINT16_MAX + * because we only use this for a percentile limit of 2 sig figs, and + * space is scare in the machineinfo struct. */ + mi->nonpadding_sent++; + if (mi->nonpadding_sent == UINT16_MAX) { + mi->padding_sent /= 2; + mi->nonpadding_sent /= 2; + } + + /* Dont remove any tokens if there was no padding scheduled */ + if (!mi->padding_scheduled_at_usec) { + return CIRCPAD_STATE_UNCHANGED; + } + + state = circpad_machine_current_state(mi); + current_time = monotime_absolute_usec(); + + /* If we have scheduled padding some time in the future, we want to see what + bin we are in at the current time */ + target_bin_usec = (circpad_delay_t) + MIN((current_time - mi->padding_scheduled_at_usec), + CIRCPAD_DELAY_INFINITE-1); + + /* We are treating this non-padding cell as a padding cell, so we cancel + padding timer, if present. */ + mi->padding_scheduled_at_usec = 0; + if (mi->is_padding_timer_scheduled) { + mi->is_padding_timer_scheduled = 0; + timer_disable(mi->padding_timer); + } + + /* If we are not in a padding state (like start or end), we're done */ + if (!state) + return CIRCPAD_STATE_UNCHANGED; + + /* If we're enforcing a state length on non-padding packets, + * decrement it */ + if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE && + state->length_includes_nonpadding && + mi->state_length > 0) { + mi->state_length--; + } + + /* Perform the specified token removal strategy */ + switch (state->token_removal) { + case CIRCPAD_TOKEN_REMOVAL_NONE: + break; + case CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC: + circpad_machine_remove_closest_token(mi, target_bin_usec, 1); + break; + case CIRCPAD_TOKEN_REMOVAL_CLOSEST: + circpad_machine_remove_closest_token(mi, target_bin_usec, 0); + break; + case CIRCPAD_TOKEN_REMOVAL_LOWER: + circpad_machine_remove_lower_token(mi, target_bin_usec); + break; + case CIRCPAD_TOKEN_REMOVAL_HIGHER: + circpad_machine_remove_higher_token(mi, target_bin_usec); + break; + case CIRCPAD_TOKEN_REMOVAL_EXACT: + circpad_machine_remove_exact(mi, target_bin_usec); + break; + } + + /* Check our token and state length limits */ + return check_machine_token_supply(mi); +} + +/** + * Send a relay command with a relay cell payload on a circuit to + * the particular hopnum. + * + * Hopnum starts at 1 (1=guard, 2=middle, 3=exit, etc). + * + * Payload may be null. + * + * Returns negative on error, 0 on success. + */ +MOCK_IMPL(STATIC signed_error_t, +circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum, + uint8_t relay_command, const uint8_t *payload, + ssize_t payload_len)) +{ + crypt_path_t *target_hop = circuit_get_cpath_hop(circ, hopnum); + signed_error_t ret; + + /* Check that the cpath has the target hop */ + if (!target_hop) { + log_fn(LOG_WARN, LD_BUG, "Padding circuit %u has %d hops, not %d", + circ->global_identifier, circuit_get_cpath_len(circ), hopnum); + return -1; + } + + /* Check that the target hop is opened */ + if (target_hop->state != CPATH_STATE_OPEN) { + log_fn(LOG_WARN,LD_CIRC, + "Padding circuit %u has %d hops, not %d", + circ->global_identifier, + circuit_get_cpath_opened_len(circ), hopnum); + return -1; + } + + /* Send the drop command to the second hop */ + ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ), relay_command, + (const char*)payload, payload_len, + target_hop); + return ret; +} + +/** + * Callback helper to send a padding cell. + * + * This helper is called after our histogram-sampled delay period passes + * without another packet being sent first. If a packet is sent before this + * callback happens, it is canceled. So when we're called here, send padding + * right away. + * + * If sending this padding cell forced us to transition states return + * CIRCPAD_STATE_CHANGED. Otherwise return CIRCPAD_STATE_UNCHANGED. + */ +circpad_decision_t +circpad_send_padding_cell_for_callback(circpad_machine_state_t *mi) +{ + circuit_t *circ = mi->on_circ; + int machine_idx = mi->machine_index; + mi->padding_scheduled_at_usec = 0; + circpad_statenum_t state = mi->current_state; + + // Make sure circuit didn't close on us + if (mi->on_circ->marked_for_close) { + log_fn(LOG_INFO,LD_CIRC, + "Padding callback on a circuit marked for close. Ignoring."); + return CIRCPAD_STATE_CHANGED; + } + + /* If it's a histogram, reduce the token count */ + if (mi->histogram && mi->histogram_len) { + /* Basic sanity check on the histogram before removing anything */ + if (BUG(mi->chosen_bin >= mi->histogram_len) || + BUG(mi->histogram[mi->chosen_bin] == 0)) { + return CIRCPAD_STATE_CHANGED; + } + + mi->histogram[mi->chosen_bin]--; + } + + /* If we have a valid state length bound, consider it */ + if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE && + !BUG(mi->state_length <= 0)) { + mi->state_length--; + } + + /* + * Update non-padding counts for rate limiting: We scale at UINT16_MAX + * because we only use this for a percentile limit of 2 sig figs, and + * space is scare in the machineinfo struct. + */ + mi->padding_sent++; + if (mi->padding_sent == UINT16_MAX) { + mi->padding_sent /= 2; + mi->nonpadding_sent /= 2; + } + circpad_global_padding_sent++; + + if (CIRCUIT_IS_ORIGIN(mi->on_circ)) { + circpad_send_command_to_hop(TO_ORIGIN_CIRCUIT(mi->on_circ), + CIRCPAD_GET_MACHINE(mi)->target_hopnum, + RELAY_COMMAND_DROP, NULL, 0); + log_fn(LOG_INFO,LD_CIRC, "Callback: Sending padding to origin circuit %u.", + TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier); + } else { + // If we're a non-origin circ, we can just send from here as if we're the + // edge. + if (TO_OR_CIRCUIT(circ)->p_chan_cells.n <= circpad_max_circ_queued_cells) { + log_fn(LOG_INFO,LD_CIRC, + "Callback: Sending padding to non-origin circuit."); + relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL, + 0, NULL); + } else { + static ratelim_t cell_lim = RATELIM_INIT(600); + log_fn_ratelim(&cell_lim,LOG_NOTICE,LD_CIRC, + "Too many cells (%d) in circ queue to send padding.", + TO_OR_CIRCUIT(circ)->p_chan_cells.n); + } + } + + rep_hist_padding_count_write(PADDING_TYPE_DROP); + /* This is a padding cell sent from the client or from the middle node, + * (because it's invoked from circuitpadding.c) */ + circpad_cell_event_padding_sent(circ); + + /* The circpad_cell_event_padding_sent() could cause us to transition. + * Check that we still have a padding machineinfo, and then check our token + * supply. */ + if (circ->padding_info[machine_idx] != NULL) { + if (state != circ->padding_info[machine_idx]->current_state) + return CIRCPAD_STATE_CHANGED; + else + return check_machine_token_supply(circ->padding_info[machine_idx]); + } else { + return CIRCPAD_STATE_CHANGED; + } +} + +/** + * Tor-timer compatible callback that tells us to send a padding cell. + * + * Timers are associated with circpad_machine_state_t's. When the machineinfo + * is freed on a circuit, the timers are cancelled. Since the lifetime + * of machineinfo is always longer than the timers, handles are not + * needed. + */ +static void +circpad_send_padding_callback(tor_timer_t *timer, void *args, + const struct monotime_t *time) +{ + circpad_machine_state_t *mi = ((circpad_machine_state_t*)args); + (void)timer; (void)time; + + if (mi && mi->on_circ) { + assert_circuit_ok(mi->on_circ); + circpad_send_padding_cell_for_callback(mi); + } else { + // This shouldn't happen (represents a timer leak) + log_fn(LOG_WARN,LD_CIRC, + "Circuit closed while waiting for padding timer."); + tor_fragile_assert(); + } + + // TODO-MP-AP: Unify this counter with channelpadding for rephist stats + //total_timers_pending--; +} + +/** + * Cache our consensus parameters upon consensus update. + */ +void +circpad_new_consensus_params(const networkstatus_t *ns) +{ + circpad_global_allowed_cells = + networkstatus_get_param(ns, "circpad_global_allowed_cells", + 0, 0, UINT16_MAX-1); + + circpad_global_max_padding_percent = + networkstatus_get_param(ns, "circpad_global_max_padding_pct", + 0, 0, 100); + + circpad_max_circ_queued_cells = + networkstatus_get_param(ns, "circpad_max_circ_queued_cells", + CIRCWINDOW_START_MAX, 0, 50*CIRCWINDOW_START_MAX); +} + +/** + * Check this machine against its padding limits, as well as global + * consensus limits. + * + * We have two limits: a percent and a cell count. The cell count + * limit must be reached before the percent is enforced (this is to + * optionally allow very light padding of things like circuit setup + * while there is no other traffic on the circuit). + * + * TODO: Don't apply limits to machines form torrc. + * + * Returns 1 if limits are set and we've hit them. Otherwise returns 0. + */ +STATIC bool +circpad_machine_reached_padding_limit(circpad_machine_state_t *mi) +{ + const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); + + /* If machine_padding_pct is non-zero, and we've sent more + * than the allowed count of padding cells, then check our + * percent limits for this machine. */ + if (machine->max_padding_percent && + mi->padding_sent >= machine->allowed_padding_count) { + uint32_t total_cells = mi->padding_sent + mi->nonpadding_sent; + + /* Check the percent */ + if ((100*(uint32_t)mi->padding_sent) / total_cells > + machine->max_padding_percent) { + return 1; // limit is reached. Stop. + } + } + + /* If circpad_max_global_padding_pct is non-zero, and we've + * sent more than the global padding cell limit, then check our + * gloabl tor process percentage limit on padding. */ + if (circpad_global_max_padding_percent && + circpad_global_padding_sent >= circpad_global_allowed_cells) { + uint64_t total_cells = circpad_global_padding_sent + + circpad_global_nonpadding_sent; + + /* Check the percent */ + if ((100*circpad_global_padding_sent) / total_cells > + circpad_global_max_padding_percent) { + return 1; // global limit reached. Stop. + } + } + + return 0; // All good! +} + +/** + * Schedule the next padding time according to the machineinfo on a + * circuit. + * + * The histograms represent inter-packet-delay. Whenever you get an packet + * event you should be scheduling your next timer (after cancelling any old + * ones and updating tokens accordingly). + * + * Returns 1 if we decide to transition states (due to infinity bin), + * 0 otherwise. + */ +MOCK_IMPL(circpad_decision_t, +circpad_machine_schedule_padding,(circpad_machine_state_t *mi)) +{ + circpad_delay_t in_usec = 0; + struct timeval timeout; + tor_assert(mi); + + // Don't pad in end (but also don't cancel any previously + // scheduled padding either). + if (mi->current_state == CIRCPAD_STATE_END) { + log_fn(LOG_INFO, LD_CIRC, "Padding end state"); + return CIRCPAD_STATE_UNCHANGED; + } + + /* Check our padding limits */ + if (circpad_machine_reached_padding_limit(mi)) { + if (CIRCUIT_IS_ORIGIN(mi->on_circ)) { + log_fn(LOG_INFO, LD_CIRC, + "Padding machine has reached padding limit on circuit %u", + TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier); + } else { + log_fn(LOG_INFO, LD_CIRC, + "Padding machine has reached padding limit on circuit %"PRIu64 + ", %d", + mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0, + mi->on_circ->n_circ_id); + } + return CIRCPAD_STATE_UNCHANGED; + } + + if (mi->is_padding_timer_scheduled) { + /* Cancel current timer (if any) */ + timer_disable(mi->padding_timer); + mi->is_padding_timer_scheduled = 0; + } + + /* in_usec = in microseconds */ + in_usec = circpad_machine_sample_delay(mi); + mi->padding_scheduled_at_usec = monotime_absolute_usec(); + log_fn(LOG_INFO,LD_CIRC,"\tPadding in %u usec", in_usec); + + // Don't schedule if we have infinite delay. + if (in_usec == CIRCPAD_DELAY_INFINITE) { + return circpad_internal_event_infinity(mi); + } + + if (mi->state_length == 0) { + /* If we're at length 0, that means we hit 0 after sending + * a cell earlier, and emitted an event for it, but + * for whatever reason we did not decide to change states then. + * So maybe the machine is waiting for bins empty, or for an + * infinity event later? That would be a strange machine, + * but there's no reason to make it impossible. */ + return CIRCPAD_STATE_UNCHANGED; + } + + if (in_usec <= 0) { + return circpad_send_padding_cell_for_callback(mi); + } + + timeout.tv_sec = in_usec/TOR_USEC_PER_SEC; + timeout.tv_usec = (in_usec%TOR_USEC_PER_SEC); + + log_fn(LOG_INFO, LD_CIRC, "\tPadding in %u sec, %u usec", + (unsigned)timeout.tv_sec, (unsigned)timeout.tv_usec); + + if (mi->padding_timer) { + timer_set_cb(mi->padding_timer, circpad_send_padding_callback, mi); + } else { + mi->padding_timer = + timer_new(circpad_send_padding_callback, mi); + } + timer_schedule(mi->padding_timer, &timeout); + mi->is_padding_timer_scheduled = 1; + + // TODO-MP-AP: Unify with channelpadding counter + //rep_hist_padding_count_timers(++total_timers_pending); + + return CIRCPAD_STATE_UNCHANGED; +} + +/** + * If the machine transitioned to the END state, we need + * to check to see if it wants us to shut it down immediately. + * If it does, then we need to send the appropate negotation commands + * depending on which side it is. + * + * After this function is called, mi may point to freed memory. Do + * not access it. + */ +static void +circpad_machine_spec_transitioned_to_end(circpad_machine_state_t *mi) +{ + const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi); + + /* + * We allow machines to shut down and delete themselves as opposed + * to just going back to START or waiting forever in END so that + * we can handle the case where this machine started while it was + * the only machine that matched conditions, but *since* then more + * "higher ranking" machines now match the conditions, and would + * be given a chance to take precidence over this one in + * circpad_add_matching_machines(). + * + * Returning to START or waiting forever in END would not give those + * other machines a chance to be launched, where as shutting down + * here does. + */ + if (machine->should_negotiate_end) { + circuit_t *on_circ = mi->on_circ; + if (machine->is_origin_side) { + /* We free the machine info here so that we can be replaced + * by a different machine. But we must leave the padding_machine + * in place to wait for the negotiated response */ + circpad_circuit_machineinfo_free_idx(on_circ, + machine->machine_index); + circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(on_circ), + machine->machine_num, + machine->target_hopnum, + CIRCPAD_COMMAND_STOP); + } else { + circpad_circuit_machineinfo_free_idx(on_circ, + machine->machine_index); + circpad_padding_negotiated(on_circ, + machine->machine_num, + CIRCPAD_COMMAND_STOP, + CIRCPAD_RESPONSE_OK); + on_circ->padding_machine[machine->machine_index] = NULL; + } + } +} + +/** + * Generic state transition function for padding state machines. + * + * Given an event and our mutable machine info, decide if/how to + * transition to a different state, and perform actions accordingly. + * + * Returns 1 if we transition states, 0 otherwise. + */ +MOCK_IMPL(circpad_decision_t, +circpad_machine_spec_transition,(circpad_machine_state_t *mi, + circpad_event_t event)) +{ + const circpad_state_t *state = + circpad_machine_current_state(mi); + + /* If state is null we are in the end state. */ + if (!state) { + /* If we in end state we don't pad no matter what. */ + return CIRCPAD_STATE_UNCHANGED; + } + + /* Check if this event is ignored or causes a cancel */ + if (state->next_state[event] == CIRCPAD_STATE_IGNORE) { + return CIRCPAD_STATE_UNCHANGED; + } else if (state->next_state[event] == CIRCPAD_STATE_CANCEL) { + /* Check cancel events and cancel any pending padding */ + mi->padding_scheduled_at_usec = 0; + if (mi->is_padding_timer_scheduled) { + mi->is_padding_timer_scheduled = 0; + /* Cancel current timer (if any) */ + timer_disable(mi->padding_timer); + } + return CIRCPAD_STATE_UNCHANGED; + } else { + circpad_statenum_t s = state->next_state[event]; + /* See if we need to transition to any other states based on this event. + * Whenever a transition happens, even to our own state, we schedule + * padding. + * + * So if a state only wants to schedule padding for an event, it specifies + * a transition to itself. All non-specified events are ignored. + */ + log_fn(LOG_INFO, LD_CIRC, + "Circpad machine %d transitioning from %s to %s", + mi->machine_index, circpad_state_to_string(mi->current_state), + circpad_state_to_string(s)); + + /* If this is not the same state, switch and init tokens, + * otherwise just reschedule padding. */ + if (mi->current_state != s) { + mi->current_state = s; + circpad_machine_setup_tokens(mi); + circpad_choose_state_length(mi); + + /* If we transition to the end state, check to see + * if this machine wants to be shut down at end */ + if (s == CIRCPAD_STATE_END) { + circpad_machine_spec_transitioned_to_end(mi); + /* We transitioned but we don't pad in end. Also, mi + * may be freed. Returning STATE_CHANGED prevents us + * from accessing it in any callers of this function. */ + return CIRCPAD_STATE_CHANGED; + } + + /* We transitioned to a new state, schedule padding */ + circpad_machine_schedule_padding(mi); + return CIRCPAD_STATE_CHANGED; + } + + /* We transitioned back to the same state. Schedule padding, + * and inform if that causes a state transition. */ + return circpad_machine_schedule_padding(mi); + } + + return CIRCPAD_STATE_UNCHANGED; +} + +/** + * Estimate the circuit RTT from the current middle hop out to the + * end of the circuit. + * + * We estimate RTT by calculating the time between "receive" and + * "send" at a middle hop. This is because we "receive" a cell + * from the origin, and then relay it towards the exit before a + * response comes back. It is that response time from the exit side + * that we want to measure, so that we can make use of it for synthetic + * response delays. + */ +static void +circpad_estimate_circ_rtt_on_received(circuit_t *circ, + circpad_machine_state_t *mi) +{ + /* Origin circuits don't estimate RTT. They could do it easily enough, + * but they have no reason to use it in any delay calculations. */ + if (CIRCUIT_IS_ORIGIN(circ) || mi->stop_rtt_update) + return; + + /* If we already have a last receieved packet time, that means we + * did not get a response before this packet. The RTT estimate + * only makes sense if we do not have multiple packets on the + * wire, so stop estimating if this is the second packet + * back to back. However, for the first set of back-to-back + * packets, we can wait until the very first response comes back + * to us, to measure that RTT (for the response to optimistic + * data, for example). Hence stop_rtt_update is only checked + * in this received side function, and not in send side below. + */ + if (mi->last_received_time_usec) { + /* We also allow multiple back-to-back packets if the circuit is not + * opened, to handle var cells. + * XXX: Will this work with out var cell plans? Maybe not, + * since we're opened at the middle hop as soon as we process + * one var extend2 :/ */ + if (circ->state == CIRCUIT_STATE_OPEN) { + log_fn(LOG_INFO, LD_CIRC, + "Stopping padding RTT estimation on circuit (%"PRIu64 + ", %d) after two back to back packets. Current RTT: %d", + circ->n_chan ? circ->n_chan->global_identifier : 0, + circ->n_circ_id, mi->rtt_estimate_usec); + mi->stop_rtt_update = 1; + } + } else { + mi->last_received_time_usec = monotime_absolute_usec(); + } +} + +/** + * Handles the "send" side of RTT calculation at middle nodes. + * + * This function calculates the RTT from the middle to the end + * of the circuit by subtracting the last received cell timestamp + * from the current time. It allows back-to-back cells until + * the circuit is opened, to allow for var cell handshakes. + * XXX: Check our var cell plans to make sure this will work. + */ +static void +circpad_estimate_circ_rtt_on_send(circuit_t *circ, + circpad_machine_state_t *mi) +{ + /* Origin circuits don't estimate RTT. They could do it easily enough, + * but they have no reason to use it in any delay calculations. */ + if (CIRCUIT_IS_ORIGIN(circ)) + return; + + /* If last_received_time_usec is non-zero, we are waiting for a response + * from the exit side. Calculate the time delta and use it as RTT. */ + if (mi->last_received_time_usec) { + circpad_time_t rtt_time = monotime_absolute_usec() - + mi->last_received_time_usec; + + /* Reset the last RTT packet time, so we can tell if two cells + * arrive back to back */ + mi->last_received_time_usec = 0; + + /* Use INT32_MAX to ensure the addition doesn't overflow */ + if (rtt_time >= INT32_MAX) { + log_fn(LOG_WARN,LD_CIRC, + "Circuit padding RTT estimate overflowed: %"PRIu64 + " vs %"PRIu64, monotime_absolute_usec(), + mi->last_received_time_usec); + return; + } + + /* If the old RTT estimate is lower than this one, use this one, because + * the circuit is getting longer. If this estimate is somehow + * faster than the previous, then maybe that was network jitter, or a + * bad monotonic clock source (so our ratchet returned a zero delta). + * In that case, average them. */ + if (mi->rtt_estimate_usec < (circpad_delay_t)rtt_time) { + mi->rtt_estimate_usec = (circpad_delay_t)rtt_time; + } else { + mi->rtt_estimate_usec += (circpad_delay_t)rtt_time; + mi->rtt_estimate_usec /= 2; + } + } else if (circ->state == CIRCUIT_STATE_OPEN) { + /* If last_received_time_usec is zero, then we have gotten two cells back + * to back. Stop estimating RTT in this case. Note that we only + * stop RTT update if the circuit is opened, to allow for RTT estimates + * of var cells during circ setup. */ + mi->stop_rtt_update = 1; + + if (!mi->rtt_estimate_usec) { + log_fn(LOG_NOTICE, LD_CIRC, + "Got two cells back to back on a circuit before estimating RTT."); + } + } +} + +/** + * A "non-padding" cell has been sent from this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we sent a cell into the network. + * For middle relay circuits, this means we sent a cell towards the + * origin. + */ +void +circpad_cell_event_nonpadding_sent(circuit_t *on_circ) +{ + /* Update global cell count */ + circpad_global_nonpadding_sent++; + + /* If there are no machines then this loop should not iterate */ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + /* First, update any RTT estimate */ + circpad_estimate_circ_rtt_on_send(on_circ, on_circ->padding_info[i]); + + /* Remove a token: this is the idea of adaptive padding, since we have an + * ideal distribution that we want our distribution to look like. */ + if (!circpad_machine_remove_token(on_circ->padding_info[i])) { + /* If removing a token did not cause a transition, check if + * non-padding sent event should */ + circpad_machine_spec_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_NONPADDING_SENT); + } + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * A "non-padding" cell has been received by this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we read a cell from the network. + * For middle relay circuits, this means we received a cell from the + * origin. + */ +void +circpad_cell_event_nonpadding_received(circuit_t *on_circ) +{ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + /* First, update any RTT estimate */ + circpad_estimate_circ_rtt_on_received(on_circ, on_circ->padding_info[i]); + + circpad_machine_spec_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_NONPADDING_RECV); + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * A padding cell has been sent from this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we sent a cell into the network. + * For middle relay circuits, this means we sent a cell towards the + * origin. + */ +void +circpad_cell_event_padding_sent(circuit_t *on_circ) +{ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + circpad_machine_spec_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_PADDING_SENT); + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * A padding cell has been received by this endpoint. React + * according to any padding state machines on the circuit. + * + * For origin circuits, this means we read a cell from the network. + * For middle relay circuits, this means we received a cell from the + * origin. + */ +void +circpad_cell_event_padding_received(circuit_t *on_circ) +{ + /* identical to padding sent */ + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) { + circpad_machine_spec_transition(on_circ->padding_info[i], + CIRCPAD_EVENT_PADDING_RECV); + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * An "infinite" delay has ben chosen from one of our histograms. + * + * "Infinite" delays mean don't send padding -- but they can also + * mean transition to another state depending on the state machine + * definitions. Check the rules and react accordingly. + * + * Return 1 if we decide to transition, 0 otherwise. + */ +circpad_decision_t +circpad_internal_event_infinity(circpad_machine_state_t *mi) +{ + return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_INFINITY); +} + +/** + * All of the bins of our current state's histogram's are empty. + * + * Check to see if this means transition to another state, and if + * not, refill the tokens. + * + * Return 1 if we decide to transition, 0 otherwise. + */ +circpad_decision_t +circpad_internal_event_bins_empty(circpad_machine_state_t *mi) +{ + if (circpad_machine_spec_transition(mi, CIRCPAD_EVENT_BINS_EMPTY) + == CIRCPAD_STATE_CHANGED) { + return CIRCPAD_STATE_CHANGED; + } else { + /* If we dont transition, then we refill the tokens */ + circpad_machine_setup_tokens(mi); + return CIRCPAD_STATE_UNCHANGED; + } +} + +/** + * This state has used up its cell count. Emit the event and + * see if we transition. + * + * Return 1 if we decide to transition, 0 otherwise. + */ +circpad_decision_t +circpad_internal_event_state_length_up(circpad_machine_state_t *mi) +{ + return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT); +} + +/** + * Returns true if the circuit matches the conditions. + */ +static inline bool +circpad_machine_conditions_met(origin_circuit_t *circ, + const circpad_machine_spec_t *machine) +{ + if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose) + & machine->conditions.purpose_mask)) + return 0; + + if (machine->conditions.requires_vanguards) { + const or_options_t *options = get_options(); + + /* Pinned middles are effectively vanguards */ + if (!(options->HSLayer2Nodes || options->HSLayer3Nodes)) + return 0; + } + + /* We check for any bits set in the circuit state mask so that machines + * can say any of the following through their state bitmask: + * "I want to apply to circuits with either streams or no streams"; OR + * "I only want to apply to circuits with streams"; OR + * "I only want to apply to circuits without streams". */ + if (!(circpad_circuit_state(circ) & machine->conditions.state_mask)) + return 0; + + if (circuit_get_cpath_opened_len(circ) < machine->conditions.min_hops) + return 0; + + return 1; +} + +/** + * Returns a minimized representation of the circuit state. + * + * The padding code only cares if the circuit is building, + * opened, used for streams, and/or still has relay early cells. + * This returns a bitmask of all state properities that apply to + * this circuit. + */ +static inline +circpad_circuit_state_t +circpad_circuit_state(origin_circuit_t *circ) +{ + circpad_circuit_state_t retmask = 0; + + if (circ->p_streams) + retmask |= CIRCPAD_CIRC_STREAMS; + else + retmask |= CIRCPAD_CIRC_NO_STREAMS; + + /* We use has_opened to prevent cannibialized circs from flapping. */ + if (circ->has_opened) + retmask |= CIRCPAD_CIRC_OPENED; + else + retmask |= CIRCPAD_CIRC_BUILDING; + + if (circ->remaining_relay_early_cells > 0) + retmask |= CIRCPAD_CIRC_HAS_RELAY_EARLY; + else + retmask |= CIRCPAD_CIRC_HAS_NO_RELAY_EARLY; + + return retmask; +} + +/** + * Convert a normal circuit purpose into a bitmask that we can + * use for determining matching circuits. + */ +static inline +circpad_purpose_mask_t +circpad_circ_purpose_to_mask(uint8_t circ_purpose) +{ + /* Treat OR circ purposes as ignored. They should not be passed here*/ + if (BUG(circ_purpose <= CIRCUIT_PURPOSE_OR_MAX_)) { + return 0; + } + + /* Treat new client circuit purposes as "OMG ITS EVERYTHING". + * This also should not happen */ + if (BUG(circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1 > 32)) { + return CIRCPAD_PURPOSE_ALL; + } + + /* Convert the purpose to a bit position */ + return 1 << (circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1); +} + +/** + * Shut down any machines whose conditions no longer match + * the current circuit. + */ +static void +circpad_shutdown_old_machines(origin_circuit_t *on_circ) +{ + circuit_t *circ = TO_CIRCUIT(on_circ); + + FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) { + if (!circpad_machine_conditions_met(on_circ, + circ->padding_machine[i])) { + // Clear machineinfo (frees timers) + circpad_circuit_machineinfo_free_idx(circ, i); + // Send padding negotiate stop + circpad_negotiate_padding(on_circ, + circ->padding_machine[i]->machine_num, + circ->padding_machine[i]->target_hopnum, + CIRCPAD_COMMAND_STOP); + } + } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END; +} + +/** + * Negotiate new machines that would apply to this circuit. + * + * This function checks to see if we have any free machine indexes, + * and for each free machine index, it initializes the most recently + * added origin-side padding machine that matches the target machine + * index and circuit conditions, and negotiates it with the appropriate + * middle relay. + */ +static void +circpad_add_matching_machines(origin_circuit_t *on_circ) +{ + circuit_t *circ = TO_CIRCUIT(on_circ); + +#ifdef TOR_UNIT_TESTS + /* Tests don't have to init our padding machines */ + if (!origin_padding_machines) + return; +#endif + + /* If padding negotiation failed before, do not try again */ + if (on_circ->padding_negotiation_failed) + return; + + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + /* If there is a padding machine info, this index is occupied. + * No need to check conditions for this index. */ + if (circ->padding_info[i]) + continue; + + /* We have a free machine index. Check the origin padding + * machines in reverse order, so that more recently added + * machines take priority over older ones. */ + SMARTLIST_FOREACH_REVERSE_BEGIN(origin_padding_machines, + circpad_machine_spec_t *, + machine) { + /* Machine definitions have a specific target machine index. + * This is so event ordering is deterministic with respect + * to which machine gets events first when there are two + * machines installed on a circuit. Make sure we only + * add this machine if its target machine index is free. */ + if (machine->machine_index == i && + circpad_machine_conditions_met(on_circ, machine)) { + + // We can only replace this machine if the target hopnum + // is the same, otherwise we'll get invalid data + if (circ->padding_machine[i]) { + if (circ->padding_machine[i]->target_hopnum != + machine->target_hopnum) + continue; + /* Replace it. (Don't free - is global). */ + circ->padding_machine[i] = NULL; + } + + /* Set up the machine immediately so that the slot is occupied. + * We will tear it down on error return, or if there is an error + * response from the relay. */ + circpad_setup_machine_on_circ(circ, machine); + if (circpad_negotiate_padding(on_circ, machine->machine_num, + machine->target_hopnum, + CIRCPAD_COMMAND_START) < 0) { + circpad_circuit_machineinfo_free_idx(circ, i); + circ->padding_machine[i] = NULL; + on_circ->padding_negotiation_failed = 1; + } else { + /* Success. Don't try any more machines */ + return; + } + } + } SMARTLIST_FOREACH_END(machine); + } FOR_EACH_CIRCUIT_MACHINE_END; +} + +/** + * Event that tells us we added a hop to an origin circuit. + * + * This event is used to decide if we should create a padding machine + * on a circuit. + */ +void +circpad_machine_event_circ_added_hop(origin_circuit_t *on_circ) +{ + /* Since our padding conditions do not specify a max_hops, + * all we can do is add machines here */ + circpad_add_matching_machines(on_circ); +} + +/** + * Event that tells us that an origin circuit is now built. + * + * Shut down any machines that only applied to un-built circuits. + * Activate any new ones. + */ +void +circpad_machine_event_circ_built(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Circpad purpose changed event. + * + * Shut down any machines that don't apply to our circ purpose. + * Activate any new ones that do. + */ +void +circpad_machine_event_circ_purpose_changed(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Event that tells us that an origin circuit is out of RELAY_EARLY + * cells. + * + * Shut down any machines that only applied to RELAY_EARLY circuits. + * Activate any new ones. + */ +void +circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Streams attached event. + * + * Called from link_apconn_to_circ() and handle_hs_exit_conn() + * + * Shut down any machines that only applied to machines without + * streams. Activate any new ones. + */ +void +circpad_machine_event_circ_has_streams(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Streams detached event. + * + * Called from circuit_detach_stream() + * + * Shut down any machines that only applied to machines without + * streams. Activate any new ones. + */ +void +circpad_machine_event_circ_has_no_streams(origin_circuit_t *circ) +{ + circpad_shutdown_old_machines(circ); + circpad_add_matching_machines(circ); +} + +/** + * Verify that padding is coming from the expected hop. + * + * Returns true if from_hop matches the target hop from + * one of our padding machines. + * + * Returns false if we're not an origin circuit, or if from_hop + * does not match one of the padding machines. + */ +bool +circpad_padding_is_from_expected_hop(circuit_t *circ, + crypt_path_t *from_hop) +{ + crypt_path_t *target_hop = NULL; + if (!CIRCUIT_IS_ORIGIN(circ)) + return 0; + + FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) { + /* We have to check padding_machine and not padding_info/active + * machines here because padding may arrive after we shut down a + * machine. The info is gone, but the padding_machine waits + * for the padding_negotiated response to come back. */ + if (!circ->padding_machine[i]) + continue; + + target_hop = circuit_get_cpath_hop(TO_ORIGIN_CIRCUIT(circ), + circ->padding_machine[i]->target_hopnum); + + if (target_hop == from_hop) + return 1; + } FOR_EACH_CIRCUIT_MACHINE_END; + + return 0; +} + +/** + * Deliver circpad events for an "unrecognized cell". + * + * Unrecognized cells are sent to relays and are forwarded + * onto the next hop of their circuits. Unrecognized cells + * are by definition not padding. We need to tell relay-side + * state machines that a non-padding cell was sent or received, + * depending on the direction, so they can update their histograms + * and decide to pad or not. + */ +void +circpad_deliver_unrecognized_cell_events(circuit_t *circ, + cell_direction_t dir) +{ + // We should never see unrecognized cells at origin. + // Our caller emits a warn when this happens. + if (CIRCUIT_IS_ORIGIN(circ)) { + return; + } + + if (dir == CELL_DIRECTION_OUT) { + /* When direction is out (away from origin), then we received non-padding + cell coming from the origin to us. */ + circpad_cell_event_nonpadding_received(circ); + } else if (dir == CELL_DIRECTION_IN) { + /* It's in and not origin, so the cell is going away from us. + * So we are relaying a non-padding cell towards the origin. */ + circpad_cell_event_nonpadding_sent(circ); + } +} + +/** + * Deliver circpad events for "recognized" relay cells. + * + * Recognized cells are destined for this hop, either client or middle. + * Check if this is a padding cell or not, and send the appropiate + * received event. + */ +void +circpad_deliver_recognized_relay_cell_events(circuit_t *circ, + uint8_t relay_command, + crypt_path_t *layer_hint) +{ + /* Padding negotiate cells are ignored by the state machines + * for simplicity. */ + if (relay_command == RELAY_COMMAND_PADDING_NEGOTIATE || + relay_command == RELAY_COMMAND_PADDING_NEGOTIATED) { + return; + } + + if (relay_command == RELAY_COMMAND_DROP) { + rep_hist_padding_count_read(PADDING_TYPE_DROP); + + if (CIRCUIT_IS_ORIGIN(circ)) { + if (circpad_padding_is_from_expected_hop(circ, layer_hint)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), 0); + } else { + /* This is unexpected padding. Ignore it for now. */ + return; + } + } + + /* The cell should be recognized by now, which means that we are on the + destination, which means that we received a padding cell. We might be + the client or the Middle node, still, because leaky-pipe. */ + circpad_cell_event_padding_received(circ); + log_fn(LOG_INFO, LD_CIRC, "Got padding cell on %s circuit %u.", + CIRCUIT_IS_ORIGIN(circ) ? "origin" : "non-origin", + CIRCUIT_IS_ORIGIN(circ) ? + TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0); + } else { + /* We received a non-padding cell on the edge */ + circpad_cell_event_nonpadding_received(circ); + } +} + +/** + * Deliver circpad events for relay cells sent from us. + * + * If this is a padding cell, update our padding stats + * and deliver the event. Otherwise just deliver the event. + */ +void +circpad_deliver_sent_relay_cell_events(circuit_t *circ, + uint8_t relay_command) +{ + /* Padding negotiate cells are ignored by the state machines + * for simplicity. */ + if (relay_command == RELAY_COMMAND_PADDING_NEGOTIATE || + relay_command == RELAY_COMMAND_PADDING_NEGOTIATED) { + return; + } + + /* RELAY_COMMAND_DROP is the multi-hop (aka circuit-level) padding cell in + * tor. (CELL_PADDING is a channel-level padding cell, which is not relayed + * or processed here) */ + if (relay_command == RELAY_COMMAND_DROP) { + /* Optimization: The event for RELAY_COMMAND_DROP is sent directly + * from circpad_send_padding_cell_for_callback(). This is to avoid + * putting a cell_t and a relay_header_t on the stack repeatedly + * if we decide to send a long train of padidng cells back-to-back + * with 0 delay. So we do nothing here. */ + return; + } else { + /* This is a non-padding cell sent from the client or from + * this node. */ + circpad_cell_event_nonpadding_sent(circ); + } +} + +/** + * Initialize the states array for a circpad machine. + */ +void +circpad_machine_states_init(circpad_machine_spec_t *machine, + circpad_statenum_t num_states) +{ + if (BUG(num_states > CIRCPAD_MAX_MACHINE_STATES)) { + num_states = CIRCPAD_MAX_MACHINE_STATES; + } + + machine->num_states = num_states; + machine->states = tor_malloc_zero(sizeof(circpad_state_t)*num_states); + + /* Initialize the default next state for all events to + * "ignore" -- if events aren't specified, they are ignored. */ + for (circpad_statenum_t s = 0; s < num_states; s++) { + for (int e = 0; e < CIRCPAD_NUM_EVENTS; e++) { + machine->states[s].next_state[e] = CIRCPAD_STATE_IGNORE; + } + } +} + +static void +circpad_setup_machine_on_circ(circuit_t *on_circ, + const circpad_machine_spec_t *machine) +{ + if (CIRCUIT_IS_ORIGIN(on_circ) && !machine->is_origin_side) { + log_fn(LOG_WARN, LD_BUG, + "Can't set up non-origin machine on origin circuit!"); + return; + } + + if (!CIRCUIT_IS_ORIGIN(on_circ) && machine->is_origin_side) { + log_fn(LOG_WARN, LD_BUG, + "Can't set up origin machine on non-origin circuit!"); + return; + } + + tor_assert_nonfatal(on_circ->padding_machine[machine->machine_index] + == NULL); + tor_assert_nonfatal(on_circ->padding_info[machine->machine_index] == NULL); + + on_circ->padding_info[machine->machine_index] = + circpad_circuit_machineinfo_new(on_circ, machine->machine_index); + on_circ->padding_machine[machine->machine_index] = machine; +} + +/* These padding machines are only used for tests pending #28634. */ +#ifdef TOR_UNIT_TESTS +static void +circpad_circ_client_machine_init(void) +{ + circpad_machine_spec_t *circ_client_machine + = tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + // XXX: Better conditions for merge.. Or disable this machine in + // merge? + circ_client_machine->conditions.min_hops = 2; + circ_client_machine->conditions.state_mask = + CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED|CIRCPAD_CIRC_HAS_RELAY_EARLY; + circ_client_machine->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; + + circ_client_machine->target_hopnum = 2; + circ_client_machine->is_origin_side = 1; + + /* Start, gap, burst */ + circpad_machine_states_init(circ_client_machine, 3); + + circ_client_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + + /* If we are in burst state, and we send a non-padding cell, then we cancel + the timer for the next padding cell: + We dont want to send fake extends when actual extends are going on */ + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL; + + circ_client_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END; + + circ_client_machine->states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_CLOSEST; + + // FIXME: Tune this histogram + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2; + circ_client_machine->states[CIRCPAD_STATE_BURST].start_usec = 500; + circ_client_machine->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + /* We have 5 tokens in the histogram, which means that all circuits will look + * like they have 7 hops (since we start this machine after the second hop, + * and tokens are decremented for any valid hops, and fake extends are + * used after that -- 2+5==7). */ + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 5; + circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 5; + + circ_client_machine->machine_num = smartlist_len(origin_padding_machines); + smartlist_add(origin_padding_machines, circ_client_machine); +} + +static void +circpad_circ_responder_machine_init(void) +{ + circpad_machine_spec_t *circ_responder_machine + = tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + /* Shut down the machine after we've sent enough packets */ + circ_responder_machine->should_negotiate_end = 1; + + /* The relay-side doesn't care what hopnum it is, but for consistency, + * let's match the client */ + circ_responder_machine->target_hopnum = 2; + circ_responder_machine->is_origin_side = 0; + + /* Start, gap, burst */ + circpad_machine_states_init(circ_responder_machine, 3); + + /* This is the settings of the state machine. In the future we are gonna + serialize this into the consensus or the torrc */ + + /* We transition to the burst state on padding receive and on non-padding + * recieve */ + circ_responder_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + circ_responder_machine->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + /* Inside the burst state we _stay_ in the burst state when a non-padding + * is sent */ + circ_responder_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_BURST; + + /* Inside the burst state we transition to the gap state when we receive a + * padding cell */ + circ_responder_machine->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP; + + /* These describe the padding charasteristics when in burst state */ + + /* use_rtt_estimate tries to estimate how long padding cells take to go from + C->M, and uses that as what as the base of the histogram */ + circ_responder_machine->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1; + /* The histogram is 2 bins: an empty one, and infinity */ + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2; + circ_responder_machine->states[CIRCPAD_STATE_BURST].start_usec = 5000; + circ_responder_machine->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + /* During burst state we wait forever for padding to arrive. + + We are waiting for a padding cell from the client to come in, so that we + respond, and we immitate how extend looks like */ + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 0; + // Only infinity bin: + circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[1] = 1; + circ_responder_machine->states[CIRCPAD_STATE_BURST]. + histogram_total_tokens = 1; + + /* From the gap state, we _stay_ in the gap state, when we receive padding + * or non padding */ + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP; + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_GAP; + + /* And from the gap state, we go to the end, when the bins are empty or a + * non-padding cell is sent */ + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END; + circ_responder_machine->states[CIRCPAD_STATE_GAP]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_END; + + // FIXME: Tune this histogram + + /* The gap state is the delay you wait after you receive a padding cell + before you send a padding response */ + circ_responder_machine->states[CIRCPAD_STATE_GAP].use_rtt_estimate = 1; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_len = 6; + circ_responder_machine->states[CIRCPAD_STATE_GAP].start_usec = 5000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].range_usec = 1000000; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[0] = 0; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[1] = 1; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[2] = 2; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[3] = 2; + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[4] = 1; + /* Total number of tokens */ + circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_total_tokens = 6; + circ_responder_machine->states[CIRCPAD_STATE_GAP].token_removal = + CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC; + + circ_responder_machine->machine_num = smartlist_len(relay_padding_machines); + smartlist_add(relay_padding_machines, circ_responder_machine); +} +#endif + +/** + * Initialize all of our padding machines. + * + * This is called at startup. It sets up some global machines, and then + * loads some from torrc, and from the tor consensus. + */ +void +circpad_machines_init(void) +{ + tor_assert_nonfatal(origin_padding_machines == NULL); + tor_assert_nonfatal(relay_padding_machines == NULL); + + origin_padding_machines = smartlist_new(); + relay_padding_machines = smartlist_new(); + + // TODO: Parse machines from consensus and torrc +#ifdef TOR_UNIT_TESTS + circpad_circ_client_machine_init(); + circpad_circ_responder_machine_init(); +#endif +} + +/** + * Free our padding machines + */ +void +circpad_machines_free(void) +{ + if (origin_padding_machines) { + SMARTLIST_FOREACH(origin_padding_machines, + circpad_machine_spec_t *, + m, tor_free(m->states); tor_free(m)); + smartlist_free(origin_padding_machines); + } + + if (relay_padding_machines) { + SMARTLIST_FOREACH(relay_padding_machines, + circpad_machine_spec_t *, + m, tor_free(m->states); tor_free(m)); + smartlist_free(relay_padding_machines); + } +} + +/** + * Check the Protover info to see if a node supports padding. + */ +static bool +circpad_node_supports_padding(const node_t *node) +{ + if (node->rs) { + log_fn(LOG_INFO, LD_CIRC, "Checking padding: %s", + node->rs->pv.supports_padding ? "supported" : "unsupported"); + return node->rs->pv.supports_padding; + } + + log_fn(LOG_INFO, LD_CIRC, "Empty routerstatus in padding check"); + return 0; +} + +/** + * Get a node_t for the nth hop in our circuit, starting from 1. + * + * Returns node_t from the consensus for that hop, if it is opened. + * Otherwise returns NULL. + */ +static const node_t * +circuit_get_nth_node(origin_circuit_t *circ, int hop) +{ + crypt_path_t *iter = circuit_get_cpath_hop(circ, hop); + + if (!iter || iter->state != CPATH_STATE_OPEN) + return NULL; + + return node_get_by_id(iter->extend_info->identity_digest); +} + +/** + * Return true if a particular circuit supports padding + * at the desired hop. + */ +static bool +circpad_circuit_supports_padding(origin_circuit_t *circ, + int target_hopnum) +{ + const node_t *hop; + + if (!(hop = circuit_get_nth_node(circ, target_hopnum))) { + return 0; + } + + return circpad_node_supports_padding(hop); +} + +/** + * Try to negotiate padding. + * + * Returns -1 on error, 0 on success. + */ +signed_error_t +circpad_negotiate_padding(origin_circuit_t *circ, + circpad_machine_num_t machine, + uint8_t target_hopnum, + uint8_t command) +{ + circpad_negotiate_t type; + cell_t cell; + ssize_t len; + + /* Check that the target hop lists support for padding in + * its ProtoVer fields */ + if (!circpad_circuit_supports_padding(circ, target_hopnum)) { + return -1; + } + + memset(&cell, 0, sizeof(cell_t)); + memset(&type, 0, sizeof(circpad_negotiate_t)); + // This gets reset to RELAY_EARLY appropriately by + // relay_send_command_from_edge_. At least, it looks that way. + // QQQ-MP-AP: Verify that. + cell.command = CELL_RELAY; + + circpad_negotiate_set_command(&type, command); + circpad_negotiate_set_version(&type, 0); + circpad_negotiate_set_machine_type(&type, machine); + + if ((len = circpad_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE, + &type)) < 0) + return -1; + + log_fn(LOG_INFO,LD_CIRC, "Negotiating padding on circuit %u", + circ->global_identifier); + + return circpad_send_command_to_hop(circ, target_hopnum, + RELAY_COMMAND_PADDING_NEGOTIATE, + cell.payload, len); +} + +/** + * Try to negotiate padding. + * + * Returns 1 if successful (or already set up), 0 otherwise. + */ +bool +circpad_padding_negotiated(circuit_t *circ, + circpad_machine_num_t machine, + uint8_t command, + uint8_t response) +{ + circpad_negotiated_t type; + cell_t cell; + ssize_t len; + + memset(&cell, 0, sizeof(cell_t)); + memset(&type, 0, sizeof(circpad_negotiated_t)); + // This gets reset to RELAY_EARLY appropriately by + // relay_send_command_from_edge_. At least, it looks that way. + // QQQ-MP-AP: Verify that. + cell.command = CELL_RELAY; + + circpad_negotiated_set_command(&type, command); + circpad_negotiated_set_response(&type, response); + circpad_negotiated_set_version(&type, 0); + circpad_negotiated_set_machine_type(&type, machine); + + if ((len = circpad_negotiated_encode(cell.payload, CELL_PAYLOAD_SIZE, + &type)) < 0) + return 0; + + /* Use relay_send because we're from the middle to the origin. We don't + * need to specify a target hop or layer_hint. */ + return relay_send_command_from_edge(0, circ, + RELAY_COMMAND_PADDING_NEGOTIATED, + (void*)cell.payload, + (size_t)len, NULL) == 0; +} + +/** + * Parse and react to a padding_negotiate cell. + * + * This is called at the middle node upon receipt of the client's choice of + * state machine, so that it can use the requested state machine index, if + * it is available. + * + * Returns -1 on error, 0 on success. + */ +signed_error_t +circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell) +{ + int retval = 0; + circpad_negotiate_t *negotiate; + + if (CIRCUIT_IS_ORIGIN(circ)) { + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Padding negotiate cell unsupported at origin."); + return -1; + } + + if (circpad_negotiate_parse(&negotiate, cell->payload+RELAY_HEADER_SIZE, + CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Received malformed PADDING_NEGOTIATE cell; dropping."); + return -1; + } + + if (negotiate->command == CIRCPAD_COMMAND_STOP) { + /* Free the machine corresponding to this machine type */ + if (free_circ_machineinfos_with_machine_num(circ, + negotiate->machine_type)) { + log_info(LD_CIRC, "Received STOP command for machine %u", + negotiate->machine_type); + goto done; + } + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Received circuit padding stop command for unknown machine."); + goto err; + } else if (negotiate->command == CIRCPAD_COMMAND_START) { + SMARTLIST_FOREACH_BEGIN(relay_padding_machines, + const circpad_machine_spec_t *, m) { + if (m->machine_num == negotiate->machine_type) { + circpad_setup_machine_on_circ(circ, m); + goto done; + } + } SMARTLIST_FOREACH_END(m); + } + + err: + retval = -1; + + done: + circpad_padding_negotiated(circ, negotiate->machine_type, + negotiate->command, + (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR); + circpad_negotiate_free(negotiate); + + return retval; +} + +/** + * Parse and react to a padding_negotiated cell. + * + * This is called at the origin upon receipt of the middle's response + * to our choice of state machine. + * + * Returns -1 on error, 0 on success. + */ +signed_error_t +circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell, + crypt_path_t *layer_hint) +{ + circpad_negotiated_t *negotiated; + + if (!CIRCUIT_IS_ORIGIN(circ)) { + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Padding negotiated cell unsupported at non-origin."); + return -1; + } + + /* Verify this came from the expected hop */ + if (!circpad_padding_is_from_expected_hop(circ, layer_hint)) { + log_fn(LOG_WARN, LD_CIRC, + "Padding negotiated cell from wrong hop!"); + return -1; + } + + if (circpad_negotiated_parse(&negotiated, cell->payload+RELAY_HEADER_SIZE, + CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) { + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Received malformed PADDING_NEGOTIATED cell; " + "dropping."); + return -1; + } + + if (negotiated->command == CIRCPAD_COMMAND_STOP) { + /* There may not be a padding_info here if we shut down the + * machine in circpad_shutdown_old_machines(). Or, if + * circpad_add_matching_matchines() added a new machine, + * there may be a padding_machine for a different machine num + * than this response. */ + free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type); + } else if (negotiated->command == CIRCPAD_COMMAND_START && + negotiated->response == CIRCPAD_RESPONSE_ERR) { + // This can happen due to consensus drift.. free the machines + // and be sad + free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type); + TO_ORIGIN_CIRCUIT(circ)->padding_negotiation_failed = 1; + log_fn(LOG_PROTOCOL_WARN, LD_CIRC, + "Middle node did not accept our padding request."); + } + + circpad_negotiated_free(negotiated); + return 0; +} + +/* Serialization */ +// TODO: Should we use keyword=value here? Are there helpers for that? +#if 0 +static void +circpad_state_serialize(const circpad_state_t *state, + smartlist_t *chunks) +{ + smartlist_add_asprintf(chunks, " %u", state->histogram[0]); + for (int i = 1; i < state->histogram_len; i++) { + smartlist_add_asprintf(chunks, ",%u", + state->histogram[i]); + } + + smartlist_add_asprintf(chunks, " 0x%x", + state->transition_cancel_events); + + for (int i = 0; i < CIRCPAD_NUM_STATES; i++) { + smartlist_add_asprintf(chunks, ",0x%x", + state->transition_events[i]); + } + + smartlist_add_asprintf(chunks, " %u %u", + state->use_rtt_estimate, + state->token_removal); +} + +char * +circpad_machine_spec_to_string(const circpad_machine_spec_t *machine) +{ + smartlist_t *chunks = smartlist_new(); + char *out; + (void)machine; + + circpad_state_serialize(&machine->start, chunks); + circpad_state_serialize(&machine->gap, chunks); + circpad_state_serialize(&machine->burst, chunks); + + out = smartlist_join_strings(chunks, "", 0, NULL); + + SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); + smartlist_free(chunks); + return out; +} + +// XXX: Writeme +const circpad_machine_spec_t * +circpad_string_to_machine(const char *str) +{ + (void)str; + return NULL; +} + +#endif diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h new file mode 100644 index 0000000000..92fd4fc2d5 --- /dev/null +++ b/src/core/or/circuitpadding.h @@ -0,0 +1,696 @@ +/* + * Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file circuitpadding.h + * \brief Header file for circuitpadding.c. + **/ + +#ifndef TOR_CIRCUITPADDING_H +#define TOR_CIRCUITPADDING_H + +#include "src/trunnel/circpad_negotiation.h" +#include "lib/evloop/timers.h" + +struct circuit_t; +struct origin_circuit_t; +struct cell_t; + +/** + * Signed error return with the specific property that negative + * values mean error codes of various semantics, 0 means success, + * and positive values are unused. + * + * XXX: Tor uses this concept a lot but just calls it int. Should we move + * this somewhere centralized? Where? + */ +typedef int signed_error_t; + +/** + * These constants specify the types of events that can cause + * transitions between state machine states. + * + * Note that SENT and RECV are relative to this endpoint. For + * relays, SENT means packets destined towards the client and + * RECV means packets destined towards the relay. On the client, + * SENT means packets destined towards the relay, where as RECV + * means packets destined towards the client. + */ +typedef enum { + /* A non-padding cell was received. */ + CIRCPAD_EVENT_NONPADDING_RECV = 0, + /* A non-padding cell was sent. */ + CIRCPAD_EVENT_NONPADDING_SENT = 1, + /* A padding cell (RELAY_COMMAND_DROP) was sent. */ + CIRCPAD_EVENT_PADDING_SENT = 2, + /* A padding cell was received. */ + CIRCPAD_EVENT_PADDING_RECV = 3, + /* We tried to schedule padding but we ended up picking the infinity bin + * which means that padding was delayed infinitely */ + CIRCPAD_EVENT_INFINITY = 4, + /* All histogram bins are empty (we are out of tokens) */ + CIRCPAD_EVENT_BINS_EMPTY = 5, + /* just a counter of the events above */ + CIRCPAD_EVENT_LENGTH_COUNT = 6 +} circpad_event_t; +#define CIRCPAD_NUM_EVENTS ((int)CIRCPAD_EVENT_LENGTH_COUNT+1) + +/** Boolean type that says if we decided to transition states or not */ +typedef enum { + CIRCPAD_STATE_UNCHANGED = 0, + CIRCPAD_STATE_CHANGED = 1 +} circpad_decision_t; + +/** The type for the things in histogram bins (aka tokens) */ +typedef uint32_t circpad_hist_token_t; + +/** The type for histogram indexes (needs to be negative for errors) */ +typedef int8_t circpad_hist_index_t; + +/** The type for absolute time, from monotime_absolute_usec() */ +typedef uint64_t circpad_time_t; + +/** The type for timer delays, in microseconds */ +typedef uint32_t circpad_delay_t; + +/** + * An infinite padding cell delay means don't schedule any padding -- + * simply wait until a different event triggers a transition. + * + * This means that the maximum delay we can scedule is UINT32_MAX-1 + * microseconds, or about 4300 seconds (1.25 hours). + * XXX: Is this enough if we want to simulate light, intermittent + * activity on an onion service? + */ +#define CIRCPAD_DELAY_INFINITE (UINT32_MAX) + +/** + * Macro to clarify when we're checking the infinity bin. + * + * Works with either circpad_state_t or circpad_machine_state_t + */ +#define CIRCPAD_INFINITY_BIN(mi) ((mi)->histogram_len-1) + +/** + * These constants form a bitfield that specifies when a state machine + * should be applied to a circuit. + * + * If any of these elements is set, then the circuit will be tested against + * that specific condition. If an element is unset, then we don't test it. + * (E.g. If neither NO_STREAMS or STREAMS are set, then we will not care + * whether a circuit has streams attached when we apply a state machine) + * + * The helper function circpad_circuit_state() converts circuit state + * flags into this more compact representation. + */ +typedef enum { + /* Only apply machine if the circuit is still building */ + CIRCPAD_CIRC_BUILDING = 1<<0, + /* Only apply machine if the circuit is open */ + CIRCPAD_CIRC_OPENED = 1<<1, + /* Only apply machine if the circuit has no attached streams */ + CIRCPAD_CIRC_NO_STREAMS = 1<<2, + /* Only apply machine if the circuit has attached streams */ + CIRCPAD_CIRC_STREAMS = 1<<3, + /* Only apply machine if the circuit still allows RELAY_EARLY cells */ + CIRCPAD_CIRC_HAS_RELAY_EARLY = 1<<4, + /* Only apply machine if the circuit has depleted its RELAY_EARLY cells + * allowance. */ + CIRCPAD_CIRC_HAS_NO_RELAY_EARLY = 1<<5 +} circpad_circuit_state_t; + +/** Bitmask that says "apply this machine to all states" */ +#define CIRCPAD_STATE_ALL \ + (CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED| \ + CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_NO_STREAMS| \ + CIRCPAD_CIRC_HAS_RELAY_EARLY|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY) + +/** + * A compact circuit purpose bitfield mask that allows us to compactly + * specify which circuit purposes a machine should apply to. + * + * The helper function circpad_circ_purpose_to_mask() converts circuit + * purposes into bit positions in this bitmask. + */ +typedef uint32_t circpad_purpose_mask_t; + +/** Bitmask that says "apply this machine to all purposes". */ +#define CIRCPAD_PURPOSE_ALL (0xFFFFFFFF) + +/** + * This type specifies all of the conditions that must be met before + * a client decides to initiate padding on a circuit. + * + * A circuit must satisfy every sub-field in this type in order + * to be considered to match the conditions. + */ +typedef struct circpad_machine_conditions_t { + /** Only apply the machine *if* the circuit has at least this many hops */ + unsigned min_hops : 3; + + /** Only apply the machine *if* vanguards are enabled */ + unsigned requires_vanguards : 1; + + /** Only apply the machine *if* the circuit's state matches any of + * the bits set in this bitmask. */ + circpad_circuit_state_t state_mask; + + /** Only apply a machine *if* the circuit's purpose matches one + * of the bits set in this bitmask */ + circpad_purpose_mask_t purpose_mask; + +} circpad_machine_conditions_t; + +/** + * Token removal strategy options. + * + * The WTF-PAD histograms are meant to specify a target distribution to shape + * traffic towards. This is accomplished by removing tokens from the histogram + * when either padding or non-padding cells are sent. + * + * When we see a non-padding cell at a particular time since the last cell, you + * remove a token from the corresponding delay bin. These flags specify + * which bin to choose if that bin is already empty. + */ +typedef enum { + /** Don't remove any tokens */ + CIRCPAD_TOKEN_REMOVAL_NONE = 0, + /** + * Remove from the first non-zero higher bin index when current is zero. + * This is the recommended strategy from the Adaptive Padding paper. */ + CIRCPAD_TOKEN_REMOVAL_HIGHER = 1, + /** Remove from the first non-zero lower bin index when current is empty. */ + CIRCPAD_TOKEN_REMOVAL_LOWER = 2, + /** Remove from the closest non-zero bin index when current is empty. */ + CIRCPAD_TOKEN_REMOVAL_CLOSEST = 3, + /** Remove from the closest bin by time value (since bins are + * exponentially spaced). */ + CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC = 4, + /** Only remove from the exact bin corresponding to this delay. If + * the bin is 0, simply do nothing. Don't pick another bin. */ + CIRCPAD_TOKEN_REMOVAL_EXACT = 5 +} circpad_removal_t; + +/** + * Distribution types supported by circpad_distribution_sample(). + * + * These can be used instead of histograms for the inter-packet + * timing distribution, or to specify a distribution on the number + * of cells that can be sent while in a specific state of the state + * machine. */ +typedef enum { + CIRCPAD_DIST_NONE = 0, + CIRCPAD_DIST_UNIFORM = 1, + CIRCPAD_DIST_LOGISTIC = 2, + CIRCPAD_DIST_LOG_LOGISTIC = 3, + CIRCPAD_DIST_GEOMETRIC = 4, + CIRCPAD_DIST_WEIBULL = 5, + CIRCPAD_DIST_PARETO = 6 +} circpad_distribution_type_t; + +/** + * Distribution information. + * + * This type specifies a specific distribution above, as well as + * up to two parameters for that distribution. The specific + * per-distribution meaning of these parameters is specified + * in circpad_distribution_sample(). + */ +typedef struct circpad_distribution_t { + circpad_distribution_type_t type; + double param1; + double param2; +} circpad_distribution_t; + +/** State number type. Represents current state of state machine. */ +typedef uint16_t circpad_statenum_t; +#define CIRCPAD_STATENUM_MAX (UINT16_MAX) + +/** A histogram is used to sample padding delays given a machine state. This + * constant defines the maximum histogram width (i.e. the max number of bins) + * + * Each histogram bin is twice as large as the previous. Two exceptions: The + * first bin has zero width (which means that minimum delay is applied to the + * next padding cell), and the last bin (infinity bin) has infinite width + * (which means that the next padding cell will be delayed infinitely). */ +#define CIRCPAD_MAX_HISTOGRAM_LEN (sizeof(circpad_delay_t)*8 + 1) + +/** + * A state of a padding state machine. The information here are immutable and + * represent the initial form of the state; it does not get updated as things + * happen. The mutable information that gets updated in runtime are carried in + * a circpad_machine_state_t. + * + * This struct describes the histograms and parameters of a single + * state in the adaptive padding machine. Instances of this struct + * exist in global circpad machine definitions that come from torrc + * or the consensus. + */ +typedef struct circpad_state_t { + /** If a histogram is used for this state, this specifies the number of bins + * of this histogram. Histograms must have at least 2 bins. + * + * If a delay probability distribution is used for this state, this is set + * to 0. */ + circpad_hist_index_t histogram_len; + /** The histogram itself: an array of uint16s of tokens, whose + * widths are exponentially spaced, in microseconds */ + circpad_hist_token_t histogram[CIRCPAD_MAX_HISTOGRAM_LEN]; + /** Total number of tokens in this histogram. This is a constant and is *not* + * decremented every time we spend a token. It's used for initializing and + * refilling the histogram. */ + uint32_t histogram_total_tokens; + + /** Minimum padding delay of this state in microseconds. + * + * If histograms are used, this is the left (and right) bound of the first + * bin (since it has zero width). + * + * If a delay probability distribution is used, this represents the minimum + * delay we can sample from the distribution. + */ + circpad_delay_t start_usec; + + /** If histograms are used, this is the width of the whole histogram in + * microseconds, and it's used to calculate individual bin width. + * + * If a delay probability distribution is used, this is used as the max + * delay we can sample from the distribution. + */ + circpad_delay_t range_usec; + + /** + * Represents a delay probability distribution (aka IAT distribution). It's a + * parametrized way of encoding inter-packet delay information in + * microseconds. It can be used instead of histograms. + * + * If it is used, token_removal below must be set to + * CIRCPAD_TOKEN_REMOVAL_NONE. + * + * Start_usec, range_sec, and rtt_estimates are still applied to the + * results of sampling from this distribution (range_sec is used as a max). + */ + circpad_distribution_t iat_dist; + + /** + * The length dist is a parameterized way of encoding how long this + * state machine runs in terms of sent padding cells or all + * sent cells. Values are sampled from this distribution, clamped + * to max_len, and then start_len is added to that value. + * + * It may be specified instead of or in addition to + * the infinity bins and bins empty conditions. */ + circpad_distribution_t length_dist; + /** A minimum length value, added to the output of length_dist */ + uint16_t start_length; + /** A cap on the length value that can be sampled from the length_dist */ + uint64_t max_length; + + /** Should we decrement length when we see a nonpadding packet? + * XXX: Are there any machines that actually want to set this to 0? There may + * not be. OTOH, it's only a bit.. */ + unsigned length_includes_nonpadding : 1; + + /** + * This is an array that specifies the next state to transition to upon + * receipt an event matching the indicated array index. + * + * This aborts our scheduled packet and switches to the state + * corresponding to the index of the array. Tokens are filled upon + * this transition. + * + * States are allowed to transition to themselves, which means re-schedule + * a new padding timer. They are also allowed to temporarily "transition" + * to the "IGNORE" and "CANCEL" pseudo-states. See #defines below + * for details on state behavior and meaning. + */ + circpad_statenum_t next_state[CIRCPAD_NUM_EVENTS]; + + /** + * If true, estimate the RTT from this relay to the exit/website and add that + * to start_usec for use as the histogram bin 0 start delay. + * + * Right now this is only supported for relay-side state machines. + */ + unsigned use_rtt_estimate : 1; + + /** This specifies the token removal strategy to use upon padding and + * non-padding activity. */ + circpad_removal_t token_removal; +} circpad_state_t; + +/** + * The start state for this machine. + * + * In the original WTF-PAD, this is only used for transition to/from + * the burst state. All other fields are not used. But to simplify the + * code we've made it a first-class state. This has no performance + * consequences, but may make naive serialization of the state machine + * large, if we're not careful about how we represent empty fields. + */ +#define CIRCPAD_STATE_START 0 + +/** + * The burst state for this machine. + * + * In the original Adaptive Padding algorithm and in WTF-PAD + * (https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf and + * https://www.cs.kau.se/pulls/hot/thebasketcase-wtfpad/), the burst + * state serves to detect bursts in traffic. This is done by using longer + * delays in its histogram, which represent the expected delays between + * bursts of packets in the target stream. If this delay expires without a + * real packet being sent, the burst state sends a padding packet and then + * immediately transitions to the gap state, which is used to generate + * a synthetic padding packet train. In this implementation, this transition + * needs to be explicitly specified in the burst state's transition events. + * + * Because of this flexibility, other padding mechanisms can transition + * between these two states arbitrarily, to encode other dynamics of + * target traffic. + */ +#define CIRCPAD_STATE_BURST 1 + +/** + * The gap state for this machine. + * + * In the original Adaptive Padding algorithm and in WTF-PAD, the gap + * state serves to simulate an artificial packet train composed of padding + * packets. It does this by specifying much lower inter-packet delays than + * the burst state, and transitioning back to itself after padding is sent + * if these timers expire before real traffic is sent. If real traffic is + * sent, it transitions back to the burst state. + * + * Again, in this implementation, these transitions must be specified + * explicitly, and other transitions are also permitted. + */ +#define CIRCPAD_STATE_GAP 2 + +/** + * End is a pseudo-state that causes the machine to go completely + * idle, and optionally get torn down (depending on the + * value of circpad_machine_spec_t.should_negotiate_end) + * + * End MUST NOT occupy a slot in the machine state array. + */ +#define CIRCPAD_STATE_END CIRCPAD_STATENUM_MAX + +/** + * "Ignore" is a pseudo-state that means "do not react to this + * event". + * + * "Ignore" MUST NOT occupy a slot in the machine state array. + */ +#define CIRCPAD_STATE_IGNORE (CIRCPAD_STATENUM_MAX-1) + +/** + * "Cancel" is a pseudo-state that means "cancel pending timers, + * but remain in your current state". + * + * Cancel MUST NOT occupy a slot in the machine state array. + */ +#define CIRCPAD_STATE_CANCEL (CIRCPAD_STATENUM_MAX-2) + +/** + * Since we have 3 pseudo-states, the max state array length is + * up to one less than cancel's statenum. + */ +#define CIRCPAD_MAX_MACHINE_STATES (CIRCPAD_STATE_CANCEL-1) + +/** + * Mutable padding machine info. + * + * This structure contains mutable information about a padding + * machine. The mutable information must be kept separate because + * it exists per-circuit, where as the machines themselves are global. + * This separation is done to conserve space in the circuit structure. + * + * This is the per-circuit state that changes regarding the global state + * machine. Some parts of it are optional (ie NULL). + * + * XXX: Play with layout to minimize space on x64 Linux (most common relay). + */ +typedef struct circpad_machine_state_t { + /** The callback pointer for the padding callbacks. + * + * These timers stick around the machineinfo until the machineinfo's circuit + * is closed, at which point the timer is cancelled. For this reason it's + * safe to assume that the machineinfo exists if this timer gets + * triggered. */ + tor_timer_t *padding_timer; + + /** The circuit for this machine */ + struct circuit_t *on_circ; + + /** A mutable copy of the histogram for the current state. + * NULL if remove_tokens is false for that state */ + circpad_hist_token_t *histogram; + /** Length of the above histogram. + * XXX: This field *could* be removed at the expense of added + * complexity+overhead for reaching back into the immutable machine + * state every time we need to inspect the histogram. It's only a byte, + * though, so it seemed worth it. + */ + circpad_hist_index_t histogram_len; + /** Remove token from this index upon sending padding */ + circpad_hist_index_t chosen_bin; + + /** Stop padding/transition if this many cells sent */ + uint64_t state_length; +#define CIRCPAD_STATE_LENGTH_INFINITE UINT64_MAX + + /** A scaled count of padding packets sent, used to limit padding overhead. + * When this reaches UINT16_MAX, we cut it and nonpadding_sent in half. */ + uint16_t padding_sent; + /** A scaled count of non-padding packets sent, used to limit padding + * overhead. When this reaches UINT16_MAX, we cut it and padding_sent in + * half. */ + uint16_t nonpadding_sent; + + /** + * EWMA estimate of the RTT of the circuit from this hop + * to the exit end, in microseconds. */ + circpad_delay_t rtt_estimate_usec; + + /** + * The last time we got an event relevant to estimating + * the RTT. Monotonic time in microseconds since system + * start. + */ + circpad_time_t last_received_time_usec; + + /** + * The time at which we scheduled a non-padding packet, + * or selected an infinite delay. + * + * Monotonic time in microseconds since system start. + * This is 0 if we haven't chosen a padding delay. + */ + circpad_time_t padding_scheduled_at_usec; + + /** What state is this machine in? */ + circpad_statenum_t current_state; + + /** + * True if we have scheduled a timer for padding. + * + * This is 1 if a timer is pending. It is 0 if + * no timer is scheduled. (It can be 0 even when + * padding_was_scheduled_at_usec is non-zero). + */ + unsigned is_padding_timer_scheduled : 1; + + /** + * If this is true, we have seen full duplex behavior. + * Stop updating the RTT. + */ + unsigned stop_rtt_update : 1; + +/** Max number of padding machines on each circuit. If changed, + * also ensure the machine_index bitwith supports the new size. */ +#define CIRCPAD_MAX_MACHINES (2) + /** Which padding machine index was this for. + * (make sure changes to the bitwidth can support the + * CIRCPAD_MAX_MACHINES define). */ + unsigned machine_index : 1; + +} circpad_machine_state_t; + +/** Helper macro to get an actual state machine from a machineinfo */ +#define CIRCPAD_GET_MACHINE(machineinfo) \ + ((machineinfo)->on_circ->padding_machine[(machineinfo)->machine_index]) + +/** + * This specifies a particular padding machine to use after negotiation. + * + * The constants for machine_num_t are in trunnel. + * We want to be able to define extra numbers in the consensus/torrc, though. + */ +typedef uint8_t circpad_machine_num_t; + +/** Global state machine structure from the consensus */ +typedef struct circpad_machine_spec_t { + /** Global machine number */ + circpad_machine_num_t machine_num; + + /** Which machine index slot should this machine go into in + * the array on the circuit_t */ + unsigned machine_index : 1; + + /** Send a padding negotiate to shut down machine at end state? */ + unsigned should_negotiate_end : 1; + + // These next three fields are origin machine-only... + /** Origin side or relay side */ + unsigned is_origin_side : 1; + + /** Which hop in the circuit should we send padding to/from? + * 1-indexed (ie: hop #1 is guard, #2 middle, #3 exit). */ + unsigned target_hopnum : 3; + + /** This machine only kills fascists if the following conditions are met. */ + circpad_machine_conditions_t conditions; + + /** How many padding cells can be sent before we apply overhead limits? + * XXX: Note that we can only allow up to 64k of padding cells on an + * otherwise quiet circuit. Is this enough? It's 33MB. */ + uint16_t allowed_padding_count; + + /** Padding percent cap: Stop padding if we exceed this percent overhead. + * 0 means no limit. Overhead is defined as percent of total traffic, so + * that we can use 0..100 here. This is the same definition as used in + * Prop#265. */ + uint8_t max_padding_percent; + + /** State array: indexed by circpad_statenum_t */ + circpad_state_t *states; + + /** + * Number of states this machine has (ie: length of the states array). + * XXX: This field is not needed other than for safety. */ + circpad_statenum_t num_states; +} circpad_machine_spec_t; + +void circpad_new_consensus_params(const networkstatus_t *ns); + +/** + * The following are event call-in points that are of interest to + * the state machines. They are called during cell processing. */ +void circpad_deliver_unrecognized_cell_events(struct circuit_t *circ, + cell_direction_t dir); +void circpad_deliver_sent_relay_cell_events(struct circuit_t *circ, + uint8_t relay_command); +void circpad_deliver_recognized_relay_cell_events(struct circuit_t *circ, + uint8_t relay_command, + crypt_path_t *layer_hint); + +/** Cell events are delivered by the above delivery functions */ +void circpad_cell_event_nonpadding_sent(struct circuit_t *on_circ); +void circpad_cell_event_nonpadding_received(struct circuit_t *on_circ); +void circpad_cell_event_padding_sent(struct circuit_t *on_circ); +void circpad_cell_event_padding_received(struct circuit_t *on_circ); + +/** Internal events are events the machines send to themselves */ +circpad_decision_t +circpad_internal_event_infinity(circpad_machine_state_t *mi); +circpad_decision_t +circpad_internal_event_bins_empty(circpad_machine_state_t *); +circpad_decision_t circpad_internal_event_state_length_up( + circpad_machine_state_t *); + +/** Machine creation events are events that cause us to set up or + * tear down padding state machines. */ +void circpad_machine_event_circ_added_hop(struct origin_circuit_t *on_circ); +void circpad_machine_event_circ_built(struct origin_circuit_t *circ); +void circpad_machine_event_circ_purpose_changed(struct origin_circuit_t *circ); +void circpad_machine_event_circ_has_streams(struct origin_circuit_t *circ); +void circpad_machine_event_circ_has_no_streams(struct origin_circuit_t *circ); +void +circpad_machine_event_circ_has_no_relay_early(struct origin_circuit_t *circ); + +void circpad_machines_init(void); +void circpad_machines_free(void); + +void circpad_machine_states_init(circpad_machine_spec_t *machine, + circpad_statenum_t num_states); + +void circpad_circuit_free_all_machineinfos(struct circuit_t *circ); + +bool circpad_padding_is_from_expected_hop(struct circuit_t *circ, + crypt_path_t *from_hop); + +/** Serializaton functions for writing to/from torrc and consensus */ +char *circpad_machine_spec_to_string(const circpad_machine_spec_t *machine); +const circpad_machine_spec_t *circpad_string_to_machine(const char *str); + +/* Padding negotiation between client and middle */ +signed_error_t circpad_handle_padding_negotiate(struct circuit_t *circ, + struct cell_t *cell); +signed_error_t circpad_handle_padding_negotiated(struct circuit_t *circ, + struct cell_t *cell, + crypt_path_t *layer_hint); +signed_error_t circpad_negotiate_padding(struct origin_circuit_t *circ, + circpad_machine_num_t machine, + uint8_t target_hopnum, + uint8_t command); +bool circpad_padding_negotiated(struct circuit_t *circ, + circpad_machine_num_t machine, + uint8_t command, + uint8_t response); + +MOCK_DECL(circpad_decision_t, +circpad_machine_schedule_padding,(circpad_machine_state_t *)); + +MOCK_DECL(circpad_decision_t, +circpad_machine_spec_transition, (circpad_machine_state_t *mi, + circpad_event_t event)); + +circpad_decision_t circpad_send_padding_cell_for_callback( + circpad_machine_state_t *mi); + +#ifdef CIRCUITPADDING_PRIVATE +STATIC circpad_delay_t +circpad_machine_sample_delay(circpad_machine_state_t *mi); + +STATIC bool +circpad_machine_reached_padding_limit(circpad_machine_state_t *mi); + +STATIC +circpad_decision_t circpad_machine_remove_token(circpad_machine_state_t *mi); + +STATIC circpad_delay_t +circpad_histogram_bin_to_usec(const circpad_machine_state_t *mi, + circpad_hist_index_t bin); + +STATIC const circpad_state_t * +circpad_machine_current_state(const circpad_machine_state_t *mi); + +STATIC circpad_hist_index_t circpad_histogram_usec_to_bin( + const circpad_machine_state_t *mi, + circpad_delay_t us); + +STATIC circpad_machine_state_t *circpad_circuit_machineinfo_new( + struct circuit_t *on_circ, + int machine_index); +STATIC void circpad_machine_remove_higher_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_us); +STATIC void circpad_machine_remove_lower_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_us); +STATIC void circpad_machine_remove_closest_token(circpad_machine_state_t *mi, + circpad_delay_t target_bin_us, + bool use_usec); +STATIC void circpad_machine_setup_tokens(circpad_machine_state_t *mi); + +MOCK_DECL(STATIC signed_error_t, +circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum, + uint8_t relay_command, const uint8_t *payload, + ssize_t payload_len)); + +#ifdef TOR_UNIT_TESTS +extern smartlist_t *origin_padding_machines; +extern smartlist_t *relay_padding_machines; +#endif + +#endif + +#endif diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c index 2cde21fa1f..c6ea2fff97 100644 --- a/src/core/or/circuitstats.c +++ b/src/core/or/circuitstats.c @@ -639,9 +639,9 @@ circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n) void circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ) { - control_event_circuit_status(circ, - CIRC_EVENT_FAILED, - END_CIRC_REASON_TIMEOUT); + circuit_event_status(circ, + CIRC_EVENT_FAILED, + END_CIRC_REASON_TIMEOUT); circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT); /* Record this event to check for too many timeouts diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c index 000a7c36da..3e604adcfc 100644 --- a/src/core/or/circuituse.c +++ b/src/core/or/circuituse.c @@ -35,6 +35,7 @@ #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "core/or/connection_edge.h" #include "core/or/policies.h" #include "feature/client/addressmap.h" @@ -1425,6 +1426,11 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn) if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) { hs_dec_rdv_stream_counter(origin_circ); } + + /* If there are no more streams on this circ, tell circpad */ + if (!origin_circ->p_streams) + circpad_machine_event_circ_has_no_streams(origin_circ); + return; } } else { @@ -1674,7 +1680,7 @@ circuit_testing_failed(origin_circuit_t *circ, int at_last_hop) void circuit_has_opened(origin_circuit_t *circ) { - control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0); + circuit_event_status(circ, CIRC_EVENT_BUILT, 0); /* Remember that this circuit has finished building. Now if we start * it building again later (e.g. by extending it), we will know not @@ -2596,6 +2602,12 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ, /* add it into the linked list of streams on this circuit */ log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.", (unsigned)circ->base_.n_circ_id); + + /* If this is the first stream on this circuit, tell circpad + * that streams are attached */ + if (!circ->p_streams) + circpad_machine_event_circ_has_streams(circ); + /* reset it, so we can measure circ timeouts */ ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL); ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams; @@ -3080,6 +3092,8 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose) if (CIRCUIT_IS_ORIGIN(circ)) { control_event_circuit_purpose_changed(TO_ORIGIN_CIRCUIT(circ), old_purpose); + + circpad_machine_event_circ_purpose_changed(TO_ORIGIN_CIRCUIT(circ)); } } diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c index 90991107dc..20c48bcedd 100644 --- a/src/core/or/connection_edge.c +++ b/src/core/or/connection_edge.c @@ -62,10 +62,12 @@ #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" #include "core/or/policies.h" @@ -97,7 +99,7 @@ #include "feature/rend/rendservice.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_util.h" #include "core/or/cell_st.h" @@ -300,6 +302,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial) } return 0; case AP_CONN_STATE_OPEN: + if (! conn->base_.linked) { + note_user_activity(approx_time()); + } + + /* falls through. */ case EXIT_CONN_STATE_OPEN: if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) { /* (We already sent an end cell if possible) */ @@ -754,6 +761,11 @@ connection_edge_flushed_some(edge_connection_t *conn) { switch (conn->base_.state) { case AP_CONN_STATE_OPEN: + if (! conn->base_.linked) { + note_user_activity(approx_time()); + } + + /* falls through. */ case EXIT_CONN_STATE_OPEN: connection_edge_consider_sending_sendme(conn); break; @@ -3706,6 +3718,10 @@ handle_hs_exit_conn(circuit_t *circ, edge_connection_t *conn) /* Link the circuit and the connection crypt path. */ conn->cpath_layer = origin_circ->cpath->prev; + /* If this is the first stream on this circuit, tell circpad */ + if (!origin_circ->p_streams) + circpad_machine_event_circ_has_streams(origin_circ); + /* Add it into the linked list of p_streams on this circuit */ conn->next_stream = origin_circ->p_streams; origin_circ->p_streams = conn; diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c index 67157d8d0b..debf482cb3 100644 --- a/src/core/or/connection_or.c +++ b/src/core/or/connection_or.c @@ -22,13 +22,14 @@ **/ #include "core/or/or.h" #include "feature/client/bridges.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" /* * Define this so we get channel internal functions, since we're implementing * part of a subclass (channel_tls_t). */ #define TOR_CHANNEL_INTERNAL_ #define CONNECTION_OR_PRIVATE +#define ORCONN_EVENT_PRIVATE #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/or/circuitbuild.h" @@ -46,6 +47,7 @@ #include "lib/geoip/geoip.h" #include "core/mainloop/mainloop.h" #include "trunnel/link_handshake.h" +#include "trunnel/netinfo.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nodelist.h" @@ -78,6 +80,8 @@ #include "lib/tls/tortls.h" #include "lib/tls/x509.h" +#include "core/or/orconn_event.h" + static int connection_tls_finish_handshake(or_connection_t *conn); static int connection_or_launch_v3_or_handshake(or_connection_t *conn); static int connection_or_process_cells_from_inbuf(or_connection_t *conn); @@ -400,6 +404,57 @@ connection_or_report_broken_states(int severity, int domain) smartlist_free(items); } +/** + * Helper function to publish an OR connection status event + * + * Publishes a messages to subscribers of ORCONN messages, and sends + * the control event. + **/ +void +connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp, + int reason) +{ + orconn_event_msg_t msg; + + msg.type = ORCONN_MSGTYPE_STATUS; + msg.u.status.gid = conn->base_.global_identifier; + msg.u.status.status = tp; + msg.u.status.reason = reason; + orconn_event_publish(&msg); + control_event_or_conn_status(conn, tp, reason); +} + +/** + * Helper function to publish a state change message + * + * connection_or_change_state() calls this to notify subscribers about + * a change of an OR connection state. + **/ +static void +connection_or_state_publish(const or_connection_t *conn, uint8_t state) +{ + orconn_event_msg_t msg; + + msg.type = ORCONN_MSGTYPE_STATE; + msg.u.state.gid = conn->base_.global_identifier; + if (conn->is_pt) { + /* Do extra decoding because conn->proxy_type indicates the proxy + * protocol that tor uses to talk with the transport plugin, + * instead of PROXY_PLUGGABLE. */ + tor_assert_nonfatal(conn->proxy_type != PROXY_NONE); + msg.u.state.proxy_type = PROXY_PLUGGABLE; + } else { + msg.u.state.proxy_type = conn->proxy_type; + } + msg.u.state.state = state; + if (conn->chan) { + msg.u.state.chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier; + } else { + msg.u.state.chan = 0; + } + orconn_event_publish(&msg); +} + /** Call this to change or_connection_t states, so the owning channel_tls_t can * be notified. */ @@ -407,16 +462,13 @@ connection_or_report_broken_states(int severity, int domain) static void connection_or_change_state(or_connection_t *conn, uint8_t state) { - uint8_t old_state; - tor_assert(conn); - old_state = conn->base_.state; conn->base_.state = state; + connection_or_state_publish(conn, state); if (conn->chan) - channel_tls_handle_state_change_on_orconn(conn->chan, conn, - old_state, state); + channel_tls_handle_state_change_on_orconn(conn->chan, conn, state); } /** Return the number of circuits using an or_connection_t; this used to @@ -707,8 +759,6 @@ connection_or_finished_connecting(or_connection_t *or_conn) log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.", conn->address,conn->port); - control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - control_event_boot_first_orconn(); if (proxy_type != PROXY_NONE) { /* start proxy handshake */ @@ -758,8 +808,8 @@ connection_or_about_to_close(or_connection_t *or_conn) entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan)); if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) { int reason = tls_error_to_orconn_end_reason(or_conn->tls_error); - control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED, - reason); + connection_or_event_status(or_conn, OR_CONN_EVENT_FAILED, + reason); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_prob_or( orconn_end_reason_to_control_string(reason), @@ -769,10 +819,10 @@ connection_or_about_to_close(or_connection_t *or_conn) } else if (conn->hold_open_until_flushed) { /* We only set hold_open_until_flushed when we're intentionally * closing a connection. */ - control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, + connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); } else if (!tor_digest_is_zero(or_conn->identity_digest)) { - control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED, + connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED, tls_error_to_orconn_end_reason(or_conn->tls_error)); } } @@ -1364,7 +1414,7 @@ void connection_or_connect_failed(or_connection_t *conn, int reason, const char *msg) { - control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason); + connection_or_event_status(conn, OR_CONN_EVENT_FAILED, reason); if (!authdir_mode_tests_reachability(get_options())) control_event_bootstrap_prob_or(msg, reason, conn); note_or_connect_failed(conn); @@ -1430,7 +1480,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, int r; tor_addr_t proxy_addr; uint16_t proxy_port; - int proxy_type; + int proxy_type, is_pt = 0; tor_assert(_addr); tor_assert(id_digest); @@ -1471,21 +1521,27 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, return NULL; } - connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); - control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0); - conn->is_outgoing = 1; /* If we are using a proxy server, find it and use it. */ - r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, TO_CONN(conn)); + r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt, + TO_CONN(conn)); if (r == 0) { conn->proxy_type = proxy_type; if (proxy_type != PROXY_NONE) { tor_addr_copy(&addr, &proxy_addr); port = proxy_port; conn->base_.proxy_state = PROXY_INFANT; + conn->is_pt = is_pt; } + connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); + connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0); } else { + /* This duplication of state change calls is necessary in case we + * run into an error condition below */ + connection_or_change_state(conn, OR_CONN_STATE_CONNECTING); + connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0); + /* get_proxy_addrport() might fail if we have a Bridge line that references a transport, but no ClientTransportPlugin lines defining its transport proxy. If this is the case, let's try to @@ -1981,8 +2037,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn, /* Tell the new guard API about the channel failure */ entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan)); - control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, - END_OR_CONN_REASON_OR_IDENTITY); + connection_or_event_status(conn, OR_CONN_EVENT_FAILED, + END_OR_CONN_REASON_OR_IDENTITY); if (!authdir_mode_tests_reachability(options)) control_event_bootstrap_prob_or( "Unexpected identity in router certificate", @@ -2222,7 +2278,7 @@ int connection_or_set_state_open(or_connection_t *conn) { connection_or_change_state(conn, OR_CONN_STATE_OPEN); - control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0); + connection_or_event_status(conn, OR_CONN_EVENT_CONNECTED, 0); /* Link protocol 3 appeared in Tor 0.2.3.6-alpha, so any connection * that uses an earlier link protocol should not be treated as a relay. */ @@ -2428,6 +2484,31 @@ connection_or_send_versions(or_connection_t *conn, int v3_plus) return 0; } +static netinfo_addr_t * +netinfo_addr_from_tor_addr(const tor_addr_t *tor_addr) +{ + sa_family_t addr_family = tor_addr_family(tor_addr); + + if (BUG(addr_family != AF_INET && addr_family != AF_INET6)) + return NULL; + + netinfo_addr_t *netinfo_addr = netinfo_addr_new(); + + if (addr_family == AF_INET) { + netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV4); + netinfo_addr_set_len(netinfo_addr, 4); + netinfo_addr_set_addr_ipv4(netinfo_addr, tor_addr_to_ipv4h(tor_addr)); + } else if (addr_family == AF_INET6) { + netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV6); + netinfo_addr_set_len(netinfo_addr, 16); + uint8_t *ipv6_buf = netinfo_addr_getarray_addr_ipv6(netinfo_addr); + const uint8_t *in6_addr = tor_addr_to_in6_addr8(tor_addr); + memcpy(ipv6_buf, in6_addr, 16); + } + + return netinfo_addr; +} + /** Send a NETINFO cell on <b>conn</b>, telling the other server what we know * about their address, our address, and the current time. */ MOCK_IMPL(int, @@ -2436,8 +2517,7 @@ connection_or_send_netinfo,(or_connection_t *conn)) cell_t cell; time_t now = time(NULL); const routerinfo_t *me; - int len; - uint8_t *out; + int r = -1; tor_assert(conn->handshake_state); @@ -2450,20 +2530,21 @@ connection_or_send_netinfo,(or_connection_t *conn)) memset(&cell, 0, sizeof(cell_t)); cell.command = CELL_NETINFO; + netinfo_cell_t *netinfo_cell = netinfo_cell_new(); + /* Timestamp, if we're a relay. */ if (public_server_mode(get_options()) || ! conn->is_outgoing) - set_uint32(cell.payload, htonl((uint32_t)now)); + netinfo_cell_set_timestamp(netinfo_cell, (uint32_t)now); /* Their address. */ - out = cell.payload + 4; + const tor_addr_t *remote_tor_addr = + !tor_addr_is_null(&conn->real_addr) ? &conn->real_addr : &conn->base_.addr; /* We use &conn->real_addr below, unless it hasn't yet been set. If it * hasn't yet been set, we know that base_.addr hasn't been tampered with * yet either. */ - len = append_address_to_payload(out, !tor_addr_is_null(&conn->real_addr) - ? &conn->real_addr : &conn->base_.addr); - if (len<0) - return -1; - out += len; + netinfo_addr_t *their_addr = netinfo_addr_from_tor_addr(remote_tor_addr); + + netinfo_cell_set_other_addr(netinfo_cell, their_addr); /* My address -- only include it if I'm a public relay, or if I'm a * bridge and this is an incoming connection. If I'm a bridge and this @@ -2471,28 +2552,42 @@ connection_or_send_netinfo,(or_connection_t *conn)) if ((public_server_mode(get_options()) || !conn->is_outgoing) && (me = router_get_my_routerinfo())) { tor_addr_t my_addr; - *out++ = 1 + !tor_addr_is_null(&me->ipv6_addr); - tor_addr_from_ipv4h(&my_addr, me->addr); - len = append_address_to_payload(out, &my_addr); - if (len < 0) - return -1; - out += len; + + uint8_t n_my_addrs = 1 + !tor_addr_is_null(&me->ipv6_addr); + netinfo_cell_set_n_my_addrs(netinfo_cell, n_my_addrs); + + netinfo_cell_add_my_addrs(netinfo_cell, + netinfo_addr_from_tor_addr(&my_addr)); if (!tor_addr_is_null(&me->ipv6_addr)) { - len = append_address_to_payload(out, &me->ipv6_addr); - if (len < 0) - return -1; + netinfo_cell_add_my_addrs(netinfo_cell, + netinfo_addr_from_tor_addr(&me->ipv6_addr)); } - } else { - *out = 0; + } + + const char *errmsg = NULL; + if ((errmsg = netinfo_cell_check(netinfo_cell))) { + log_warn(LD_OR, "Failed to validate NETINFO cell with error: %s", + errmsg); + goto cleanup; + } + + if (netinfo_cell_encode(cell.payload, CELL_PAYLOAD_SIZE, + netinfo_cell) < 0) { + log_warn(LD_OR, "Failed generating NETINFO cell"); + goto cleanup; } conn->handshake_state->digest_sent_data = 0; conn->handshake_state->sent_netinfo = 1; connection_or_write_cell_to_buf(&cell, conn); - return 0; + r = 0; + cleanup: + netinfo_cell_free(netinfo_cell); + + return r; } /** Helper used to add an encoded certs to a cert cell */ diff --git a/src/core/or/connection_or.h b/src/core/or/connection_or.h index 817bcdd317..272f536b83 100644 --- a/src/core/or/connection_or.h +++ b/src/core/or/connection_or.h @@ -17,32 +17,7 @@ struct ed25519_keypair_t; or_connection_t *TO_OR_CONN(connection_t *); -#define OR_CONN_STATE_MIN_ 1 -/** State for a connection to an OR: waiting for connect() to finish. */ -#define OR_CONN_STATE_CONNECTING 1 -/** State for a connection to an OR: waiting for proxy handshake to complete */ -#define OR_CONN_STATE_PROXY_HANDSHAKING 2 -/** State for an OR connection client: SSL is handshaking, not done - * yet. */ -#define OR_CONN_STATE_TLS_HANDSHAKING 3 -/** State for a connection to an OR: We're doing a second SSL handshake for - * renegotiation purposes. (V2 handshake only.) */ -#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 -/** State for a connection at an OR: We're waiting for the client to - * renegotiate (to indicate a v2 handshake) or send a versions cell (to - * indicate a v3 handshake) */ -#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 -/** State for an OR connection: We're done with our SSL handshake, we've done - * renegotiation, but we haven't yet negotiated link protocol versions and - * sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING_V2 6 -/** State for an OR connection: We're done with our SSL handshake, but we - * haven't yet negotiated link protocol versions, done a V3 handshake, and - * sent a netinfo cell. */ -#define OR_CONN_STATE_OR_HANDSHAKING_V3 7 -/** State for an OR connection: Ready to send/receive cells. */ -#define OR_CONN_STATE_OPEN 8 -#define OR_CONN_STATE_MAX_ 8 +#include "core/or/orconn_event.h" void connection_or_clear_identity(or_connection_t *conn); void connection_or_clear_identity_map(void); @@ -81,6 +56,9 @@ MOCK_DECL(void,connection_or_close_for_error, void connection_or_report_broken_states(int severity, int domain); +void connection_or_event_status(or_connection_t *conn, + or_conn_status_event_t tp, int reason); + MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn, int receiving)); int connection_tls_continue_handshake(or_connection_t *conn); diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c new file mode 100644 index 0000000000..4a6fc748c9 --- /dev/null +++ b/src/core/or/ocirc_event.c @@ -0,0 +1,84 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ocirc_event.c + * \brief Publish state change messages for origin circuits + * + * Implements a basic publish-subscribe framework for messages about + * the state of origin circuits. The publisher calls the subscriber + * callback functions synchronously. + * + * Although the synchronous calls might not simplify the call graph, + * this approach improves data isolation because the publisher doesn't + * need knowledge about the internals of subscribing subsystems. It + * also avoids race conditions that might occur in asynchronous + * frameworks. + **/ + +#include "core/or/or.h" + +#define OCIRC_EVENT_PRIVATE + +#include "core/or/cpath_build_state_st.h" +#include "core/or/ocirc_event.h" +#include "core/or/ocirc_event_sys.h" +#include "core/or/origin_circuit_st.h" +#include "lib/subsys/subsys.h" + +/** List of subscribers */ +static smartlist_t *ocirc_event_rcvrs; + +/** Initialize subscriber list */ +static int +ocirc_event_init(void) +{ + ocirc_event_rcvrs = smartlist_new(); + return 0; +} + +/** Free subscriber list */ +static void +ocirc_event_fini(void) +{ + smartlist_free(ocirc_event_rcvrs); +} + +/** + * Subscribe to messages about origin circuit events + * + * Register a callback function to receive messages about origin + * circuits. The publisher calls this function synchronously. + **/ +void +ocirc_event_subscribe(ocirc_event_rcvr_t fn) +{ + tor_assert(fn); + /* Don't duplicate subscriptions. */ + if (smartlist_contains(ocirc_event_rcvrs, fn)) + return; + + smartlist_add(ocirc_event_rcvrs, fn); +} + +/** + * Publish a message about OR connection events + * + * This calls the subscriber receiver function synchronously. + **/ +void +ocirc_event_publish(const ocirc_event_msg_t *msg) +{ + SMARTLIST_FOREACH_BEGIN(ocirc_event_rcvrs, ocirc_event_rcvr_t, fn) { + tor_assert(fn); + (*fn)(msg); + } SMARTLIST_FOREACH_END(fn); +} + +const subsys_fns_t sys_ocirc_event = { + .name = "ocirc_event", + .supported = true, + .level = -32, + .initialize = ocirc_event_init, + .shutdown = ocirc_event_fini, +}; diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h new file mode 100644 index 0000000000..0b125c2898 --- /dev/null +++ b/src/core/or/ocirc_event.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ocirc_event.h + * \brief Header file for ocirc_event.c + **/ + +#ifndef TOR_OCIRC_EVENT_H +#define TOR_OCIRC_EVENT_H + +#include <stdbool.h> + +#include "lib/cc/torint.h" + +/** Used to indicate the type of a circuit event passed to the controller. + * The various types are defined in control-spec.txt */ +typedef enum circuit_status_event_t { + CIRC_EVENT_LAUNCHED = 0, + CIRC_EVENT_BUILT = 1, + CIRC_EVENT_EXTENDED = 2, + CIRC_EVENT_FAILED = 3, + CIRC_EVENT_CLOSED = 4, +} circuit_status_event_t; + +/** Message for origin circuit state update */ +typedef struct ocirc_state_msg_t { + uint32_t gid; /**< global ID (only origin circuits have them) */ + int state; /**< new circuit state */ + bool onehop; /**< one-hop circuit? */ +} ocirc_state_msg_t; + +/** + * Message when a channel gets associated to a circuit. + * + * This doesn't always correspond to something in circuitbuild.c + * setting the n_chan field in the circuit. For some reason, if + * circuit_handle_first_hop() launches a new circuit, it doesn't set + * the n_chan field. + */ +typedef struct ocirc_chan_msg_t { + uint32_t gid; /**< global ID */ + uint64_t chan; /**< channel ID */ + bool onehop; /**< one-hop circuit? */ +} ocirc_chan_msg_t; + +/** + * Message for origin circuit status event + * + * This contains information that ends up in CIRC control protocol events. + */ +typedef struct ocirc_cevent_msg_t { + uint32_t gid; /**< global ID */ + int evtype; /**< event type */ + int reason; /**< reason */ + bool onehop; /**< one-hop circuit? */ +} ocirc_cevent_msg_t; + +/** Discriminant values for origin circuit event message */ +typedef enum ocirc_msgtype_t { + OCIRC_MSGTYPE_STATE, + OCIRC_MSGTYPE_CHAN, + OCIRC_MSGTYPE_CEVENT, +} ocirc_msgtype_t; + +/** Discriminated union for the actual message */ +typedef struct ocirc_event_msg_t { + int type; + union { + ocirc_state_msg_t state; + ocirc_chan_msg_t chan; + ocirc_cevent_msg_t cevent; + } u; +} ocirc_event_msg_t; + +/** + * Receiver function pointer for origin circuit subscribers + * + * This function gets called synchronously by the publisher. + **/ +typedef void (*ocirc_event_rcvr_t)(const ocirc_event_msg_t *); + +void ocirc_event_subscribe(ocirc_event_rcvr_t fn); + +#ifdef OCIRC_EVENT_PRIVATE +void ocirc_event_publish(const ocirc_event_msg_t *msg); +#endif + +#endif /* defined(TOR_OCIRC_EVENT_STATE_H) */ diff --git a/src/core/or/ocirc_event_sys.h b/src/core/or/ocirc_event_sys.h new file mode 100644 index 0000000000..9d4bfe5333 --- /dev/null +++ b/src/core/or/ocirc_event_sys.h @@ -0,0 +1,13 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ + +/** + * \file ocirc_event_sys.h + * \brief Declare subsystem object for the origin circuit event module. + **/ + +#ifndef TOR_OCIRC_EVENT_SYS_H +#define TOR_OCIRC_EVENT_SYS_H + +extern const struct subsys_fns_t sys_ocirc_event; + +#endif /* defined(TOR_OCIRC_EVENT_H) */ diff --git a/src/core/or/or.h b/src/core/or/or.h index 7c601e49b3..db6d089582 100644 --- a/src/core/or/or.h +++ b/src/core/or/or.h @@ -26,7 +26,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "lib/container/map.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/container/smartlist.h" #include "lib/crypt_ops/crypto_cipher.h" #include "lib/crypt_ops/crypto_rsa.h" @@ -97,6 +97,8 @@ struct curve25519_public_key_t; #define SIGNEWNYM 129 #define SIGCLEARDNSCACHE 130 #define SIGHEARTBEAT 131 +#define SIGACTIVE 132 +#define SIGDORMANT 133 #if (SIZEOF_CELL_T != 0) /* On Irix, stdlib.h defines a cell_t type, so we need to make sure @@ -205,6 +207,9 @@ struct curve25519_public_key_t; #define RELAY_COMMAND_RENDEZVOUS_ESTABLISHED 39 #define RELAY_COMMAND_INTRODUCE_ACK 40 +#define RELAY_COMMAND_PADDING_NEGOTIATE 41 +#define RELAY_COMMAND_PADDING_NEGOTIATED 42 + /* Reasons why an OR connection is closed. */ #define END_OR_CONN_REASON_DONE 1 #define END_OR_CONN_REASON_REFUSED 2 /* connection refused */ @@ -834,6 +839,10 @@ typedef struct protover_summary_flags_t { * service rendezvous point supporting version 3 as seen in proposal 224. * This requires HSRend=2. */ unsigned int supports_v3_rendezvous_point: 1; + + /** True iff this router has a protocol list that allows clients to + * negotiate link-level padding. Requires Padding>=1. */ + unsigned int supports_padding : 1; } protover_summary_flags_t; typedef struct routerinfo_t routerinfo_t; diff --git a/src/core/or/or_connection_st.h b/src/core/or/or_connection_st.h index d5db5e8694..a5ce844bff 100644 --- a/src/core/or/or_connection_st.h +++ b/src/core/or/or_connection_st.h @@ -67,6 +67,8 @@ struct or_connection_t { * geoip cache and handled by the DoS mitigation subsystem. We use this to * insure we have a coherent count of concurrent connection. */ unsigned int tracked_for_dos_mitigation : 1; + /** True iff this connection is using a pluggable transport */ + unsigned int is_pt : 1; uint16_t link_proto; /**< What protocol version are we using? 0 for * "none negotiated yet." */ diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c new file mode 100644 index 0000000000..9fb34bd1ff --- /dev/null +++ b/src/core/or/orconn_event.c @@ -0,0 +1,81 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file orconn_event.c + * \brief Publish state change messages for OR connections + * + * Implements a basic publish-subscribe framework for messages about + * the state of OR connections. The publisher calls the subscriber + * callback functions synchronously. + * + * Although the synchronous calls might not simplify the call graph, + * this approach improves data isolation because the publisher doesn't + * need knowledge about the internals of subscribing subsystems. It + * also avoids race conditions that might occur in asynchronous + * frameworks. + **/ + +#include "core/or/or.h" +#include "lib/subsys/subsys.h" + +#define ORCONN_EVENT_PRIVATE +#include "core/or/orconn_event.h" +#include "core/or/orconn_event_sys.h" + +/** List of subscribers */ +static smartlist_t *orconn_event_rcvrs; + +/** Initialize subscriber list */ +static int +orconn_event_init(void) +{ + orconn_event_rcvrs = smartlist_new(); + return 0; +} + +/** Free subscriber list */ +static void +orconn_event_fini(void) +{ + smartlist_free(orconn_event_rcvrs); +} + +/** + * Subscribe to messages about OR connection events + * + * Register a callback function to receive messages about ORCONNs. + * The publisher calls this function synchronously. + **/ +void +orconn_event_subscribe(orconn_event_rcvr_t fn) +{ + tor_assert(fn); + /* Don't duplicate subscriptions. */ + if (smartlist_contains(orconn_event_rcvrs, fn)) + return; + + smartlist_add(orconn_event_rcvrs, fn); +} + +/** + * Publish a message about OR connection events + * + * This calls the subscriber receiver function synchronously. + **/ +void +orconn_event_publish(const orconn_event_msg_t *msg) +{ + SMARTLIST_FOREACH_BEGIN(orconn_event_rcvrs, orconn_event_rcvr_t, fn) { + tor_assert(fn); + (*fn)(msg); + } SMARTLIST_FOREACH_END(fn); +} + +const subsys_fns_t sys_orconn_event = { + .name = "orconn_event", + .supported = true, + .level = -33, + .initialize = orconn_event_init, + .shutdown = orconn_event_fini, +}; diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h new file mode 100644 index 0000000000..80289d53e6 --- /dev/null +++ b/src/core/or/orconn_event.h @@ -0,0 +1,120 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file orconn_event.h + * \brief Header file for orconn_event.c + * + * The OR_CONN_STATE_* symbols are here to make it easier for + * subscribers to make decisions based on the messages that they + * receive. + **/ + +#ifndef TOR_ORCONN_EVENT_H +#define TOR_ORCONN_EVENT_H + +/** + * @name States of OR connections + * + * These must be in a partial ordering such that usually no OR + * connection will transition from a higher-numbered state to a + * lower-numbered one. Code such as bto_update_best() depends on this + * ordering to determine the best state it's seen so far. + * @{ */ +#define OR_CONN_STATE_MIN_ 1 +/** State for a connection to an OR: waiting for connect() to finish. */ +#define OR_CONN_STATE_CONNECTING 1 +/** State for a connection to an OR: waiting for proxy handshake to complete */ +#define OR_CONN_STATE_PROXY_HANDSHAKING 2 +/** State for an OR connection client: SSL is handshaking, not done + * yet. */ +#define OR_CONN_STATE_TLS_HANDSHAKING 3 +/** State for a connection to an OR: We're doing a second SSL handshake for + * renegotiation purposes. (V2 handshake only.) */ +#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4 +/** State for a connection at an OR: We're waiting for the client to + * renegotiate (to indicate a v2 handshake) or send a versions cell (to + * indicate a v3 handshake) */ +#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5 +/** State for an OR connection: We're done with our SSL handshake, we've done + * renegotiation, but we haven't yet negotiated link protocol versions and + * sent a netinfo cell. */ +#define OR_CONN_STATE_OR_HANDSHAKING_V2 6 +/** State for an OR connection: We're done with our SSL handshake, but we + * haven't yet negotiated link protocol versions, done a V3 handshake, and + * sent a netinfo cell. */ +#define OR_CONN_STATE_OR_HANDSHAKING_V3 7 +/** State for an OR connection: Ready to send/receive cells. */ +#define OR_CONN_STATE_OPEN 8 +#define OR_CONN_STATE_MAX_ 8 +/** @} */ + +/** Used to indicate the type of an OR connection event passed to the + * controller. The various types are defined in control-spec.txt */ +typedef enum or_conn_status_event_t { + OR_CONN_EVENT_LAUNCHED = 0, + OR_CONN_EVENT_CONNECTED = 1, + OR_CONN_EVENT_FAILED = 2, + OR_CONN_EVENT_CLOSED = 3, + OR_CONN_EVENT_NEW = 4, +} or_conn_status_event_t; + +/** Discriminant values for orconn event message */ +typedef enum orconn_msgtype_t { + ORCONN_MSGTYPE_STATE, + ORCONN_MSGTYPE_STATUS, +} orconn_msgtype_t; + +/** + * Message for orconn state update + * + * This contains information about internal state changes of + * or_connection_t objects. The chan and proxy_type fields are + * additional information that a subscriber may need to make + * decisions. + **/ +typedef struct orconn_state_msg_t { + uint64_t gid; /**< connection's global ID */ + uint64_t chan; /**< associated channel ID */ + int proxy_type; /**< connection's proxy type */ + uint8_t state; /**< new connection state */ +} orconn_state_msg_t; + +/** + * Message for orconn status event + * + * This contains information that ends up in ORCONN control protocol + * events. + **/ +typedef struct orconn_status_msg_t { + uint64_t gid; /**< connection's global ID */ + int status; /**< or_conn_status_event_t */ + int reason; /**< reason */ +} orconn_status_msg_t; + +/** Discriminated union for the actual message */ +typedef struct orconn_event_msg_t { + int type; + union { + orconn_state_msg_t state; + orconn_status_msg_t status; + } u; +} orconn_event_msg_t; + +/** + * Receiver function pointer for OR subscribers + * + * This function gets called synchronously by the publisher. + **/ +typedef void (*orconn_event_rcvr_t)(const orconn_event_msg_t *); + +void orconn_event_subscribe(orconn_event_rcvr_t); + +#ifdef ORCONN_EVENT_PRIVATE +void orconn_event_publish(const orconn_event_msg_t *); +#endif + +#endif /* defined(TOR_ORCONN_EVENT_H) */ diff --git a/src/core/or/orconn_event_sys.h b/src/core/or/orconn_event_sys.h new file mode 100644 index 0000000000..bfb0a3ac4a --- /dev/null +++ b/src/core/or/orconn_event_sys.h @@ -0,0 +1,12 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ + +/** + * \file orconn_event_sys.h + * \brief Declare subsystem object for the OR connection event module. + **/ +#ifndef TOR_ORCONN_EVENT_SYS_H +#define TOR_ORCONN_EVENT_SYS_H + +extern const struct subsys_fns_t sys_orconn_event; + +#endif /* defined(TOR_ORCONN_SYS_H) */ diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h index f55416db14..daa5f41dad 100644 --- a/src/core/or/origin_circuit_st.h +++ b/src/core/or/origin_circuit_st.h @@ -161,6 +161,10 @@ struct origin_circuit_t { * connections to this circuit. */ unsigned int unusable_for_new_conns : 1; + /* If this flag is set (due to padding negotiation failure), we should + * not try to negotiate further circuit padding. */ + unsigned padding_negotiation_failed : 1; + /** * Tristate variable to guard against pathbias miscounting * due to circuit purpose transitions changing the decision diff --git a/src/core/or/policies.c b/src/core/or/policies.c index 3ed282d785..a6d66d36de 100644 --- a/src/core/or/policies.c +++ b/src/core/or/policies.c @@ -29,6 +29,7 @@ #include "feature/relay/routermode.h" #include "lib/geoip/geoip.h" #include "ht.h" +#include "lib/crypt_ops/crypto_rand.h" #include "lib/encoding/confline.h" #include "core/or/addr_policy_st.h" @@ -461,7 +462,8 @@ fascist_firewall_use_ipv6(const or_options_t *options) * ClientPreferIPv6DirPort is deprecated, but check it anyway. */ return (options->ClientUseIPv6 == 1 || options->ClientUseIPv4 == 0 || options->ClientPreferIPv6ORPort == 1 || - options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1); + options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1 || + options->ClientAutoIPv6ORPort == 1); } /** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and @@ -488,6 +490,15 @@ fascist_firewall_prefer_ipv6_impl(const or_options_t *options) return -1; } +/* Choose whether we prefer IPv4 or IPv6 by randomly choosing an address + * family. Return 0 for IPv4, and 1 for IPv6. */ +MOCK_IMPL(int, +fascist_firewall_rand_prefer_ipv6_addr, (void)) +{ + /* TODO: Check for failures, and infer our preference based on this. */ + return crypto_rand_int(2); +} + /** Do we prefer to connect to IPv6 ORPorts? * Use node_ipv6_or_preferred() whenever possible: it supports bridge client * per-node IPv6 preferences. @@ -502,7 +513,10 @@ fascist_firewall_prefer_ipv6_orport(const or_options_t *options) } /* We can use both IPv4 and IPv6 - which do we prefer? */ - if (options->ClientPreferIPv6ORPort == 1) { + if (options->ClientAutoIPv6ORPort == 1) { + /* If ClientAutoIPv6ORPort is 1, we prefer IPv4 or IPv6 at random. */ + return fascist_firewall_rand_prefer_ipv6_addr(); + } else if (options->ClientPreferIPv6ORPort == 1) { return 1; } @@ -2706,7 +2720,7 @@ parse_short_policy(const char *summary) int is_accept; int n_entries; short_policy_entry_t entries[MAX_EXITPOLICY_SUMMARY_LEN]; /* overkill */ - const char *next; + char *next; if (!strcmpstart(summary, "accept ")) { is_accept = 1; @@ -2721,57 +2735,56 @@ parse_short_policy(const char *summary) n_entries = 0; for ( ; *summary; summary = next) { - const char *comma = strchr(summary, ','); - unsigned low, high; - char dummy; - char ent_buf[32]; - size_t len; - - next = comma ? comma+1 : strchr(summary, '\0'); - len = comma ? (size_t)(comma - summary) : strlen(summary); - if (n_entries == MAX_EXITPOLICY_SUMMARY_LEN) { log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Impossibly long policy summary %s", escaped(orig_summary)); return NULL; } - if (! TOR_ISDIGIT(*summary) || len > (sizeof(ent_buf)-1)) { - /* unrecognized entry format. skip it. */ - continue; - } - if (len < 1) { - /* empty; skip it. */ - /* XXX This happens to be unreachable, since if len==0, then *summary is - * ',' or '\0', and the TOR_ISDIGIT test above would have failed. */ - continue; + unsigned low, high; + int ok; + low = (unsigned) tor_parse_ulong(summary, 10, 1, 65535, &ok, &next); + if (!ok) { + if (! TOR_ISDIGIT(*summary) || *summary == ',') { + /* Unrecognized format: skip it. */ + goto skip_ent; + } else { + goto bad_ent; + } } - memcpy(ent_buf, summary, len); - ent_buf[len] = '\0'; + switch (*next) { + case ',': + ++next; + /* fall through */ + case '\0': + high = low; + break; + case '-': + high = (unsigned) tor_parse_ulong(next+1, 10, low, 65535, &ok, &next); + if (!ok) + goto bad_ent; - if (tor_sscanf(ent_buf, "%u-%u%c", &low, &high, &dummy) == 2) { - if (low<1 || low>65535 || high<1 || high>65535 || low>high) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Found bad entry in policy summary %s", escaped(orig_summary)); - return NULL; - } - } else if (tor_sscanf(ent_buf, "%u%c", &low, &dummy) == 1) { - if (low<1 || low>65535) { - log_fn(LOG_PROTOCOL_WARN, LD_DIR, - "Found bad entry in policy summary %s", escaped(orig_summary)); - return NULL; - } - high = low; - } else { - log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s", - escaped(orig_summary)); - return NULL; + if (*next == ',') + ++next; + else if (*next != '\0') + goto bad_ent; + + break; + default: + goto bad_ent; } entries[n_entries].min_port = low; entries[n_entries].max_port = high; n_entries++; + + continue; + skip_ent: + next = strchr(next, ','); + if (!next) + break; + ++next; } if (n_entries == 0) { @@ -2792,6 +2805,11 @@ parse_short_policy(const char *summary) result->n_entries = n_entries; memcpy(result->entries, entries, sizeof(short_policy_entry_t)*n_entries); return result; + + bad_ent: + log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s", + escaped(orig_summary)); + return NULL; } /** Write <b>policy</b> back out into a string. */ diff --git a/src/core/or/policies.h b/src/core/or/policies.h index 2c38de362f..324c1c2dd1 100644 --- a/src/core/or/policies.h +++ b/src/core/or/policies.h @@ -70,6 +70,7 @@ typedef struct short_policy_t { int firewall_is_fascist_or(void); int firewall_is_fascist_dir(void); int fascist_firewall_use_ipv6(const or_options_t *options); +MOCK_DECL(int, fascist_firewall_rand_prefer_ipv6_addr, (void)); int fascist_firewall_prefer_ipv6_orport(const or_options_t *options); int fascist_firewall_prefer_ipv6_dirport(const or_options_t *options); diff --git a/src/core/or/protover.c b/src/core/or/protover.c index 17979d04ea..53709ad002 100644 --- a/src/core/or/protover.c +++ b/src/core/or/protover.c @@ -39,6 +39,9 @@ static int protocol_list_contains(const smartlist_t *protos, static const struct { protocol_type_t protover_type; const char *name; +/* If you add a new protocol here, you probably also want to add + * parsing for it in routerstatus_parse_entry_from_string() so that + * it is set in routerstatus_t */ } PROTOCOL_NAMES[] = { { PRT_LINK, "Link" }, { PRT_LINKAUTH, "LinkAuth" }, @@ -49,6 +52,7 @@ static const struct { { PRT_HSREND, "HSRend" }, { PRT_DESC, "Desc" }, { PRT_MICRODESC, "Microdesc"}, + { PRT_PADDING, "Padding"}, { PRT_CONS, "Cons" } }; @@ -396,7 +400,8 @@ protover_get_supported_protocols(void) "LinkAuth=3 " #endif "Microdesc=1-2 " - "Relay=1-2"; + "Relay=1-2 " + "Padding=1"; } /** The protocols from protover_get_supported_protocols(), as parsed into a diff --git a/src/core/or/protover.h b/src/core/or/protover.h index 7e181ba97a..567b94a168 100644 --- a/src/core/or/protover.h +++ b/src/core/or/protover.h @@ -33,16 +33,17 @@ struct smartlist_t; /// C_RUST_COUPLED: src/rust/protover/ffi.rs `translate_to_rust` /// C_RUST_COUPLED: src/rust/protover/protover.rs `Proto` typedef enum protocol_type_t { - PRT_LINK, - PRT_LINKAUTH, - PRT_RELAY, - PRT_DIRCACHE, - PRT_HSDIR, - PRT_HSINTRO, - PRT_HSREND, - PRT_DESC, - PRT_MICRODESC, - PRT_CONS, + PRT_LINK = 0, + PRT_LINKAUTH = 1, + PRT_RELAY = 2, + PRT_DIRCACHE = 3, + PRT_HSDIR = 4, + PRT_HSINTRO = 5, + PRT_HSREND = 6, + PRT_DESC = 7, + PRT_MICRODESC = 8, + PRT_CONS = 9, + PRT_PADDING = 10, } protocol_type_t; bool protover_contains_long_protocol_names(const char *s); diff --git a/src/core/or/relay.c b/src/core/or/relay.c index dc88a6f649..706a6e05cb 100644 --- a/src/core/or/relay.c +++ b/src/core/or/relay.c @@ -49,12 +49,13 @@ #include "core/or/or.h" #include "feature/client/addressmap.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/channel.h" #include "feature/client/circpathbias.h" #include "core/or/circuitbuild.h" #include "core/or/circuitlist.h" #include "core/or/circuituse.h" +#include "core/or/circuitpadding.h" #include "lib/compress/compress.h" #include "app/config/config.h" #include "core/mainloop/connection.h" @@ -80,7 +81,6 @@ #include "feature/nodelist/describe.h" #include "feature/nodelist/routerlist.h" #include "core/or/scheduler.h" -#include "feature/stats/rephist.h" #include "core/or/cell_st.h" #include "core/or/cell_queue_st.h" @@ -293,7 +293,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, return 0; } - /* not recognized. pass it on. */ + /* not recognized. inform circpad and pass it on. */ + circpad_deliver_unrecognized_cell_events(circ, cell_direction); + if (cell_direction == CELL_DIRECTION_OUT) { cell->circ_id = circ->n_circ_id; /* switch it */ chan = circ->n_chan; @@ -353,11 +355,11 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, * - Encrypt it to the right layer * - Append it to the appropriate cell_queue on <b>circ</b>. */ -static int -circuit_package_relay_cell(cell_t *cell, circuit_t *circ, +MOCK_IMPL(int, +circuit_package_relay_cell, (cell_t *cell, circuit_t *circ, cell_direction_t cell_direction, crypt_path_t *layer_hint, streamid_t on_stream, - const char *filename, int lineno) + const char *filename, int lineno)) { channel_t *chan; /* where to send the cell */ @@ -524,6 +526,8 @@ relay_command_to_string(uint8_t command) case RELAY_COMMAND_INTRODUCE_ACK: return "INTRODUCE_ACK"; case RELAY_COMMAND_EXTEND2: return "EXTEND2"; case RELAY_COMMAND_EXTENDED2: return "EXTENDED2"; + case RELAY_COMMAND_PADDING_NEGOTIATE: return "PADDING_NEGOTIATE"; + case RELAY_COMMAND_PADDING_NEGOTIATED: return "PADDING_NEGOTIATED"; default: tor_snprintf(buf, sizeof(buf), "Unrecognized relay command %u", (unsigned)command); @@ -577,8 +581,8 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, log_debug(LD_OR,"delivering %d cell %s.", relay_command, cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward"); - if (relay_command == RELAY_COMMAND_DROP) - rep_hist_padding_count_write(PADDING_TYPE_DROP); + /* Tell circpad we're sending a relay cell */ + circpad_deliver_sent_relay_cell_events(circ, relay_command); /* If we are sending an END cell and this circuit is used for a tunneled * directory request, advance its state. */ @@ -602,7 +606,9 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, * one of them. Don't worry about the conn protocol version: * append_cell_to_circuit_queue will fix it up. */ cell.command = CELL_RELAY_EARLY; - --origin_circ->remaining_relay_early_cells; + /* If we're out of relay early cells, tell circpad */ + if (--origin_circ->remaining_relay_early_cells == 0) + circpad_machine_event_circ_has_no_relay_early(origin_circ); log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.", (int)origin_circ->remaining_relay_early_cells); /* Memorize the command that is sent as RELAY_EARLY cell; helps debug @@ -1481,9 +1487,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, } } + /* Tell circpad that we've recieved a recognized cell */ + circpad_deliver_recognized_relay_cell_events(circ, rh.command, layer_hint); + /* either conn is NULL, in which case we've got a control cell, or else * conn points to the recognized stream. */ - if (conn && !connection_state_is_open(TO_CONN(conn))) { if (conn->base_.type == CONN_TYPE_EXIT && (conn->base_.state == EXIT_CONN_STATE_CONNECTING || @@ -1504,8 +1512,14 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, switch (rh.command) { case RELAY_COMMAND_DROP: - rep_hist_padding_count_read(PADDING_TYPE_DROP); -// log_info(domain,"Got a relay-level padding cell. Dropping."); + /* Already examined in circpad_deliver_recognized_relay_cell_events */ + return 0; + case RELAY_COMMAND_PADDING_NEGOTIATE: + circpad_handle_padding_negotiate(circ, cell); + return 0; + case RELAY_COMMAND_PADDING_NEGOTIATED: + if (circpad_handle_padding_negotiated(circ, cell, layer_hint) == 0) + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); return 0; case RELAY_COMMAND_BEGIN: case RELAY_COMMAND_BEGIN_DIR: diff --git a/src/core/or/relay.h b/src/core/or/relay.h index 7cc3c43e43..044f6be156 100644 --- a/src/core/or/relay.h +++ b/src/core/or/relay.h @@ -78,6 +78,11 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue, void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); MOCK_DECL(int, channel_flush_from_first_active_circuit, (channel_t *chan, int max)); +MOCK_DECL(int, circuit_package_relay_cell, (cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction, + crypt_path_t *layer_hint, streamid_t on_stream, + const char *filename, int lineno)); + void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, const char *file, int lineno); #define update_circuit_on_cmux(circ, direction) \ diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c index f85201b7d5..ee22a38142 100644 --- a/src/core/or/scheduler.c +++ b/src/core/or/scheduler.c @@ -9,7 +9,7 @@ #define SCHEDULER_KIST_PRIVATE #include "core/or/scheduler.h" #include "core/mainloop/mainloop.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #define TOR_CHANNEL_INTERNAL_ #include "core/or/channeltls.h" #include "lib/evloop/compat_libevent.h" diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c index 79ecb0bc7e..1ec1c49c73 100644 --- a/src/core/or/scheduler_kist.c +++ b/src/core/or/scheduler_kist.c @@ -4,7 +4,7 @@ #define SCHEDULER_KIST_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/nodelist/networkstatus.h" @@ -724,7 +724,7 @@ kist_scheduler_run(void) SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) { scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING); if (!smartlist_contains(cp, readd_chan)) { - if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) { + if (!SCHED_BUG(readd_chan->sched_heap_idx != -1, readd_chan)) { /* XXXX Note that the check above is in theory redundant with * the smartlist_contains check. But let's make sure we're * not messing anything up, and leave them both for now. */ diff --git a/src/core/or/versions.c b/src/core/or/versions.c index 33273a5294..2a572d4704 100644 --- a/src/core/or/versions.c +++ b/src/core/or/versions.c @@ -16,6 +16,25 @@ #include "core/or/tor_version_st.h" +/** + * Return the approximate date when this release came out, or was + * scheduled to come out, according to the APPROX_RELEASE_DATE set in + * configure.ac + **/ +time_t +tor_get_approx_release_date(void) +{ + char tbuf[ISO_TIME_LEN+1]; + tor_snprintf(tbuf, sizeof(tbuf), + "%s 00:00:00", APPROX_RELEASE_DATE); + time_t result = 0; + int r = parse_iso_time(tbuf, &result); + if (BUG(r < 0)) { + result = 0; + } + return result; +} + /** Return VS_RECOMMENDED if <b>myversion</b> is contained in * <b>versionlist</b>. Else, return VS_EMPTY if versionlist has no * entries. Else, return VS_OLD if every member of @@ -377,6 +396,66 @@ sort_version_list(smartlist_t *versions, int remove_duplicates) smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_); } +/** If there are more than this many entries, we're probably under + * some kind of weird DoS. */ +static const int MAX_PROTOVER_SUMMARY_MAP_LEN = 1024; + +/** + * Map from protover string to protover_summary_flags_t. + */ +static strmap_t *protover_summary_map = NULL; + +/** + * Helper. Given a non-NULL protover string <b>protocols</b>, set <b>out</b> + * to its summary, and memoize the result in <b>protover_summary_map</b>. + */ +static void +memoize_protover_summary(protover_summary_flags_t *out, + const char *protocols) +{ + if (!protover_summary_map) + protover_summary_map = strmap_new(); + + if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) { + protover_summary_cache_free_all(); + tor_assert(protover_summary_map == NULL); + protover_summary_map = strmap_new(); + } + + const protover_summary_flags_t *cached = + strmap_get(protover_summary_map, protocols); + + if (cached != NULL) { + /* We found a cached entry; no need to parse this one. */ + memcpy(out, cached, sizeof(protover_summary_flags_t)); + tor_assert(out->protocols_known); + return; + } + + memset(out, 0, sizeof(*out)); + out->protocols_known = 1; + out->supports_extend2_cells = + protocol_list_supports_protocol(protocols, PRT_RELAY, 2); + out->supports_ed25519_link_handshake_compat = + protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3); + out->supports_ed25519_link_handshake_any = + protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3); + out->supports_ed25519_hs_intro = + protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4); + out->supports_v3_hsdir = + protocol_list_supports_protocol(protocols, PRT_HSDIR, + PROTOVER_HSDIR_V3); + out->supports_v3_rendezvous_point = + protocol_list_supports_protocol(protocols, PRT_HSREND, + PROTOVER_HS_RENDEZVOUS_POINT_V3); + out->supports_padding = + protocol_list_supports_protocol(protocols, PRT_PADDING, 1); + + protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out)); + cached = strmap_set(protover_summary_map, protocols, new_cached); + tor_assert(!cached); +} + /** Summarize the protocols listed in <b>protocols</b> into <b>out</b>, * falling back or correcting them based on <b>version</b> as appropriate. */ @@ -388,21 +467,7 @@ summarize_protover_flags(protover_summary_flags_t *out, tor_assert(out); memset(out, 0, sizeof(*out)); if (protocols) { - out->protocols_known = 1; - out->supports_extend2_cells = - protocol_list_supports_protocol(protocols, PRT_RELAY, 2); - out->supports_ed25519_link_handshake_compat = - protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3); - out->supports_ed25519_link_handshake_any = - protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3); - out->supports_ed25519_hs_intro = - protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4); - out->supports_v3_hsdir = - protocol_list_supports_protocol(protocols, PRT_HSDIR, - PROTOVER_HSDIR_V3); - out->supports_v3_rendezvous_point = - protocol_list_supports_protocol(protocols, PRT_HSREND, - PROTOVER_HS_RENDEZVOUS_POINT_V3); + memoize_protover_summary(out, protocols); } if (version && !strcmpstart(version, "Tor ")) { if (!out->protocols_known) { @@ -420,3 +485,13 @@ summarize_protover_flags(protover_summary_flags_t *out, } } } + +/** + * Free all space held in the protover_summary_map. + */ +void +protover_summary_cache_free_all(void) +{ + strmap_free(protover_summary_map, tor_free_); + protover_summary_map = NULL; +} diff --git a/src/core/or/versions.h b/src/core/or/versions.h index 22f3be176f..9aa7a0db87 100644 --- a/src/core/or/versions.h +++ b/src/core/or/versions.h @@ -26,6 +26,8 @@ typedef enum version_status_t { VS_UNKNOWN, /**< We have no idea. */ } version_status_t; +time_t tor_get_approx_release_date(void); + version_status_t tor_version_is_obsolete(const char *myversion, const char *versionlist); int tor_version_parse_platform(const char *platform, @@ -41,4 +43,6 @@ void summarize_protover_flags(protover_summary_flags_t *out, const char *protocols, const char *version); +void protover_summary_cache_free_all(void); + #endif /* !defined(TOR_VERSIONS_H) */ diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c index 0442e2c6ee..697fed29e1 100644 --- a/src/core/proto/proto_cell.c +++ b/src/core/proto/proto_cell.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_cell.h" #include "core/or/connection_or.h" diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c index 21fa328f02..d741f28f09 100644 --- a/src/core/proto/proto_control0.c +++ b/src/core/proto/proto_control0.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_control0.h" /** Return 1 iff buf looks more like it has an (obsolete) v0 controller diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c index edbc51b10c..4213bc14dd 100644 --- a/src/core/proto/proto_ext_or.c +++ b/src/core/proto/proto_ext_or.c @@ -5,7 +5,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "feature/relay/ext_orport.h" #include "core/proto/proto_ext_or.h" diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c index 5c86fc4979..88c59ef561 100644 --- a/src/core/proto/proto_http.c +++ b/src/core/proto/proto_http.c @@ -6,7 +6,7 @@ #define PROTO_HTTP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_http.h" /** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */ diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c index 8b78ed44c2..ac0c9e911b 100644 --- a/src/core/proto/proto_socks.c +++ b/src/core/proto/proto_socks.c @@ -6,7 +6,7 @@ #include "core/or/or.h" #include "feature/client/addressmap.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "app/config/config.h" diff --git a/src/ext/.may_include b/src/ext/.may_include new file mode 100644 index 0000000000..1eafff2eeb --- /dev/null +++ b/src/ext/.may_include @@ -0,0 +1,10 @@ + +orconfig.h + +lib/err/*.h +lib/cc/*.h + +tinytest*.h +ext/siphash.h +ext/byteorder.h +ext/tor_readpassphrase.h
\ No newline at end of file diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c index a6a9846db4..af8559a476 100644 --- a/src/ext/csiphash.c +++ b/src/ext/csiphash.c @@ -30,12 +30,12 @@ */ #include "lib/cc/torint.h" -#include "lib/log/util_bug.h" +#include "lib/err/torerr.h" -#include "siphash.h" +#include "ext/siphash.h" #include <string.h> #include <stdlib.h> -#include "byteorder.h" +#include "ext/byteorder.h" #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) ) @@ -112,13 +112,13 @@ static int the_siphash_key_is_set = 0; static struct sipkey the_siphash_key; uint64_t siphash24g(const void *src, unsigned long src_sz) { - tor_assert(the_siphash_key_is_set); + raw_assert(the_siphash_key_is_set); return siphash24(src, src_sz, &the_siphash_key); } void siphash_set_global_key(const struct sipkey *key) { - tor_assert(! the_siphash_key_is_set); + raw_assert(! the_siphash_key_is_set); the_siphash_key.k0 = key->k0; the_siphash_key.k1 = key->k1; the_siphash_key_is_set = 1; diff --git a/src/ext/readpassphrase.c b/src/ext/readpassphrase.c index e0df05d7b7..16611af1e2 100644 --- a/src/ext/readpassphrase.c +++ b/src/ext/readpassphrase.c @@ -30,7 +30,7 @@ #include <signal.h> #include <ctype.h> #include <fcntl.h> -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #include <errno.h> #include <string.h> #include <unistd.h> diff --git a/src/ext/timeouts/.may_include b/src/ext/timeouts/.may_include new file mode 100644 index 0000000000..42f44befd4 --- /dev/null +++ b/src/ext/timeouts/.may_include @@ -0,0 +1,6 @@ +orconfig.h + +ext/tor_queue.h +timeout-bitops.c +timeout-debug.h +timeout.h diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c index d4b514d2c5..07d06772c5 100644 --- a/src/ext/timeouts/timeout.c +++ b/src/ext/timeouts/timeout.c @@ -38,7 +38,7 @@ #include <errno.h> /* errno */ -#include "tor_queue.h" /* TAILQ(3) */ +#include "ext/tor_queue.h" /* TAILQ(3) */ #include "timeout.h" @@ -531,7 +531,7 @@ static timeout_t timeouts_int(struct timeouts *T) { timeout = MIN(_timeout, timeout); } - relmask <<= WHEEL_BIT; + relmask <<= WHEEL_BIT; relmask |= WHEEL_MASK; } @@ -751,4 +751,3 @@ TIMEOUT_PUBLIC int timeout_v_abi(void) { TIMEOUT_PUBLIC int timeout_v_api(void) { return TIMEOUT_V_API; } /* timeout_version() */ - diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h index b35874e153..1ed309fd08 100644 --- a/src/ext/timeouts/timeout.h +++ b/src/ext/timeouts/timeout.h @@ -31,7 +31,7 @@ #include <inttypes.h> /* PRIu64 PRIx64 PRIX64 uint64_t */ -#include "tor_queue.h" /* TAILQ(3) */ +#include "ext/tor_queue.h" /* TAILQ(3) */ /* @@ -147,7 +147,7 @@ TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int); #ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS TIMEOUT_PUBLIC bool timeout_pending(struct timeout *); /* true if on timing wheel, false otherwise */ - + TIMEOUT_PUBLIC bool timeout_expired(struct timeout *); /* true if on expired queue, false otherwise */ diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c index 626c5efcae..05f89ad36c 100644 --- a/src/feature/client/bridges.c +++ b/src/feature/client/bridges.c @@ -844,7 +844,8 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } } - if (options->ClientPreferIPv6ORPort == -1) { + if (options->ClientPreferIPv6ORPort == -1 || + options->ClientAutoIPv6ORPort == 0) { /* Mark which address to use based on which bridge_t we got. */ node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 && !tor_addr_is_null(&node->ri->ipv6_addr)); diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c index cf593cfb68..44e0caaafa 100644 --- a/src/feature/client/dnsserv.c +++ b/src/feature/client/dnsserv.c @@ -28,6 +28,7 @@ #include "core/or/connection_edge.h" #include "feature/control/control.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/or/policies.h" #include "feature/control/control_connection_st.h" @@ -213,6 +214,9 @@ dnsserv_launch_request(const char *name, int reverse, edge_connection_t *conn; char *q_name; + /* Launching a request for a user counts as user activity. */ + note_user_activity(approx_time()); + /* Make a new dummy AP connection, and attach the request to it. */ entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET); entry_conn->entry_cfg.dns_request = 1; diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 0d1342c87b..f34d4af480 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -101,11 +101,13 @@ #include "core/or/connection_or.h" #include "feature/relay/ext_orport.h" #include "feature/control/control.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" +#include "lib/process/process.h" #include "lib/process/env.h" -#include "lib/process/subprocess.h" -static process_environment_t * +static smartlist_t * create_managed_proxy_environment(const managed_proxy_t *mp); static inline int proxy_configuration_finished(const managed_proxy_t *mp); @@ -127,6 +129,8 @@ static void parse_method_error(const char *line, int is_server_method); #define PROTO_SMETHODS_DONE "SMETHODS DONE" #define PROTO_PROXY_DONE "PROXY DONE" #define PROTO_PROXY_ERROR "PROXY-ERROR" +#define PROTO_LOG "LOG" +#define PROTO_STATUS "STATUS" /** The first and only supported - at the moment - configuration protocol version. */ @@ -490,8 +494,8 @@ proxy_prepare_for_restart(managed_proxy_t *mp) tor_assert(mp->conf_state == PT_PROTO_COMPLETED); /* destroy the process handle and terminate the process. */ - tor_process_handle_destroy(mp->process_handle, 1); - mp->process_handle = NULL; + process_set_data(mp->process, NULL); + process_terminate(mp->process); /* destroy all its registered transports, since we will no longer use them. */ @@ -520,34 +524,35 @@ proxy_prepare_for_restart(managed_proxy_t *mp) static int launch_managed_proxy(managed_proxy_t *mp) { - int retval; - - process_environment_t *env = create_managed_proxy_environment(mp); - -#ifdef _WIN32 - /* Passing NULL as lpApplicationName makes Windows search for the .exe */ - retval = tor_spawn_background(NULL, - (const char **)mp->argv, - env, - &mp->process_handle); -#else /* !(defined(_WIN32)) */ - retval = tor_spawn_background(mp->argv[0], - (const char **)mp->argv, - env, - &mp->process_handle); -#endif /* defined(_WIN32) */ - - process_environment_free(env); - - if (retval == PROCESS_STATUS_ERROR) { - log_warn(LD_GENERAL, "Managed proxy at '%s' failed at launch.", + tor_assert(mp); + + smartlist_t *env = create_managed_proxy_environment(mp); + + /* Configure our process. */ + process_set_data(mp->process, mp); + process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback); + process_set_stderr_read_callback(mp->process, managed_proxy_stderr_callback); + process_set_exit_callback(mp->process, managed_proxy_exit_callback); + process_set_protocol(mp->process, PROCESS_PROTOCOL_LINE); + process_reset_environment(mp->process, env); + + /* Cleanup our env. */ + SMARTLIST_FOREACH(env, char *, x, tor_free(x)); + smartlist_free(env); + + /* Skip the argv[0] as we get that from process_new(argv[0]). */ + for (int i = 1; mp->argv[i] != NULL; ++i) + process_append_argument(mp->process, mp->argv[i]); + + if (process_exec(mp->process) != PROCESS_STATUS_RUNNING) { + log_warn(LD_CONFIG, "Managed proxy at '%s' failed at launch.", mp->argv[0]); return -1; } - log_info(LD_CONFIG, "Managed proxy at '%s' has spawned with PID '%d'.", - mp->argv[0], tor_process_get_pid(mp->process_handle)); - + log_info(LD_CONFIG, + "Managed proxy at '%s' has spawned with PID '%" PRIu64 "'.", + mp->argv[0], process_get_pid(mp->process)); mp->conf_state = PT_PROTO_LAUNCHED; return 0; @@ -615,10 +620,6 @@ pt_configure_remaining_proxies(void) STATIC int configure_proxy(managed_proxy_t *mp) { - int configuration_finished = 0; - smartlist_t *proxy_output = NULL; - enum stream_status stream_status = 0; - /* if we haven't launched the proxy yet, do it now */ if (mp->conf_state == PT_PROTO_INFANT) { if (launch_managed_proxy(mp) < 0) { /* launch fail */ @@ -629,45 +630,8 @@ configure_proxy(managed_proxy_t *mp) } tor_assert(mp->conf_state != PT_PROTO_INFANT); - tor_assert(mp->process_handle); - - proxy_output = - tor_get_lines_from_handle(tor_process_get_stdout_pipe(mp->process_handle), - &stream_status); - if (!proxy_output) { /* failed to get input from proxy */ - if (stream_status != IO_STREAM_EAGAIN) { /* bad stream status! */ - mp->conf_state = PT_PROTO_BROKEN; - log_warn(LD_GENERAL, "The communication stream of managed proxy '%s' " - "is '%s'. Most probably the managed proxy stopped running. " - "This might be a bug of the managed proxy, a bug of Tor, or " - "a misconfiguration. Please enable logging on your managed " - "proxy and check the logs for errors.", - mp->argv[0], stream_status_to_string(stream_status)); - } - - goto done; - } - - /* Handle lines. */ - SMARTLIST_FOREACH_BEGIN(proxy_output, const char *, line) { - handle_proxy_line(line, mp); - if (proxy_configuration_finished(mp)) - goto done; - } SMARTLIST_FOREACH_END(line); - - done: - /* if the proxy finished configuring, exit the loop. */ - if (proxy_configuration_finished(mp)) { - handle_finished_proxy(mp); - configuration_finished = 1; - } - - if (proxy_output) { - SMARTLIST_FOREACH(proxy_output, char *, cp, tor_free(cp)); - smartlist_free(proxy_output); - } - - return configuration_finished; + tor_assert(mp->process); + return mp->conf_state == PT_PROTO_COMPLETED; } /** Register server managed proxy <b>mp</b> transports to state */ @@ -748,8 +712,14 @@ managed_proxy_destroy(managed_proxy_t *mp, /* free the outgoing proxy URI */ tor_free(mp->proxy_uri); - tor_process_handle_destroy(mp->process_handle, also_terminate_process); - mp->process_handle = NULL; + /* do we want to terminate our process if it's still running? */ + if (also_terminate_process && mp->process) { + /* Note that we do not call process_free(mp->process) here because we let + * the exit handler in managed_proxy_exit_callback() return `true` which + * makes the process subsystem deallocate the process_t. */ + process_set_data(mp->process, NULL); + process_terminate(mp->process); + } tor_free(mp); } @@ -945,21 +915,16 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) parse_proxy_error(line); goto err; - } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) { - /* managed proxy launch failed: parse error message to learn why. */ - int retval, child_state, saved_errno; - retval = tor_sscanf(line, SPAWN_ERROR_MESSAGE "%x/%x", - &child_state, &saved_errno); - if (retval == 2) { - log_warn(LD_GENERAL, - "Could not launch managed proxy executable at '%s' ('%s').", - mp->argv[0], strerror(saved_errno)); - } else { /* failed to parse error message */ - log_warn(LD_GENERAL,"Could not launch managed proxy executable at '%s'.", - mp->argv[0]); - } - mp->conf_state = PT_PROTO_FAILED_LAUNCH; + /* We check for the additional " " after the PROTO_LOG * PROTO_STATUS + * string to make sure we can later extend this big if/else-if table with + * something that begins with "LOG" without having to get the order right. + * */ + } else if (!strcmpstart(line, PROTO_LOG " ")) { + parse_log_line(line, mp); + return; + } else if (!strcmpstart(line, PROTO_STATUS " ")) { + parse_status_line(line, mp); return; } @@ -1182,6 +1147,121 @@ parse_proxy_error(const char *line) line+strlen(PROTO_PROXY_ERROR)+1); } +/** Parses a LOG <b>line</b> and emit log events accordingly. */ +STATIC void +parse_log_line(const char *line, managed_proxy_t *mp) +{ + tor_assert(line); + tor_assert(mp); + + config_line_t *values = NULL; + char *log_message = NULL; + + if (strlen(line) < (strlen(PROTO_LOG) + 1)) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with missing argument.", PROTO_LOG); + goto done; + } + + const char *data = line + strlen(PROTO_LOG) + 1; + values = kvline_parse(data, KV_QUOTED); + + if (! values) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid LOG message: %s", + mp->argv[0], data); + goto done; + } + + const config_line_t *severity = config_line_find(values, "SEVERITY"); + const config_line_t *message = config_line_find(values, "MESSAGE"); + + /* Check if we got a message. */ + if (! message) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without " + "MESSAGE: %s", mp->argv[0], escaped(data)); + goto done; + } + + /* Check if severity is there and whether it's valid. */ + if (! severity) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without " + "SEVERITY: %s", mp->argv[0], escaped(data)); + goto done; + } + + int log_severity = managed_proxy_severity_parse(severity->value); + + if (log_severity == -1) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line with an " + "invalid severity level: %s", + mp->argv[0], severity->value); + goto done; + } + + tor_log(log_severity, LD_PT, "Managed proxy \"%s\": %s", + mp->argv[0], message->value); + + /* Prepend the PT name. */ + config_line_prepend(&values, "PT", mp->argv[0]); + log_message = kvline_encode(values, KV_QUOTED); + + /* Emit control port event. */ + control_event_pt_log(log_message); + + done: + config_free_lines(values); + tor_free(log_message); +} + +/** Parses a STATUS <b>line</b> and emit control events accordingly. */ +STATIC void +parse_status_line(const char *line, managed_proxy_t *mp) +{ + tor_assert(line); + tor_assert(mp); + + config_line_t *values = NULL; + char *status_message = NULL; + + if (strlen(line) < (strlen(PROTO_STATUS) + 1)) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with missing argument.", PROTO_STATUS); + goto done; + } + + const char *data = line + strlen(PROTO_STATUS) + 1; + + values = kvline_parse(data, KV_QUOTED); + + if (! values) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid " + "STATUS message: %s", mp->argv[0], escaped(data)); + goto done; + } + + /* We check if we received the TRANSPORT parameter, which is the only + * *required* value. */ + const config_line_t *type = config_line_find(values, "TRANSPORT"); + + if (! type) { + log_warn(LD_PT, "Managed proxy \"%s\" wrote a STATUS line without " + "TRANSPORT: %s", mp->argv[0], escaped(data)); + goto done; + } + + /* Prepend the PT name. */ + config_line_prepend(&values, "PT", mp->argv[0]); + status_message = kvline_encode(values, KV_QUOTED); + + /* We have checked that TRANSPORT is there, we can now emit the STATUS event + * via the control port. */ + control_event_pt_status(status_message); + + done: + config_free_lines(values); + tor_free(status_message); +} + /** Return a newly allocated string that tor should place in * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server * manged proxy in <b>mp</b>. Return NULL if no such options are found. */ @@ -1257,7 +1337,7 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp) /** Return a newly allocated process_environment_t * for <b>mp</b>'s * process. */ -static process_environment_t * +static smartlist_t * create_managed_proxy_environment(const managed_proxy_t *mp) { const or_options_t *options = get_options(); @@ -1272,8 +1352,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp) /* The final environment to be passed to mp. */ smartlist_t *merged_env_vars = get_current_process_environment_variables(); - process_environment_t *env; - { char *state_tmp = get_datadir_fname("pt_state/"); /* XXX temp */ smartlist_add_asprintf(envs, "TOR_PT_STATE_LOCATION=%s", state_tmp); @@ -1366,14 +1444,9 @@ create_managed_proxy_environment(const managed_proxy_t *mp) tor_free_, 1); } SMARTLIST_FOREACH_END(env_var); - env = process_environment_make(merged_env_vars); - smartlist_free(envs); - SMARTLIST_FOREACH(merged_env_vars, void *, x, tor_free(x)); - smartlist_free(merged_env_vars); - - return env; + return merged_env_vars; } /** Create and return a new managed proxy for <b>transport</b> using @@ -1392,6 +1465,7 @@ managed_proxy_create(const smartlist_t *with_transport_list, mp->argv = proxy_argv; mp->transports = smartlist_new(); mp->proxy_uri = get_pt_proxy_uri(); + mp->process = process_new(proxy_argv[0]); mp->transports_to_launch = smartlist_new(); SMARTLIST_FOREACH(with_transport_list, const char *, transport, @@ -1736,3 +1810,93 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape) return new_string; } + +/** Callback function that is called when our PT process have data on its + * stdout. Our process can be found in <b>process</b>, the data can be found in + * <b>line</b> and the length of our line is given in <b>size</b>. */ +STATIC void +managed_proxy_stdout_callback(process_t *process, + const char *line, + size_t size) +{ + tor_assert(process); + tor_assert(line); + + (void)size; + + managed_proxy_t *mp = process_get_data(process); + + if (mp == NULL) + return; + + handle_proxy_line(line, mp); + + if (proxy_configuration_finished(mp)) + handle_finished_proxy(mp); +} + +/** Callback function that is called when our PT process have data on its + * stderr. Our process can be found in <b>process</b>, the data can be found in + * <b>line</b> and the length of our line is given in <b>size</b>. */ +STATIC void +managed_proxy_stderr_callback(process_t *process, + const char *line, + size_t size) +{ + tor_assert(process); + tor_assert(line); + + (void)size; + + managed_proxy_t *mp = process_get_data(process); + + if (BUG(mp == NULL)) + return; + + log_warn(LD_PT, "Managed proxy at '%s' reported: %s", mp->argv[0], line); +} + +/** Callback function that is called when our PT process terminates. The + * process exit code can be found in <b>exit_code</b> and our process can be + * found in <b>process</b>. Returns true iff we want the process subsystem to + * free our process_t handle for us. */ +STATIC bool +managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_warn(LD_PT, + "Pluggable Transport process terminated with status code %" PRIu64, + exit_code); + + /* Returning true here means that the process subsystem will take care of + * calling process_free() on our process_t. */ + return true; +} + +/** Returns a valid integer log severity level from <b>severity</b> that + * is compatible with Tor's logging functions. Returns <b>-1</b> on + * error. */ +STATIC int +managed_proxy_severity_parse(const char *severity) +{ + tor_assert(severity); + + /* Slightly different than log.c's parse_log_level :-( */ + if (! strcmp(severity, "debug")) + return LOG_DEBUG; + + if (! strcmp(severity, "info")) + return LOG_INFO; + + if (! strcmp(severity, "notice")) + return LOG_NOTICE; + + if (! strcmp(severity, "warning")) + return LOG_WARN; + + if (! strcmp(severity, "error")) + return LOG_ERR; + + return -1; +} diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index e2fa45828f..900dd9288e 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -11,6 +11,8 @@ #ifndef TOR_TRANSPORTS_H #define TOR_TRANSPORTS_H +#include "lib/process/process.h" + /** Represents a pluggable transport used by a bridge. */ typedef struct transport_t { /** SOCKS version: One of PROXY_SOCKS4, PROXY_SOCKS5. */ @@ -81,7 +83,7 @@ enum pt_proto_state { PT_PROTO_FAILED_LAUNCH /* failed while launching */ }; -struct process_handle_t; +struct process_t; /** Structure containing information of a managed proxy. */ typedef struct { @@ -94,10 +96,8 @@ typedef struct { int is_server; /* is it a server proxy? */ - /* A pointer to the process handle of this managed proxy. */ - struct process_handle_t *process_handle; - - int pid; /* The Process ID this managed proxy is using. */ + /* A pointer to the process of this managed proxy. */ + struct process_t *process; /** Boolean: We are re-parsing our config, and we are going to * remove this managed proxy if we don't find it any transport @@ -128,6 +128,8 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp); STATIC void parse_env_error(const char *line); STATIC void parse_proxy_error(const char *line); STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC void parse_log_line(const char *line, managed_proxy_t *mp); +STATIC void parse_status_line(const char *line, managed_proxy_t *mp); STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); STATIC void managed_proxy_destroy(managed_proxy_t *mp, @@ -142,6 +144,12 @@ STATIC char* get_pt_proxy_uri(void); STATIC void free_execve_args(char **arg); +STATIC void managed_proxy_stdout_callback(process_t *, const char *, size_t); +STATIC void managed_proxy_stderr_callback(process_t *, const char *, size_t); +STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t); + +STATIC int managed_proxy_severity_parse(const char *); + #endif /* defined(PT_PRIVATE) */ #endif /* !defined(TOR_TRANSPORTS_H) */ diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c new file mode 100644 index 0000000000..d3d12cb2b7 --- /dev/null +++ b/src/feature/control/btrack.c @@ -0,0 +1,53 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack.c + * \brief Bootstrap trackers + * + * Initializes and shuts down the specific bootstrap trackers. These + * trackers help the reporting of bootstrap progress by maintaining + * state information about various subsystems within tor. When the + * correct state changes happen, these trackers emit controller + * events. + * + * These trackers avoid referring directly to the internals of state + * objects of other subsystems. + * + * btrack_circuit.c contains the tracker for origin circuits. + * + * btrack_orconn.c contains the tracker for OR connections. + * + * Eventually there will be a tracker for directory downloads as well. + **/ + +#include "feature/control/btrack_circuit.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_sys.h" +#include "lib/subsys/subsys.h" + +static int +btrack_init(void) +{ + if (btrack_orconn_init()) + return -1; + if (btrack_circ_init()) + return -1; + + return 0; +} + +static void +btrack_fini(void) +{ + btrack_orconn_fini(); + btrack_circ_fini(); +} + +const subsys_fns_t sys_btrack = { + .name = "btrack", + .supported = true, + .level = -30, + .initialize = btrack_init, + .shutdown = btrack_fini, +}; diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c new file mode 100644 index 0000000000..dcee9e460e --- /dev/null +++ b/src/feature/control/btrack_circuit.c @@ -0,0 +1,164 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_circuit.c + * \brief Bootstrap tracker for origin circuits + * + * Track state changes of origin circuits, as published by the circuit + * subsystem. + **/ + +#include "core/or/or.h" + +#include "core/or/ocirc_event.h" + +#include "feature/control/btrack_circuit.h" +#include "feature/control/control.h" +#include "lib/log/log.h" + +/** Pair of a best origin circuit GID with its state or status */ +typedef struct btc_best_t { + uint32_t gid; + int val; +} btc_best_t; + +/** GID and state of the best origin circuit we've seen so far */ +static btc_best_t best_any_state = { 0, -1 }; +/** GID and state of the best application circuit we've seen so far */ +static btc_best_t best_ap_state = { 0, -1 }; +/** GID and status of the best origin circuit we've seen so far */ +static btc_best_t best_any_evtype = { 0, -1 }; +/** GID and status of the best application circuit we've seen so far */ +static btc_best_t best_ap_evtype = { 0, -1 }; + +/** Reset cached "best" values */ +static void +btc_reset_bests(void) +{ + best_any_state.gid = best_ap_state.gid = 0; + best_any_state.val = best_ap_state.val = -1; + best_any_evtype.gid = best_ap_state.gid = 0; + best_any_evtype.val = best_ap_evtype.val = -1; +} + +/** True if @a state is a "better" origin circuit state than @a best->val */ +static bool +btc_state_better(int state, const btc_best_t *best) +{ + return state > best->val; +} + +/** + * Definine an ordering on circuit status events + * + * The CIRC_EVENT_ constants aren't sorted in a useful order, so this + * array helps to decode them. This approach depends on the statuses + * being nonnegative and dense. + **/ +static int circ_event_order[] = { + [CIRC_EVENT_FAILED] = -1, + [CIRC_EVENT_CLOSED] = -1, + [CIRC_EVENT_LAUNCHED] = 1, + [CIRC_EVENT_EXTENDED] = 2, + [CIRC_EVENT_BUILT] = 3, +}; +#define N_CIRC_EVENT_ORDER \ + (sizeof(circ_event_order) / sizeof(circ_event_order[0])) + +/** True if @a state is a "better" origin circuit event status than @a + best->val */ +static bool +btc_evtype_better(int state, const btc_best_t *best) +{ + if (state < 0) + return false; + if (best->val < 0) + return true; + + tor_assert(state >= 0 && (unsigned)state < N_CIRC_EVENT_ORDER); + tor_assert(best->val >= 0 && (unsigned)best->val < N_CIRC_EVENT_ORDER); + return circ_event_order[state] > circ_event_order[best->val]; +} + +static bool +btc_update_state(const ocirc_state_msg_t *msg, btc_best_t *best, + const char *type) +{ + if (btc_state_better(msg->state, best)) { + log_info(LD_BTRACK, "CIRC BEST_%s state %d->%d gid=%"PRIu32, type, + best->val, msg->state, msg->gid); + best->gid = msg->gid; + best->val = msg->state; + return true; + } + return false; +} + +static bool +btc_update_evtype(const ocirc_cevent_msg_t *msg, btc_best_t *best, + const char *type) +{ + if (btc_evtype_better(msg->evtype, best)) { + log_info(LD_BTRACK, "CIRC BEST_%s evtype %d->%d gid=%"PRIu32, type, + best->val, msg->evtype, msg->gid); + best->gid = msg->gid; + best->val = msg->evtype; + return true; + } + return false; +} + +static void +btc_state_rcvr(const ocirc_state_msg_t *msg) +{ + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" state=%d onehop=%d", + msg->gid, msg->state, msg->onehop); + + btc_update_state(msg, &best_any_state, "ANY"); + if (msg->onehop) + return; + btc_update_state(msg, &best_ap_state, "AP"); +} + +static void +btc_cevent_rcvr(const ocirc_cevent_msg_t *msg) +{ + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" evtype=%d reason=%d onehop=%d", + msg->gid, msg->evtype, msg->reason, msg->onehop); + + btc_update_evtype(msg, &best_any_evtype, "ANY"); + if (msg->onehop) + return; + btc_update_evtype(msg, &best_ap_evtype, "AP"); +} + +static void +btc_event_rcvr(const ocirc_event_msg_t *msg) +{ + switch (msg->type) { + case OCIRC_MSGTYPE_STATE: + return btc_state_rcvr(&msg->u.state); + case OCIRC_MSGTYPE_CHAN: + log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d", + msg->u.chan.gid, msg->u.chan.chan, msg->u.chan.onehop); + break; + case OCIRC_MSGTYPE_CEVENT: + return btc_cevent_rcvr(&msg->u.cevent); + default: + break; + } +} + +int +btrack_circ_init(void) +{ + ocirc_event_subscribe(btc_event_rcvr); + return 0; +} + +void +btrack_circ_fini(void) +{ + btc_reset_bests(); +} diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h new file mode 100644 index 0000000000..c40822f1f1 --- /dev/null +++ b/src/feature/control/btrack_circuit.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_circuit.h + * \brief Header file for btrack_circuit.c + **/ + +#ifndef TOR_BTRACK_CIRCUIT_H +#define TOR_BTRACK_CIRCUIT_H + +int btrack_circ_init(void); +void btrack_circ_fini(void); + +#endif /* defined(TOR_BTRACK_CIRCUIT_H) */ diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c new file mode 100644 index 0000000000..93ebe8d9cc --- /dev/null +++ b/src/feature/control/btrack_orconn.c @@ -0,0 +1,206 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn.c + * \brief Bootstrap tracker for OR connections + * + * Track state changes of OR connections, as published by the + * connection subsystem. Also track circuit launch events, because + * they're one of the few ways to discover the association between a + * channel (and OR connection) and a circuit. + * + * We track all OR connections that we receive events for, whether or + * not they're carrying origin circuits. (An OR connection might + * carry origin circuits only after we first find out about that + * connection.) + * + * All origin ORCONN events update the "any" state variables, while + * only application ORCONN events update the "ap" state variables (and + * also update the "any") variables. + * + * We do this because we want to report the first increments of + * connection progress as the earliest bootstrap phases. This results + * in a better user experience because failures here translate into + * zero or very small amounts of displayed progress, instead of + * progress stuck near completion. The first connection to a relay + * might be a one-hop circuit for directory lookups, or it might be a + * connection for an application circuit because we already have + * enough directory info to build an application circuit. + * + * We call functions in btrack_orconn_cevent.c to generate the actual + * controller events, because some of the state decoding we need to do + * is complicated. + **/ + +#include <stdbool.h> + +#include "core/or/or.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_cevent.h" +#include "feature/control/btrack_orconn_maps.h" +#include "lib/log/log.h" + +/** Pair of a best ORCONN GID and with its state */ +typedef struct bto_best_t { + uint64_t gid; + int state; +} bto_best_t; + +/** GID and state of the best ORCONN we've seen so far */ +static bto_best_t best_any = { 0, -1 }; +/** GID and state of the best application circuit ORCONN we've seen so far */ +static bto_best_t best_ap = { 0, -1 }; + +/** + * Update a cached state of a best ORCONN progress we've seen so far. + * + * Return true if the new state is better than the old. + **/ +static bool +bto_update_best(const bt_orconn_t *bto, bto_best_t *best, const char *type) +{ + if (bto->state < best->state) + return false; + /* Update even if we won't change best->state, because it's more + * recent information that a particular connection transitioned to + * that state. */ + best->gid = bto->gid; + if (bto->state > best->state) { + log_info(LD_BTRACK, "ORCONN BEST_%s state %d->%d gid=%"PRIu64, type, + best->state, bto->state, bto->gid); + best->state = bto->state; + return true; + } + return false; +} + +/** + * Update cached states of best ORCONN progress we've seen + * + * Only update the application ORCONN state if we know it's carrying + * an application circuit. + **/ +static void +bto_update_bests(const bt_orconn_t *bto) +{ + tor_assert(bto->is_orig); + + if (bto_update_best(bto, &best_any, "ANY")) + bto_cevent_anyconn(bto); + if (!bto->is_onehop && bto_update_best(bto, &best_ap, "AP")) + bto_cevent_apconn(bto); +} + +/** Reset cached "best" values */ +static void +bto_reset_bests(void) +{ + best_any.gid = best_ap.gid = 0; + best_any.state = best_ap.state = -1; +} + +/** + * Update cached states of ORCONNs from the incoming message. This + * message comes from code in connection_or.c. + **/ +static void +bto_state_rcvr(const orconn_state_msg_t *msg) +{ + bt_orconn_t *bto; + + bto = bto_find_or_new(msg->gid, msg->chan); + log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" chan=%"PRIu64 + " proxy_type=%d state=%d", + msg->gid, msg->chan, msg->proxy_type, msg->state); + bto->proxy_type = msg->proxy_type; + bto->state = msg->state; + if (bto->is_orig) + bto_update_bests(bto); +} + +/** + * Delete a cached ORCONN state if we get an incoming message saying + * the ORCONN is failed or closed. This message comes from code in + * control.c. + **/ +static void +bto_status_rcvr(const orconn_status_msg_t *msg) +{ + switch (msg->status) { + case OR_CONN_EVENT_FAILED: + case OR_CONN_EVENT_CLOSED: + log_info(LD_BTRACK, "ORCONN DELETE gid=%"PRIu64" status=%d reason=%d", + msg->gid, msg->status, msg->reason); + return bto_delete(msg->gid); + default: + break; + } +} + +/** Dispatch to individual ORCONN message handlers */ +static void +bto_event_rcvr(const orconn_event_msg_t *msg) +{ + switch (msg->type) { + case ORCONN_MSGTYPE_STATE: + return bto_state_rcvr(&msg->u.state); + case ORCONN_MSGTYPE_STATUS: + return bto_status_rcvr(&msg->u.status); + default: + tor_assert(false); + } +} + +/** + * Create or update a cached ORCONN state for a newly launched + * connection, including whether it's launched by an origin circuit + * and whether it's a one-hop circuit. + **/ +static void +bto_chan_rcvr(const ocirc_event_msg_t *msg) +{ + bt_orconn_t *bto; + + /* Ignore other kinds of origin circuit events; we don't need them */ + if (msg->type != OCIRC_MSGTYPE_CHAN) + return; + + bto = bto_find_or_new(0, msg->u.chan.chan); + if (!bto->is_orig || (bto->is_onehop && !msg->u.chan.onehop)) { + log_debug(LD_BTRACK, "ORCONN LAUNCH chan=%"PRIu64" onehop=%d", + msg->u.chan.chan, msg->u.chan.onehop); + } + bto->is_orig = true; + if (!msg->u.chan.onehop) + bto->is_onehop = false; + bto_update_bests(bto); +} + +/** + * Initialize the hash maps and subscribe to ORCONN and origin + * circuit events. + **/ +int +btrack_orconn_init(void) +{ + bto_init_maps(); + orconn_event_subscribe(bto_event_rcvr); + ocirc_event_subscribe(bto_chan_rcvr); + + return 0; +} + +/** Clear the hash maps and reset the "best" states */ +void +btrack_orconn_fini(void) +{ + bto_clear_maps(); + bto_reset_bests(); + bto_cevent_reset(); +} diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h new file mode 100644 index 0000000000..6ab4892a78 --- /dev/null +++ b/src/feature/control/btrack_orconn.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn.h + * \brief Header file for btrack_orconn.c + **/ + +#ifndef TOR_BTRACK_ORCONN_H +#define TOR_BTRACK_ORCONN_H + +#ifdef BTRACK_ORCONN_PRIVATE + +#include "ht.h" + +/** + * Structure for tracking OR connection states + * + * This gets linked into two hash maps: one with connection IDs, and + * another with channel IDs. + **/ +typedef struct bt_orconn_t { + HT_ENTRY(bt_orconn_t) node; /**< Hash map entry indexed by gid */ + HT_ENTRY(bt_orconn_t) chan_node; /**< Hash map entry indexed by channel ID */ + uint64_t gid; /**< Global ID of this ORCONN */ + uint64_t chan; /**< Channel ID, if known */ + int proxy_type; /**< Proxy type */ + uint8_t state; /**< State of this ORCONN */ + bool is_orig; /**< Does this carry an origin circuit? */ + bool is_onehop; /**< Is this for a one-hop circuit? */ +} bt_orconn_t; + +#endif /* defined(BTRACK_ORCONN_PRIVATE) */ + +int btrack_orconn_init(void); +void btrack_orconn_fini(void); + +#endif /* defined(TOR_BTRACK_ORCONN_H) */ diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c new file mode 100644 index 0000000000..ee142f2873 --- /dev/null +++ b/src/feature/control/btrack_orconn_cevent.c @@ -0,0 +1,159 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_cevent.c + * \brief Emit bootstrap status events for OR connections + * + * We do some decoding of the raw OR_CONN_STATE_* values. For + * example, OR_CONN_STATE_CONNECTING means the first TCP connect() + * completing, regardless of whether it's directly to a relay instead + * of a proxy or a PT. + **/ + +#include <stdbool.h> + +#include "core/or/or.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "core/or/orconn_event.h" +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_cevent.h" +#include "feature/control/control.h" + +/** + * Have we completed our first OR connection? + * + * Block display of application circuit progress until we do, to avoid + * some misleading behavior of jumping to high progress. + **/ +static bool bto_first_orconn = false; + +/** Is the ORCONN using a pluggable transport? */ +static bool +using_pt(const bt_orconn_t *bto) +{ + return bto->proxy_type == PROXY_PLUGGABLE; +} + +/** Is the ORCONN using a non-PT proxy? */ +static bool +using_proxy(const bt_orconn_t *bto) +{ + switch (bto->proxy_type) { + case PROXY_CONNECT: + case PROXY_SOCKS4: + case PROXY_SOCKS5: + return true; + default: + return false; + } +} + +/** + * Emit control events when we have updated our idea of the best state + * that any OR connection has reached. + * + * Do some decoding of the ORCONN states depending on whether a PT or + * a proxy is in use. + **/ +void +bto_cevent_anyconn(const bt_orconn_t *bto) +{ + switch (bto->state) { + case OR_CONN_STATE_CONNECTING: + /* Exactly what kind of thing we're connecting to isn't + * information we directly get from the states in connection_or.c, + * so decode it here. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PROXY, 0); + else + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + break; + case OR_CONN_STATE_PROXY_HANDSHAKING: + /* Similarly, starting a proxy handshake means the TCP connect() + * succeeded to the proxy. Let's be specific about what kind of + * proxy. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PROXY, 0); + break; + case OR_CONN_STATE_TLS_HANDSHAKING: + control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE, 0); + break; + case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: + case OR_CONN_STATE_OR_HANDSHAKING_V2: + case OR_CONN_STATE_OR_HANDSHAKING_V3: + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); + break; + case OR_CONN_STATE_OPEN: + control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DONE, 0); + /* Unblock directory progress display */ + control_event_boot_first_orconn(); + /* Unblock apconn progress display */ + bto_first_orconn = true; + break; + default: + break; + } +} + +/** + * Emit control events when we have updated our idea of the best state + * that any application circuit OR connection has reached. + * + * Do some decoding of the ORCONN states depending on whether a PT or + * a proxy is in use. + **/ +void +bto_cevent_apconn(const bt_orconn_t *bto) +{ + if (!bto_first_orconn) + return; + + switch (bto->state) { + case OR_CONN_STATE_CONNECTING: + /* Exactly what kind of thing we're connecting to isn't + * information we directly get from the states in connection_or.c, + * so decode it here. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PROXY, 0); + else + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN, 0); + break; + case OR_CONN_STATE_PROXY_HANDSHAKING: + /* Similarly, starting a proxy handshake means the TCP connect() + * succeeded to the proxy. Let's be specific about what kind of + * proxy. */ + if (using_pt(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PT, 0); + else if (using_proxy(bto)) + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, 0); + break; + case OR_CONN_STATE_TLS_HANDSHAKING: + control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE, 0); + break; + case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING: + case OR_CONN_STATE_OR_HANDSHAKING_V2: + case OR_CONN_STATE_OR_HANDSHAKING_V3: + control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE, 0); + break; + case OR_CONN_STATE_OPEN: + control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, 0); + default: + break; + } +} + +/** Forget that we completed our first OR connection */ +void +bto_cevent_reset(void) +{ + bto_first_orconn = false; +} diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h new file mode 100644 index 0000000000..f9d24633aa --- /dev/null +++ b/src/feature/control/btrack_orconn_cevent.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_cevent.h + * \brief Header file for btrack_orconn_cevent.c + **/ + +#ifndef TOR_BTRACK_ORCONN_CEVENT_H + +#include "feature/control/btrack_orconn.h" + +void bto_cevent_anyconn(const bt_orconn_t *); +void bto_cevent_apconn(const bt_orconn_t *); +void bto_cevent_reset(void); + +#endif /* defined(TOR_BTRACK_ORCONN_CEVENT_H) */ diff --git a/src/feature/control/btrack_orconn_maps.c b/src/feature/control/btrack_orconn_maps.c new file mode 100644 index 0000000000..e64bd3f0fe --- /dev/null +++ b/src/feature/control/btrack_orconn_maps.c @@ -0,0 +1,223 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_maps.c + * \brief Hash map implementation for btrack_orconn.c + * + * These functions manipulate the hash maps that contain bt_orconn + * objects. + **/ + +#include <stdbool.h> + +#include "core/or/or.h" + +#include "ht.h" +#include "siphash.h" + +#define BTRACK_ORCONN_PRIVATE + +#include "feature/control/btrack_orconn.h" +#include "feature/control/btrack_orconn_maps.h" +#include "lib/log/log.h" + +static inline unsigned int +bto_gid_hash_(bt_orconn_t *elm) +{ + return (unsigned)siphash24g(&elm->gid, sizeof(elm->gid)); +} + +static inline int +bto_gid_eq_(bt_orconn_t *a, bt_orconn_t *b) +{ + return a->gid == b->gid; +} + +static inline unsigned int +bto_chan_hash_(bt_orconn_t *elm) +{ + return (unsigned)siphash24g(&elm->chan, sizeof(elm->chan)); +} + +static inline int +bto_chan_eq_(bt_orconn_t *a, bt_orconn_t *b) +{ + return a->chan == b->chan; +} + +HT_HEAD(bto_gid_ht, bt_orconn_t); +HT_PROTOTYPE(bto_gid_ht, bt_orconn_t, node, bto_gid_hash_, bto_gid_eq_) +HT_GENERATE2(bto_gid_ht, bt_orconn_t, node, + bto_gid_hash_, bto_gid_eq_, 0.6, + tor_reallocarray_, tor_free_) +static struct bto_gid_ht *bto_gid_map; + +HT_HEAD(bto_chan_ht, bt_orconn_t); +HT_PROTOTYPE(bto_chan_ht, bt_orconn_t, chan_node, bto_chan_hash_, bto_chan_eq_) +HT_GENERATE2(bto_chan_ht, bt_orconn_t, chan_node, + bto_chan_hash_, bto_chan_eq_, 0.6, + tor_reallocarray_, tor_free_) +static struct bto_chan_ht *bto_chan_map; + +/** Clear the GID hash map, freeing any bt_orconn_t objects that become + * unreferenced */ +static void +bto_gid_clear_map(void) +{ + bt_orconn_t **elt, **next, *c; + + for (elt = HT_START(bto_gid_ht, bto_gid_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(bto_gid_ht, bto_gid_map, elt); + + c->gid = 0; + /* Don't delete if chan ID isn't zero: it's still in the chan hash map */ + if (!c->chan) + tor_free(c); + } + HT_CLEAR(bto_gid_ht, bto_gid_map); + tor_free(bto_gid_map); +} + +/** Clear the chan ID hash map, freeing any bt_orconn_t objects that + * become unreferenced */ +static void +bto_chan_clear_map(void) +{ + bt_orconn_t **elt, **next, *c; + + for (elt = HT_START(bto_chan_ht, bto_chan_map); + elt; + elt = next) { + c = *elt; + next = HT_NEXT_RMV(bto_chan_ht, bto_chan_map, elt); + + c->chan = 0; + /* Don't delete if GID isn't zero, it's still in the GID hash map */ + if (!c->gid) + tor_free(c); + } + HT_CLEAR(bto_chan_ht, bto_chan_map); + tor_free(bto_chan_map); +} + +/** Delete a bt_orconn from the hash maps by GID */ +void +bto_delete(uint64_t gid) +{ + bt_orconn_t key, *bto; + + key.gid = gid; + key.chan = 0; + bto = HT_FIND(bto_gid_ht, bto_gid_map, &key); + if (!bto) { + /* The orconn might be unregistered because it's an EXT_OR_CONN? */ + log_debug(LD_BTRACK, "tried to delete unregistered ORCONN gid=%"PRIu64, + gid); + return; + } + HT_REMOVE(bto_gid_ht, bto_gid_map, &key); + if (bto->chan) { + key.chan = bto->chan; + HT_REMOVE(bto_chan_ht, bto_chan_map, &key); + } + tor_free(bto); +} + +/** + * Helper for bto_find_or_new(). + * + * Update GID and chan ID of an existing bt_orconn object if needed, + * given a search key previously used within bto_find_or_new(). + **/ +static bt_orconn_t * +bto_update(bt_orconn_t *bto, const bt_orconn_t *key) +{ + /* ORCONN GIDs shouldn't change once assigned */ + tor_assert(!bto->gid || !key->gid || bto->gid == key->gid); + if (!bto->gid && key->gid) { + /* Got a gid when we didn't already have one; insert into gid map */ + log_debug(LD_BTRACK, "ORCONN chan=%"PRIu64" newgid=%"PRIu64, key->chan, + key->gid); + bto->gid = key->gid; + HT_INSERT(bto_gid_ht, bto_gid_map, bto); + } + /* association of ORCONN with channel shouldn't change */ + tor_assert(!bto->chan || !key->chan || bto->chan == key->chan); + if (!bto->chan && key->chan) { + /* Got a chan when we didn't already have one; insert into chan map */ + log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" newchan=%"PRIu64, + bto->gid, key->chan); + bto->chan = key->chan; + HT_INSERT(bto_chan_ht, bto_chan_map, bto); + } + return bto; +} + +/** Helper for bto_find_or_new() */ +static bt_orconn_t * +bto_new(const bt_orconn_t *key) +{ + struct bt_orconn_t *bto = tor_malloc(sizeof(*bto)); + + bto->gid = key->gid; + bto->chan = key->chan; + bto->state = 0; + bto->proxy_type = 0; + bto->is_orig = false; + bto->is_onehop = true; + + if (bto->gid) + HT_INSERT(bto_gid_ht, bto_gid_map, bto); + if (bto->chan) + HT_INSERT(bto_chan_ht, bto_chan_map, bto); + + return bto; +} + +/** + * Insert a new bt_orconn with the given GID and chan ID, or update + * the GID and chan ID if one already exists. + * + * Return the found or allocated bt_orconn. + **/ +bt_orconn_t * +bto_find_or_new(uint64_t gid, uint64_t chan) +{ + bt_orconn_t key, *bto = NULL; + + tor_assert(gid || chan); + key.gid = gid; + key.chan = chan; + if (key.gid) + bto = HT_FIND(bto_gid_ht, bto_gid_map, &key); + if (!bto && key.chan) { + /* Not found by GID; look up by chan ID */ + bto = HT_FIND(bto_chan_ht, bto_chan_map, &key); + } + if (bto) + return bto_update(bto, &key); + else + return bto_new(&key); +} + +/** Initialize the hash maps */ +void +bto_init_maps(void) +{ + bto_gid_map = tor_malloc(sizeof(*bto_gid_map)); + HT_INIT(bto_gid_ht, bto_gid_map); + bto_chan_map = tor_malloc(sizeof(*bto_chan_map)); + HT_INIT(bto_chan_ht, bto_chan_map); +} + +/** Clear the hash maps, freeing all associated storage */ +void +bto_clear_maps(void) +{ + bto_gid_clear_map(); + bto_chan_clear_map(); +} diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h new file mode 100644 index 0000000000..3ead40984c --- /dev/null +++ b/src/feature/control/btrack_orconn_maps.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_orconn_maps.h + * \brief Header file for btrack_orconn_maps.c + **/ + +#ifndef TOR_BTRACK_ORCONN_MAPS_H + +void bto_delete(uint64_t); +bt_orconn_t *bto_find_or_new(uint64_t, uint64_t); + +void bto_init_maps(void); +void bto_clear_maps(void); + +#endif /* defined(TOR_BTRACK_ORCONN_MAPS_H) */ diff --git a/src/feature/control/btrack_sys.h b/src/feature/control/btrack_sys.h new file mode 100644 index 0000000000..fad35b41db --- /dev/null +++ b/src/feature/control/btrack_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file btrack_sys.h + * \brief Declare subsystem object for the bootstrap tracker susbystem. + **/ + +#ifndef TOR_BTRACK_SYS_H +#define TOR_BTRACK_SYS_H + +extern const struct subsys_fns_t sys_btrack; + +#endif /* defined(TOR_BTRACK_SYS_H) */ diff --git a/src/feature/control/control.c b/src/feature/control/control.c index cc7ecff2ff..6f8cd8f0aa 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -34,6 +34,7 @@ **/ #define CONTROL_PRIVATE +#define OCIRC_EVENT_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -50,6 +51,7 @@ #include "core/or/command.h" #include "core/or/connection_edge.h" #include "core/or/connection_or.h" +#include "core/or/ocirc_event.h" #include "core/or/policies.h" #include "core/or/reasons.h" #include "core/or/versions.h" @@ -87,11 +89,12 @@ #include "feature/rend/rendservice.h" #include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_util.h" #include "lib/encoding/confline.h" #include "lib/evloop/compat_libevent.h" +#include "lib/version/torversion.h" #include "feature/dircache/cached_dir_st.h" #include "feature/control/control_connection_st.h" @@ -178,13 +181,6 @@ static uint8_t *authentication_cookie = NULL; */ static smartlist_t *detached_onion_services = NULL; -/** A sufficiently large size to record the last bootstrap phase string. */ -#define BOOTSTRAP_MSG_LEN 1024 - -/** What was the last bootstrap phase message we sent? We keep track - * of this so we can respond to getinfo status/bootstrap-phase queries. */ -static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; - static void connection_printf_to_buf(control_connection_t *conn, const char *format, ...) CHECK_PRINTF(2,3); @@ -367,7 +363,7 @@ control_update_global_event_mask(void) control_get_bytes_rw_last_sec(&r, &w); } if (any_old_per_sec_events != control_any_per_second_event_enabled()) { - reschedule_per_second_timer(); + rescan_periodic_events(get_options()); } #undef NEWLY_ENABLED @@ -1680,6 +1676,8 @@ static const struct signal_t signal_table[] = { { SIGNEWNYM, "NEWNYM" }, { SIGCLEARDNSCACHE, "CLEARDNSCACHE"}, { SIGHEARTBEAT, "HEARTBEAT"}, + { SIGACTIVE, "ACTIVE" }, + { SIGDORMANT, "DORMANT" }, { 0, NULL }, }; @@ -1745,6 +1743,26 @@ handle_control_takeownership(control_connection_t *conn, uint32_t len, return 0; } +/** Called when we get a DROPOWNERSHIP command. Mark this connection + * as a non-owning connection, so that we will not exit if the connection + * closes. */ +static int +handle_control_dropownership(control_connection_t *conn, uint32_t len, + const char *body) +{ + (void)len; + (void)body; + + conn->is_owning_control_connection = 0; + + log_info(LD_CONTROL, "Control connection %d has dropped ownership of this " + "Tor instance.", + (int)(conn->base_.s)); + + send_control_done(conn); + return 0; +} + /** Return true iff <b>addr</b> is unusable as a mapaddress target because of * containing funny characters. */ static int @@ -2352,7 +2370,11 @@ getinfo_helper_dir(control_connection_t *control_conn, *answer = tor_strdup(consensus->dir); } if (!*answer) { /* try loading it from disk */ - *answer = networkstatus_read_cached_consensus("ns"); + tor_mmap_t *mapped = networkstatus_map_cached_consensus("ns"); + if (mapped) { + *answer = tor_memdup_nulterm(mapped->data, mapped->size); + tor_munmap_file(mapped); + } if (!*answer) { /* generate an error */ *errmsg = "Could not open cached consensus. " "Make sure FetchUselessDescriptors is set to 1."; @@ -3037,7 +3059,7 @@ getinfo_helper_events(control_connection_t *control_conn, check_whether_orport_reachable(options) ? 1 : 0, check_whether_dirport_reachable(options) ? 1 : 0); } else if (!strcmp(question, "status/bootstrap-phase")) { - *answer = tor_strdup(last_sent_bootstrap_message); + *answer = control_event_boot_last_msg(); } else if (!strcmpstart(question, "status/version/")) { int is_server = server_mode(options); networkstatus_t *c = networkstatus_get_latest_consensus(); @@ -3067,11 +3089,6 @@ getinfo_helper_events(control_connection_t *control_conn, case VS_UNKNOWN: *answer = tor_strdup("unknown"); break; default: tor_fragile_assert(); } - } else if (!strcmp(question, "status/version/num-versioning") || - !strcmp(question, "status/version/num-concurring")) { - tor_asprintf(answer, "%d", get_n_authorities(V3_DIRINFO)); - log_warn(LD_GENERAL, "%s is deprecated; it no longer gives useful " - "information", question); } } else if (!strcmp(question, "status/clients-seen")) { char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL)); @@ -3364,10 +3381,6 @@ static const getinfo_item_t getinfo_items[] = { "A fresh relay/ei descriptor pair for Tor's current state. Not stored."), DOC("status/version/recommended", "List of currently recommended versions."), DOC("status/version/current", "Status of the current version."), - DOC("status/version/num-versioning", "Number of versioning authorities."), - DOC("status/version/num-concurring", - "Number of versioning authorities agreeing on the status of the " - "current version"), ITEM("address", misc, "IP address of this Tor host, if we can guess it."), ITEM("traffic/read", misc,"Bytes read since the process was started."), ITEM("traffic/written", misc, @@ -3749,7 +3762,7 @@ handle_control_extendcircuit(control_connection_t *conn, uint32_t len, connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n", (unsigned long)circ->global_identifier); if (zero_circ) /* send a 'launched' event, for completeness */ - control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0); + circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0); done: SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n)); smartlist_free(router_nicknames); @@ -5548,6 +5561,9 @@ connection_control_process_inbuf(control_connection_t *conn) } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) { if (handle_control_takeownership(conn, cmd_data_len, args)) return -1; + } else if (!strcasecmp(conn->incoming_cmd, "DROPOWNERSHIP")) { + if (handle_control_dropownership(conn, cmd_data_len, args)) + return -1; } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) { if (handle_control_mapaddress(conn, cmd_data_len, args)) return -1; @@ -5625,6 +5641,7 @@ control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp, { const char *status; char reasons[64] = ""; + if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS)) return 0; tor_assert(circ); @@ -7008,361 +7025,6 @@ monitor_owning_controller_process(const char *process_spec) } } -/** Convert the name of a bootstrapping phase <b>s</b> into strings - * <b>tag</b> and <b>summary</b> suitable for display by the controller. */ -static int -bootstrap_status_to_string(bootstrap_status_t s, const char **tag, - const char **summary) -{ - switch (s) { - case BOOTSTRAP_STATUS_UNDEF: - *tag = "undef"; - *summary = "Undefined"; - break; - case BOOTSTRAP_STATUS_STARTING: - *tag = "starting"; - *summary = "Starting"; - break; - case BOOTSTRAP_STATUS_CONN_DIR: - *tag = "conn_dir"; - *summary = "Connecting to directory server"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE: - *tag = "status_handshake"; - *summary = "Finishing handshake"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_DIR: - *tag = "handshake_dir"; - *summary = "Finishing handshake with directory server"; - break; - case BOOTSTRAP_STATUS_ONEHOP_CREATE: - *tag = "onehop_create"; - *summary = "Establishing an encrypted directory connection"; - break; - case BOOTSTRAP_STATUS_REQUESTING_STATUS: - *tag = "requesting_status"; - *summary = "Asking for networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_STATUS: - *tag = "loading_status"; - *summary = "Loading networkstatus consensus"; - break; - case BOOTSTRAP_STATUS_LOADING_KEYS: - *tag = "loading_keys"; - *summary = "Loading authority key certs"; - break; - case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS: - *tag = "requesting_descriptors"; - /* XXXX this appears to incorrectly report internal on most loads */ - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Asking for relay descriptors for internal paths" : - "Asking for relay descriptors"; - break; - /* If we're sure there are no exits in the consensus, - * inform the controller by adding "internal" - * to the status summaries. - * (We only check this while loading descriptors, - * so we may not know in the earlier stages.) - * But if there are exits, we can't be sure whether - * we're creating internal or exit paths/circuits. - * XXXX Or should be use different tags or statuses - * for internal and exit/all? */ - case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS: - *tag = "loading_descriptors"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Loading relay descriptors for internal paths" : - "Loading relay descriptors"; - break; - case BOOTSTRAP_STATUS_CONN_OR: - *tag = "conn_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Connecting to the Tor network internally" : - "Connecting to the Tor network"; - break; - case BOOTSTRAP_STATUS_HANDSHAKE_OR: - *tag = "handshake_or"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Finishing handshake with first hop of internal circuit" : - "Finishing handshake with first hop"; - break; - case BOOTSTRAP_STATUS_CIRCUIT_CREATE: - *tag = "circuit_create"; - *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ? - "Establishing an internal Tor circuit" : - "Establishing a Tor circuit"; - break; - case BOOTSTRAP_STATUS_DONE: - *tag = "done"; - *summary = "Done"; - break; - default: -// log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s); - *tag = *summary = "unknown"; - return -1; - } - return 0; -} - -/** What percentage through the bootstrap process are we? We remember - * this so we can avoid sending redundant bootstrap status events, and - * so we can guess context for the bootstrap messages which are - * ambiguous. It starts at 'undef', but gets set to 'starting' while - * Tor initializes. */ -static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; - -/** Like bootstrap_percent, but only takes on the enumerated values in - * bootstrap_status_t. - */ -static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; - -/** As bootstrap_percent, but holds the bootstrapping level at which we last - * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, - * to avoid flooding the log with a new message every time we get a few more - * microdescriptors */ -static int notice_bootstrap_percent = 0; - -/** How many problems have we had getting to the next bootstrapping phase? - * These include failure to establish a connection to a Tor relay, - * failures to finish the TLS handshake, failures to validate the - * consensus document, etc. */ -static int bootstrap_problems = 0; - -/** We only tell the controller once we've hit a threshold of problems - * for the current phase. */ -#define BOOTSTRAP_PROBLEM_THRESHOLD 10 - -/** When our bootstrapping progress level changes, but our bootstrapping - * status has not advanced, we only log at NOTICE when we have made at least - * this much progress. - */ -#define BOOTSTRAP_PCT_INCREMENT 5 - -/** Do the actual logging and notifications for - * control_event_bootstrap(). Doesn't change any state beyond that. - */ -static void -control_event_bootstrap_core(int loglevel, bootstrap_status_t status, - int progress) -{ - char buf[BOOTSTRAP_MSG_LEN]; - const char *tag, *summary; - - bootstrap_status_to_string(status, &tag, &summary); - /* Locally reset status if there's incremental progress */ - if (progress) - status = progress; - - tor_log(loglevel, LD_CONTROL, - "Bootstrapped %d%%: %s", status, summary); - tor_snprintf(buf, sizeof(buf), - "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", - status, tag, summary); - tor_snprintf(last_sent_bootstrap_message, - sizeof(last_sent_bootstrap_message), - "NOTICE %s", buf); - control_event_client_status(LOG_NOTICE, "%s", buf); -} - -/** Called when Tor has made progress at bootstrapping its directory - * information and initial circuits. - * - * <b>status</b> is the new status, that is, what task we will be doing - * next. <b>progress</b> is zero if we just started this task, else it - * represents progress on the task. - */ -void -control_event_bootstrap(bootstrap_status_t status, int progress) -{ - int loglevel = LOG_NOTICE; - - if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) - return; /* already bootstrapped; nothing to be done here. */ - - /* special case for handshaking status, since our TLS handshaking code - * can't distinguish what the connection is going to be for. */ - if (status == BOOTSTRAP_STATUS_HANDSHAKE) { - if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) { - status = BOOTSTRAP_STATUS_HANDSHAKE_DIR; - } else { - status = BOOTSTRAP_STATUS_HANDSHAKE_OR; - } - } - - if (status <= bootstrap_percent) { - /* If there's no new progress, return early. */ - if (!progress || progress <= bootstrap_percent) - return; - /* Log at INFO if not enough progress happened. */ - if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT) - loglevel = LOG_INFO; - } - - control_event_bootstrap_core(loglevel, status, progress); - - if (status > bootstrap_percent) { - bootstrap_phase = status; /* new milestone reached */ - bootstrap_percent = status; - } - if (progress > bootstrap_percent) { - /* incremental progress within a milestone */ - bootstrap_percent = progress; - bootstrap_problems = 0; /* Progress! Reset our problem counter. */ - } - if (loglevel == LOG_NOTICE && - bootstrap_percent > notice_bootstrap_percent) { - /* Remember that we gave a notice at this level. */ - notice_bootstrap_percent = bootstrap_percent; - } -} - -/** Flag whether we've opened an OR_CONN yet */ -static int bootstrap_first_orconn = 0; - -/** Like bootstrap_phase, but for (possibly deferred) directory progress */ -static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; - -/** Like bootstrap_problems, but for (possibly deferred) directory progress */ -static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; - -/** Defer directory info bootstrap events until we have successfully - * completed our first connection to a router. */ -void -control_event_boot_dir(bootstrap_status_t status, int progress) -{ - if (status > bootstrap_dir_progress) { - bootstrap_dir_progress = status; - bootstrap_dir_phase = status; - } - if (progress && progress >= bootstrap_dir_progress) { - bootstrap_dir_progress = progress; - } - - /* Don't report unless we have successfully opened at least one OR_CONN */ - if (!bootstrap_first_orconn) - return; - - control_event_bootstrap(status, progress); -} - -/** Set a flag to allow reporting of directory bootstrap progress. - * (Code that reports completion of an OR_CONN calls this.) Also, - * report directory progress so far. */ -void -control_event_boot_first_orconn(void) -{ - bootstrap_first_orconn = 1; - control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress); -} - -/** Called when Tor has failed to make bootstrapping progress in a way - * that indicates a problem. <b>warn</b> gives a human-readable hint - * as to why, and <b>reason</b> provides a controller-facing short - * tag. <b>conn</b> is the connection that caused this problem and - * can be NULL if a connection cannot be easily identified. - */ -void -control_event_bootstrap_problem(const char *warn, const char *reason, - const connection_t *conn, int dowarn) -{ - int status = bootstrap_percent; - const char *tag = "", *summary = ""; - char buf[BOOTSTRAP_MSG_LEN]; - const char *recommendation = "ignore"; - int severity; - char *or_id = NULL, *hostaddr = NULL; - or_connection_t *or_conn = NULL; - - /* bootstrap_percent must not be in "undefined" state here. */ - tor_assert(status >= 0); - - if (bootstrap_percent == 100) - return; /* already bootstrapped; nothing to be done here. */ - - bootstrap_problems++; - - if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) - dowarn = 1; - - /* Don't warn about our bootstrapping status if we are hibernating or - * shutting down. */ - if (we_are_hibernating()) - dowarn = 0; - - tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0); - - severity = dowarn ? LOG_WARN : LOG_INFO; - - if (dowarn) - recommendation = "warn"; - - if (conn && conn->type == CONN_TYPE_OR) { - /* XXX TO_OR_CONN can't deal with const */ - or_conn = TO_OR_CONN((connection_t *)conn); - or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN)); - } else { - or_id = tor_strdup("?"); - } - - if (conn) - tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port); - else - hostaddr = tor_strdup("?"); - - log_fn(severity, - LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; " - "count %d; recommendation %s; host %s at %s)", - status, summary, warn, reason, - bootstrap_problems, recommendation, - or_id, hostaddr); - - connection_or_report_broken_states(severity, LD_HANDSHAKE); - - tor_snprintf(buf, sizeof(buf), - "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " - "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"", - bootstrap_percent, tag, summary, warn, reason, bootstrap_problems, - recommendation, - or_id, hostaddr); - - tor_snprintf(last_sent_bootstrap_message, - sizeof(last_sent_bootstrap_message), - "WARN %s", buf); - control_event_client_status(LOG_WARN, "%s", buf); - - tor_free(hostaddr); - tor_free(or_id); -} - -/** Called when Tor has failed to make bootstrapping progress in a way - * that indicates a problem. <b>warn</b> gives a hint as to why, and - * <b>reason</b> provides an "or_conn_end_reason" tag. <b>or_conn</b> - * is the connection that caused this problem. - */ -MOCK_IMPL(void, -control_event_bootstrap_prob_or, (const char *warn, int reason, - or_connection_t *or_conn)) -{ - int dowarn = 0; - - if (or_conn->have_noted_bootstrap_problem) - return; - - or_conn->have_noted_bootstrap_problem = 1; - - if (reason == END_OR_CONN_REASON_NO_ROUTE) - dowarn = 1; - - /* If we are using bridges and all our OR connections are now - closed, it means that we totally failed to connect to our - bridges. Throw a warning. */ - if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) - dowarn = 1; - - control_event_bootstrap_problem(warn, - orconn_end_reason_to_control_string(reason), - TO_CONN(or_conn), dowarn); -} - /** We just generated a new summary of which countries we've seen clients * from recently. Send a copy to the controller in case it wants to * display it for the user. */ @@ -7388,6 +7050,26 @@ control_event_transport_launched(const char *mode, const char *transport_name, mode, transport_name, fmt_addr(addr), port); } +/** A pluggable transport called <b>pt_name</b> has emitted a log message + * found in <b>message</b> at <b>severity</b> log level. */ +void +control_event_pt_log(const char *log) +{ + send_control_event(EVENT_PT_LOG, + "650 PT_LOG %s\r\n", + log); +} + +/** A pluggable transport has emitted a STATUS message found in + * <b>status</b>. */ +void +control_event_pt_status(const char *status) +{ + send_control_event(EVENT_PT_STATUS, + "650 PT_STATUS %s\r\n", + status); +} + /** Convert rendezvous auth type to string for HS_DESC control events */ const char * @@ -7879,17 +7561,10 @@ control_free_all(void) mainloop_event_free(flush_queued_events_event); flush_queued_events_event = NULL; } - bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; - bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; - notice_bootstrap_percent = 0; - bootstrap_problems = 0; - bootstrap_first_orconn = 0; - bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; - bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + control_event_bootstrap_reset(); authentication_cookie_is_set = 0; global_event_mask = 0; disable_log_messages = 0; - memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); } #ifdef TOR_UNIT_TESTS diff --git a/src/feature/control/control.h b/src/feature/control/control.h index a09c1cd11b..b2ab4c1997 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -12,15 +12,7 @@ #ifndef TOR_CONTROL_H #define TOR_CONTROL_H -/** Used to indicate the type of a circuit event passed to the controller. - * The various types are defined in control-spec.txt */ -typedef enum circuit_status_event_t { - CIRC_EVENT_LAUNCHED = 0, - CIRC_EVENT_BUILT = 1, - CIRC_EVENT_EXTENDED = 2, - CIRC_EVENT_FAILED = 3, - CIRC_EVENT_CLOSED = 4, -} circuit_status_event_t; +#include "core/or/ocirc_event.h" /** Used to indicate the type of a CIRC_MINOR event passed to the controller. * The various types are defined in control-spec.txt . */ @@ -29,6 +21,8 @@ typedef enum circuit_status_minor_event_t { CIRC_MINOR_EVENT_CANNIBALIZED, } circuit_status_minor_event_t; +#include "core/or/orconn_event.h" + /** Used to indicate the type of a stream event passed to the controller. * The various types are defined in control-spec.txt */ typedef enum stream_status_event_t { @@ -43,16 +37,6 @@ typedef enum stream_status_event_t { STREAM_EVENT_REMAP = 8 } stream_status_event_t; -/** Used to indicate the type of an OR connection event passed to the - * controller. The various types are defined in control-spec.txt */ -typedef enum or_conn_status_event_t { - OR_CONN_EVENT_LAUNCHED = 0, - OR_CONN_EVENT_CONNECTED = 1, - OR_CONN_EVENT_FAILED = 2, - OR_CONN_EVENT_CLOSED = 3, - OR_CONN_EVENT_NEW = 4, -} or_conn_status_event_t; - /** Used to indicate the type of a buildtime event */ typedef enum buildtimeout_set_event_t { BUILDTIMEOUT_SET_EVENT_COMPUTED = 0, @@ -67,18 +51,42 @@ typedef enum buildtimeout_set_event_t { typedef enum { BOOTSTRAP_STATUS_UNDEF=-1, BOOTSTRAP_STATUS_STARTING=0, - BOOTSTRAP_STATUS_CONN_DIR=5, - BOOTSTRAP_STATUS_HANDSHAKE=-2, - BOOTSTRAP_STATUS_HANDSHAKE_DIR=10, - BOOTSTRAP_STATUS_ONEHOP_CREATE=15, - BOOTSTRAP_STATUS_REQUESTING_STATUS=20, - BOOTSTRAP_STATUS_LOADING_STATUS=25, + + /* Initial connection to any relay */ + + BOOTSTRAP_STATUS_CONN_PT=1, + BOOTSTRAP_STATUS_CONN_DONE_PT=2, + BOOTSTRAP_STATUS_CONN_PROXY=3, + BOOTSTRAP_STATUS_CONN_DONE_PROXY=4, + BOOTSTRAP_STATUS_CONN=5, + BOOTSTRAP_STATUS_CONN_DONE=10, + BOOTSTRAP_STATUS_HANDSHAKE=14, + BOOTSTRAP_STATUS_HANDSHAKE_DONE=15, + + /* Loading directory info */ + + BOOTSTRAP_STATUS_ONEHOP_CREATE=20, + BOOTSTRAP_STATUS_REQUESTING_STATUS=25, + BOOTSTRAP_STATUS_LOADING_STATUS=30, BOOTSTRAP_STATUS_LOADING_KEYS=40, BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45, BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50, - BOOTSTRAP_STATUS_CONN_OR=80, - BOOTSTRAP_STATUS_HANDSHAKE_OR=85, - BOOTSTRAP_STATUS_CIRCUIT_CREATE=90, + BOOTSTRAP_STATUS_ENOUGH_DIRINFO=75, + + /* Connecting to a relay for AP circuits */ + + BOOTSTRAP_STATUS_AP_CONN_PT=76, + BOOTSTRAP_STATUS_AP_CONN_DONE_PT=77, + BOOTSTRAP_STATUS_AP_CONN_PROXY=78, + BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY=79, + BOOTSTRAP_STATUS_AP_CONN=80, + BOOTSTRAP_STATUS_AP_CONN_DONE=85, + BOOTSTRAP_STATUS_AP_HANDSHAKE=89, + BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE=90, + + /* Creating AP circuits */ + + BOOTSTRAP_STATUS_CIRCUIT_CREATE=95, BOOTSTRAP_STATUS_DONE=100 } bootstrap_status_t; @@ -200,11 +208,15 @@ void control_event_boot_dir(bootstrap_status_t status, int progress); void control_event_boot_first_orconn(void); void control_event_bootstrap_problem(const char *warn, const char *reason, const connection_t *conn, int dowarn); +char *control_event_boot_last_msg(void); +void control_event_bootstrap_reset(void); void control_event_clients_seen(const char *controller_str); void control_event_transport_launched(const char *mode, const char *transport_name, tor_addr_t *addr, uint16_t port); +void control_event_pt_log(const char *log); +void control_event_pt_status(const char *status); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_event_hs_descriptor_requested(const char *onion_address, @@ -293,7 +305,9 @@ void control_free_all(void); #define EVENT_HS_DESC 0x0021 #define EVENT_HS_DESC_CONTENT 0x0022 #define EVENT_NETWORK_LIVENESS 0x0023 -#define EVENT_MAX_ 0x0023 +#define EVENT_PT_LOG 0x0024 +#define EVENT_PT_STATUS 0x0025 +#define EVENT_MAX_ 0x0025 /* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ #define EVENT_CAPACITY_ 0x0040 diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c new file mode 100644 index 0000000000..8153d7595a --- /dev/null +++ b/src/feature/control/control_bootstrap.c @@ -0,0 +1,383 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file control_bootstrap.c + * \brief Provide bootstrap progress events for the control port. + */ +#include "core/or/or.h" + +#include "app/config/config.h" +#include "core/mainloop/connection.h" +#include "core/or/connection_or.h" +#include "core/or/connection_st.h" +#include "core/or/or_connection_st.h" +#include "core/or/reasons.h" +#include "feature/control/control.h" +#include "feature/hibernate/hibernate.h" +#include "lib/malloc/malloc.h" + +/** A sufficiently large size to record the last bootstrap phase string. */ +#define BOOTSTRAP_MSG_LEN 1024 + +/** What was the last bootstrap phase message we sent? We keep track + * of this so we can respond to getinfo status/bootstrap-phase queries. */ +static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN]; + +/** Table to convert bootstrap statuses to strings. */ +static const struct { + bootstrap_status_t status; + const char *tag; + const char *summary; +} boot_to_str_tab[] = { + { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" }, + { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" }, + + /* Initial connection to any relay */ + + { BOOTSTRAP_STATUS_CONN_PT, "conn_pt", "Connecting to pluggable transport" }, + { BOOTSTRAP_STATUS_CONN_DONE_PT, "conn_done_pt", + "Connected to pluggable transport" }, + { BOOTSTRAP_STATUS_CONN_PROXY, "conn_proxy", "Connecting to proxy" }, + { BOOTSTRAP_STATUS_CONN_DONE_PROXY, "conn_done_proxy", + "Connected to proxy" }, + { BOOTSTRAP_STATUS_CONN, "conn", "Connecting to a relay" }, + { BOOTSTRAP_STATUS_CONN_DONE, "conn_done", "Connected to a relay" }, + { BOOTSTRAP_STATUS_HANDSHAKE, "handshake", + "Handshaking with a relay" }, + { BOOTSTRAP_STATUS_HANDSHAKE_DONE, "handshake_done", + "Handshake with a relay done" }, + + /* Loading directory info */ + + { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create", + "Establishing an encrypted directory connection" }, + { BOOTSTRAP_STATUS_REQUESTING_STATUS, "requesting_status", + "Asking for networkstatus consensus" }, + { BOOTSTRAP_STATUS_LOADING_STATUS, "loading_status", + "Loading networkstatus consensus" }, + { BOOTSTRAP_STATUS_LOADING_KEYS, "loading_keys", + "Loading authority key certs" }, + { BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, "requesting_descriptors", + "Asking for relay descriptors" }, + { BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, "loading_descriptors", + "Loading relay descriptors" }, + { BOOTSTRAP_STATUS_ENOUGH_DIRINFO, "enough_dirinfo", + "Loaded enough directory info to build circuits" }, + + /* Connecting to a relay for AP circuits */ + + { BOOTSTRAP_STATUS_AP_CONN_PT, "ap_conn_pt", + "Connecting to pluggable transport to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_DONE_PT, "ap_conn_done_pt", + "Connected to pluggable transport to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_PROXY, "ap_conn_proxy", + "Connecting to proxy to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, "ap_conn_done_proxy", + "Connected to proxy to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN, "ap_conn", + "Connecting to a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_CONN_DONE, "ap_conn_done", + "Connected to a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_HANDSHAKE, "ap_handshake", + "Finishing handshake with a relay to build circuits" }, + { BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, "ap_handshake_done", + "Handshake finished with a relay to build circuits" }, + + /* Creating AP circuits */ + + { BOOTSTRAP_STATUS_CIRCUIT_CREATE, "circuit_create", + "Establishing a Tor circuit" }, + { BOOTSTRAP_STATUS_DONE, "done", "Done" }, +}; +#define N_BOOT_TO_STR (sizeof(boot_to_str_tab)/sizeof(boot_to_str_tab[0])) + +/** Convert the name of a bootstrapping phase <b>s</b> into strings + * <b>tag</b> and <b>summary</b> suitable for display by the controller. */ +static int +bootstrap_status_to_string(bootstrap_status_t s, const char **tag, + const char **summary) +{ + for (size_t i = 0; i < N_BOOT_TO_STR; i++) { + if (s == boot_to_str_tab[i].status) { + *tag = boot_to_str_tab[i].tag; + *summary = boot_to_str_tab[i].summary; + return 0; + } + } + + *tag = *summary = "unknown"; + return -1; +} + +/** What percentage through the bootstrap process are we? We remember + * this so we can avoid sending redundant bootstrap status events, and + * so we can guess context for the bootstrap messages which are + * ambiguous. It starts at 'undef', but gets set to 'starting' while + * Tor initializes. */ +static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; + +/** Like bootstrap_percent, but only takes on the enumerated values in + * bootstrap_status_t. + */ +static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; + +/** As bootstrap_percent, but holds the bootstrapping level at which we last + * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT, + * to avoid flooding the log with a new message every time we get a few more + * microdescriptors */ +static int notice_bootstrap_percent = 0; + +/** How many problems have we had getting to the next bootstrapping phase? + * These include failure to establish a connection to a Tor relay, + * failures to finish the TLS handshake, failures to validate the + * consensus document, etc. */ +static int bootstrap_problems = 0; + +/** We only tell the controller once we've hit a threshold of problems + * for the current phase. */ +#define BOOTSTRAP_PROBLEM_THRESHOLD 10 + +/** When our bootstrapping progress level changes, but our bootstrapping + * status has not advanced, we only log at NOTICE when we have made at least + * this much progress. + */ +#define BOOTSTRAP_PCT_INCREMENT 5 + +/** Do the actual logging and notifications for + * control_event_bootstrap(). Doesn't change any state beyond that. + */ +static void +control_event_bootstrap_core(int loglevel, bootstrap_status_t status, + int progress) +{ + char buf[BOOTSTRAP_MSG_LEN]; + const char *tag, *summary; + + bootstrap_status_to_string(status, &tag, &summary); + /* Locally reset status if there's incremental progress */ + if (progress) + status = progress; + + tor_log(loglevel, LD_CONTROL, + "Bootstrapped %d%% (%s): %s", status, tag, summary); + tor_snprintf(buf, sizeof(buf), + "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"", + status, tag, summary); + tor_snprintf(last_sent_bootstrap_message, + sizeof(last_sent_bootstrap_message), + "NOTICE %s", buf); + control_event_client_status(LOG_NOTICE, "%s", buf); +} + +/** Called when Tor has made progress at bootstrapping its directory + * information and initial circuits. + * + * <b>status</b> is the new status, that is, what task we will be doing + * next. <b>progress</b> is zero if we just started this task, else it + * represents progress on the task. + */ +void +control_event_bootstrap(bootstrap_status_t status, int progress) +{ + int loglevel = LOG_NOTICE; + + if (bootstrap_percent == BOOTSTRAP_STATUS_DONE) + return; /* already bootstrapped; nothing to be done here. */ + + if (status <= bootstrap_percent) { + /* If there's no new progress, return early. */ + if (!progress || progress <= bootstrap_percent) + return; + /* Log at INFO if not enough progress happened. */ + if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT) + loglevel = LOG_INFO; + } + + control_event_bootstrap_core(loglevel, status, progress); + + if (status > bootstrap_percent) { + bootstrap_phase = status; /* new milestone reached */ + bootstrap_percent = status; + } + if (progress > bootstrap_percent) { + /* incremental progress within a milestone */ + bootstrap_percent = progress; + bootstrap_problems = 0; /* Progress! Reset our problem counter. */ + } + if (loglevel == LOG_NOTICE && + bootstrap_percent > notice_bootstrap_percent) { + /* Remember that we gave a notice at this level. */ + notice_bootstrap_percent = bootstrap_percent; + } +} + +/** Flag whether we've opened an OR_CONN yet */ +static int bootstrap_first_orconn = 0; + +/** Like bootstrap_phase, but for (possibly deferred) directory progress */ +static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + +/** Like bootstrap_problems, but for (possibly deferred) directory progress */ +static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; + +/** Defer directory info bootstrap events until we have successfully + * completed our first connection to a router. */ +void +control_event_boot_dir(bootstrap_status_t status, int progress) +{ + if (status > bootstrap_dir_progress) { + bootstrap_dir_progress = status; + bootstrap_dir_phase = status; + } + if (progress && progress >= bootstrap_dir_progress) { + bootstrap_dir_progress = progress; + } + + /* Don't report unless we have successfully opened at least one OR_CONN */ + if (!bootstrap_first_orconn) + return; + + control_event_bootstrap(status, progress); +} + +/** Set a flag to allow reporting of directory bootstrap progress. + * (Code that reports completion of an OR_CONN calls this.) Also, + * report directory progress so far. */ +void +control_event_boot_first_orconn(void) +{ + bootstrap_first_orconn = 1; + control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress); +} + +/** Called when Tor has failed to make bootstrapping progress in a way + * that indicates a problem. <b>warn</b> gives a human-readable hint + * as to why, and <b>reason</b> provides a controller-facing short + * tag. <b>conn</b> is the connection that caused this problem and + * can be NULL if a connection cannot be easily identified. + */ +void +control_event_bootstrap_problem(const char *warn, const char *reason, + const connection_t *conn, int dowarn) +{ + int status = bootstrap_percent; + const char *tag = "", *summary = ""; + char buf[BOOTSTRAP_MSG_LEN]; + const char *recommendation = "ignore"; + int severity; + char *or_id = NULL, *hostaddr = NULL; + or_connection_t *or_conn = NULL; + + /* bootstrap_percent must not be in "undefined" state here. */ + tor_assert(status >= 0); + + if (bootstrap_percent == 100) + return; /* already bootstrapped; nothing to be done here. */ + + bootstrap_problems++; + + if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) + dowarn = 1; + + /* Don't warn about our bootstrapping status if we are hibernating or + * shutting down. */ + if (we_are_hibernating()) + dowarn = 0; + + tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0); + + severity = dowarn ? LOG_WARN : LOG_INFO; + + if (dowarn) + recommendation = "warn"; + + if (conn && conn->type == CONN_TYPE_OR) { + /* XXX TO_OR_CONN can't deal with const */ + or_conn = TO_OR_CONN((connection_t *)conn); + or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN)); + } else { + or_id = tor_strdup("?"); + } + + if (conn) + tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port); + else + hostaddr = tor_strdup("?"); + + log_fn(severity, + LD_CONTROL, "Problem bootstrapping. Stuck at %d%% (%s): %s. (%s; %s; " + "count %d; recommendation %s; host %s at %s)", + status, tag, summary, warn, reason, + bootstrap_problems, recommendation, + or_id, hostaddr); + + connection_or_report_broken_states(severity, LD_HANDSHAKE); + + tor_snprintf(buf, sizeof(buf), + "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s " + "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"", + bootstrap_percent, tag, summary, warn, reason, bootstrap_problems, + recommendation, + or_id, hostaddr); + + tor_snprintf(last_sent_bootstrap_message, + sizeof(last_sent_bootstrap_message), + "WARN %s", buf); + control_event_client_status(LOG_WARN, "%s", buf); + + tor_free(hostaddr); + tor_free(or_id); +} + +/** Called when Tor has failed to make bootstrapping progress in a way + * that indicates a problem. <b>warn</b> gives a hint as to why, and + * <b>reason</b> provides an "or_conn_end_reason" tag. <b>or_conn</b> + * is the connection that caused this problem. + */ +MOCK_IMPL(void, +control_event_bootstrap_prob_or, (const char *warn, int reason, + or_connection_t *or_conn)) +{ + int dowarn = 0; + + if (or_conn->have_noted_bootstrap_problem) + return; + + or_conn->have_noted_bootstrap_problem = 1; + + if (reason == END_OR_CONN_REASON_NO_ROUTE) + dowarn = 1; + + /* If we are using bridges and all our OR connections are now + closed, it means that we totally failed to connect to our + bridges. Throw a warning. */ + if (get_options()->UseBridges && !any_other_active_or_conns(or_conn)) + dowarn = 1; + + control_event_bootstrap_problem(warn, + orconn_end_reason_to_control_string(reason), + TO_CONN(or_conn), dowarn); +} + +/** Return a copy of the last sent bootstrap message. */ +char * +control_event_boot_last_msg(void) +{ + return tor_strdup(last_sent_bootstrap_message); +} + +/** Reset bootstrap tracking state. */ +void +control_event_bootstrap_reset(void) +{ + bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; + bootstrap_phase = BOOTSTRAP_STATUS_UNDEF; + notice_bootstrap_percent = 0; + bootstrap_problems = 0; + bootstrap_first_orconn = 0; + bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF; + bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF; + memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); +} diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c index 12f9399e9f..1cfd8119df 100644 --- a/src/feature/dirauth/bwauth.c +++ b/src/feature/dirauth/bwauth.c @@ -20,6 +20,7 @@ #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/vote_routerstatus_st.h" +#include "lib/crypt_ops/crypto_format.h" #include "lib/encoding/keyval.h" /** Total number of routers with measured bandwidth; this is set by @@ -205,7 +206,8 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri) int dirserv_read_measured_bandwidths(const char *from_file, smartlist_t *routerstatuses, - smartlist_t *bw_file_headers) + smartlist_t *bw_file_headers, + uint8_t *digest_out) { FILE *fp = tor_fopen_cloexec(from_file, "r"); int applied_lines = 0; @@ -219,6 +221,7 @@ dirserv_read_measured_bandwidths(const char *from_file, int rv = -1; char *line = NULL; size_t n = 0; + crypto_digest_t *digest = crypto_digest256_new(DIGEST_SHA256); /* Initialise line, so that we can't possibly run off the end. */ @@ -233,11 +236,14 @@ dirserv_read_measured_bandwidths(const char *from_file, log_warn(LD_DIRSERV, "Empty bandwidth file"); goto err; } + /* If the line could be gotten, add it to the digest */ + crypto_digest_add_bytes(digest, (const char *) line, strlen(line)); if (!strlen(line) || line[strlen(line)-1] != '\n') { log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s", escaped(line)); - goto err; + /* Continue adding lines to the digest. */ + goto continue_digest; } line[strlen(line)-1] = '\0'; @@ -245,14 +251,14 @@ dirserv_read_measured_bandwidths(const char *from_file, if (!ok) { log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s", escaped(line)); - goto err; + goto continue_digest; } - now = time(NULL); + now = approx_time(); if ((now - file_time) > MAX_MEASUREMENT_AGE) { log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u", (unsigned)(time(NULL) - file_time)); - goto err; + goto continue_digest; } /* If timestamp was correct and bw_file_headers is not NULL, @@ -267,6 +273,7 @@ dirserv_read_measured_bandwidths(const char *from_file, while (!feof(fp)) { measured_bw_line_t parsed_line; if (tor_getline(&line, &n, fp) >= 0) { + crypto_digest_add_bytes(digest, (const char *) line, strlen(line)); if (measured_bw_line_parse(&parsed_line, line, line_is_after_headers) != -1) { /* This condition will be true when the first complete valid bw line @@ -305,6 +312,14 @@ dirserv_read_measured_bandwidths(const char *from_file, "Applied %d measurements.", applied_lines); rv = 0; + continue_digest: + /* Continue parsing lines to return the digest of the Bandwidth File. */ + while (!feof(fp)) { + if (tor_getline(&line, &n, fp) >= 0) { + crypto_digest_add_bytes(digest, (const char *) line, strlen(line)); + } + } + err: if (line) { // we need to raw_free this buffer because we got it from tor_getdelim() @@ -312,6 +327,9 @@ dirserv_read_measured_bandwidths(const char *from_file, } if (fp) fclose(fp); + if (digest_out) + crypto_digest_get_digest(digest, (char *) digest_out, DIGEST256_LEN); + crypto_digest_free(digest); return rv; } diff --git a/src/feature/dirauth/bwauth.h b/src/feature/dirauth/bwauth.h index 4507728458..8b7acc4a1c 100644 --- a/src/feature/dirauth/bwauth.h +++ b/src/feature/dirauth/bwauth.h @@ -21,8 +21,8 @@ int dirserv_read_measured_bandwidths(const char *from_file, smartlist_t *routerstatuses, - smartlist_t *bw_file_headers); - + smartlist_t *bw_file_headers, + uint8_t *digest_out); int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_out, time_t *as_of_out); diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c index af8b3dc207..755b99bae2 100644 --- a/src/feature/dirauth/dirvote.c +++ b/src/feature/dirauth/dirvote.c @@ -28,6 +28,7 @@ #include "feature/nodelist/fmt_routerstatus.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" @@ -60,6 +61,9 @@ #include "lib/encoding/confline.h" #include "lib/crypt_ops/crypto_format.h" +/* Algorithm to use for the bandwidth file digest. */ +#define DIGEST_ALG_BW_FILE DIGEST_SHA256 + /** * \file dirvote.c * \brief Functions to compute directory consensus, and schedule voting. @@ -268,6 +272,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, char *flag_thresholds = dirserv_get_flag_thresholds_line(); char *params; char *bw_headers_line = NULL; + char *bw_file_digest = NULL; authority_cert_t *cert = v3_ns->cert; char *methods = make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD, @@ -307,6 +312,27 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, tor_free(bw_file_headers); } + /* Create bandwidth-file-digest if applicable. + * v3_ns->b64_digest_bw_file will contain the digest when V3BandwidthsFile + * is configured and the bandwidth file could be read, even if it was not + * parseable. + */ + if (!tor_digest256_is_zero((const char *)v3_ns->bw_file_digest256)) { + /* Encode the digest. */ + char b64_digest_bw_file[BASE64_DIGEST256_LEN+1] = {0}; + digest256_to_base64(b64_digest_bw_file, + (const char *)v3_ns->bw_file_digest256); + /* "bandwidth-file-digest" 1*(SP algorithm "=" digest) NL */ + char *digest_algo_b64_digest_bw_file = NULL; + tor_asprintf(&digest_algo_b64_digest_bw_file, "%s=%s", + crypto_digest_algorithm_get_name(DIGEST_ALG_BW_FILE), + b64_digest_bw_file); + /* No need for tor_strdup(""), format_line_if_present does it. */ + bw_file_digest = format_line_if_present( + "bandwidth-file-digest", digest_algo_b64_digest_bw_file); + tor_free(digest_algo_b64_digest_bw_file); + } + smartlist_add_asprintf(chunks, "network-status-version 3\n" "vote-status %s\n" @@ -323,6 +349,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, "flag-thresholds %s\n" "params %s\n" "%s" /* bandwidth file headers */ + "%s" /* bandwidth file digest */ "dir-source %s %s %s %s %d %d\n" "contact %s\n" "%s" /* shared randomness information */ @@ -339,6 +366,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, flag_thresholds, params, bw_headers_line ? bw_headers_line : "", + bw_file_digest ? bw_file_digest: "", voter->nickname, fingerprint, voter->address, fmt_addr32(addr), voter->dir_port, voter->or_port, voter->contact, @@ -351,6 +379,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, tor_free(methods); tor_free(shared_random_vote_str); tor_free(bw_headers_line); + tor_free(bw_file_digest); if (!tor_digest_is_zero(voter->legacy_id_digest)) { char fpbuf[HEX_DIGEST_LEN+1]; @@ -412,7 +441,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, { networkstatus_t *v; - if (!(v = networkstatus_parse_vote_from_string(status, NULL, + if (!(v = networkstatus_parse_vote_from_string(status, strlen(status), + NULL, v3_ns->type))) { log_err(LD_BUG,"Generated a networkstatus %s we couldn't parse: " "<<%s>>", @@ -2409,7 +2439,8 @@ networkstatus_compute_consensus(smartlist_t *votes, { networkstatus_t *c; - if (!(c = networkstatus_parse_vote_from_string(result, NULL, + if (!(c = networkstatus_parse_vote_from_string(result, strlen(result), + NULL, NS_TYPE_CONSENSUS))) { log_err(LD_BUG, "Generated a networkstatus consensus we couldn't " "parse."); @@ -3132,7 +3163,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) *msg_out = NULL; again: - vote = networkstatus_parse_vote_from_string(vote_body, &end_of_vote, + vote = networkstatus_parse_vote_from_string(vote_body, strlen(vote_body), + &end_of_vote, NS_TYPE_VOTE); if (!end_of_vote) end_of_vote = vote_body + strlen(vote_body); @@ -3390,7 +3422,9 @@ dirvote_compute_consensuses(void) flavor_name); continue; } - consensus = networkstatus_parse_vote_from_string(consensus_body, NULL, + consensus = networkstatus_parse_vote_from_string(consensus_body, + strlen(consensus_body), + NULL, NS_TYPE_CONSENSUS); if (!consensus) { log_warn(LD_DIR, "Couldn't parse %s consensus we generated!", @@ -3529,7 +3563,7 @@ dirvote_add_signatures_to_pending_consensus( * just in case we break detached signature processing at some point. */ { networkstatus_t *v = networkstatus_parse_vote_from_string( - pc->body, NULL, + pc->body, strlen(pc->body), NULL, NS_TYPE_CONSENSUS); tor_assert(v); networkstatus_vote_free(v); @@ -3654,7 +3688,9 @@ dirvote_publish_consensus(void) continue; } - if (networkstatus_set_current_consensus(pending->body, name, 0, NULL)) + if (networkstatus_set_current_consensus(pending->body, + strlen(pending->body), + name, 0, NULL)) log_warn(LD_DIR, "Error publishing %s consensus", name); else log_notice(LD_DIR, "Published %s consensus", name); @@ -3791,8 +3827,16 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) smartlist_add_asprintf(chunks, "a %s\n", fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport)); - if (family) - smartlist_add_asprintf(chunks, "family %s\n", family); + if (family) { + if (consensus_method < MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS) { + smartlist_add_asprintf(chunks, "family %s\n", family); + } else { + const uint8_t *id = (const uint8_t *)ri->cache_info.identity_digest; + char *canonical_family = nodefamily_canonicalize(family, id, 0); + smartlist_add_asprintf(chunks, "family %s\n", canonical_family); + tor_free(canonical_family); + } + } if (summary && strcmp(summary, "reject 1-65535")) smartlist_add_asprintf(chunks, "p %s\n", summary); @@ -3890,7 +3934,10 @@ static const struct consensus_method_range_t { int high; } microdesc_consensus_methods[] = { {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1}, - {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD}, + {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, + MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS - 1}, + {MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS, + MAX_SUPPORTED_CONSENSUS_METHOD}, {-1, -1} }; @@ -4364,6 +4411,23 @@ clear_status_flags_on_sybil(routerstatus_t *rs) * forget to add it to this clause. */ } +/** Space-separated list of all the flags that we will always vote on. */ +const char DIRVOTE_UNIVERSAL_FLAGS[] = + "Authority " + "Exit " + "Fast " + "Guard " + "HSDir " + "Stable " + "StaleDesc " + "V2Dir " + "Valid"; +/** Space-separated list of all flags that we may or may not vote on, + * depending on our configuration. */ +const char DIRVOTE_OPTIONAL_FLAGS[] = + "BadExit " + "Running"; + /** Return a new networkstatus_t* containing our current opinion. (For v3 * authorities) */ networkstatus_t * @@ -4388,6 +4452,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, const int vote_on_reachability = running_long_enough_to_decide_unreachable(); smartlist_t *microdescriptors = NULL; smartlist_t *bw_file_headers = NULL; + uint8_t bw_file_digest256[DIGEST256_LEN] = {0}; tor_assert(private_key); tor_assert(cert); @@ -4425,7 +4490,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, * set_routerstatus_from_routerinfo() see up-to-date bandwidth info. */ if (options->V3BandwidthsFile) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL); + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL, + NULL); } else { /* * No bandwidths file; clear the measured bandwidth cache in case we had @@ -4530,7 +4596,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, /* Only set bw_file_headers when V3BandwidthsFile is configured */ bw_file_headers = smartlist_new(); dirserv_read_measured_bandwidths(options->V3BandwidthsFile, - routerstatuses, bw_file_headers); + routerstatuses, bw_file_headers, + bw_file_digest256); + } else { /* * No bandwidths file; clear the measured bandwidth cache in case we had @@ -4612,7 +4680,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, v3_out->known_flags = smartlist_new(); smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid HSDir", + DIRVOTE_UNIVERSAL_FLAGS, 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); if (vote_on_reachability) smartlist_add_strdup(v3_out->known_flags, "Running"); @@ -4627,6 +4695,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, smartlist_sort_strings(v3_out->net_params); } v3_out->bw_file_headers = bw_file_headers; + memcpy(v3_out->bw_file_digest256, bw_file_digest256, DIGEST256_LEN); voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); voter->nickname = tor_strdup(options->Nickname); diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h index 02d88d19d1..f9de5ebc41 100644 --- a/src/feature/dirauth/dirvote.h +++ b/src/feature/dirauth/dirvote.h @@ -57,7 +57,7 @@ #define MIN_SUPPORTED_CONSENSUS_METHOD 25 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 28 +#define MAX_SUPPORTED_CONSENSUS_METHOD 29 /** Lowest consensus method where authorities vote on required/recommended * protocols. */ @@ -79,6 +79,12 @@ * addresses. See #23828 and #20916. */ #define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28 +/** + * Lowest consensus method where microdescriptor lines are put in canonical + * form for improved compressibility and ease of storage. See proposal 298. + **/ +#define MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS 29 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not * get confused with the above macros.) */ @@ -92,6 +98,9 @@ /** Maximum size of a line in a vote. */ #define MAX_BW_FILE_HEADERS_LINE_LEN 1024 +extern const char DIRVOTE_UNIVERSAL_FLAGS[]; +extern const char DIRVOTE_OPTIONAL_FLAGS[]; + /* * Public API. Used outside of the dirauth subsystem. * diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c index 21b8e239ec..656922233e 100644 --- a/src/feature/dirauth/process_descs.c +++ b/src/feature/dirauth/process_descs.c @@ -519,7 +519,8 @@ WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b) /** As for dirserv_add_descriptor(), but accepts multiple documents, and * returns the most severe error that occurred for any one of them. */ was_router_added_t -dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, +dirserv_add_multiple_descriptors(const char *desc, size_t desclen, + uint8_t purpose, const char *source, const char **msg) { @@ -536,6 +537,11 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */ + if (!string_is_utf8_no_bom(desc, desclen)) { + *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM."; + return ROUTER_AUTHDIR_REJECTS; + } + format_iso_time(time_buf, now); if (tor_snprintf(annotation_buf, sizeof(annotation_buf), "@uploaded-at %s\n" @@ -552,7 +558,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, s = desc; list = smartlist_new(); - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0, + if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 0, 0, annotation_buf, NULL)) { SMARTLIST_FOREACH(list, routerinfo_t *, ri, { msg_out = NULL; @@ -568,7 +574,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose, smartlist_clear(list); s = desc; - if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0, + if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 1, 0, NULL, NULL)) { SMARTLIST_FOREACH(list, extrainfo_t *, ei, { msg_out = NULL; diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h index ae2d6ad25d..510e54f813 100644 --- a/src/feature/dirauth/process_descs.h +++ b/src/feature/dirauth/process_descs.h @@ -17,7 +17,8 @@ void dirserv_free_fingerprint_list(void); int dirserv_add_own_fingerprint(crypto_pk_t *pk); enum was_router_added_t dirserv_add_multiple_descriptors( - const char *desc, uint8_t purpose, + const char *desc, size_t desclen, + uint8_t purpose, const char *source, const char **msg); enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c index 34b2283250..137c49800f 100644 --- a/src/feature/dirauth/shared_random.c +++ b/src/feature/dirauth/shared_random.c @@ -120,8 +120,8 @@ static const char sr_flag_ns_str[] = "shared-rand-participate"; static int32_t num_srv_agreements_from_vote; /* Return a heap allocated copy of the SRV <b>orig</b>. */ -STATIC sr_srv_t * -srv_dup(const sr_srv_t *orig) +sr_srv_t * +sr_srv_dup(const sr_srv_t *orig) { sr_srv_t *duplicate = NULL; @@ -1253,8 +1253,8 @@ sr_act_post_consensus(const networkstatus_t *consensus) * decided by the majority. */ sr_state_unset_fresh_srv(); /* Set the SR values from the given consensus. */ - sr_state_set_previous_srv(srv_dup(consensus->sr_info.previous_srv)); - sr_state_set_current_srv(srv_dup(consensus->sr_info.current_srv)); + sr_state_set_previous_srv(sr_srv_dup(consensus->sr_info.previous_srv)); + sr_state_set_current_srv(sr_srv_dup(consensus->sr_info.current_srv)); } /* Prepare our state so that it's ready for the next voting period. */ diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h index 25d95ebbc7..0b45ad1ed7 100644 --- a/src/feature/dirauth/shared_random.h +++ b/src/feature/dirauth/shared_random.h @@ -154,6 +154,7 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit) void sr_compute_srv(void); sr_commit_t *sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert); +sr_srv_t *sr_srv_dup(const sr_srv_t *orig); #ifdef SHARED_RANDOM_PRIVATE @@ -172,7 +173,6 @@ STATIC sr_srv_t *get_majority_srv_from_votes(const smartlist_t *votes, int current); STATIC void save_commit_to_state(sr_commit_t *commit); -STATIC sr_srv_t *srv_dup(const sr_srv_t *orig); STATIC int commitments_are_the_same(const sr_commit_t *commit_one, const sr_commit_t *commit_two); STATIC int commit_is_authoritative(const sr_commit_t *commit, diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c index b3e4a4ef92..a7b7480edd 100644 --- a/src/feature/dirauth/shared_random_state.c +++ b/src/feature/dirauth/shared_random_state.c @@ -22,6 +22,7 @@ #include "feature/dirauth/shared_random_state.h" #include "feature/dircommon/voting_schedule.h" #include "lib/encoding/confline.h" +#include "lib/version/torversion.h" #include "app/config/or_state_st.h" @@ -101,6 +102,8 @@ static const config_format_t state_format = { &state_extra_var, }; +static void state_query_del_(sr_state_object_t obj_type, void *data); + /* Return a string representation of a protocol phase. */ STATIC const char * get_phase_str(sr_phase_t phase) @@ -833,6 +836,9 @@ state_query_get_commit(const char *rsa_fpr) static void * state_query_get_(sr_state_object_t obj_type, const void *data) { + if (BUG(!sr_state)) + return NULL; + void *obj = NULL; switch (obj_type) { @@ -861,23 +867,44 @@ state_query_get_(sr_state_object_t obj_type, const void *data) } /* Helper function: This handles the PUT state action using an - * <b>obj_type</b> and <b>data</b> needed for the action. */ + * <b>obj_type</b> and <b>data</b> needed for the action. + * PUT frees the previous data before replacing it, if needed. */ static void state_query_put_(sr_state_object_t obj_type, void *data) { + if (BUG(!sr_state)) + return; + switch (obj_type) { case SR_STATE_OBJ_COMMIT: { sr_commit_t *commit = data; tor_assert(commit); + /* commit_add_to_state() frees the old commit, if there is one */ commit_add_to_state(commit, sr_state); break; } case SR_STATE_OBJ_CURSRV: - sr_state->current_srv = (sr_srv_t *) data; + /* Check if the new pointer is the same as the old one: if it is, it's + * probably a bug. The caller may have confused current and previous, + * or they may have forgotten to sr_srv_dup(). + * Putting NULL multiple times is allowed. */ + if (!BUG(data && sr_state->current_srv == (sr_srv_t *) data)) { + /* We own the old SRV, so we need to free it. */ + state_query_del_(SR_STATE_OBJ_CURSRV, NULL); + sr_state->current_srv = (sr_srv_t *) data; + } break; case SR_STATE_OBJ_PREVSRV: - sr_state->previous_srv = (sr_srv_t *) data; + /* Check if the new pointer is the same as the old one: if it is, it's + * probably a bug. The caller may have confused current and previous, + * or they may have forgotten to sr_srv_dup(). + * Putting NULL multiple times is allowed. */ + if (!BUG(data && sr_state->previous_srv == (sr_srv_t *) data)) { + /* We own the old SRV, so we need to free it. */ + state_query_del_(SR_STATE_OBJ_PREVSRV, NULL); + sr_state->previous_srv = (sr_srv_t *) data; + } break; case SR_STATE_OBJ_VALID_AFTER: sr_state->valid_after = *((time_t *) data); @@ -897,6 +924,9 @@ state_query_put_(sr_state_object_t obj_type, void *data) static void state_query_del_all_(sr_state_object_t obj_type) { + if (BUG(!sr_state)) + return; + switch (obj_type) { case SR_STATE_OBJ_COMMIT: { @@ -925,6 +955,9 @@ state_query_del_(sr_state_object_t obj_type, void *data) { (void) data; + if (BUG(!sr_state)) + return; + switch (obj_type) { case SR_STATE_OBJ_PREVSRV: tor_free(sr_state->previous_srv); @@ -999,16 +1032,16 @@ state_del_previous_srv(void) state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_PREVSRV, NULL, NULL); } -/* Rotate SRV value by freeing the previous value, assigning the current - * value to the previous one and nullifying the current one. */ +/* Rotate SRV value by setting the previous SRV to the current SRV, and + * clearing the current SRV. */ STATIC void state_rotate_srv(void) { /* First delete previous SRV from the state. Object will be freed. */ state_del_previous_srv(); - /* Set previous SRV with the current one. */ - sr_state_set_previous_srv(sr_state_get_current_srv()); - /* Nullify the current srv. */ + /* Set previous SRV to a copy of the current one. */ + sr_state_set_previous_srv(sr_srv_dup(sr_state_get_current_srv())); + /* Free and NULL the current srv. */ sr_state_set_current_srv(NULL); } @@ -1029,7 +1062,9 @@ sr_state_get_phase(void) return *(sr_phase_t *) ptr; } -/* Return the previous SRV value from our state. Value CAN be NULL. */ +/* Return the previous SRV value from our state. Value CAN be NULL. + * The state object owns the SRV, so the calling code should not free the SRV. + * Use sr_srv_dup() if you want to keep a copy of the SRV. */ const sr_srv_t * sr_state_get_previous_srv(void) { @@ -1048,7 +1083,9 @@ sr_state_set_previous_srv(const sr_srv_t *srv) NULL); } -/* Return the current SRV value from our state. Value CAN be NULL. */ +/* Return the current SRV value from our state. Value CAN be NULL. + * The state object owns the SRV, so the calling code should not free the SRV. + * Use sr_srv_dup() if you want to keep a copy of the SRV. */ const sr_srv_t * sr_state_get_current_srv(void) { diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c index 54c70b989a..4f7593a3e1 100644 --- a/src/feature/dirauth/voteflags.c +++ b/src/feature/dirauth/voteflags.c @@ -95,7 +95,7 @@ real_uptime(const routerinfo_t *router, time_t now) */ static int dirserv_thinks_router_is_unreliable(time_t now, - routerinfo_t *router, + const routerinfo_t *router, int need_uptime, int need_capacity) { if (need_uptime) { @@ -541,7 +541,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, - routerinfo_t *ri, + const routerinfo_t *ri, time_t now, int listbadexits) { @@ -593,6 +593,10 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->or_port = ri->or_port; rs->dir_port = ri->dir_port; rs->is_v2_dir = ri->supports_tunnelled_dir_requests; + + rs->is_staledesc = + (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now; + if (options->AuthDirHasIPv6Connectivity == 1 && !tor_addr_is_null(&ri->ipv6_addr) && node->last_reachable6 >= now - REACHABLE_TIMEOUT) { diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h index aa7b6ed082..cca6f53746 100644 --- a/src/feature/dirauth/voteflags.h +++ b/src/feature/dirauth/voteflags.h @@ -19,12 +19,16 @@ int running_long_enough_to_decide_unreachable(void); void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, - routerinfo_t *ri, time_t now, + const routerinfo_t *ri, + time_t now, int listbadexits); void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); #ifdef VOTEFLAGS_PRIVATE +/** Any descriptor older than this age causes the authorities to set the + * StaleDesc flag. */ +#define DESC_IS_STALE_INTERVAL (18*60*60) STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs); #endif diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c index 025361fa60..6b16307e3c 100644 --- a/src/feature/dircache/consdiffmgr.c +++ b/src/feature/dircache/consdiffmgr.c @@ -189,6 +189,7 @@ static consdiff_cfg_t consdiff_cfg = { static int consdiffmgr_ensure_space_for_files(int n); static int consensus_queue_compression_work(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed); static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from, consensus_cache_entry_t *diff_to); @@ -509,8 +510,25 @@ get_max_age_to_cache(void) MAX_MAX_AGE_TO_CACHE); } +#ifdef TOR_UNIT_TESTS +/** As consdiffmgr_add_consensus, but requires a nul-terminated input. For + * testing. */ +int +consdiffmgr_add_consensus_nulterm(const char *consensus, + const networkstatus_t *as_parsed) +{ + size_t len = strlen(consensus); + /* make a non-nul-terminated copy so that we can have a better chance + * of catching errors. */ + char *ctmp = tor_memdup(consensus, len); + int r = consdiffmgr_add_consensus(ctmp, len, as_parsed); + tor_free(ctmp); + return r; +} +#endif + /** - * Given a string containing a networkstatus consensus, and the results of + * Given a buffer containing a networkstatus consensus, and the results of * having parsed that consensus, add that consensus to the cache if it is not * already present and not too old. Create new consensus diffs from or to * that consensus as appropriate. @@ -519,6 +537,7 @@ get_max_age_to_cache(void) */ int consdiffmgr_add_consensus(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed) { if (BUG(consensus == NULL) || BUG(as_parsed == NULL)) @@ -544,7 +563,7 @@ consdiffmgr_add_consensus(const char *consensus, } /* We don't have it. Add it to the cache. */ - return consensus_queue_compression_work(consensus, as_parsed); + return consensus_queue_compression_work(consensus, consensus_len, as_parsed); } /** @@ -1387,19 +1406,21 @@ typedef struct consensus_diff_worker_job_t { } consensus_diff_worker_job_t; /** Given a consensus_cache_entry_t, check whether it has a label claiming - * that it was compressed. If so, uncompress its contents into <b>out</b> and - * set <b>outlen</b> to hold their size. If not, just copy the body into - * <b>out</b> and set <b>outlen</b> to its length. Return 0 on success, - * -1 on failure. - * - * In all cases, the output is nul-terminated. */ + * that it was compressed. If so, uncompress its contents into *<b>out</b> and + * set <b>outlen</b> to hold their size, and set *<b>owned_out</b> to a pointer + * that the caller will need to free. If not, just set *<b>out</b> and + * <b>outlen</b> to its extent in memory. Return 0 on success, -1 on failure. + **/ STATIC int -uncompress_or_copy(char **out, size_t *outlen, - consensus_cache_entry_t *ent) +uncompress_or_set_ptr(const char **out, size_t *outlen, + char **owned_out, + consensus_cache_entry_t *ent) { const uint8_t *body; size_t bodylen; + *owned_out = NULL; + if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0) return -1; @@ -1410,8 +1431,17 @@ uncompress_or_copy(char **out, size_t *outlen, if (lv_compression) method = compression_method_get_by_name(lv_compression); - return tor_uncompress(out, outlen, (const char *)body, bodylen, + int rv; + if (method == NO_METHOD) { + *out = (const char *)body; + *outlen = bodylen; + rv = 0; + } else { + rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen, method, 1, LOG_WARN); + *out = *owned_out; + } + return rv; } /** @@ -1478,16 +1508,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_) char *consensus_diff; { - char *diff_from_nt = NULL, *diff_to_nt = NULL; + const char *diff_from_nt = NULL, *diff_to_nt = NULL; + char *owned1 = NULL, *owned2 = NULL; size_t diff_from_nt_len, diff_to_nt_len; - if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len, - job->diff_from) < 0) { + if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1, + job->diff_from) < 0) { return WQ_RPL_REPLY; } - if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len, - job->diff_to) < 0) { - tor_free(diff_from_nt); + if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2, + job->diff_to) < 0) { + tor_free(owned1); return WQ_RPL_REPLY; } tor_assert(diff_from_nt); @@ -1496,9 +1527,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_) // XXXX ugh; this is going to calculate the SHA3 of both its // XXXX inputs again, even though we already have that. Maybe it's time // XXXX to change the API here? - consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt); - tor_free(diff_from_nt); - tor_free(diff_to_nt); + consensus_diff = consensus_diff_generate(diff_from_nt, + diff_from_nt_len, + diff_to_nt, + diff_to_nt_len); + tor_free(owned1); + tor_free(owned2); } if (!consensus_diff) { /* Couldn't generate consensus; we'll leave the reply blank. */ @@ -1746,8 +1780,8 @@ consensus_compress_worker_threadfn(void *state_, void *work_) (const uint8_t *)consensus, bodylen); { const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(consensus, - &start, &end) < 0) { + if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen, + &start, &end) < 0) { start = consensus; end = consensus+bodylen; } @@ -1811,14 +1845,15 @@ static int background_compression = 0; */ static int consensus_queue_compression_work(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed) { tor_assert(consensus); tor_assert(as_parsed); consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job)); - job->consensus = tor_strdup(consensus); - job->consensus_len = strlen(consensus); + job->consensus = tor_memdup_nulterm(consensus, consensus_len); + job->consensus_len = strlen(job->consensus); job->flavor = as_parsed->flavor; char va_str[ISO_TIME_LEN+1]; diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h index 39e8fa31cb..b1b3323b6c 100644 --- a/src/feature/dircache/consdiffmgr.h +++ b/src/feature/dircache/consdiffmgr.h @@ -22,6 +22,7 @@ typedef struct consdiff_cfg_t { struct consensus_cache_entry_t; // from conscache.h int consdiffmgr_add_consensus(const char *consensus, + size_t consensus_len, const networkstatus_t *as_parsed); consdiff_status_t consdiffmgr_find_consensus( @@ -68,8 +69,14 @@ STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus( STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out, consensus_cache_entry_t *ent, const char *label); -STATIC int uncompress_or_copy(char **out, size_t *outlen, - consensus_cache_entry_t *ent); +STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen, + char **owned_out, + consensus_cache_entry_t *ent); #endif /* defined(CONSDIFFMGR_PRIVATE) */ +#ifdef TOR_UNIT_TESTS +int consdiffmgr_add_consensus_nulterm(const char *consensus, + const networkstatus_t *as_parsed); +#endif + #endif /* !defined(TOR_CONSDIFFMGR_H) */ diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c index e8cb284165..eece1e6503 100644 --- a/src/feature/dircache/dircache.c +++ b/src/feature/dircache/dircache.c @@ -49,7 +49,8 @@ #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60) #define ROBOTS_CACHE_LIFETIME (24*60*60) #define MICRODESC_CACHE_LIFETIME (48*60*60) - +/* Bandwidth files change every hour. */ +#define BANDWIDTH_CACHE_LIFETIME (30*60) /** Parse an HTTP request string <b>headers</b> of the form * \verbatim * "\%s [http[s]://]\%s HTTP/1..." @@ -123,7 +124,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, long cache_lifetime) { char date[RFC1123_TIME_LEN+1]; - time_t now = time(NULL); + time_t now = approx_time(); buf_t *buf = buf_new_with_capacity(1024); tor_assert(conn); @@ -166,22 +167,16 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length, buf_free(buf); } -/** As write_http_response_header_impl, but sets encoding and content-typed - * based on whether the response will be <b>compressed</b> or not. */ +/** As write_http_response_header_impl, but translates method into + * encoding */ static void write_http_response_headers(dir_connection_t *conn, ssize_t length, compress_method_t method, const char *extra_headers, long cache_lifetime) { - const char *methodname = compression_method_get_name(method); - const char *doctype; - if (method == NO_METHOD) - doctype = "text/plain"; - else - doctype = "application/octet-stream"; write_http_response_header_impl(conn, length, - doctype, - methodname, + "text/plain", + compression_method_get_name(method), extra_headers, cache_lifetime); } @@ -357,12 +352,15 @@ static int handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args); static int handle_get_networkstatus_bridges(dir_connection_t *conn, const get_handler_args_t *args); +static int handle_get_next_bandwidth(dir_connection_t *conn, + const get_handler_args_t *args); /** Table for handling GET requests. */ static const url_table_ent_t url_table[] = { { "/tor/", 0, handle_get_frontpage }, { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus }, { "/tor/status-vote/current/", 1, handle_get_status_vote }, + { "/tor/status-vote/next/bandwidth", 0, handle_get_next_bandwidth }, { "/tor/status-vote/next/", 1, handle_get_status_vote }, { "/tor/micro/d/", 1, handle_get_microdesc }, { "/tor/server/", 1, handle_get_descriptor }, @@ -495,28 +493,47 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) } /** Warn that the cached consensus <b>consensus</b> of type - * <b>flavor</b> is too old and will not be served to clients. Rate-limit the - * warning to avoid logging an entry on every request. + * <b>flavor</b> too new or too old, based on <b>is_too_new</b>, + * and will not be served to clients. Rate-limit the warning to avoid logging + * an entry on every request. */ static void -warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus, - const char *flavor, time_t now) +warn_consensus_is_not_reasonably_live( + const struct consensus_cache_entry_t *consensus, + const char *flavor, time_t now, bool is_too_new) { -#define TOO_OLD_WARNING_INTERVAL (60*60) - static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL); +#define NOT_REASONABLY_LIVE_WARNING_INTERVAL (60*60) + static ratelim_t warned[2] = { RATELIM_INIT( + NOT_REASONABLY_LIVE_WARNING_INTERVAL), + RATELIM_INIT( + NOT_REASONABLY_LIVE_WARNING_INTERVAL) }; char timestamp[ISO_TIME_LEN+1]; - time_t valid_until; - char *dupes; - - if (consensus_cache_entry_get_valid_until(consensus, &valid_until)) - return; + /* valid_after if is_too_new, valid_until if !is_too_new */ + time_t valid_time = 0; + char *dupes = NULL; - if ((dupes = rate_limit_log(&warned, now))) { - format_local_iso_time(timestamp, valid_until); - log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not " - "serve it to clients. It was valid until %s local time and we " - "continued to serve it for up to 24 hours after it expired.%s", - flavor ? flavor : "", flavor ? " " : "", timestamp, dupes); + if (is_too_new) { + if (consensus_cache_entry_get_valid_after(consensus, &valid_time)) + return; + dupes = rate_limit_log(&warned[1], now); + } else { + if (consensus_cache_entry_get_valid_until(consensus, &valid_time)) + return; + dupes = rate_limit_log(&warned[0], now); + } + + if (dupes) { + format_local_iso_time(timestamp, valid_time); + log_warn(LD_DIRSERV, "Our %s%sconsensus is too %s, so we will not " + "serve it to clients. It was valid %s %s local time and we " + "continued to serve it for up to 24 hours %s.%s", + flavor ? flavor : "", + flavor ? " " : "", + is_too_new ? "new" : "old", + is_too_new ? "after" : "until", + timestamp, + is_too_new ? "before it was valid" : "after it expired", + dupes); tor_free(dupes); } } @@ -859,7 +876,6 @@ handle_get_current_consensus(dir_connection_t *conn, if (req.diff_only && !cached_consensus) { write_short_http_response(conn, 404, "No such diff available"); - // XXXX warn_consensus_is_too_old(v, req.flavor, now); geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); goto done; } @@ -870,19 +886,30 @@ handle_get_current_consensus(dir_connection_t *conn, &compression_used); } - time_t fresh_until, valid_until; - int have_fresh_until = 0, have_valid_until = 0; + time_t valid_after, fresh_until, valid_until; + int have_valid_after = 0, have_fresh_until = 0, have_valid_until = 0; if (cached_consensus) { + have_valid_after = + !consensus_cache_entry_get_valid_after(cached_consensus, &valid_after); have_fresh_until = !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until); have_valid_until = !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until); } - if (cached_consensus && have_valid_until && + if (cached_consensus && have_valid_after && + !networkstatus_valid_after_is_reasonably_live(valid_after, now)) { + write_short_http_response(conn, 404, "Consensus is too new"); + warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now, + 1); + geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); + goto done; + } else if ( + cached_consensus && have_valid_until && !networkstatus_valid_until_is_reasonably_live(valid_until, now)) { write_short_http_response(conn, 404, "Consensus is too old"); - warn_consensus_is_too_old(cached_consensus, req.flavor, now); + warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now, + 0); geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND); goto done; } @@ -1438,6 +1465,39 @@ handle_get_networkstatus_bridges(dir_connection_t *conn, return 0; } +/** Helper function for GET the bandwidth file used for the next vote */ +static int +handle_get_next_bandwidth(dir_connection_t *conn, + const get_handler_args_t *args) +{ + log_debug(LD_DIR, "Getting next bandwidth."); + const or_options_t *options = get_options(); + const compress_method_t compress_method = + find_best_compression_method(args->compression_supported, 1); + + if (options->V3BandwidthsFile) { + char *bandwidth = read_file_to_str(options->V3BandwidthsFile, + RFTS_IGNORE_MISSING, NULL); + if (bandwidth != NULL) { + ssize_t len = strlen(bandwidth); + write_http_response_header(conn, compress_method != NO_METHOD ? -1 : len, + compress_method, BANDWIDTH_CACHE_LIFETIME); + if (compress_method != NO_METHOD) { + conn->compress_state = tor_compress_new(1, compress_method, + choose_compression_level(len/2)); + log_debug(LD_DIR, "Compressing bandwidth file."); + } else { + log_debug(LD_DIR, "Not compressing bandwidth file."); + } + connection_dir_buf_add((const char*)bandwidth, len, conn, 1); + tor_free(bandwidth); + return 0; + } + } + write_short_http_response(conn, 404, "Not found"); + return 0; +} + /** Helper function for GET robots.txt or /tor/robots.txt */ static int handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) @@ -1608,8 +1668,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers, const char *msg = "[None]"; uint8_t purpose = authdir_mode_bridge(options) ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL; - was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose, - conn->base_.address, &msg); + was_router_added_t r = dirserv_add_multiple_descriptors(body, body_len, + purpose, conn->base_.address, &msg); tor_assert(msg); if (r == ROUTER_ADDED_SUCCESSFULLY) { diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c index 213c490314..4be6836fe1 100644 --- a/src/feature/dircache/dirserv.c +++ b/src/feature/dircache/dirserv.c @@ -234,6 +234,7 @@ free_cached_dir_(void *_d) * validation is performed. */ void dirserv_set_cached_consensus_networkstatus(const char *networkstatus, + size_t networkstatus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, @@ -244,7 +245,9 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus, if (!cached_consensuses) cached_consensuses = strmap_new(); - new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published); + new_networkstatus = + new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len), + published); memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t)); memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed, DIGEST256_LEN); diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h index 890b10fd80..7f944459da 100644 --- a/src/feature/dircache/dirserv.h +++ b/src/feature/dircache/dirserv.h @@ -84,6 +84,7 @@ int directory_too_idle_to_fetch_descriptors(const or_options_t *options, cached_dir_t *dirserv_get_consensus(const char *flavor_name); void dirserv_set_cached_consensus_networkstatus(const char *consensus, + size_t consensus_len, const char *flavor_name, const common_digests_t *digests, const uint8_t *sha3_as_signed, diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c index 0fd1a47017..70b6a20028 100644 --- a/src/feature/dirclient/dirclient.c +++ b/src/feature/dirclient/dirclient.c @@ -2205,24 +2205,31 @@ handle_response_fetch_consensus(dir_connection_t *conn, if (looks_like_a_consensus_diff(body, body_len)) { /* First find our previous consensus. Maybe it's in ram, maybe not. */ cached_dir_t *cd = dirserv_get_consensus(flavname); - const char *consensus_body; - char *owned_consensus = NULL; + const char *consensus_body = NULL; + size_t consensus_body_len; + tor_mmap_t *mapped_consensus = NULL; if (cd) { consensus_body = cd->dir; + consensus_body_len = cd->dir_len; } else { - owned_consensus = networkstatus_read_cached_consensus(flavname); - consensus_body = owned_consensus; + mapped_consensus = networkstatus_map_cached_consensus(flavname); + if (mapped_consensus) { + consensus_body = mapped_consensus->data; + consensus_body_len = mapped_consensus->size; + } } if (!consensus_body) { log_warn(LD_DIR, "Received a consensus diff, but we can't find " "any %s-flavored consensus in our current cache.",flavname); + tor_munmap_file(mapped_consensus); networkstatus_consensus_download_failed(0, flavname); // XXXX if this happens too much, see below return -1; } - new_consensus = consensus_diff_apply(consensus_body, body); - tor_free(owned_consensus); + new_consensus = consensus_diff_apply(consensus_body, consensus_body_len, + body, body_len); + tor_munmap_file(mapped_consensus); if (new_consensus == NULL) { log_warn(LD_DIR, "Could not apply consensus diff received from server " "'%s:%d'", conn->base_.address, conn->base_.port); @@ -2244,7 +2251,9 @@ handle_response_fetch_consensus(dir_connection_t *conn, sourcename = "downloaded"; } - if ((r=networkstatus_set_current_consensus(consensus, flavname, 0, + if ((r=networkstatus_set_current_consensus(consensus, + strlen(consensus), + flavname, 0, conn->identity_digest))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load %s consensus directory %s from " diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c index d0f7594ce3..8e93953f73 100644 --- a/src/feature/dircommon/consdiff.c +++ b/src/feature/dircommon/consdiff.c @@ -101,11 +101,11 @@ smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s) /* This is a separate, mockable function so that we can override it when * fuzzing. */ MOCK_IMPL(STATIC int, -consensus_compute_digest,(const char *cons, +consensus_compute_digest,(const char *cons, size_t len, consensus_digest_t *digest_out)) { int r = crypto_digest256((char*)digest_out->sha3_256, - cons, strlen(cons), DIGEST_SHA3_256); + cons, len, DIGEST_SHA3_256); return r; } @@ -114,11 +114,11 @@ consensus_compute_digest,(const char *cons, /* This is a separate, mockable function so that we can override it when * fuzzing. */ MOCK_IMPL(STATIC int, -consensus_compute_digest_as_signed,(const char *cons, +consensus_compute_digest_as_signed,(const char *cons, size_t len, consensus_digest_t *digest_out)) { return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256, - cons); + cons, len); } /** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */ @@ -1229,7 +1229,8 @@ consdiff_apply_diff(const smartlist_t *cons1, cons2_str = consensus_join_lines(cons2); consensus_digest_t cons2_digests; - if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) { + if (consensus_compute_digest(cons2_str, strlen(cons2_str), + &cons2_digests) < 0) { /* LCOV_EXCL_START -- digest can't fail */ log_warn(LD_CONSDIFF, "Could not compute digests of the consensus " "resulting from applying a consensus diff."); @@ -1283,12 +1284,13 @@ consdiff_apply_diff(const smartlist_t *cons1, * generated cdlines will become invalid. */ STATIC int -consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area) +consensus_split_lines(smartlist_t *out, + const char *s, size_t len, + memarea_t *area) { - const char *end_of_str = s + strlen(s); - tor_assert(*end_of_str == '\0'); + const char *end_of_str = s + len; - while (*s) { + while (s < end_of_str) { const char *eol = memchr(s, '\n', end_of_str - s); if (!eol) { /* File doesn't end with newline. */ @@ -1334,25 +1336,25 @@ consensus_join_lines(const smartlist_t *inp) * success, retun a newly allocated string containing that diff. On failure, * return NULL. */ char * -consensus_diff_generate(const char *cons1, - const char *cons2) +consensus_diff_generate(const char *cons1, size_t cons1len, + const char *cons2, size_t cons2len) { consensus_digest_t d1, d2; smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL; int r1, r2; char *result = NULL; - r1 = consensus_compute_digest_as_signed(cons1, &d1); - r2 = consensus_compute_digest(cons2, &d2); + r1 = consensus_compute_digest_as_signed(cons1, cons1len, &d1); + r2 = consensus_compute_digest(cons2, cons2len, &d2); if (BUG(r1 < 0 || r2 < 0)) return NULL; // LCOV_EXCL_LINE memarea_t *area = memarea_new(); lines1 = smartlist_new(); lines2 = smartlist_new(); - if (consensus_split_lines(lines1, cons1, area) < 0) + if (consensus_split_lines(lines1, cons1, cons1len, area) < 0) goto done; - if (consensus_split_lines(lines2, cons2, area) < 0) + if (consensus_split_lines(lines2, cons2, cons2len, area) < 0) goto done; result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area); @@ -1375,7 +1377,9 @@ consensus_diff_generate(const char *cons1, * consensus. On failure, return NULL. */ char * consensus_diff_apply(const char *consensus, - const char *diff) + size_t consensus_len, + const char *diff, + size_t diff_len) { consensus_digest_t d1; smartlist_t *lines1 = NULL, *lines2 = NULL; @@ -1383,15 +1387,15 @@ consensus_diff_apply(const char *consensus, char *result = NULL; memarea_t *area = memarea_new(); - r1 = consensus_compute_digest_as_signed(consensus, &d1); + r1 = consensus_compute_digest_as_signed(consensus, consensus_len, &d1); if (BUG(r1 < 0)) goto done; lines1 = smartlist_new(); lines2 = smartlist_new(); - if (consensus_split_lines(lines1, consensus, area) < 0) + if (consensus_split_lines(lines1, consensus, consensus_len, area) < 0) goto done; - if (consensus_split_lines(lines2, diff, area) < 0) + if (consensus_split_lines(lines2, diff, diff_len, area) < 0) goto done; result = consdiff_apply_diff(lines1, lines2, &d1); diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h index 98217e6d46..b63fcb2cc6 100644 --- a/src/feature/dircommon/consdiff.h +++ b/src/feature/dircommon/consdiff.h @@ -7,10 +7,10 @@ #include "core/or/or.h" -char *consensus_diff_generate(const char *cons1, - const char *cons2); -char *consensus_diff_apply(const char *consensus, - const char *diff); +char *consensus_diff_generate(const char *cons1, size_t cons1len, + const char *cons2, size_t cons2len); +char *consensus_diff_apply(const char *consensus, size_t consensus_len, + const char *diff, size_t diff_len); int looks_like_a_consensus_diff(const char *document, size_t len); @@ -78,7 +78,8 @@ STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice, STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2, const smartlist_slice_t *slice1, const smartlist_slice_t *slice2); -STATIC int consensus_split_lines(smartlist_t *out, const char *s, +STATIC int consensus_split_lines(smartlist_t *out, + const char *s, size_t len, struct memarea_t *area); STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area, const char *s); @@ -86,10 +87,10 @@ STATIC int lines_eq(const cdline_t *a, const cdline_t *b); STATIC int line_str_eq(const cdline_t *a, const char *b); MOCK_DECL(STATIC int, - consensus_compute_digest,(const char *cons, + consensus_compute_digest,(const char *cons, size_t len, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, - consensus_compute_digest_as_signed,(const char *cons, + consensus_compute_digest_as_signed,(const char *cons, size_t len, consensus_digest_t *digest_out)); MOCK_DECL(STATIC int, consensus_digest_eq,(const uint8_t *d1, diff --git a/src/feature/dirparse/authcert_parse.c b/src/feature/dirparse/authcert_parse.c index 1680bdbf30..8ba5a53981 100644 --- a/src/feature/dirparse/authcert_parse.c +++ b/src/feature/dirparse/authcert_parse.c @@ -24,7 +24,8 @@ static token_rule_t dir_key_certificate_table[] = { /** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to * the first character after the certificate. */ authority_cert_t * -authority_cert_parse_from_string(const char *s, const char **end_of_string) +authority_cert_parse_from_string(const char *s, size_t maxlen, + const char **end_of_string) { /** Reject any certificate at least this big; it is probably an overflow, an * attack, a bug, or some other nonsense. */ @@ -35,24 +36,25 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) char digest[DIGEST_LEN]; directory_token_t *tok; char fp_declared[DIGEST_LEN]; - char *eos; + const char *eos; size_t len; int found; memarea_t *area = NULL; + const char *end_of_s = s + maxlen; const char *s_dup = s; - s = eat_whitespace(s); - eos = strstr(s, "\ndir-key-certification"); + s = eat_whitespace_eos(s, end_of_s); + eos = tor_memstr(s, end_of_s - s, "\ndir-key-certification"); if (! eos) { log_warn(LD_DIR, "No signature found on key certificate"); return NULL; } - eos = strstr(eos, "\n-----END SIGNATURE-----\n"); + eos = tor_memstr(eos, end_of_s - eos, "\n-----END SIGNATURE-----\n"); if (! eos) { log_warn(LD_DIR, "No end-of-signature found on key certificate"); return NULL; } - eos = strchr(eos+2, '\n'); + eos = memchr(eos+2, '\n', end_of_s - (eos+2)); tor_assert(eos); ++eos; len = eos - s; @@ -69,7 +71,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) log_warn(LD_DIR, "Error tokenizing key certificate"); goto err; } - if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version", + if (router_get_hash_impl(s, eos - s, digest, "dir-key-certificate-version", "\ndir-key-certification", '\n', DIGEST_SHA1) < 0) goto err; tok = smartlist_get(tokens, 0); diff --git a/src/feature/dirparse/authcert_parse.h b/src/feature/dirparse/authcert_parse.h index ca475ad0e3..800631c3de 100644 --- a/src/feature/dirparse/authcert_parse.h +++ b/src/feature/dirparse/authcert_parse.h @@ -13,6 +13,7 @@ #define TOR_AUTHCERT_PARSE_H authority_cert_t *authority_cert_parse_from_string(const char *s, + size_t maxlen, const char **end_of_string); #endif /* !defined(TOR_AUTHCERT_PARSE_H) */ diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c index 5a75af3994..22cc1e272e 100644 --- a/src/feature/dirparse/microdesc_parse.c +++ b/src/feature/dirparse/microdesc_parse.c @@ -18,6 +18,7 @@ #include "feature/dirparse/routerparse.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" #include "feature/relay/router.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" @@ -32,7 +33,7 @@ static token_rule_t microdesc_token_table[] = { T01("ntor-onion-key", K_ONION_KEY_NTOR, GE(1), NO_OBJ ), T0N("id", K_ID, GE(2), NO_OBJ ), T0N("a", K_A, GE(1), NO_OBJ ), - T01("family", K_FAMILY, ARGS, NO_OBJ ), + T01("family", K_FAMILY, CONCAT_ARGS, NO_OBJ ), T01("p", K_P, CONCAT_ARGS, NO_OBJ ), T01("p6", K_P6, CONCAT_ARGS, NO_OBJ ), A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ ), @@ -159,7 +160,22 @@ microdescs_parse_from_string(const char *s, const char *eos, if (tokenize_string(area, s, start_of_next_microdesc, tokens, microdesc_token_table, flags)) { - log_warn(LD_DIR, "Unparseable microdescriptor"); + const char *location; + switch (where) { + case SAVED_NOWHERE: + location = "download or generated string"; + break; + case SAVED_IN_CACHE: + location = "cache"; + break; + case SAVED_IN_JOURNAL: + location = "journal"; + break; + default: + location = "unknown location"; + break; + } + log_warn(LD_DIR, "Unparseable microdescriptor found in %s", location); goto next; } @@ -176,8 +192,8 @@ microdescs_parse_from_string(const char *s, const char *eos, "Relay's onion key had invalid exponent."); goto next; } - router_set_rsa_onion_pkey(tok->key, &md->onion_pkey, - &md->onion_pkey_len); + md->onion_pkey = tor_memdup(tok->object_body, tok->object_size); + md->onion_pkey_len = tok->object_size; crypto_pk_free(tok->key); if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) { @@ -222,16 +238,9 @@ microdescs_parse_from_string(const char *s, const char *eos, } if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) { - int i; - md->family = smartlist_new(); - for (i=0;i<tok->n_args;++i) { - if (!is_legal_nickname_or_hexdigest(tok->args[i])) { - log_warn(LD_DIR, "Illegal nickname %s in family line", - escaped(tok->args[i])); - goto next; - } - smartlist_add_strdup(md->family, tok->args[i]); - } + md->family = nodefamily_parse(tok->args[0], + NULL, + NF_WARN_MALFORMED); } if ((tok = find_opt_by_keyword(tokens, K_P))) { diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c index 109eebeb66..d653a59826 100644 --- a/src/feature/dirparse/ns_parse.c +++ b/src/feature/dirparse/ns_parse.c @@ -151,10 +151,11 @@ static token_rule_t networkstatus_vote_footer_token_table[] = { * -1. */ int router_get_networkstatus_v3_signed_boundaries(const char *s, + size_t len, const char **start_out, const char **end_out) { - return router_get_hash_impl_helper(s, strlen(s), + return router_get_hash_impl_helper(s, len, "network-status-version", "\ndirectory-signature", ' ', LOG_INFO, @@ -166,12 +167,13 @@ router_get_networkstatus_v3_signed_boundaries(const char *s, * signed portion can be identified. Return 0 on success, -1 on failure. */ int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, - const char *s) + const char *s, size_t len) { const char *start, *end; - if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) { + if (router_get_networkstatus_v3_signed_boundaries(s, len, + &start, &end) < 0) { start = s; - end = s + strlen(s); + end = s + len; } tor_assert(start); tor_assert(end); @@ -182,9 +184,10 @@ router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, /** Set <b>digests</b> to all the digests of the consensus document in * <b>s</b> */ int -router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests) +router_get_networkstatus_v3_hashes(const char *s, size_t len, + common_digests_t *digests) { - return router_get_hashes_impl(s,strlen(s),digests, + return router_get_hashes_impl(s, len, digests, "network-status-version", "\ndirectory-signature", ' '); @@ -195,13 +198,13 @@ router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests) * return the start of the directory footer, or the next directory signature. * If none is found, return the end of the string. */ static inline const char * -find_start_of_next_routerstatus(const char *s) +find_start_of_next_routerstatus(const char *s, const char *s_eos) { const char *eos, *footer, *sig; - if ((eos = strstr(s, "\nr "))) + if ((eos = tor_memstr(s, s_eos - s, "\nr "))) ++eos; else - eos = s + strlen(s); + eos = s_eos; footer = tor_memstr(s, eos-s, "\ndirectory-footer"); sig = tor_memstr(s, eos-s, "\ndirectory-signature"); @@ -289,7 +292,8 @@ routerstatus_parse_guardfraction(const char *guardfraction_str, **/ STATIC routerstatus_t * routerstatus_parse_entry_from_string(memarea_t *area, - const char **s, smartlist_t *tokens, + const char **s, const char *s_eos, + smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, int consensus_method, @@ -308,7 +312,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, flav = FLAV_NS; tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC); - eos = find_start_of_next_routerstatus(*s); + eos = find_start_of_next_routerstatus(*s, s_eos); if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) { log_warn(LD_DIR, "Error tokenizing router status"); @@ -430,6 +434,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, rs->is_hs_dir = 1; } else if (!strcmp(tok->args[i], "V2Dir")) { rs->is_v2_dir = 1; + } else if (!strcmp(tok->args[i], "StaleDesc")) { + rs->is_staledesc = 1; } } /* These are implied true by having been included in a consensus made @@ -1051,7 +1057,9 @@ extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens) /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from <b>s</b>, and return the result. Return NULL on failure. */ networkstatus_t * -networkstatus_parse_vote_from_string(const char *s, const char **eos_out, +networkstatus_parse_vote_from_string(const char *s, + size_t s_len, + const char **eos_out, networkstatus_type_t ns_type) { smartlist_t *tokens = smartlist_new(); @@ -1067,20 +1075,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, memarea_t *area = NULL, *rs_area = NULL; consensus_flavor_t flav = FLAV_NS; char *last_kwd=NULL; + const char *eos = s + s_len; tor_assert(s); if (eos_out) *eos_out = NULL; - if (router_get_networkstatus_v3_hashes(s, &ns_digests) || - router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) { + if (router_get_networkstatus_v3_hashes(s, s_len, &ns_digests) || + router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, + s, s_len)<0) { log_warn(LD_DIR, "Unable to compute digest of network-status"); goto err; } area = memarea_new(); - end_of_header = find_start_of_next_routerstatus(s); + end_of_header = find_start_of_next_routerstatus(s, eos); if (tokenize_string(area, s, end_of_header, tokens, (ns_type == NS_TYPE_CONSENSUS) ? networkstatus_consensus_token_table : @@ -1111,10 +1121,12 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (ns_type != NS_TYPE_CONSENSUS) { const char *end_of_cert = NULL; - if (!(cert = strstr(s, "\ndir-key-certificate-version"))) + if (!(cert = tor_memstr(s, end_of_header - s, + "\ndir-key-certificate-version"))) goto err; ++cert; - ns->cert = authority_cert_parse_from_string(cert, &end_of_cert); + ns->cert = authority_cert_parse_from_string(cert, end_of_header - cert, + &end_of_cert); if (!ns->cert || !end_of_cert || end_of_cert > end_of_header) goto err; } @@ -1424,10 +1436,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, s = end_of_header; ns->routerstatus_list = smartlist_new(); - while (!strcmpstart(s, "r ")) { + while (eos - s >= 2 && fast_memeq(s, "r ", 2)) { if (ns->type != NS_TYPE_CONSENSUS) { vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns, + if (routerstatus_parse_entry_from_string(rs_area, &s, eos, rs_tokens, ns, rs, 0, 0)) { smartlist_add(ns->routerstatus_list, rs); } else { @@ -1435,7 +1447,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } } else { routerstatus_t *rs; - if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, + if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, eos, + rs_tokens, NULL, NULL, ns->consensus_method, flav))) { @@ -1480,10 +1493,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, /* Parse footer; check signature. */ footer_tokens = smartlist_new(); - if ((end_of_footer = strstr(s, "\nnetwork-status-version "))) + if ((end_of_footer = tor_memstr(s, eos-s, "\nnetwork-status-version "))) ++end_of_footer; else - end_of_footer = s + strlen(s); + end_of_footer = eos; if (tokenize_string(area,s, end_of_footer, footer_tokens, networkstatus_vote_footer_token_table, 0)) { log_warn(LD_DIR, "Error tokenizing network-status vote footer."); diff --git a/src/feature/dirparse/ns_parse.h b/src/feature/dirparse/ns_parse.h index 10a6f9cefc..dedfa6fc88 100644 --- a/src/feature/dirparse/ns_parse.h +++ b/src/feature/dirparse/ns_parse.h @@ -12,18 +12,19 @@ #ifndef TOR_NS_PARSE_H #define TOR_NS_PARSE_H -int router_get_networkstatus_v3_hashes(const char *s, +int router_get_networkstatus_v3_hashes(const char *s, size_t len, common_digests_t *digests); -int router_get_networkstatus_v3_signed_boundaries(const char *s, +int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len, const char **start_out, const char **end_out); int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out, - const char *s); + const char *s, size_t len); int compare_vote_routerstatus_entries(const void **_a, const void **_b); int networkstatus_verify_bw_weights(networkstatus_t *ns, int); enum networkstatus_type_t; networkstatus_t *networkstatus_parse_vote_from_string(const char *s, + size_t len, const char **eos_out, enum networkstatus_type_t ns_type); @@ -35,7 +36,8 @@ STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str, struct memarea_t; STATIC routerstatus_t *routerstatus_parse_entry_from_string( struct memarea_t *area, - const char **s, smartlist_t *tokens, + const char **s, const char *eos, + smartlist_t *tokens, networkstatus_t *vote, vote_routerstatus_t *vote_rs, int consensus_method, diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c index 632f5adff0..036a51689c 100644 --- a/src/feature/dirparse/parsecommon.c +++ b/src/feature/dirparse/parsecommon.c @@ -15,6 +15,7 @@ #include "lib/string/printf.h" #include "lib/memarea/memarea.h" #include "lib/crypt_ops/crypto_rsa.h" +#include "lib/ctime/di_ops.h" #include <string.h> @@ -169,7 +170,6 @@ get_token_arguments(memarea_t *area, directory_token_t *tok, char *cp = mem; int j = 0; char *args[MAX_ARGS]; - memset(args, 0, sizeof(args)); while (*cp) { if (j == MAX_ARGS) return -1; @@ -251,6 +251,16 @@ token_check_object(memarea_t *area, const char *kwd, return tok; } +/** Return true iff the <b>memlen</b>-byte chunk of memory at + * <b>memlen</b> is the same length as <b>token</b>, and their + * contents are equal. */ +static bool +mem_eq_token(const void *mem, size_t memlen, const char *token) +{ + size_t len = strlen(token); + return memlen == len && fast_memeq(mem, token, len); +} + /** Helper function: read the next token from *s, advance *s to the end of the * token, and return the parsed token. Parse *<b>s</b> according to the list * of tokens in <b>table</b>. @@ -266,7 +276,7 @@ get_next_token(memarea_t *area, * attack, a bug, or some other nonsense. */ #define MAX_LINE_LENGTH (128*1024) - const char *next, *eol, *obstart; + const char *next, *eol; size_t obname_len; int i; directory_token_t *tok; @@ -290,7 +300,7 @@ get_next_token(memarea_t *area, next = find_whitespace_eos(*s, eol); - if (!strcmp_len(*s, "opt", next-*s)) { + if (mem_eq_token(*s, next-*s, "opt")) { /* Skip past an "opt" at the start of the line. */ *s = eat_whitespace_eos_no_nl(next, eol); next = find_whitespace_eos(*s, eol); @@ -301,7 +311,7 @@ get_next_token(memarea_t *area, /* Search the table for the appropriate entry. (I tried a binary search * instead, but it wasn't any faster.) */ for (i = 0; table[i].t ; ++i) { - if (!strcmp_len(*s, table[i].t, next-*s)) { + if (mem_eq_token(*s, next-*s, table[i].t)) { /* We've found the keyword. */ kwd = table[i].t; tok->tp = table[i].v; @@ -352,9 +362,8 @@ get_next_token(memarea_t *area, if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */ goto check_object; - obstart = *s; /* Set obstart to start of object spec */ if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */ - strcmp_len(eol-5, "-----", 5) || /* nuls or invalid endings */ + !mem_eq_token(eol-5, 5, "-----") || /* nuls or invalid endings */ (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) { /* name too long */ RET_ERR("Malformed object: bad begin line"); } @@ -373,8 +382,8 @@ get_next_token(memarea_t *area, eol = eos; /* Validate the ending tag, which should be 9 + NAME + 5 + eol */ if ((size_t)(eol-next) != 9+obname_len+5 || - strcmp_len(next+9, tok->object_type, obname_len) || - strcmp_len(eol-5, "-----", 5)) { + !mem_eq_token(next+9, obname_len, tok->object_type) || + !mem_eq_token(eol-5, 5, "-----")) { tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s", tok->object_type); ebuf[sizeof(ebuf)-1] = '\0'; @@ -383,22 +392,26 @@ get_next_token(memarea_t *area, if (next - *s > MAX_UNPARSED_OBJECT_SIZE) RET_ERR("Couldn't parse object: missing footer or object much too big."); - if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */ - tok->key = crypto_pk_new(); - if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart)) - RET_ERR("Couldn't parse public key."); - } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */ - tok->key = crypto_pk_new(); - if (crypto_pk_read_private_key_from_string(tok->key, obstart, eol-obstart)) - RET_ERR("Couldn't parse private key."); - } else { /* If it's something else, try to base64-decode it */ + { int r; - tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */ - r = base64_decode(tok->object_body, next-*s, *s, next-*s); + size_t maxsize = base64_decode_maxsize(next-*s); + tok->object_body = ALLOC(maxsize); + r = base64_decode(tok->object_body, maxsize, *s, next-*s); if (r<0) RET_ERR("Malformed object: bad base64-encoded data"); tok->object_size = r; } + + if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */ + tok->key = crypto_pk_asn1_decode(tok->object_body, tok->object_size); + if (! tok->key) + RET_ERR("Couldn't parse public key."); + } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */ + tok->key = crypto_pk_asn1_decode_private(tok->object_body, + tok->object_size); + if (! tok->key) + RET_ERR("Couldn't parse private key."); + } *s = eol; check_object: diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c index e44fbf77f9..f78c46f186 100644 --- a/src/feature/dirparse/routerparse.c +++ b/src/feature/dirparse/routerparse.c @@ -591,8 +591,8 @@ router_parse_entry_from_string(const char *s, const char *end, "Relay's onion key had invalid exponent."); goto err; } - router_set_rsa_onion_pkey(tok->key, &router->onion_pkey, - &router->onion_pkey_len); + router->onion_pkey = tor_memdup(tok->object_body, tok->object_size); + router->onion_pkey_len = tok->object_size; crypto_pk_free(tok->key); if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) { diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c index 09932c97ac..70c2b4f69f 100644 --- a/src/feature/hibernate/hibernate.c +++ b/src/feature/hibernate/hibernate.c @@ -37,6 +37,7 @@ hibernating, phase 2: #include "core/or/connection_or.h" #include "feature/control/control.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/defs/time.h" #include "feature/hibernate/hibernate.h" #include "core/mainloop/mainloop.h" #include "feature/relay/router.h" @@ -66,8 +67,9 @@ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL; /** If are hibernating, when do we plan to wake up? Set to 0 if we * aren't hibernating. */ static time_t hibernate_end_time = 0; -/** If we are shutting down, when do we plan finally exit? Set to 0 if - * we aren't shutting down. */ +/** If we are shutting down, when do we plan finally exit? Set to 0 if we + * aren't shutting down. (This is obsolete; scheduled shutdowns are supposed + * to happen from mainloop_schedule_shutdown() now.) */ static time_t shutdown_time = 0; /** A timed event that we'll use when it's time to wake up from @@ -831,8 +833,6 @@ hibernate_soft_limit_reached(void) return get_accounting_bytes() >= soft_limit; } -#define TOR_USEC_PER_SEC (1000000) - /** Called when we get a SIGINT, or when bandwidth soft limit is * reached. Puts us into "loose hibernation": we don't accept new * connections, but we continue handling old ones. */ @@ -867,7 +867,13 @@ hibernate_begin(hibernate_state_t new_state, time_t now) log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new " "connections, and will shut down in %d seconds. Interrupt " "again to exit now.", options->ShutdownWaitLength); - shutdown_time = time(NULL) + options->ShutdownWaitLength; + /* We add an arbitrary delay here so that even if something goes wrong + * with the mainloop shutdown code, we can still shutdown from + * consider_hibernation() if we call it... but so that the + * mainloop_schedule_shutdown() mechanism will be the first one called. + */ + shutdown_time = time(NULL) + options->ShutdownWaitLength + 5; + mainloop_schedule_shutdown(options->ShutdownWaitLength); #ifdef HAVE_SYSTEMD /* tell systemd that we may need more than the default 90 seconds to shut * down so they don't kill us. add some extra time to actually finish @@ -1096,11 +1102,12 @@ consider_hibernation(time_t now) hibernate_state_t prev_state = hibernate_state; /* If we're in 'exiting' mode, then we just shut down after the interval - * elapses. */ + * elapses. The mainloop was supposed to catch this via + * mainloop_schedule_shutdown(), but apparently it didn't. */ if (hibernate_state == HIBERNATE_STATE_EXITING) { tor_assert(shutdown_time); if (shutdown_time <= now) { - log_notice(LD_GENERAL, "Clean shutdown finished. Exiting."); + log_notice(LD_BUG, "Mainloop did not catch shutdown event; exiting."); tor_shutdown_event_loop_and_exit(0); } return; /* if exiting soon, don't worry about bandwidth limits */ @@ -1112,7 +1119,7 @@ consider_hibernation(time_t now) if (hibernate_end_time > now && accounting_enabled) { /* If we're hibernating, don't wake up until it's time, regardless of * whether we're in a new interval. */ - return ; + return; } else { hibernate_end_time_elapsed(now); } @@ -1240,8 +1247,6 @@ on_hibernate_state_change(hibernate_state_t prev_state) if (prev_state != HIBERNATE_STATE_INITIAL) { rescan_periodic_events(get_options()); } - - reschedule_per_second_timer(); } /** Free all resources held by the accounting module */ diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c index b6abf14a11..b09d50e010 100644 --- a/src/feature/hs/hs_descriptor.c +++ b/src/feature/hs/hs_descriptor.c @@ -1400,6 +1400,50 @@ encrypted_data_length_is_valid(size_t len) return 0; } +/* Build the KEYS component for the authorized client computation. The format + * of the construction is: + * + * SECRET_SEED = x25519(sk, pk) + * KEYS = KDF(subcredential | SECRET_SEED, 40) + * + * Set the <b>keys_out</b> argument to point to the buffer containing the KEYS, + * and return the buffer's length. The caller should wipe and free its content + * once done with it. This function can't fail. */ +static size_t +build_descriptor_cookie_keys(const uint8_t *subcredential, + size_t subcredential_len, + const curve25519_secret_key_t *sk, + const curve25519_public_key_t *pk, + uint8_t **keys_out) +{ + uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; + uint8_t *keystream; + size_t keystream_len = HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN; + crypto_xof_t *xof; + + tor_assert(subcredential); + tor_assert(sk); + tor_assert(pk); + tor_assert(keys_out); + + keystream = tor_malloc_zero(keystream_len); + + /* Calculate x25519(sk, pk) to get the secret seed. */ + curve25519_handshake(secret_seed, sk, pk); + + /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ + xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, subcredential, subcredential_len); + crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); + crypto_xof_squeeze_bytes(xof, keystream, keystream_len); + crypto_xof_free(xof); + + memwipe(secret_seed, 0, sizeof(secret_seed)); + + *keys_out = keystream; + return keystream_len; +} + /* Decrypt the descriptor cookie given the descriptor, the auth client, * and the client secret key. On sucess, return 0 and a newly allocated * descriptor cookie descriptor_cookie_out. On error or if the client id @@ -1412,12 +1456,11 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, uint8_t **descriptor_cookie_out) { int ret = -1; - uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; - uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN]; - uint8_t *cookie_key = NULL; + uint8_t *keystream = NULL; + size_t keystream_length = 0; uint8_t *descriptor_cookie = NULL; + const uint8_t *cookie_key = NULL; crypto_cipher_t *cipher = NULL; - crypto_xof_t *xof = NULL; tor_assert(desc); tor_assert(client); @@ -1429,16 +1472,13 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, sizeof(*client_auth_sk))); tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN)); - /* Calculate x25519(client_x, hs_Y) */ - curve25519_handshake(secret_seed, client_auth_sk, - &desc->superencrypted_data.auth_ephemeral_pubkey); - - /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, desc->subcredential, DIGEST256_LEN); - crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)); - crypto_xof_free(xof); + /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */ + keystream_length = + build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN, + client_auth_sk, + &desc->superencrypted_data.auth_ephemeral_pubkey, + &keystream); + tor_assert(keystream_length > 0); /* If the client id of auth client is not the same as the calculcated * client id, it means that this auth client is invaild according to the @@ -1464,8 +1504,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc, if (cipher) { crypto_cipher_free(cipher); } - memwipe(secret_seed, 0, sizeof(secret_seed)); - memwipe(keystream, 0, sizeof(keystream)); + memwipe(keystream, 0, keystream_length); + tor_free(keystream); return ret; } @@ -2878,11 +2918,10 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, const uint8_t *descriptor_cookie, hs_desc_authorized_client_t *client_out) { - uint8_t secret_seed[CURVE25519_OUTPUT_LEN]; - uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN]; - uint8_t *cookie_key; + uint8_t *keystream = NULL; + size_t keystream_length = 0; + const uint8_t *cookie_key; crypto_cipher_t *cipher; - crypto_xof_t *xof; tor_assert(client_auth_pk); tor_assert(auth_ephemeral_sk); @@ -2898,18 +2937,14 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, tor_assert(!tor_mem_is_zero((char *) subcredential, DIGEST256_LEN)); - /* Calculate x25519(hs_y, client_X) */ - curve25519_handshake(secret_seed, - auth_ephemeral_sk, - client_auth_pk); - - /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */ - xof = crypto_xof_new(); - crypto_xof_add_bytes(xof, subcredential, DIGEST256_LEN); - crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed)); - crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)); - crypto_xof_free(xof); + /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */ + keystream_length = + build_descriptor_cookie_keys(subcredential, DIGEST256_LEN, + auth_ephemeral_sk, client_auth_pk, + &keystream); + tor_assert(keystream_length > 0); + /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */ memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN); cookie_key = keystream + HS_DESC_CLIENT_ID_LEN; @@ -2924,8 +2959,8 @@ hs_desc_build_authorized_client(const uint8_t *subcredential, (const char *) descriptor_cookie, HS_DESC_DESCRIPTOR_COOKIE_LEN); - memwipe(secret_seed, 0, sizeof(secret_seed)); - memwipe(keystream, 0, sizeof(keystream)); + memwipe(keystream, 0, keystream_length); + tor_free(keystream); crypto_cipher_free(cipher); } diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c index 7e150599fc..160fb87397 100644 --- a/src/feature/hs/hs_service.c +++ b/src/feature/hs/hs_service.c @@ -2956,8 +2956,8 @@ set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc, time_t now, /* The OPE module returns CRYPTO_OPE_ERROR in case of errors. */ tor_assert_nonfatal(rev_counter < CRYPTO_OPE_ERROR); - log_info(LD_REND, "Encrypted revision counter %d to %ld", - (int) seconds_since_start_of_srv, (long int) rev_counter); + log_info(LD_REND, "Encrypted revision counter %d to %" PRIu64, + (int) seconds_since_start_of_srv, rev_counter); hs_desc->desc->plaintext_data.revision_counter = rev_counter; } @@ -3694,8 +3694,8 @@ hs_service_lookup_current_desc(const ed25519_public_key_t *pk) } /* Return the number of service we have configured and usable. */ -unsigned int -hs_service_get_num_services(void) +MOCK_IMPL(unsigned int, +hs_service_get_num_services,(void)) { if (hs_service_map == NULL) { return 0; diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h index 5f43233ea1..ec53f2f23b 100644 --- a/src/feature/hs/hs_service.h +++ b/src/feature/hs/hs_service.h @@ -310,7 +310,7 @@ hs_service_t *hs_service_new(const or_options_t *options); void hs_service_free_(hs_service_t *service); #define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s)) -unsigned int hs_service_get_num_services(void); +MOCK_DECL(unsigned int, hs_service_get_num_services,(void)); void hs_service_stage_services(const smartlist_t *service_list); int hs_service_load_all_keys(void); int hs_service_get_version_from_key(const hs_service_t *service); diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c index 7a065662a7..9fc3b62525 100644 --- a/src/feature/nodelist/authcert.c +++ b/src/feature/nodelist/authcert.c @@ -380,7 +380,8 @@ trusted_dirs_load_certs_from_string(const char *contents, int source, int added_trusted_cert = 0; for (s = contents; *s; s = eos) { - authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); + authority_cert_t *cert = authority_cert_parse_from_string(s, strlen(s), + &eos); cert_list_t *cl; if (!cert) { failure_code = -1; diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c index 75cab7a0af..8c9212e05c 100644 --- a/src/feature/nodelist/fmt_routerstatus.c +++ b/src/feature/nodelist/fmt_routerstatus.c @@ -135,7 +135,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, goto done; smartlist_add_asprintf(chunks, - "s%s%s%s%s%s%s%s%s%s%s\n", + "s%s%s%s%s%s%s%s%s%s%s%s\n", /* These must stay in alphabetical order. */ rs->is_authority?" Authority":"", rs->is_bad_exit?" BadExit":"", @@ -145,6 +145,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, rs->is_hs_dir?" HSDir":"", rs->is_flagged_running?" Running":"", rs->is_stable?" Stable":"", + rs->is_staledesc?" StaleDesc":"", rs->is_v2_dir?" V2Dir":"", rs->is_valid?" Valid":""); diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c index dafaabb5e5..36922561a0 100644 --- a/src/feature/nodelist/microdesc.c +++ b/src/feature/nodelist/microdesc.c @@ -23,6 +23,7 @@ #include "feature/nodelist/dirlist.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/router.h" @@ -69,6 +70,8 @@ struct microdesc_cache_t { }; static microdesc_cache_t *get_microdesc_cache_noload(void); +static void warn_if_nul_found(const char *inp, size_t len, int64_t offset, + const char *activity); /** Helper: computes a hash of <b>md</b> to place it in a hash table. */ static inline unsigned int @@ -110,8 +113,9 @@ microdesc_note_outdated_dirserver(const char *relay_digest) /* If we have a reasonably live consensus, then most of our dirservers should * still be caching all the microdescriptors in it. Reasonably live - * consensuses are up to a day old. But microdescriptors expire 7 days after - * the last consensus that referenced them. */ + * consensuses are up to a day old (or a day in the future). But + * microdescriptors expire 7 days after the last consensus that referenced + * them. */ if (!networkstatus_get_reasonably_live_consensus(approx_time(), FLAV_MICRODESC)) { return; @@ -221,6 +225,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out) } md->off = tor_fd_getpos(fd); + warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off, + "dumping a microdescriptor"); written = write_all_to_fd(fd, md->body, md->bodylen); if (written != (ssize_t)md->bodylen) { written = written < 0 ? 0 : written; @@ -480,6 +486,27 @@ microdesc_cache_clear(microdesc_cache_t *cache) cache->bytes_dropped = 0; } +static void +warn_if_nul_found(const char *inp, size_t len, int64_t offset, + const char *activity) +{ + const char *nul_found = memchr(inp, 0, len); + if (BUG(nul_found)) { + log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64 + "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".", + activity, offset, (nul_found - inp), len); + const char *start_excerpt_at, *eos = inp + len; + if ((nul_found - inp) >= 16) + start_excerpt_at = nul_found - 16; + else + start_excerpt_at = inp; + size_t excerpt_len = MIN(32, eos - start_excerpt_at); + char tmp[65]; + base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len); + log_warn(LD_BUG, " surrounding string: %s", tmp); + } +} + /** Reload the contents of <b>cache</b> from disk. If it is empty, load it * for the first time. Return 0 on success, -1 on failure. */ int @@ -497,6 +524,7 @@ microdesc_cache_reload(microdesc_cache_t *cache) mm = cache->cache_content = tor_mmap_file(cache->cache_fname); if (mm) { + warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache"); added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size, SAVED_IN_CACHE, 0, -1, NULL); if (added) { @@ -509,6 +537,8 @@ microdesc_cache_reload(microdesc_cache_t *cache) RFTS_IGNORE_MISSING, &st); if (journal_content) { cache->journal_len = (size_t) st.st_size; + warn_if_nul_found(journal_content, cache->journal_len, 0, + "reading microdesc journal"); added = microdescs_add_to_cache(cache, journal_content, journal_content+st.st_size, SAVED_IN_JOURNAL, 0, -1, NULL); @@ -544,8 +574,8 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force) size_t bytes_dropped = 0; time_t now = time(NULL); - /* If we don't know a live consensus, don't believe last_listed values: we - * might be starting up after being down for a while. */ + /* If we don't know a reasonably live consensus, don't believe last_listed + * values: we might be starting up after being down for a while. */ if (! force && ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC)) return; @@ -884,10 +914,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno) if (md->body && md->saved_location != SAVED_IN_CACHE) tor_free(md->body); - if (md->family) { - SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp)); - smartlist_free(md->family); - } + nodefamily_free(md->family); short_policy_free(md->exit_policy); short_policy_free(md->ipv6_exit_policy); @@ -973,6 +1000,7 @@ update_microdesc_downloads(time_t now) if (directory_too_idle_to_fetch_descriptors(options, now)) return; + /* Give up if we don't have a reasonably live consensus. */ consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC); if (!consensus) return; diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h index bb8b23d664..367e6a3ef6 100644 --- a/src/feature/nodelist/microdesc_st.h +++ b/src/feature/nodelist/microdesc_st.h @@ -9,6 +9,7 @@ struct curve25519_public_key_t; struct ed25519_public_key_t; +struct nodefamily_t; struct short_policy_t; /** A microdescriptor is the smallest amount of information needed to build a @@ -69,8 +70,8 @@ struct microdesc_t { tor_addr_t ipv6_addr; /** As routerinfo_t.ipv6_orport */ uint16_t ipv6_orport; - /** As routerinfo_t.family */ - smartlist_t *family; + /** As routerinfo_t.family, with readable members parsed. */ + struct nodefamily_t *family; /** IPv4 exit policy summary */ struct short_policy_t *exit_policy; /** IPv6 exit policy summary */ diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c index c74acd8b74..24e3b212f0 100644 --- a/src/feature/nodelist/networkstatus.c +++ b/src/feature/nodelist/networkstatus.c @@ -44,6 +44,7 @@ #include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/channelpadding.h" +#include "core/or/circuitpadding.h" #include "core/or/circuitmux.h" #include "core/or/circuitmux_ewma.h" #include "core/or/circuitstats.h" @@ -116,8 +117,6 @@ STATIC networkstatus_t *current_md_consensus = NULL; typedef struct consensus_waiting_for_certs_t { /** The consensus itself. */ networkstatus_t *consensus; - /** The encoded version of the consensus, nul-terminated. */ - char *body; /** When did we set the current value of consensus_waiting_for_certs? If * this is too recent, we shouldn't try to fetch a new consensus for a * little while, to give ourselves time to get certificates for this one. */ @@ -179,6 +178,10 @@ static void update_consensus_bootstrap_multiple_downloads( static int networkstatus_check_required_protocols(const networkstatus_t *ns, int client_mode, char **warning_out); +static int reload_consensus_from_file(const char *fname, + const char *flavor, + unsigned flags, + const char *source_dir); /** Forget that we've warned about anything networkstatus-related, so we will * give fresh warnings if the same behavior happens again. */ @@ -210,14 +213,11 @@ networkstatus_reset_download_failures(void) download_status_reset(&consensus_bootstrap_dl_status[i]); } -/** - * Read and and return the cached consensus of type <b>flavorname</b>. If - * <b>unverified</b> is true, get the one we haven't verified. Return NULL if - * the file isn't there. */ +/** Return the filename used to cache the consensus of a given flavor */ static char * -networkstatus_read_cached_consensus_impl(int flav, - const char *flavorname, - int unverified_consensus) +networkstatus_get_cache_fname(int flav, + const char *flavorname, + int unverified_consensus) { char buf[128]; const char *prefix; @@ -232,21 +232,35 @@ networkstatus_read_cached_consensus_impl(int flav, tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname); } - char *filename = get_cachedir_fname(buf); - char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); + return get_cachedir_fname(buf); +} + +/** + * Read and and return the cached consensus of type <b>flavorname</b>. If + * <b>unverified</b> is false, get the one we haven't verified. Return NULL if + * the file isn't there. */ +static tor_mmap_t * +networkstatus_map_cached_consensus_impl(int flav, + const char *flavorname, + int unverified_consensus) +{ + char *filename = networkstatus_get_cache_fname(flav, + flavorname, + unverified_consensus); + tor_mmap_t *result = tor_mmap_file(filename); tor_free(filename); return result; } -/** Return a new string containing the current cached consensus of flavor - * <b>flavorname</b>. */ -char * -networkstatus_read_cached_consensus(const char *flavorname) - { +/** Map the file containing the current cached consensus of flavor + * <b>flavorname</b> */ +tor_mmap_t * +networkstatus_map_cached_consensus(const char *flavorname) +{ int flav = networkstatus_parse_flavor_name(flavorname); if (flav < 0) return NULL; - return networkstatus_read_cached_consensus_impl(flav, flavorname, 0); + return networkstatus_map_cached_consensus_impl(flav, flavorname, 0); } /** Read every cached v3 consensus networkstatus from the disk. */ @@ -259,25 +273,15 @@ router_reload_consensus_networkstatus(void) /* FFFF Suppress warnings if cached consensus is bad? */ for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) { const char *flavor = networkstatus_get_flavor_name(flav); - char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0); - if (s) { - if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) { - log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache", - flavor); - } - tor_free(s); - } + char *fname = networkstatus_get_cache_fname(flav, flavor, 0); + reload_consensus_from_file(fname, flavor, flags, NULL); + tor_free(fname); - s = networkstatus_read_cached_consensus_impl(flav, flavor, 1); - if (s) { - if (networkstatus_set_current_consensus(s, flavor, - flags | NSSET_WAS_WAITING_FOR_CERTS, - NULL)) { - log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " - "from cache", flavor); - } - tor_free(s); - } + fname = networkstatus_get_cache_fname(flav, flavor, 1); + reload_consensus_from_file(fname, flavor, + flags | NSSET_WAS_WAITING_FOR_CERTS, + NULL); + tor_free(fname); } update_certificate_downloads(time(NULL)); @@ -713,8 +717,8 @@ networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest) /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or * NULL if none was found. */ -const routerstatus_t * -networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) +MOCK_IMPL(const routerstatus_t *, +networkstatus_vote_find_entry,(networkstatus_t *ns, const char *digest)) { return networkstatus_vote_find_mutable_entry(ns, digest); } @@ -1377,7 +1381,7 @@ networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor)) } /** Return the most recent consensus that we have downloaded, or NULL if we - * don't have one. */ + * don't have one. May return future or expired consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_latest_consensus,(void)) { @@ -1388,7 +1392,7 @@ networkstatus_get_latest_consensus,(void)) } /** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL - * if we don't have one. */ + * if we don't have one. May return future or expired consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f)) { @@ -1422,10 +1426,11 @@ networkstatus_is_live(const networkstatus_t *ns, time_t now) return (ns->valid_after <= now && now <= ns->valid_until); } -/** Determine if <b>consensus</b> is valid or expired recently enough that - * we can still use it. +/** Determine if <b>consensus</b> is valid, or expired recently enough, or not + * too far in the future, so that we can still use it. * - * Return 1 if the consensus is reasonably live, or 0 if it is too old. + * Return 1 if the consensus is reasonably live, or 0 if it is too old or + * too new. */ int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, @@ -1434,29 +1439,42 @@ networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, if (BUG(!consensus)) return 0; - return networkstatus_valid_until_is_reasonably_live(consensus->valid_until, + return networkstatus_valid_after_is_reasonably_live(consensus->valid_after, + now) && + networkstatus_valid_until_is_reasonably_live(consensus->valid_until, now); } +#define REASONABLY_LIVE_TIME (24*60*60) + +/** As networkstatus_consensus_reasonably_live, but takes a valid_after + * time, and checks to see if it is in the past, or not too far in the future. + */ +int +networkstatus_valid_after_is_reasonably_live(time_t valid_after, + time_t now) +{ + return (now >= valid_after - REASONABLY_LIVE_TIME); +} + /** As networkstatus_consensus_reasonably_live, but takes a valid_until - * time rather than an entire consensus. */ + * time, and checks to see if it is in the future, or not too far in the past. + */ int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now) { -#define REASONABLY_LIVE_TIME (24*60*60) return (now <= valid_until + REASONABLY_LIVE_TIME); } /** As networkstatus_get_live_consensus(), but is way more tolerant of expired - * consensuses. */ + * and future consensuses. */ MOCK_IMPL(networkstatus_t *, networkstatus_get_reasonably_live_consensus,(time_t now, int flavor)) { networkstatus_t *consensus = networkstatus_get_latest_consensus_by_flavor(flavor); if (consensus && - consensus->valid_after <= now && networkstatus_consensus_reasonably_live(consensus, now)) return consensus; else @@ -1725,6 +1743,44 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c, #endif /* defined(TOR_UNIT_TESTS) */ /** + * Helper: Read the current consensus of type <b>flavor</b> from + * <b>fname</b>. Flags and return values are as for + * networkstatus_set_current_consensus(). + **/ +static int +reload_consensus_from_file(const char *fname, + const char *flavor, + unsigned flags, + const char *source_dir) +{ + tor_mmap_t *map = tor_mmap_file(fname); + if (!map) + return 0; + + int rv = networkstatus_set_current_consensus(map->data, map->size, + flavor, flags, source_dir); +#ifdef _WIN32 + if (rv < 0 && tor_memstr(map->data, map->size, "\r\n")) { + log_notice(LD_GENERAL, "Looks like the above failures are probably " + "because of a CRLF in consensus file %s; falling back to " + "read_file_to_string. Nothing to worry about: this file " + "was probably saved by an earlier version of Tor.", + escaped(fname)); + char *content = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL); + rv = networkstatus_set_current_consensus(content, strlen(content), + flavor, flags, source_dir); + tor_free(content); + } +#endif + if (rv < -1) { + log_warn(LD_GENERAL, "Couldn't set consensus from cache file %s", + escaped(fname)); + } + tor_munmap_file(map); + return rv; +} + +/** * Helper for handle_missing_protocol_warning: handles either the * client case (if <b>is_client</b> is set) or the server case otherwise. */ @@ -1841,6 +1897,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor, */ int networkstatus_set_current_consensus(const char *consensus, + size_t consensus_len, const char *flavor, unsigned flags, const char *source_dir) @@ -1869,7 +1926,9 @@ networkstatus_set_current_consensus(const char *consensus, } /* Make sure it's parseable. */ - c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS); + c = networkstatus_parse_vote_from_string(consensus, + consensus_len, + NULL, NS_TYPE_CONSENSUS); if (!c) { log_warn(LD_DIR, "Unable to parse networkstatus consensus"); result = -2; @@ -1957,14 +2016,12 @@ networkstatus_set_current_consensus(const char *consensus, c->valid_after > current_valid_after) { waiting = &consensus_waiting_for_certs[flav]; networkstatus_vote_free(waiting->consensus); - tor_free(waiting->body); waiting->consensus = c; free_consensus = 0; - waiting->body = tor_strdup(consensus); waiting->set_at = now; waiting->dl_failed = 0; if (!from_cache) { - write_str_to_file(unverified_fname, consensus, 0); + write_bytes_to_file(unverified_fname, consensus, consensus_len, 1); } if (dl_certs) authority_certs_fetch_missing(c, now, source_dir); @@ -1976,9 +2033,9 @@ networkstatus_set_current_consensus(const char *consensus, * latest consensus. */ if (was_waiting_for_certs && from_cache) if (unlink(unverified_fname) != 0) { - log_warn(LD_FS, - "Failed to unlink %s: %s", - unverified_fname, strerror(errno)); + log_debug(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); } } goto done; @@ -1991,9 +2048,9 @@ networkstatus_set_current_consensus(const char *consensus, } if (was_waiting_for_certs && (r < -1) && from_cache) { if (unlink(unverified_fname) != 0) { - log_warn(LD_FS, - "Failed to unlink %s: %s", - unverified_fname, strerror(errno)); + log_debug(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); } } goto done; @@ -2055,16 +2112,12 @@ networkstatus_set_current_consensus(const char *consensus, waiting->consensus->valid_after <= c->valid_after) { networkstatus_vote_free(waiting->consensus); waiting->consensus = NULL; - if (consensus != waiting->body) - tor_free(waiting->body); - else - waiting->body = NULL; waiting->set_at = 0; waiting->dl_failed = 0; if (unlink(unverified_fname) != 0) { - log_warn(LD_FS, - "Failed to unlink %s: %s", - unverified_fname, strerror(errno)); + log_debug(LD_FS, + "Failed to unlink %s: %s", + unverified_fname, strerror(errno)); } } @@ -2082,7 +2135,6 @@ networkstatus_set_current_consensus(const char *consensus, nodelist_set_consensus(c); - /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/ update_consensus_networkstatus_fetch_time(now); /* Change the cell EWMA settings */ @@ -2095,6 +2147,7 @@ networkstatus_set_current_consensus(const char *consensus, circuit_build_times_new_consensus_params( get_circuit_build_times_mutable(), c); channelpadding_new_consensus_params(c); + circpad_new_consensus_params(c); } /* Reset the failure count only if this consensus is actually valid. */ @@ -2108,17 +2161,18 @@ networkstatus_set_current_consensus(const char *consensus, if (we_want_to_fetch_flavor(options, flav)) { if (dir_server_mode(get_options())) { dirserv_set_cached_consensus_networkstatus(consensus, + consensus_len, flavor, &c->digests, c->digest_sha3_as_signed, c->valid_after); - consdiffmgr_add_consensus(consensus, c); + consdiffmgr_add_consensus(consensus, consensus_len, c); } } if (!from_cache) { - write_str_to_file(consensus_fname, consensus, 0); + write_bytes_to_file(consensus_fname, consensus, consensus_len, 1); } warn_early_consensus(c, flavor, now); @@ -2154,14 +2208,10 @@ networkstatus_note_certs_arrived(const char *source_dir) if (!waiting->consensus) continue; if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) { - char *waiting_body = waiting->body; - if (!networkstatus_set_current_consensus( - waiting_body, - flavor_name, - NSSET_WAS_WAITING_FOR_CERTS, - source_dir)) { - tor_free(waiting_body); - } + char *fname = networkstatus_get_cache_fname(i, flavor_name, 1); + reload_consensus_from_file(fname, flavor_name, + NSSET_WAS_WAITING_FOR_CERTS, source_dir); + tor_free(fname); } } } @@ -2389,7 +2439,9 @@ networkstatus_dump_bridge_status_to_file(time_t now) published, thresholds, fingerprint_line ? fingerprint_line : "", status); fname = get_datadir_fname("networkstatus-bridges"); - write_str_to_file(fname,published_thresholds_and_status,0); + if (write_str_to_file(fname,published_thresholds_and_status,0)<0) { + log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file."); + } tor_free(thresholds); tor_free(published_thresholds_and_status); tor_free(fname); @@ -2668,6 +2720,9 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, const char *required, *recommended; char *missing = NULL; + const bool consensus_postdates_this_release = + ns->valid_after >= tor_get_approx_release_date(); + tor_assert(warning_out); if (client_mode) { @@ -2685,7 +2740,7 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, "%s on the Tor network. The missing protocols are: %s", func, missing); tor_free(missing); - return 1; + return consensus_postdates_this_release ? 1 : 0; } if (! protover_all_supported(recommended, &missing)) { @@ -2718,6 +2773,5 @@ networkstatus_free_all(void) networkstatus_vote_free(waiting->consensus); waiting->consensus = NULL; } - tor_free(waiting->body); } } diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h index 9e7b0f1bb0..8269fc6182 100644 --- a/src/feature/nodelist/networkstatus.h +++ b/src/feature/nodelist/networkstatus.h @@ -16,7 +16,7 @@ void networkstatus_reset_warnings(void); void networkstatus_reset_download_failures(void); -char *networkstatus_read_cached_consensus(const char *flavorname); +tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname); int router_reload_consensus_networkstatus(void); void routerstatus_free_(routerstatus_t *rs); #define routerstatus_free(rs) \ @@ -40,8 +40,9 @@ int compare_digest_to_routerstatus_entry(const void *_key, const void **_member); int compare_digest_to_vote_routerstatus_entry(const void *_key, const void **_member); -const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns, - const char *digest); +MOCK_DECL(const routerstatus_t *,networkstatus_vote_find_entry,( + networkstatus_t *ns, + const char *digest)); routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest); int networkstatus_vote_find_entry_idx(networkstatus_t *ns, @@ -87,6 +88,8 @@ MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now)); int networkstatus_is_live(const networkstatus_t *ns, time_t now); int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus, time_t now); +int networkstatus_valid_after_is_reasonably_live(time_t valid_after, + time_t now); int networkstatus_valid_until_is_reasonably_live(time_t valid_until, time_t now); MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus, @@ -106,6 +109,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options); #define NSSET_ACCEPT_OBSOLETE 8 #define NSSET_REQUIRE_FLAVOR 16 int networkstatus_set_current_consensus(const char *consensus, + size_t consensus_len, const char *flavor, unsigned flags, const char *source_dir); @@ -157,4 +161,3 @@ extern networkstatus_t *current_md_consensus; #endif /* defined(NETWORKSTATUS_PRIVATE) */ #endif /* !defined(TOR_NETWORKSTATUS_H) */ - diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h index 6160f12361..5c1eea3259 100644 --- a/src/feature/nodelist/networkstatus_st.h +++ b/src/feature/nodelist/networkstatus_st.h @@ -99,6 +99,9 @@ struct networkstatus_t { /** List of key=value strings from the headers of the bandwidth list file */ smartlist_t *bw_file_headers; + + /** A SHA256 digest of the bandwidth file used in a vote. */ + uint8_t bw_file_digest256[DIGEST256_LEN]; }; #endif diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c new file mode 100644 index 0000000000..2ec9d5fa40 --- /dev/null +++ b/src/feature/nodelist/nodefamily.c @@ -0,0 +1,416 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file nodefamily.c + * \brief Code to manipulate encoded, reference-counted node families. We + * use these tricks to save space, since these families would otherwise + * require a large number of tiny allocations. + **/ + +#include "core/or/or.h" +#include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" +#include "feature/nodelist/nodefamily_st.h" +#include "feature/nodelist/nodelist.h" +#include "feature/relay/router.h" +#include "feature/nodelist/routerlist.h" + +#include "ht.h" +#include "siphash.h" + +#include "lib/container/smartlist.h" +#include "lib/ctime/di_ops.h" +#include "lib/defs/digest_sizes.h" +#include "lib/log/util_bug.h" + +#include <stdlib.h> +#include <string.h> + +/** + * Allocate and return a blank node family with space to hold <b>n_members</b> + * members. + */ +static nodefamily_t * +nodefamily_alloc(int n_members) +{ + size_t alloc_len = offsetof(nodefamily_t, family_members) + + NODEFAMILY_ARRAY_SIZE(n_members); + nodefamily_t *nf = tor_malloc_zero(alloc_len); + nf->n_members = n_members; + return nf; +} + +/** + * Hashtable hash implementation. + */ +static inline unsigned int +nodefamily_hash(const nodefamily_t *nf) +{ + return (unsigned) siphash24g(nf->family_members, + NODEFAMILY_ARRAY_SIZE(nf->n_members)); +} + +/** + * Hashtable equality implementation. + */ +static inline unsigned int +nodefamily_eq(const nodefamily_t *a, const nodefamily_t *b) +{ + return (a->n_members == b->n_members) && + fast_memeq(a->family_members, b->family_members, + NODEFAMILY_ARRAY_SIZE(a->n_members)); +} + +static HT_HEAD(nodefamily_map, nodefamily_t) the_node_families + = HT_INITIALIZER(); + +HT_PROTOTYPE(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash, + nodefamily_eq) +HT_GENERATE2(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash, + node_family_eq, 0.6, tor_reallocarray_, tor_free_) + +/** + * Parse the family declaration in <b>s</b>, returning the canonical + * <b>nodefamily_t</b> for its members. Return NULL on error. + * + * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest + * for the router that declared this family: insert it into the + * family declaration if it is not there already. + * + * If NF_WARN_MALFORMED is set in <b>flags</b>, warn about any + * elements that we can't parse. (By default, we log at info.) + * + * If NF_REJECT_MALFORMED is set in <b>flags</b>, treat any unparseable + * elements as an error. (By default, we simply omit them.) + **/ +nodefamily_t * +nodefamily_parse(const char *s, const uint8_t *rsa_id_self, + unsigned flags) +{ + smartlist_t *sl = smartlist_new(); + smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags, NULL); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + return result; +} + +/** + * Canonicalize the family list <b>s</b>, returning a newly allocated string. + * + * The canonicalization rules are fully specified in dir-spec.txt, but, + * briefly: $hexid entries are put in caps, $hexid[=~]foo entries are + * truncated, nicknames are put into lowercase, unrecognized entries are left + * alone, and everything is sorted. + **/ +char * +nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, + unsigned flags) +{ + smartlist_t *sl = smartlist_new(); + smartlist_t *result_members = smartlist_new(); + smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + nodefamily_t *nf = nodefamily_from_members(sl, rsa_id_self, flags, + result_members); + + char *formatted = nodefamily_format(nf); + smartlist_split_string(result_members, formatted, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + smartlist_sort_strings(result_members); + char *combined = smartlist_join_strings(result_members, " ", 0, NULL); + + nodefamily_free(nf); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + SMARTLIST_FOREACH(result_members, char *, cp, tor_free(cp)); + smartlist_free(result_members); + tor_free(formatted); + + return combined; +} + +/** + * qsort helper for encoded nodefamily elements. + **/ +static int +compare_members(const void *a, const void *b) +{ + return fast_memcmp(a, b, NODEFAMILY_MEMBER_LEN); +} + +/** + * Parse the member strings in <b>members</b>, returning their canonical + * <b>nodefamily_t</b>. Return NULL on error. + * + * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest + * for the router that declared this family: insert it into the + * family declaration if it is not there already. + * + * The <b>flags</b> element is interpreted as in nodefamily_parse(). + * + * If <b>unrecognized</b> is provided, fill it copies of any unrecognized + * members. (Note that malformed $hexids are not considered unrecognized.) + **/ +nodefamily_t * +nodefamily_from_members(const smartlist_t *members, + const uint8_t *rsa_id_self, + unsigned flags, + smartlist_t *unrecognized_out) +{ + const int n_self = rsa_id_self ? 1 : 0; + int n_bad_elements = 0; + int n_members = smartlist_len(members) + n_self; + nodefamily_t *tmp = nodefamily_alloc(n_members); + uint8_t *ptr = NODEFAMILY_MEMBER_PTR(tmp, 0); + + SMARTLIST_FOREACH_BEGIN(members, const char *, cp) { + bool bad_element = true; + if (is_legal_nickname(cp)) { + ptr[0] = NODEFAMILY_BY_NICKNAME; + tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname + memcpy(ptr+1, cp, strlen(cp)); + tor_strlower((char*) ptr+1); + bad_element = false; + } else if (is_legal_hexdigest(cp)) { + char digest_buf[DIGEST_LEN]; + char nn_buf[MAX_NICKNAME_LEN+1]; + char nn_char=0; + if (hex_digest_nickname_decode(cp, digest_buf, &nn_char, nn_buf)==0) { + bad_element = false; + ptr[0] = NODEFAMILY_BY_RSA_ID; + memcpy(ptr+1, digest_buf, DIGEST_LEN); + } + } else { + if (unrecognized_out) + smartlist_add_strdup(unrecognized_out, cp); + } + + if (bad_element) { + const int severity = (flags & NF_WARN_MALFORMED) ? LOG_WARN : LOG_INFO; + log_fn(severity, LD_GENERAL, + "Bad element %s while parsing a node family.", + escaped(cp)); + ++n_bad_elements; + } else { + ptr += NODEFAMILY_MEMBER_LEN; + } + } SMARTLIST_FOREACH_END(cp); + + if (n_bad_elements && (flags & NF_REJECT_MALFORMED)) + goto err; + + if (rsa_id_self) { + /* Add self. */ + ptr[0] = NODEFAMILY_BY_RSA_ID; + memcpy(ptr+1, rsa_id_self, DIGEST_LEN); + } + + n_members -= n_bad_elements; + + /* Sort tmp into canonical order. */ + qsort(tmp->family_members, n_members, NODEFAMILY_MEMBER_LEN, + compare_members); + + /* Remove duplicates. */ + int i; + for (i = 0; i < n_members-1; ++i) { + uint8_t *thisptr = NODEFAMILY_MEMBER_PTR(tmp, i); + uint8_t *nextptr = NODEFAMILY_MEMBER_PTR(tmp, i+1); + if (fast_memeq(thisptr, nextptr, NODEFAMILY_MEMBER_LEN)) { + memmove(thisptr, nextptr, (n_members-i-1)*NODEFAMILY_MEMBER_LEN); + --n_members; + --i; + } + } + int n_members_alloc = tmp->n_members; + tmp->n_members = n_members; + + /* See if we already allocated this family. */ + nodefamily_t *found = HT_FIND(nodefamily_map, &the_node_families, tmp); + if (found) { + /* If we did, great: incref it and return it. */ + ++found->refcnt; + tor_free(tmp); + return found; + } else { + /* If not, insert it into the hashtable. */ + if (n_members_alloc != n_members) { + /* Compact the family if needed */ + nodefamily_t *tmp2 = nodefamily_alloc(n_members); + memcpy(tmp2->family_members, tmp->family_members, + n_members * NODEFAMILY_MEMBER_LEN); + tor_free(tmp); + tmp = tmp2; + } + + tmp->refcnt = 1; + HT_INSERT(nodefamily_map, &the_node_families, tmp); + return tmp; + } + + err: + tor_free(tmp); + return NULL; +} + +/** + * Drop our reference to <b>family</b>, freeing it if there are no more + * references. + */ +void +nodefamily_free_(nodefamily_t *family) +{ + if (family == NULL) + return; + + --family->refcnt; + + if (family->refcnt == 0) { + HT_REMOVE(nodefamily_map, &the_node_families, family); + tor_free(family); + } +} + +/** + * Return true iff <b>family</b> contains the SHA1 RSA1024 identity + * <b>rsa_id</b>. + */ +bool +nodefamily_contains_rsa_id(const nodefamily_t *family, + const uint8_t *rsa_id) +{ + if (family == NULL) + return false; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + if (ptr[0] == NODEFAMILY_BY_RSA_ID && + fast_memeq(ptr+1, rsa_id, DIGEST_LEN)) { + return true; + } + } + return false; +} + +/** + * Return true iff <b>family</b> contains the nickname <b>name</b>. + */ +bool +nodefamily_contains_nickname(const nodefamily_t *family, + const char *name) +{ + if (family == NULL) + return false; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + // note that the strcasecmp() is safe because there is always at least one + // NUL in the encoded nickname, because all legal nicknames are less than + // DIGEST_LEN bytes long. + if (ptr[0] == NODEFAMILY_BY_NICKNAME && !strcasecmp((char*)ptr+1, name)) { + return true; + } + } + return false; +} + +/** + * Return true if <b>family</b> contains the nickname or the RSA ID for + * <b>node</b> + **/ +bool +nodefamily_contains_node(const nodefamily_t *family, + const node_t *node) +{ + return + nodefamily_contains_nickname(family, node_get_nickname(node)) + || + nodefamily_contains_rsa_id(family, node_get_rsa_id_digest(node)); +} + +/** + * Look up every entry in <b>family</b>, and add add the corresponding + * node_t to <b>out</b>. + **/ +void +nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, + smartlist_t *out) +{ + if (!family) + return; + + unsigned i; + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + const node_t *node = NULL; + switch (ptr[0]) { + case NODEFAMILY_BY_NICKNAME: + node = node_get_by_nickname((char*)ptr+1, NNF_NO_WARN_UNNAMED); + break; + case NODEFAMILY_BY_RSA_ID: + node = node_get_by_id((char*)ptr+1); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + break; + /* LCOV_EXCL_STOP */ + } + if (node) + smartlist_add(out, (void *)node); + } +} + +/** + * Encode <b>family</b> as a space-separated string. + */ +char * +nodefamily_format(const nodefamily_t *family) +{ + if (!family) + return tor_strdup(""); + + unsigned i; + smartlist_t *sl = smartlist_new(); + for (i = 0; i < family->n_members; ++i) { + const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i); + switch (ptr[0]) { + case NODEFAMILY_BY_NICKNAME: + smartlist_add_strdup(sl, (char*)ptr+1); + break; + case NODEFAMILY_BY_RSA_ID: { + char buf[HEX_DIGEST_LEN+2]; + buf[0]='$'; + base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN); + tor_strupper(buf); + smartlist_add_strdup(sl, buf); + break; + } + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached(); + break; + /* LCOV_EXCL_STOP */ + } + } + + char *result = smartlist_join_strings(sl, " ", 0, NULL); + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); + return result; +} + +/** + * Free all storage held in the nodefamily map. + **/ +void +nodefamily_free_all(void) +{ + HT_CLEAR(nodefamily_map, &the_node_families); +} diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h new file mode 100644 index 0000000000..bc5dafce03 --- /dev/null +++ b/src/feature/nodelist/nodefamily.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file nodefamily.h + * \brief Header file for nodefamily.c. + **/ + +#ifndef TOR_NODEFAMILY_H +#define TOR_NODEFAMILY_H + +#include "lib/malloc/malloc.h" +#include <stdbool.h> + +typedef struct nodefamily_t nodefamily_t; +struct node_t; +struct smartlist_t; + +#define NF_WARN_MALFORMED (1u<<0) +#define NF_REJECT_MALFORMED (1u<<1) + +nodefamily_t *nodefamily_parse(const char *s, + const uint8_t *rsa_id_self, + unsigned flags); +nodefamily_t *nodefamily_from_members(const struct smartlist_t *members, + const uint8_t *rsa_id_self, + unsigned flags, + smartlist_t *unrecognized_out); +void nodefamily_free_(nodefamily_t *family); +#define nodefamily_free(family) \ + FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family)) + +bool nodefamily_contains_rsa_id(const nodefamily_t *family, + const uint8_t *rsa_id); +bool nodefamily_contains_nickname(const nodefamily_t *family, + const char *name); +bool nodefamily_contains_node(const nodefamily_t *family, + const struct node_t *node); +void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family, + struct smartlist_t *out); +char *nodefamily_format(const nodefamily_t *family); +char *nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self, + unsigned flags); + +void nodefamily_free_all(void); + +#endif diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h new file mode 100644 index 0000000000..be533da824 --- /dev/null +++ b/src/feature/nodelist/nodefamily_st.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_NODEFAMILY_ST_H +#define TOR_NODEFAMILY_ST_H + +#include "orconfig.h" +#include "ht.h" + +struct nodefamily_t { + /** Entry for this nodefamily_t within the hashtable. */ + HT_ENTRY(nodefamily_t) ht_ent; + /** Reference count. (The hashtable is not treated as a reference */ + uint32_t refcnt; + /** Number of items encoded in <b>family_members</b>. */ + uint32_t n_members; + /* A byte-array encoding the members of this family. We encode each member + * as one byte to indicate whether it's a nickname or a fingerprint, plus + * DIGEST_LEN bytes of data. The entries are lexically sorted. + */ + uint8_t family_members[FLEXIBLE_ARRAY_MEMBER]; +}; + +#define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN) + +/** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID. + */ +#define NODEFAMILY_BY_RSA_ID 0 +/** Tag byte, indicates that the following bytes are a NUL-padded nickname. + */ +#define NODEFAMILY_BY_NICKNAME 1 + +/** + * Number of bytes to allocate in the array for a nodefamily_t with N members. + **/ +#define NODEFAMILY_ARRAY_SIZE(n) \ + ((n) * NODEFAMILY_MEMBER_LEN) + +/** + * Pointer to the i'th member of <b>nf</b>, as encoded. + */ +#define NODEFAMILY_MEMBER_PTR(nf, i) \ + (&((nf)->family_members[(i) * NODEFAMILY_MEMBER_LEN])) + +#endif diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c index 99d7f746a8..8b02dd9c66 100644 --- a/src/feature/nodelist/nodelist.c +++ b/src/feature/nodelist/nodelist.c @@ -59,6 +59,7 @@ #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/node_select.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/routerset.h" @@ -1105,7 +1106,7 @@ node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id) /** Dummy object that should be unreturnable. Used to ensure that * node_get_protover_summary_flags() always returns non-NULL. */ static const protover_summary_flags_t zero_protover_flags = { - 0,0,0,0,0,0,0 + 0,0,0,0,0,0,0,0 }; /** Return the protover_summary_flags for a given node. */ @@ -1503,19 +1504,6 @@ node_is_me(const node_t *node) return router_digest_is_me(node->identity); } -/** Return <b>node</b> declared family (as a list of names), or NULL if - * the node didn't declare a family. */ -const smartlist_t * -node_get_declared_family(const node_t *node) -{ - if (node->ri && node->ri->declared_family) - return node->ri->declared_family; - else if (node->md && node->md->family) - return node->md->family; - else - return NULL; -} - /* Does this node have a valid IPv6 address? * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for * checking specific ports. */ @@ -1866,6 +1854,9 @@ int addrs_in_same_network_family(const tor_addr_t *a1, const tor_addr_t *a2) { + if (tor_addr_is_null(a1) || tor_addr_is_null(a2)) + return 0; + switch (tor_addr_family(a1)) { case AF_INET: return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC); @@ -1881,7 +1872,7 @@ addrs_in_same_network_family(const tor_addr_t *a1, * (case-insensitive), or if <b>node's</b> identity key digest * matches a hexadecimal value stored in <b>nickname</b>. Return * false otherwise. */ -static int +STATIC int node_nickname_matches(const node_t *node, const char *nickname) { const char *n = node_get_nickname(node); @@ -1893,7 +1884,7 @@ node_nickname_matches(const node_t *node, const char *nickname) } /** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */ -static inline int +STATIC int node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) { if (!lst) return 0; @@ -1904,6 +1895,61 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node) return 0; } +/** Return true iff n1's declared family contains n2. */ +STATIC int +node_family_contains(const node_t *n1, const node_t *n2) +{ + if (n1->ri && n1->ri->declared_family) { + return node_in_nickname_smartlist(n1->ri->declared_family, n2); + } else if (n1->md) { + return nodefamily_contains_node(n1->md->family, n2); + } else { + return 0; + } +} + +/** + * Return true iff <b>node</b> has declared a nonempty family. + **/ +STATIC bool +node_has_declared_family(const node_t *node) +{ + if (node->ri && node->ri->declared_family && + smartlist_len(node->ri->declared_family)) { + return true; + } + + if (node->md && node->md->family) { + return true; + } + + return false; +} + +/** + * Add to <b>out</b> every node_t that is listed by <b>node</b> as being in + * its family. (Note that these nodes are not in node's family unless they + * also agree that node is in their family.) + **/ +STATIC void +node_lookup_declared_family(smartlist_t *out, const node_t *node) +{ + if (node->ri && node->ri->declared_family && + smartlist_len(node->ri->declared_family)) { + SMARTLIST_FOREACH_BEGIN(node->ri->declared_family, const char *, name) { + const node_t *n2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED); + if (n2) { + smartlist_add(out, (node_t *)n2); + } + } SMARTLIST_FOREACH_END(name); + return; + } + + if (node->md && node->md->family) { + nodefamily_add_nodes_to_smartlist(node->md->family, out); + } +} + /** Return true iff r1 and r2 are in the same family, but not the same * router. */ int @@ -1916,19 +1962,20 @@ nodes_in_same_family(const node_t *node1, const node_t *node2) tor_addr_t a1, a2; node_get_addr(node1, &a1); node_get_addr(node2, &a2); - if (addrs_in_same_network_family(&a1, &a2)) + + tor_addr_port_t ap6_1, ap6_2; + node_get_pref_ipv6_orport(node1, &ap6_1); + node_get_pref_ipv6_orport(node2, &ap6_2); + + if (addrs_in_same_network_family(&a1, &a2) || + addrs_in_same_network_family(&ap6_1.addr, &ap6_2.addr)) return 1; } /* Are they in the same family because the agree they are? */ - { - const smartlist_t *f1, *f2; - f1 = node_get_declared_family(node1); - f2 = node_get_declared_family(node2); - if (f1 && f2 && - node_in_nickname_smartlist(f1, node2) && - node_in_nickname_smartlist(f2, node1)) - return 1; + if (node_family_contains(node1, node2) && + node_family_contains(node2, node1)) { + return 1; } /* Are they in the same family because the user says they are? */ @@ -1956,13 +2003,10 @@ void nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) { const smartlist_t *all_nodes = nodelist_get_list(); - const smartlist_t *declared_family; const or_options_t *options = get_options(); tor_assert(node); - declared_family = node_get_declared_family(node); - /* Let's make sure that we have the node itself, if it's a real node. */ { const node_t *real_node = node_get_by_id(node->identity); @@ -1973,35 +2017,35 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node) /* First, add any nodes with similar network addresses. */ if (options->EnforceDistinctSubnets) { tor_addr_t node_addr; + tor_addr_port_t node_ap6; node_get_addr(node, &node_addr); + node_get_pref_ipv6_orport(node, &node_ap6); SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) { tor_addr_t a; + tor_addr_port_t ap6; node_get_addr(node2, &a); - if (addrs_in_same_network_family(&a, &node_addr)) + node_get_pref_ipv6_orport(node2, &ap6); + if (addrs_in_same_network_family(&a, &node_addr) || + addrs_in_same_network_family(&ap6.addr, &node_ap6.addr)) smartlist_add(sl, (void*)node2); } SMARTLIST_FOREACH_END(node2); } - /* Now, add all nodes in the declared_family of this node, if they + /* Now, add all nodes in the declared family of this node, if they * also declare this node to be in their family. */ - if (declared_family) { + if (node_has_declared_family(node)) { + smartlist_t *declared_family = smartlist_new(); + node_lookup_declared_family(declared_family, node); + /* Add every r such that router declares familyness with node, and node * declares familyhood with router. */ - SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) { - const node_t *node2; - const smartlist_t *family2; - if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED))) - continue; - if (!(family2 = node_get_declared_family(node2))) - continue; - SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) { - if (node_nickname_matches(node, name2)) { - smartlist_add(sl, (void*)node2); - break; - } - } SMARTLIST_FOREACH_END(name2); - } SMARTLIST_FOREACH_END(name); + SMARTLIST_FOREACH_BEGIN(declared_family, const node_t *, node2) { + if (node_family_contains(node2, node)) { + smartlist_add(sl, (void*)node2); + } + } SMARTLIST_FOREACH_END(node2); + smartlist_free(declared_family); } /* If the user declared any families locally, honor those too. */ @@ -2306,7 +2350,7 @@ compute_frac_paths_available(const networkstatus_t *consensus, const int authdir = authdir_mode_v3(options); count_usable_descriptors(num_present_out, num_usable_out, - mid, consensus, now, NULL, + mid, consensus, now, options->MiddleNodes, USABLE_DESCRIPTOR_ALL); log_debug(LD_NET, "%s: %d present, %d usable", @@ -2508,7 +2552,7 @@ count_loading_descriptors_progress(void) if (fraction > 1.0) return 0; /* it's not the number of descriptors holding us back */ return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int) - (fraction*(BOOTSTRAP_STATUS_CONN_OR-1 - + (fraction*(BOOTSTRAP_STATUS_ENOUGH_DIRINFO-1 - BOOTSTRAP_STATUS_LOADING_DESCRIPTORS)); } @@ -2595,7 +2639,7 @@ update_router_have_minimum_dir_info(void) /* If paths have just become available in this update. */ if (res && !have_min_dir_info) { control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO"); - control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0); + control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0); log_info(LD_DIR, "We now have enough directory information to build circuits."); } diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h index c430f497d5..3420959618 100644 --- a/src/feature/nodelist/nodelist.h +++ b/src/feature/nodelist/nodelist.h @@ -68,7 +68,6 @@ const char *node_get_platform(const node_t *node); uint32_t node_get_prim_addr_ipv4h(const node_t *node); void node_get_address_string(const node_t *node, char *cp, size_t len); long node_get_declared_uptime(const node_t *node); -const smartlist_t *node_get_declared_family(const node_t *node); const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node); int node_ed25519_id_matches(const node_t *node, const struct ed25519_public_key_t *id); @@ -155,10 +154,16 @@ int count_loading_descriptors_progress(void); #ifdef NODELIST_PRIVATE +STATIC int node_nickname_matches(const node_t *node, const char *nickname); +STATIC int node_in_nickname_smartlist(const smartlist_t *lst, + const node_t *node); +STATIC int node_family_contains(const node_t *n1, const node_t *n2); +STATIC bool node_has_declared_family(const node_t *node); +STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node); + #ifdef TOR_UNIT_TESTS -STATIC void -node_set_hsdir_index(node_t *node, const networkstatus_t *ns); +STATIC void node_set_hsdir_index(node_t *node, const networkstatus_t *ns); #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c index e48675aada..b33dca67e3 100644 --- a/src/feature/nodelist/routerlist.c +++ b/src/feature/nodelist/routerlist.c @@ -160,7 +160,7 @@ static time_t last_descriptor_download_attempted = 0; * * From time to time, we replace "cached-descriptors" with a new file * containing only the live, non-superseded descriptors, and clear - * cached-routers.new. + * cached-descriptors.new. * * On startup, we read both files. */ @@ -3223,6 +3223,8 @@ refresh_all_country_info(void) routerset_refresh_countries(options->EntryNodes); if (options->ExitNodes) routerset_refresh_countries(options->ExitNodes); + if (options->MiddleNodes) + routerset_refresh_countries(options->MiddleNodes); if (options->ExcludeNodes) routerset_refresh_countries(options->ExcludeNodes); if (options->ExcludeExitNodes) diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h index 288edf5982..8d91b45e11 100644 --- a/src/feature/nodelist/routerstatus_st.h +++ b/src/feature/nodelist/routerstatus_st.h @@ -47,6 +47,8 @@ struct routerstatus_t { unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort * or it claims to accept tunnelled dir requests. */ + unsigned int is_staledesc:1; /** True iff the authorities think this router + * should upload a new descriptor soon. */ unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */ unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */ diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c index cc9f4cf490..feaf18e764 100644 --- a/src/feature/relay/dns.c +++ b/src/feature/relay/dns.c @@ -1360,6 +1360,42 @@ evdns_err_is_transient(int err) } } +/** + * Return number of configured nameservers in <b>the_evdns_base</b>. + */ +size_t +number_of_configured_nameservers(void) +{ + return evdns_base_count_nameservers(the_evdns_base); +} + +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR +/** + * Return address of configured nameserver in <b>the_evdns_base</b> + * at index <b>idx</b>. + */ +tor_addr_t * +configured_nameserver_address(const size_t idx) +{ + struct sockaddr_storage sa; + ev_socklen_t sa_len = sizeof(sa); + + if (evdns_base_get_nameserver_addr(the_evdns_base, (int)idx, + (struct sockaddr *)&sa, + sa_len) > 0) { + tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t)); + if (tor_addr_from_sockaddr(tor_addr, + (const struct sockaddr *)&sa, + NULL) == 0) { + return tor_addr; + } + tor_free(tor_addr); + } + + return NULL; +} +#endif + /** Configure eventdns nameservers if force is true, or if the configuration * has changed since the last time we called this function, or if we failed on * our last attempt. On Unix, this reads from /etc/resolv.conf or @@ -1391,16 +1427,23 @@ configure_nameservers(int force) evdns_set_log_fn(evdns_log_cb); if (conf_fname) { log_debug(LD_FS, "stat()ing %s", conf_fname); - if (stat(sandbox_intern_string(conf_fname), &st)) { + int missing_resolv_conf = 0; + int stat_res = stat(sandbox_intern_string(conf_fname), &st); + + if (stat_res) { log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s", conf_fname, strerror(errno)); - goto err; - } - if (!force && resolv_conf_fname && !strcmp(conf_fname,resolv_conf_fname) + missing_resolv_conf = 1; + } else if (!force && resolv_conf_fname && + !strcmp(conf_fname,resolv_conf_fname) && st.st_mtime == resolv_conf_mtime) { log_info(LD_EXIT, "No change to '%s'", conf_fname); return 0; } + + if (stat_res == 0 && st.st_size == 0) + missing_resolv_conf = 1; + if (nameservers_configured) { evdns_base_search_clear(the_evdns_base); evdns_base_clear_nameservers_and_suspend(the_evdns_base); @@ -1413,20 +1456,34 @@ configure_nameservers(int force) sandbox_intern_string("/etc/hosts")); } #endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */ - log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); - if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, - sandbox_intern_string(conf_fname)))) { - log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)", - conf_fname, conf_fname, r); - goto err; - } - if (evdns_base_count_nameservers(the_evdns_base) == 0) { - log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname); - goto err; + + if (!missing_resolv_conf) { + log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname); + if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags, + sandbox_intern_string(conf_fname)))) { + log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers " + "in '%s' (%d)", conf_fname, conf_fname, r); + + if (r != 6) // "r = 6" means "no DNS servers were in resolv.conf" - + goto err; // in which case we expect libevent to add 127.0.0.1 as + // fallback. + } + if (evdns_base_count_nameservers(the_evdns_base) == 0) { + log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", + conf_fname); + } + + tor_free(resolv_conf_fname); + resolv_conf_fname = tor_strdup(conf_fname); + resolv_conf_mtime = st.st_mtime; + } else { + log_warn(LD_EXIT, "Could not read your DNS config from '%s' - " + "please investigate your DNS configuration. " + "This is possibly a problem. Meanwhile, falling" + " back to local DNS at 127.0.0.1.", conf_fname); + evdns_base_nameserver_ip_add(the_evdns_base, "127.0.0.1"); } - tor_free(resolv_conf_fname); - resolv_conf_fname = tor_strdup(conf_fname); - resolv_conf_mtime = st.st_mtime; + if (nameservers_configured) evdns_base_resume(the_evdns_base); } diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h index e4474cdf43..7b2a31a311 100644 --- a/src/feature/relay/dns.h +++ b/src/feature/relay/dns.h @@ -45,6 +45,11 @@ size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes); #ifdef DNS_PRIVATE #include "feature/relay/dns_structs.h" +size_t number_of_configured_nameservers(void); +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR +tor_addr_t *configured_nameserver_address(const size_t idx); +#endif + MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn, int is_resolve,or_circuit_t *oncirc, char **hostname_out, int *made_connection_pending_out, cached_resolve_t **resolve_out)); diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c index 56c5bb96f5..8589efb48d 100644 --- a/src/feature/relay/ext_orport.c +++ b/src/feature/relay/ext_orport.c @@ -90,7 +90,7 @@ connection_ext_or_transition(or_connection_t *conn) conn->base_.type = CONN_TYPE_OR; TO_CONN(conn)->state = 0; // set the state to a neutral value - control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0); + connection_or_event_status(conn, OR_CONN_EVENT_NEW, 0); connection_tls_start_handshake(conn, 1); } diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c index 1dbaf2ed66..12ee1dd6e2 100644 --- a/src/feature/relay/router.c +++ b/src/feature/relay/router.c @@ -30,6 +30,7 @@ #include "feature/nodelist/dirlist.h" #include "feature/nodelist/networkstatus.h" #include "feature/nodelist/nickname.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/torcert.h" @@ -49,6 +50,7 @@ #include "lib/encoding/confline.h" #include "lib/osinfo/uname.h" #include "lib/tls/tortls.h" +#include "lib/version/torversion.h" #include "feature/dirauth/authmode.h" @@ -58,6 +60,7 @@ #include "feature/dircommon/dir_connection_st.h" #include "feature/nodelist/authority_cert_st.h" #include "feature/nodelist/extrainfo_st.h" +#include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" @@ -337,6 +340,16 @@ set_server_identity_key(crypto_pk_t *k) } } +#ifdef TOR_UNIT_TESTS +/** Testing only -- set the server's RSA identity digest to + * be <b>digest</b> */ +void +set_server_identity_key_digest_testing(const uint8_t *digest) +{ + memcpy(server_identitykey_digest, digest, DIGEST_LEN); +} +#endif + /** Make sure that we have set up our identity keys to match or not match as * appropriate, and die with an assertion if we have not. */ static void @@ -634,7 +647,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out, fname); goto done; } - parsed = authority_cert_parse_from_string(cert, &eos); + parsed = authority_cert_parse_from_string(cert, strlen(cert), &eos); if (!parsed) { log_warn(LD_DIR, "Unable to parse certificate in %s", fname); goto done; @@ -1467,9 +1480,9 @@ static extrainfo_t *desc_extrainfo = NULL; static const char *desc_gen_reason = "uninitialized reason"; /** Since when has our descriptor been "clean"? 0 if we need to regenerate it * now. */ -static time_t desc_clean_since = 0; +STATIC time_t desc_clean_since = 0; /** Why did we mark the descriptor dirty? */ -static const char *desc_dirty_reason = "Tor just started"; +STATIC const char *desc_dirty_reason = "Tor just started"; /** Boolean: do we need to regenerate the above? */ static int desc_needs_upload = 0; @@ -1688,10 +1701,6 @@ router_get_descriptor_gen_reason(void) return desc_gen_reason; } -/** A list of nicknames that we've warned about including in our family - * declaration verbatim rather than as digests. */ -static smartlist_t *warned_nonexistent_family = NULL; - static int router_guess_address_from_dir_headers(uint32_t *guess); /** Make a current best guess at our address, either because @@ -1804,6 +1813,132 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr) CONN_TYPE_DIR_LISTENER); } +/** A list of nicknames that we've warned about including in our family, + * for one reason or another. */ +static smartlist_t *warned_family = NULL; + +/** + * Return a new smartlist containing the family members configured in + * <b>options</b>. Warn about invalid or missing entries. Return NULL + * if this relay should not declare a family. + **/ +STATIC smartlist_t * +get_my_declared_family(const or_options_t *options) +{ + if (!options->MyFamily) + return NULL; + + if (options->BridgeRelay) + return NULL; + + if (!warned_family) + warned_family = smartlist_new(); + + smartlist_t *declared_family = smartlist_new(); + config_line_t *family; + + /* First we try to get the whole family in the form of hexdigests. */ + for (family = options->MyFamily; family; family = family->next) { + char *name = family->value; + const node_t *member; + if (options->Nickname && !strcasecmp(name, options->Nickname)) + continue; /* Don't list ourself by nickname, that's redundant */ + else + member = node_get_by_nickname(name, 0); + + if (!member) { + /* This node doesn't seem to exist, so warn about it if it is not + * a hexdigest. */ + int is_legal = is_legal_nickname_or_hexdigest(name); + if (!smartlist_contains_string(warned_family, name) && + !is_legal_hexdigest(name)) { + if (is_legal) + log_warn(LD_CONFIG, + "There is a router named %s in my declared family, but " + "I have no descriptor for it. I'll use the nickname " + "as is, but this may confuse clients. Please list it " + "by identity digest instead.", escaped(name)); + else + log_warn(LD_CONFIG, "There is a router named %s in my declared " + "family, but that isn't a legal digest or nickname. " + "Skipping it.", escaped(name)); + smartlist_add_strdup(warned_family, name); + } + if (is_legal) { + smartlist_add_strdup(declared_family, name); + } + } else { + /* List the node by digest. */ + char *fp = tor_malloc(HEX_DIGEST_LEN+2); + fp[0] = '$'; + base16_encode(fp+1,HEX_DIGEST_LEN+1, + member->identity, DIGEST_LEN); + smartlist_add(declared_family, fp); + + if (! is_legal_hexdigest(name) && + !smartlist_contains_string(warned_family, name)) { + /* Warn if this node was not specified by hexdigest. */ + log_warn(LD_CONFIG, "There is a router named %s in my declared " + "family, but it wasn't listed by digest. Please consider " + "saying %s instead, if that's what you meant.", + escaped(name), fp); + smartlist_add_strdup(warned_family, name); + } + } + } + + /* Now declared_family should have the closest we can come to the + * identities that the user wanted. + * + * Unlike older versions of Tor, we _do_ include our own identity: this + * helps microdescriptor compression, and helps in-memory compression + * on clients. */ + nodefamily_t *nf = nodefamily_from_members(declared_family, + router_get_my_id_digest(), + NF_WARN_MALFORMED, + NULL); + SMARTLIST_FOREACH(declared_family, char *, s, tor_free(s)); + smartlist_free(declared_family); + if (!nf) { + return NULL; + } + + char *s = nodefamily_format(nf); + nodefamily_free(nf); + + smartlist_t *result = smartlist_new(); + smartlist_split_string(result, s, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + tor_free(s); + + if (smartlist_len(result) == 1) { + /* This is a one-element list containing only ourself; instead return + * nothing */ + const char *singleton = smartlist_get(result, 0); + bool is_me = false; + if (singleton[0] == '$') { + char d[DIGEST_LEN]; + int n = base16_decode(d, sizeof(d), singleton+1, strlen(singleton+1)); + if (n == DIGEST_LEN && + fast_memeq(d, router_get_my_id_digest(), DIGEST_LEN)) { + is_me = true; + } + } + if (!is_me) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Found a singleton family list with an element " + "that wasn't us! Element was %s", escaped(singleton)); + // LCOV_EXCL_STOP + } else { + SMARTLIST_FOREACH(result, char *, cp, tor_free(cp)); + smartlist_free(result); + return NULL; + } + } + + return result; +} + /** Build a fresh routerinfo, signed server descriptor, and extra-info document * for this OR. Set r to the generated routerinfo, e to the generated * extra-info document. Return 0 on success, -1 on temporary error. Failure to @@ -1918,54 +2053,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) tor_free(p_tmp); } - if (options->MyFamily && ! options->BridgeRelay) { - if (!warned_nonexistent_family) - warned_nonexistent_family = smartlist_new(); - ri->declared_family = smartlist_new(); - config_line_t *family; - for (family = options->MyFamily; family; family = family->next) { - char *name = family->value; - const node_t *member; - if (!strcasecmp(name, options->Nickname)) - continue; /* Don't list ourself, that's redundant */ - else - member = node_get_by_nickname(name, 0); - if (!member) { - int is_legal = is_legal_nickname_or_hexdigest(name); - if (!smartlist_contains_string(warned_nonexistent_family, name) && - !is_legal_hexdigest(name)) { - if (is_legal) - log_warn(LD_CONFIG, - "I have no descriptor for the router named \"%s\" in my " - "declared family; I'll use the nickname as is, but " - "this may confuse clients.", name); - else - log_warn(LD_CONFIG, "There is a router named \"%s\" in my " - "declared family, but that isn't a legal nickname. " - "Skipping it.", escaped(name)); - smartlist_add_strdup(warned_nonexistent_family, name); - } - if (is_legal) { - smartlist_add_strdup(ri->declared_family, name); - } - } else if (router_digest_is_me(member->identity)) { - /* Don't list ourself in our own family; that's redundant */ - /* XXX shouldn't be possible */ - } else { - char *fp = tor_malloc(HEX_DIGEST_LEN+2); - fp[0] = '$'; - base16_encode(fp+1,HEX_DIGEST_LEN+1, - member->identity, DIGEST_LEN); - smartlist_add(ri->declared_family, fp); - if (smartlist_contains_string(warned_nonexistent_family, name)) - smartlist_string_remove(warned_nonexistent_family, name); - } - } - - /* remove duplicates from the list */ - smartlist_sort_strings(ri->declared_family); - smartlist_uniq_strings(ri->declared_family); - } + ri->declared_family = get_my_declared_family(options); /* Now generate the extrainfo. */ ei = tor_malloc_zero(sizeof(extrainfo_t)); @@ -2131,7 +2219,9 @@ mark_my_descriptor_dirty_if_too_old(time_t now) /* Now we see whether we want to be retrying frequently or no. The * rule here is that we'll retry frequently if we aren't listed in the * live consensus we have, or if the publication time of the - * descriptor listed for us in the consensus is very old. */ + * descriptor listed for us in the consensus is very old, or if the + * consensus lists us as "stale" and we haven't regenerated since the + * consensus was published. */ ns = networkstatus_get_live_consensus(now); if (ns) { rs = networkstatus_vote_find_entry(ns, server_identitykey_digest); @@ -2139,6 +2229,8 @@ mark_my_descriptor_dirty_if_too_old(time_t now) retry_fast_reason = "not listed in consensus"; else if (rs->published_on < slow_cutoff) retry_fast_reason = "version listed in consensus is quite old"; + else if (rs->is_staledesc && ns->valid_after > desc_clean_since) + retry_fast_reason = "listed as stale in consensus"; } if (retry_fast_reason && desc_clean_since < fast_cutoff) @@ -3055,9 +3147,9 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo, void router_reset_warnings(void) { - if (warned_nonexistent_family) { - SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); - smartlist_clear(warned_nonexistent_family); + if (warned_family) { + SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp)); + smartlist_clear(warned_family); } } @@ -3081,11 +3173,12 @@ router_free_all(void) memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key)); memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key)); - if (warned_nonexistent_family) { - SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp)); - smartlist_free(warned_nonexistent_family); + if (warned_family) { + SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp)); + smartlist_free(warned_family); } } + /* From the given RSA key object, convert it to ASN-1 encoded format and set * the newly allocated object in onion_pkey_out. The length of the key is set * in onion_pkey_len_out. */ diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h index bd6a8a012e..60bc857ceb 100644 --- a/src/feature/relay/router.h +++ b/src/feature/relay/router.h @@ -117,6 +117,14 @@ void router_free_all(void); /* Used only by router.c and test.c */ STATIC void get_platform_str(char *platform, size_t len); STATIC int router_write_fingerprint(int hashed); +STATIC smartlist_t *get_my_declared_family(const or_options_t *options); + +#ifdef TOR_UNIT_TESTS +extern time_t desc_clean_since; +extern const char *desc_dirty_reason; +void set_server_identity_key_digest_testing(const uint8_t *digest); +#endif + #endif #endif /* !defined(TOR_ROUTER_H) */ diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c index 1c3badaff3..fadfb43883 100644 --- a/src/feature/rend/rendcache.c +++ b/src/feature/rend/rendcache.c @@ -45,7 +45,7 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL; * looked up in this cache and if present, it is discarded from the fetched * descriptor. At the end, all IP(s) in the cache, for a specific service * ID, that were NOT present in the descriptor are removed from this cache. - * Which means that if at least one IP was not in this cache, thus usuable, + * Which means that if at least one IP was not in this cache, thus usable, * it's considered a new descriptor so we keep it. Else, if all IPs were in * this cache, we discard the descriptor as it's considered unusable. * diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c index 3ba48f8858..421e0f2139 100644 --- a/src/feature/rend/rendmid.c +++ b/src/feature/rend/rendmid.c @@ -237,8 +237,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request, goto err; } - /* Check if we are configured to accept established rendezvous cells from - * client or in other words Tor2Web clients. */ + /* Check if we are configured to defend ourselves from clients that + * attempt to establish rendezvous points directly to us. */ if (channel_is_client(circ->p_chan) && dos_should_refuse_single_hop_client()) { /* Note it down for the heartbeat log purposes. */ diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c index a54b589eb6..5119da19a0 100644 --- a/src/feature/stats/geoip_stats.c +++ b/src/feature/stats/geoip_stats.c @@ -30,7 +30,7 @@ #include "core/or/or.h" #include "ht.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" #include "feature/client/dnsserv.h" diff --git a/src/include.am b/src/include.am index d2f83da814..9070a69a03 100644 --- a/src/include.am +++ b/src/include.am @@ -1,5 +1,6 @@ include src/ext/include.am include src/lib/arch/include.am +include src/lib/buf/include.am include src/lib/err/include.am include src/lib/cc/include.am include src/lib/ctime/include.am @@ -25,6 +26,7 @@ include src/lib/osinfo/include.am include src/lib/process/include.am include src/lib/sandbox/include.am include src/lib/string/include.am +include src/lib/subsys/include.am include src/lib/smartlist_core/include.am include src/lib/term/include.am include src/lib/testsupport/include.am @@ -32,6 +34,7 @@ include src/lib/thread/include.am include src/lib/time/include.am include src/lib/tls/include.am include src/lib/trace/include.am +include src/lib/version/include.am include src/lib/wallclock/include.am include src/trunnel/include.am diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include new file mode 100644 index 0000000000..c4be73bce2 --- /dev/null +++ b/src/lib/buf/.may_include @@ -0,0 +1,10 @@ +orconfig.h + +lib/buf/*.h +lib/cc/*.h +lib/ctime/*.h +lib/malloc/*.h +lib/testsupport/*.h +lib/log/*.h +lib/string/*.h +lib/time/*.h diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c index 67887f2f30..88a25b8470 100644 --- a/src/lib/container/buffers.c +++ b/src/lib/buf/buffers.c @@ -25,7 +25,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include <stddef.h> -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/cc/torint.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h index c103b93a82..c103b93a82 100644 --- a/src/lib/container/buffers.h +++ b/src/lib/buf/buffers.h diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am new file mode 100644 index 0000000000..3338c3dbdb --- /dev/null +++ b/src/lib/buf/include.am @@ -0,0 +1,17 @@ + +noinst_LIBRARIES += src/lib/libtor-buf.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-buf-testing.a +endif + +src_lib_libtor_buf_a_SOURCES = \ + src/lib/buf/buffers.c + +src_lib_libtor_buf_testing_a_SOURCES = \ + $(src_lib_libtor_buf_a_SOURCES) +src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +noinst_HEADERS += \ + src/lib/buf/buffers.h diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include index 2b06e8519c..fa1478ce46 100644 --- a/src/lib/cc/.may_include +++ b/src/lib/cc/.may_include @@ -1 +1,2 @@ orconfig.h +lib/cc/*.h
\ No newline at end of file diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h new file mode 100644 index 0000000000..e42976360f --- /dev/null +++ b/src/lib/cc/ctassert.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2018 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ctassert.h + * + * \brief Compile-time assertions: CTASSERT(expression). + */ + +#ifndef TOR_CTASSERT_H +#define TOR_CTASSERT_H + +#include "lib/cc/compat_compiler.h" + +/** + * CTASSERT(expression) + * + * Trigger a compiler error if expression is false. + */ +#if __STDC_VERSION__ >= 201112L + +/* If C11 is available, just use _Static_assert. */ +#define CTASSERT(x) _Static_assert((x), #x) + +#else + +/* + * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__ + * and __LINE__, or just __LINE__, with an intermediate preprocessor + * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the + * expansions together into a unique name. + * + * We use this name as a typedef of an array type with a positive + * length if the assertion is true, and a negative length of the + * assertion is false, which is invalid and hence triggers a compiler + * error. + */ +#if defined(__COUNTER__) +#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__) +#elif defined(__INCLUDE_LEVEL__) +#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__) +#else +/* hope it's unique enough */ +#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__) +#endif + +#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b) +#define CTASSERT_DECL(x, a, b) \ + typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED + +#endif + +#endif /* !defined(TOR_CTASSERT_H) */ diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am index 2ae90f97dd..52cf8a9f72 100644 --- a/src/lib/cc/include.am +++ b/src/lib/cc/include.am @@ -1,4 +1,5 @@ noinst_HEADERS += \ src/lib/cc/compat_compiler.h \ + src/lib/cc/ctassert.h \ src/lib/cc/torint.h diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include index 68fe9f1c54..6cd80086e6 100644 --- a/src/lib/compress/.may_include +++ b/src/lib/compress/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/compress/*.h lib/container/*.h @@ -8,5 +9,6 @@ lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c index 95fd73bb32..51591410a2 100644 --- a/src/lib/compress/compress.c +++ b/src/lib/compress/compress.c @@ -29,10 +29,12 @@ #include "lib/compress/compress.h" #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_none.h" +#include "lib/compress/compress_sys.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/intmath/cmp.h" #include "lib/malloc/malloc.h" +#include "lib/subsys/subsys.h" #include "lib/thread/threads.h" /** Total number of bytes allocated for compression state overhead. */ @@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state) } /** Initialize all compression modules. */ -void +int tor_compress_init(void) { atomic_counter_init(&total_compress_allocation); @@ -668,6 +670,8 @@ tor_compress_init(void) tor_zlib_init(); tor_lzma_init(); tor_zstd_init(); + + return 0; } /** Warn if we had any problems while setting up our compression libraries. @@ -677,5 +681,20 @@ tor_compress_init(void) void tor_compress_log_init_warnings(void) { + // XXXX can we move this into tor_compress_init() after all? log.c queues + // XXXX log messages at startup. tor_zstd_warn_if_version_mismatched(); } + +static int +subsys_compress_initialize(void) +{ + return tor_compress_init(); +} + +const subsys_fns_t sys_compress = { + .name = "compress", + .supported = true, + .level = -70, + .initialize = subsys_compress_initialize, +}; diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h index 5f16a2ab27..8cea4ead60 100644 --- a/src/lib/compress/compress.h +++ b/src/lib/compress/compress.h @@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); -void tor_compress_init(void); +int tor_compress_init(void); void tor_compress_log_init_warnings(void); struct buf_t; diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c index 198128b261..2e704466f2 100644 --- a/src/lib/compress/compress_buf.c +++ b/src/lib/compress/compress_buf.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/cc/compat_compiler.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/compress/compress.h" #include "lib/log/util_bug.h" diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h new file mode 100644 index 0000000000..6181072315 --- /dev/null +++ b/src/lib/compress/compress_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_sys.h + * \brief Declare subsystem object for the compress module + **/ + +#ifndef TOR_COMPRESS_SYS_H +#define TOR_COMPRESS_SYS_H + +extern const struct subsys_fns_t sys_compress; + +#endif /* !defined(TOR_COMPRESS_SYS_H) */ diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am index 75c9032bd2..b952779578 100644 --- a/src/lib/compress/include.am +++ b/src/lib/compress/include.am @@ -22,5 +22,6 @@ noinst_HEADERS += \ src/lib/compress/compress.h \ src/lib/compress/compress_lzma.h \ src/lib/compress/compress_none.h \ + src/lib/compress/compress_sys.h \ src/lib/compress/compress_zlib.h \ src/lib/compress/compress_zstd.h diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include index 90de5eda40..81507527d3 100644 --- a/src/lib/container/.may_include +++ b/src/lib/container/.may_include @@ -7,12 +7,9 @@ lib/malloc/*.h lib/err/*.h lib/smartlist_core/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/intmath/*.h lib/log/*.h -# XXXX I am unsure about this one. It's only here for buffers.c -lib/time/*.h - -ht.h -siphash.h +ext/ht.h +ext/siphash.h diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c index 9aa9b1ee56..8c61db81d6 100644 --- a/src/lib/container/bloomfilt.c +++ b/src/lib/container/bloomfilt.c @@ -14,7 +14,7 @@ #include "lib/container/bloomfilt.h" #include "lib/intmath/bits.h" #include "lib/log/util_bug.h" -#include "siphash.h" +#include "ext/siphash.h" /** How many bloom-filter bits we set per address. This is twice the * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit diff --git a/src/lib/container/include.am b/src/lib/container/include.am index e6492098b5..032e4033da 100644 --- a/src/lib/container/include.am +++ b/src/lib/container/include.am @@ -7,7 +7,6 @@ endif src_lib_libtor_container_a_SOURCES = \ src/lib/container/bloomfilt.c \ - src/lib/container/buffers.c \ src/lib/container/map.c \ src/lib/container/order.c \ src/lib/container/smartlist.c @@ -20,7 +19,6 @@ src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/container/bitarray.h \ src/lib/container/bloomfilt.h \ - src/lib/container/buffers.h \ src/lib/container/handles.h \ src/lib/container/map.h \ src/lib/container/order.h \ diff --git a/src/lib/container/map.c b/src/lib/container/map.c index d213ad50bf..fde33d6ace 100644 --- a/src/lib/container/map.c +++ b/src/lib/container/map.c @@ -21,7 +21,7 @@ #include <stdlib.h> #include <string.h> -#include "ht.h" +#include "ext/ht.h" /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called <b>maptype</b>. The key part of each diff --git a/src/lib/container/map.h b/src/lib/container/map.h index a2d1b01d12..d61b1ec18f 100644 --- a/src/lib/container/map.h +++ b/src/lib/container/map.h @@ -15,7 +15,7 @@ #include "lib/testsupport/testsupport.h" #include "lib/cc/torint.h" -#include "siphash.h" +#include "ext/siphash.h" #define DECLARE_MAP_FNS(maptype, keytype, prefix) \ typedef struct maptype maptype; \ diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include index a0fa4ec05c..0739699686 100644 --- a/src/lib/crypt_ops/.may_include +++ b/src/lib/crypt_ops/.may_include @@ -12,7 +12,8 @@ lib/malloc/*.h lib/intmath/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/subsys/*.h +lib/testsupport/*.h lib/thread/*.h lib/log/*.h @@ -21,4 +22,4 @@ trunnel/pwbox.h keccak-tiny/*.h ed25519/*.h -siphash.h +ext/siphash.h diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 329c264af6..4040085c76 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -20,8 +20,11 @@ #include "lib/crypt_ops/crypto_openssl_mgt.h" #include "lib/crypt_ops/crypto_nss_mgt.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_sys.h" -#include "siphash.h" +#include "lib/subsys/subsys.h" + +#include "ext/siphash.h" /** Boolean: has our crypto library been initialized? (early phase) */ static int crypto_early_initialized_ = 0; @@ -202,3 +205,47 @@ tor_is_using_nss(void) return 0; #endif } + +static int +subsys_crypto_initialize(void) +{ + if (crypto_early_init() < 0) + return -1; + crypto_dh_init(); + return 0; +} + +static void +subsys_crypto_shutdown(void) +{ + crypto_global_cleanup(); +} + +static void +subsys_crypto_prefork(void) +{ + crypto_prefork(); +} + +static void +subsys_crypto_postfork(void) +{ + crypto_postfork(); +} + +static void +subsys_crypto_thread_cleanup(void) +{ + crypto_thread_cleanup(); +} + +const struct subsys_fns_t sys_crypto = { + .name = "crypto", + .supported = true, + .level = -60, + .initialize = subsys_crypto_initialize, + .shutdown = subsys_crypto_shutdown, + .prefork = subsys_crypto_prefork, + .postfork = subsys_crypto_postfork, + .thread_cleanup = subsys_crypto_thread_cleanup, +}; diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index 915fe0870d..0b1cb96c1b 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -11,7 +11,6 @@ * number generators, and working with randomness. **/ -#ifndef CRYPTO_RAND_PRIVATE #define CRYPTO_RAND_PRIVATE #include "lib/crypt_ops/crypto_rand.h" @@ -530,111 +529,14 @@ crypto_rand_unmocked(char *to, size_t n) } /** - * Return a pseudorandom integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and - * INT_MAX+1, inclusive. + * Draw an unsigned 32-bit integer uniformly at random. */ -int -crypto_rand_int(unsigned int max) -{ - unsigned int val; - unsigned int cutoff; - tor_assert(max <= ((unsigned int)INT_MAX)+1); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT_MAX - (UINT_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** - * Return a pseudorandom integer, chosen uniformly from the values i such - * that min <= i < max. - * - * <b>min</b> MUST be in range [0, <b>max</b>). - * <b>max</b> MUST be in range (min, INT_MAX]. - **/ -int -crypto_rand_int_range(unsigned int min, unsigned int max) +uint32_t +crypto_rand_u32(void) { - tor_assert(min < max); - tor_assert(max <= INT_MAX); - - /* The overflow is avoided here because crypto_rand_int() returns a value - * between 0 and (max - min) inclusive. */ - return min + crypto_rand_int(max - min); -} - -/** - * As crypto_rand_int_range, but supports uint64_t. - **/ -uint64_t -crypto_rand_uint64_range(uint64_t min, uint64_t max) -{ - tor_assert(min < max); - return min + crypto_rand_uint64(max - min); -} - -/** - * As crypto_rand_int_range, but supports time_t. - **/ -time_t -crypto_rand_time_range(time_t min, time_t max) -{ - tor_assert(min < max); - return min + (time_t)crypto_rand_uint64(max - min); -} - -/** - * Return a pseudorandom 64-bit integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. - **/ -uint64_t -crypto_rand_uint64(uint64_t max) -{ - uint64_t val; - uint64_t cutoff; - tor_assert(max < UINT64_MAX); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT64_MAX - (UINT64_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** - * Return a pseudorandom double d, chosen uniformly from the range - * 0.0 <= d < 1.0. - **/ -double -crypto_rand_double(void) -{ - /* We just use an unsigned int here; we don't really care about getting - * more than 32 bits of resolution */ - unsigned int u; - crypto_rand((char*)&u, sizeof(u)); -#if SIZEOF_INT == 4 -#define UINT_MAX_AS_DOUBLE 4294967296.0 -#elif SIZEOF_INT == 8 -#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 -#else -#error SIZEOF_INT is neither 4 nor 8 -#endif /* SIZEOF_INT == 4 || ... */ - return ((double)u) / UINT_MAX_AS_DOUBLE; + uint32_t rand; + crypto_rand((void*)&rand, sizeof(rand)); + return rand; } /** @@ -727,5 +629,3 @@ crypto_force_rand_ssleay(void) #endif return 0; } - -#endif /* !defined(CRYPTO_RAND_PRIVATE) */ diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 86fa20faa3..8a81a4acdc 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -16,6 +16,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "lib/testsupport/testsupport.h" +#include "lib/malloc/malloc.h" /* random numbers */ int crypto_seed_rng(void) ATTR_WUR; @@ -24,9 +25,11 @@ void crypto_rand_unmocked(char *to, size_t n); void crypto_strongest_rand(uint8_t *out, size_t out_len); MOCK_DECL(void,crypto_strongest_rand_,(uint8_t *out, size_t out_len)); int crypto_rand_int(unsigned int max); +unsigned crypto_rand_uint(unsigned limit); int crypto_rand_int_range(unsigned int min, unsigned int max); uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); time_t crypto_rand_time_range(time_t min, time_t max); +uint32_t crypto_rand_u32(void); uint64_t crypto_rand_uint64(uint64_t max); double crypto_rand_double(void); struct tor_weak_rng_t; @@ -40,6 +43,36 @@ void *smartlist_choose(const struct smartlist_t *sl); void smartlist_shuffle(struct smartlist_t *sl); int crypto_force_rand_ssleay(void); +/** + * A fast PRNG, for use when the PRNG provided by our crypto library isn't + * fast enough. This one _should_ be cryptographically strong, but + * has seen less auditing than the PRNGs in OpenSSL and NSS. Use with + * caution. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +typedef struct crypto_fast_rng_t crypto_fast_rng_t; +/** + * Number of bytes used to seed a crypto_rand_fast_t. + **/ +crypto_fast_rng_t *crypto_fast_rng_new(void); +#define CRYPTO_FAST_RNG_SEED_LEN 48 +crypto_fast_rng_t *crypto_fast_rng_new_from_seed(const uint8_t *seed); +void crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n); +void crypto_fast_rng_free_(crypto_fast_rng_t *); +#define crypto_fast_rng_free(c) \ + FREE_AND_NULL(crypto_fast_rng_t, crypto_fast_rng_free_, (c)) + +unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit); +uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit); +double crypto_fast_rng_get_double(crypto_fast_rng_t *rng); + +#if defined(TOR_UNIT_TESTS) +/* Used for white-box testing */ +size_t crypto_fast_rng_get_bytes_used_per_stream(void); +#endif + #ifdef CRYPTO_RAND_PRIVATE STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c new file mode 100644 index 0000000000..34e763bf51 --- /dev/null +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -0,0 +1,263 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand_fast.c + * + * \brief A fast strong PRNG for use when our underlying cryptographic + * library's PRNG isn't fast enough. + **/ + +/* This library is currently implemented to use the same implementation + * technique as libottery, using AES-CTR-256 as our underlying stream cipher. + * It's backtracking-resistant immediately, and prediction-resistant after + * a while. + * + * Here's how it works: + * + * We generate pseudorandom bytes using AES-CTR-256. We generate BUFLEN bytes + * at a time. When we do this, we keep the first SEED_LEN bytes as the key + * and the IV for our next invocation of AES_CTR, and yield the remaining + * BUFLEN - SEED_LEN bytes to the user as they invoke the PRNG. As we yield + * bytes to the user, we clear them from the buffer. + * + * After we have refilled the buffer RESEED_AFTER times, we mix in an + * additional SEED_LEN bytes from our strong PRNG into the seed. + * + * If the user ever asks for a huge number of bytes at once, we pull SEED_LEN + * bytes from the PRNG and use them with our stream cipher to fill the user's + * request. + */ + +#define CRYPTO_RAND_FAST_PRIVATE + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_cipher.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/intmath/cmp.h" +#include "lib/cc/ctassert.h" +#include "lib/malloc/map_anon.h" + +#include "lib/log/util_bug.h" + +#include <string.h> + +/* Alias for CRYPTO_FAST_RNG_SEED_LEN to make our code shorter. + */ +#define SEED_LEN (CRYPTO_FAST_RNG_SEED_LEN) + +/* The amount of space that we mmap for a crypto_fast_rng_t. + */ +#define MAPLEN 4096 + +/* The number of random bytes that we can yield to the user after each + * time we fill a crypto_fast_rng_t's buffer. + */ +#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN) + +/* The number of buffer refills after which we should fetch more + * entropy from crypto_strongest_rand(). + */ +#define RESEED_AFTER 16 + +/* The length of the stream cipher key we will use for the PRNG, in bytes. + */ +#define KEY_LEN (CRYPTO_FAST_RNG_SEED_LEN - CIPHER_IV_LEN) +/* The length of the stream cipher key we will use for the PRNG, in bits. + */ +#define KEY_BITS (KEY_LEN * 8) + +/* Make sure that we have a key length we can actually use with AES. */ +CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256); + +struct crypto_fast_rng_t { + /** How many more fills does this buffer have before we should mix + * in the output of crypto_rand()? */ + uint16_t n_till_reseed; + /** How many bytes are remaining in cbuf.bytes? */ + uint16_t bytes_left; + struct cbuf { + /** The seed (key and IV) that we will use the next time that we refill + * cbuf. */ + uint8_t seed[SEED_LEN]; + /** + * Bytes that we are yielding to the user. The next byte to be + * yielded is at bytes[BUFLEN-bytes_left]; all other bytes in this + * array are set to zero. + */ + uint8_t bytes[BUFLEN]; + } buf; +}; + +/* alignof(uint8_t) should be 1, so there shouldn't be any padding in cbuf. + */ +CTASSERT(sizeof(struct cbuf) == BUFLEN+SEED_LEN); +/* We're trying to fit all of the RNG state into a nice mmapable chunk. + */ +CTASSERT(sizeof(crypto_fast_rng_t) <= MAPLEN); + +/** + * Initialize and return a new fast PRNG, using a strong random seed. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +crypto_fast_rng_t * +crypto_fast_rng_new(void) +{ + uint8_t seed[SEED_LEN]; + crypto_strongest_rand(seed, sizeof(seed)); + crypto_fast_rng_t *result = crypto_fast_rng_new_from_seed(seed); + memwipe(seed, 0, sizeof(seed)); + return result; +} + +/** + * Initialize and return a new fast PRNG, using a seed value specified + * in <b>seed</b>. This value must be CRYPTO_FAST_RNG_SEED_LEN bytes + * long. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +crypto_fast_rng_t * +crypto_fast_rng_new_from_seed(const uint8_t *seed) +{ + /* We try to allocate this object as securely as we can, to avoid + * having it get dumped, swapped, or shared after fork. + */ + crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result), + ANONMAP_PRIVATE | ANONMAP_NOINHERIT); + + memcpy(result->buf.seed, seed, SEED_LEN); + /* Causes an immediate refill once the user asks for data. */ + result->bytes_left = 0; + result->n_till_reseed = RESEED_AFTER; + return result; +} + +/** + * Helper: create a crypto_cipher_t object from SEED_LEN bytes of + * input. The first KEY_LEN bytes are used as the stream cipher's key, + * and the remaining CIPHER_IV_LEN bytes are used as its IV. + **/ +static inline crypto_cipher_t * +cipher_from_seed(const uint8_t *seed) +{ + return crypto_cipher_new_with_iv_and_bits(seed, seed+KEY_LEN, KEY_BITS); +} + +/** + * Helper: refill the seed bytes and output buffer of <b>rng</b>, using + * the input seed bytes as input (key and IV) for the stream cipher. + * + * If the n_till_reseed counter has reached zero, mix more random bytes into + * the seed before refilling the buffer. + **/ +static void +crypto_fast_rng_refill(crypto_fast_rng_t *rng) +{ + if (rng->n_till_reseed-- == 0) { + /* It's time to reseed the RNG. We'll do this by using our XOF to mix the + * old value for the seed with some additional bytes from + * crypto_strongest_rand(). */ + crypto_xof_t *xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN); + { + uint8_t seedbuf[SEED_LEN]; + crypto_strongest_rand(seedbuf, SEED_LEN); + crypto_xof_add_bytes(xof, seedbuf, SEED_LEN); + memwipe(seedbuf, 0, SEED_LEN); + } + crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN); + crypto_xof_free(xof); + + rng->n_till_reseed = RESEED_AFTER; + } + /* Now fill rng->buf with output from our stream cipher, initialized from + * that seed value. */ + crypto_cipher_t *c = cipher_from_seed(rng->buf.seed); + memset(&rng->buf, 0, sizeof(rng->buf)); + crypto_cipher_crypt_inplace(c, (char*)&rng->buf, sizeof(rng->buf)); + crypto_cipher_free(c); + + rng->bytes_left = sizeof(rng->buf.bytes); +} + +/** + * Release all storage held by <b>rng</b>. + **/ +void +crypto_fast_rng_free_(crypto_fast_rng_t *rng) +{ + if (!rng) + return; + memwipe(rng, 0, sizeof(*rng)); + tor_munmap_anonymous(rng, sizeof(*rng)); +} + +/** + * Helper: extract bytes from the PRNG, refilling it as necessary. Does not + * optimize the case when the user has asked for a huge output. + **/ +static void +crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out, + const size_t n) +{ + size_t bytes_to_yield = n; + + while (bytes_to_yield) { + if (rng->bytes_left == 0) + crypto_fast_rng_refill(rng); + + const size_t to_copy = MIN(rng->bytes_left, bytes_to_yield); + + tor_assert(sizeof(rng->buf.bytes) >= rng->bytes_left); + uint8_t *copy_from = rng->buf.bytes + + (sizeof(rng->buf.bytes) - rng->bytes_left); + memcpy(out, copy_from, to_copy); + memset(copy_from, 0, to_copy); + + out += to_copy; + bytes_to_yield -= to_copy; + rng->bytes_left -= to_copy; + } +} + +/** + * Extract <b>n</b> bytes from <b>rng</b> into the buffer at <b>out</b>. + **/ +void +crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n) +{ + if (PREDICT_UNLIKELY(n > BUFLEN)) { + /* The user has asked for a lot of output; generate it from a stream + * cipher seeded by the PRNG rather than by pulling it out of the PRNG + * directly. + */ + uint8_t seed[SEED_LEN]; + crypto_fast_rng_getbytes_impl(rng, seed, SEED_LEN); + crypto_cipher_t *c = cipher_from_seed(seed); + memset(out, 0, n); + crypto_cipher_crypt_inplace(c, (char*)out, n); + crypto_cipher_free(c); + memwipe(seed, 0, sizeof(seed)); + return; + } + + crypto_fast_rng_getbytes_impl(rng, out, n); +} + +#if defined(TOR_UNIT_TESTS) +/** for white-box testing: return the number of bytes that are returned from + * the user for each invocation of the stream cipher in this RNG. */ +size_t +crypto_fast_rng_get_bytes_used_per_stream(void) +{ + return BUFLEN; +} +#endif diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c new file mode 100644 index 0000000000..d02c5cdcfa --- /dev/null +++ b/src/lib/crypt_ops/crypto_rand_numeric.c @@ -0,0 +1,166 @@ +/** + * \file crypto_rand_numeric.c + * + * \brief Functions for retrieving uniformly distributed numbers + * from our PRNGs. + **/ + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/log/util_bug.h" + +/** + * Implementation macro: yields code that returns a uniform unbiased + * random number between 0 and limit. "type" is the type of the number to + * return; "maxval" is the largest possible value of "type"; and "fill_stmt" + * is a code snippet that fills an object named "val" with random bits. + **/ +#define IMPLEMENT_RAND_UNSIGNED(type, maxval, limit, fill_stmt) \ + do { \ + type val; \ + type cutoff; \ + tor_assert((limit) > 0); \ + \ + /* We ignore any values that are >= 'cutoff,' to avoid biasing */ \ + /* the distribution with clipping at the upper end of the type's */ \ + /* range. */ \ + cutoff = (maxval) - ((maxval)%(limit)); \ + while (1) { \ + fill_stmt; \ + if (val < cutoff) \ + return val % (limit); \ + } \ + } while (0) + +/** + * Return a pseudorandom integer chosen uniformly from the values between 0 + * and <b>limit</b>-1 inclusive. limit must be strictly between 0 and + * UINT_MAX. */ +unsigned +crypto_rand_uint(unsigned limit) +{ + tor_assert(limit < UINT_MAX); + IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit, + crypto_rand((char*)&val, sizeof(val))); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and + * INT_MAX+1, inclusive. + */ +int +crypto_rand_int(unsigned int max) +{ + /* We can't use IMPLEMENT_RAND_UNSIGNED directly, since we're trying + * to return a signed type. Instead we make sure that the range is + * reasonable for a nonnegative int, use crypto_rand_uint(), and cast. + */ + tor_assert(max <= ((unsigned int)INT_MAX)+1); + + return (int)crypto_rand_uint(max); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values i such + * that min <= i < max. + * + * <b>min</b> MUST be in range [0, <b>max</b>). + * <b>max</b> MUST be in range (min, INT_MAX]. + **/ +int +crypto_rand_int_range(unsigned int min, unsigned int max) +{ + tor_assert(min < max); + tor_assert(max <= INT_MAX); + + /* The overflow is avoided here because crypto_rand_int() returns a value + * between 0 and (max - min) inclusive. */ + return min + crypto_rand_int(max - min); +} + +/** + * As crypto_rand_int_range, but supports uint64_t. + **/ +uint64_t +crypto_rand_uint64_range(uint64_t min, uint64_t max) +{ + tor_assert(min < max); + return min + crypto_rand_uint64(max - min); +} + +/** + * As crypto_rand_int_range, but supports time_t. + **/ +time_t +crypto_rand_time_range(time_t min, time_t max) +{ + tor_assert(min < max); + return min + (time_t)crypto_rand_uint64(max - min); +} + +/** + * Return a pseudorandom 64-bit integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. + **/ +uint64_t +crypto_rand_uint64(uint64_t max) +{ + tor_assert(max < UINT64_MAX); + IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, max, + crypto_rand((char*)&val, sizeof(val))); +} + +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif /* SIZEOF_INT == 4 || ... */ + +/** + * Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + **/ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int u; + crypto_rand((char*)&u, sizeof(u)); + return ((double)u) / UINT_MAX_AS_DOUBLE; +} + +/** + * As crypto_rand_uint, but extract the result from a crypto_fast_rng_t + */ +unsigned +crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit) +{ + tor_assert(limit < UINT_MAX); + IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit, + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val))); +} + +/** + * As crypto_rand_uint64, but extract the result from a crypto_fast_rng_t. + */ +uint64_t +crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit) +{ + tor_assert(limit < UINT64_MAX); + IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, limit, + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val))); +} + +/** + * As crypto_rand_, but extract the result from a crypto_fast_rng_t. + */ +double +crypto_fast_rng_get_double(crypto_fast_rng_t *rng) +{ + unsigned int u; + crypto_fast_rng_getbytes(rng, (void*)&u, sizeof(u)); + return ((double)u) / UINT_MAX_AS_DOUBLE; +} diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h new file mode 100644 index 0000000000..894243b175 --- /dev/null +++ b/src/lib/crypt_ops/crypto_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_crypto.h + * \brief Declare subsystem object for the crypto module. + **/ + +#ifndef TOR_CRYPTO_SYS_H +#define TOR_CRYPTO_SYS_H + +extern const struct subsys_fns_t sys_crypto; + +#endif /* !defined(TOR_CRYPTO_SYS_H) */ diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c index 0dba64d595..c931b58369 100644 --- a/src/lib/crypt_ops/digestset.c +++ b/src/lib/crypt_ops/digestset.c @@ -11,7 +11,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/defs/digest_sizes.h" #include "lib/crypt_ops/digestset.h" -#include "siphash.h" +#include "ext/siphash.h" /* Wrap our hash function to have the signature that the bloom filter * needs. */ diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index 1022096fdc..4730440143 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -17,6 +17,8 @@ src_lib_libtor_crypt_ops_a_SOURCES = \ src/lib/crypt_ops/crypto_ope.c \ src/lib/crypt_ops/crypto_pwbox.c \ src/lib/crypt_ops/crypto_rand.c \ + src/lib/crypt_ops/crypto_rand_fast.c \ + src/lib/crypt_ops/crypto_rand_numeric.c \ src/lib/crypt_ops/crypto_rsa.c \ src/lib/crypt_ops/crypto_s2k.c \ src/lib/crypt_ops/crypto_util.c \ @@ -66,5 +68,6 @@ noinst_HEADERS += \ src/lib/crypt_ops/crypto_rand.h \ src/lib/crypt_ops/crypto_rsa.h \ src/lib/crypt_ops/crypto_s2k.h \ + src/lib/crypt_ops/crypto_sys.h \ src/lib/crypt_ops/crypto_util.h \ src/lib/crypt_ops/digestset.h diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am index 48ee7f29fc..6a7f9114ea 100644 --- a/src/lib/defs/include.am +++ b/src/lib/defs/include.am @@ -2,4 +2,5 @@ noinst_HEADERS += \ src/lib/defs/dh_sizes.h \ src/lib/defs/digest_sizes.h \ + src/lib/defs/time.h \ src/lib/defs/x25519_sizes.h diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h new file mode 100644 index 0000000000..c25f5022c5 --- /dev/null +++ b/src/lib/defs/time.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_TIME_DEFS_H +#define TOR_TIME_DEFS_H + +/** + * \file time.h + * + * \brief Definitions for timing-related constants. + **/ + +/** How many microseconds per second */ +#define TOR_USEC_PER_SEC (1000000) +/** How many nanoseconds per microsecond */ +#define TOR_NSEC_PER_USEC (1000) +/* How many nanoseconds per millisecond */ +#define TOR_NSEC_PER_MSEC (1000*1000) + +#endif diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include index 7c2ef36929..c9bf4b1786 100644 --- a/src/lib/encoding/.may_include +++ b/src/lib/encoding/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/cc/*.h +lib/container/*.h lib/ctime/*.h lib/encoding/*.h lib/intmath/*.h diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c index bd063440d6..de4d1648bb 100644 --- a/src/lib/encoding/binascii.c +++ b/src/lib/encoding/binascii.c @@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags) return enclen; } +/** Return an upper bound on the number of bytes that might be needed to hold + * the data from decoding the base64 string <b>srclen</b>. This is only an + * upper bound, since some part of the base64 string might be padding or + * space. */ +size_t +base64_decode_maxsize(size_t srclen) +{ + tor_assert(srclen < INT_MAX / 3); + + return CEIL_DIV(srclen * 3, 4); +} + /** Internal table mapping 6 bit values to the Base64 alphabet. */ static const char base64_encode_table[64] = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h index 7e3cc04f09..44998bb85b 100644 --- a/src/lib/encoding/binascii.h +++ b/src/lib/encoding/binascii.h @@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen); #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); +size_t base64_decode_maxsize(size_t srclen); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int flags); int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am index 2d2aa3988a..83e9211b6f 100644 --- a/src/lib/encoding/include.am +++ b/src/lib/encoding/include.am @@ -9,6 +9,7 @@ src_lib_libtor_encoding_a_SOURCES = \ src/lib/encoding/confline.c \ src/lib/encoding/cstring.c \ src/lib/encoding/keyval.c \ + src/lib/encoding/kvline.c \ src/lib/encoding/pem.c \ src/lib/encoding/time_fmt.c @@ -22,5 +23,6 @@ noinst_HEADERS += \ src/lib/encoding/confline.h \ src/lib/encoding/cstring.h \ src/lib/encoding/keyval.h \ + src/lib/encoding/kvline.h \ src/lib/encoding/pem.h \ src/lib/encoding/time_fmt.h diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c new file mode 100644 index 0000000000..307adc3f12 --- /dev/null +++ b/src/lib/encoding/kvline.c @@ -0,0 +1,239 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.c + * + * \brief Manipulating lines of key-value pairs. + **/ + +#include "orconfig.h" + +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/malloc/malloc.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" + +#include <stdbool.h> +#include <stddef.h> +#include <string.h> + +/** Return true iff we need to quote and escape the string <b>s</b> to encode + * it. */ +static bool +needs_escape(const char *s, bool as_keyless_val) +{ + if (as_keyless_val && *s == 0) + return true; + + for (; *s; ++s) { + if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) || + *s == '\'' || *s == '\"') { + return true; + } + } + return false; +} + +/** + * Return true iff the key in <b>line</b> is not set. + **/ +static bool +line_has_no_key(const config_line_t *line) +{ + return line->key == NULL || strlen(line->key) == 0; +} + +/** + * Return true iff the all the lines in <b>line</b> can be encoded + * using <b>flags</b>. + **/ +static bool +kvline_can_encode_lines(const config_line_t *line, unsigned flags) +{ + for ( ; line; line = line->next) { + const bool keyless = line_has_no_key(line); + if (keyless) { + if (! (flags & KV_OMIT_KEYS)) { + /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */ + return false; + } + if (strchr(line->value, '=') && !( flags & KV_QUOTED)) { + /* We can't have a keyless value with = without quoting it. */ + return false; + } + } + + if (needs_escape(line->value, keyless) && ! (flags & KV_QUOTED)) { + /* If KV_QUOTED is false, we can't encode a value that needs quotes. */ + return false; + } + if (line->key && strlen(line->key) && + (needs_escape(line->key, false) || strchr(line->key, '='))) { + /* We can't handle keys that need quoting. */ + return false; + } + } + return true; +} + +/** + * Encode a linked list of lines in <b>line</b> as a series of 'Key=Value' + * pairs, using the provided <b>flags</b> to encode it. Return a newly + * allocated string on success, or NULL on failure. + * + * If KV_QUOTED is set in <b>flags</b>, then all values that contain + * spaces or unusual characters are escaped and quoted. Otherwise, such + * values are not allowed. + * + * If KV_OMIT_KEYS is set in <b>flags</b>, then pairs with empty keys are + * allowed, and are encoded as 'Value'. Otherwise, such pairs are not + * allowed. + */ +char * +kvline_encode(const config_line_t *line, + unsigned flags) +{ + if (!kvline_can_encode_lines(line, flags)) + return NULL; + + smartlist_t *elements = smartlist_new(); + + for (; line; line = line->next) { + + const char *k = ""; + const char *eq = "="; + const char *v = ""; + const bool keyless = line_has_no_key(line); + bool esc = needs_escape(line->value, keyless); + char *tmp = NULL; + + if (! keyless) { + k = line->key; + } else { + eq = ""; + if (strchr(line->value, '=')) { + esc = true; + } + } + + if (esc) { + tmp = esc_for_log(line->value); + v = tmp; + } else { + v = line->value; + } + + smartlist_add_asprintf(elements, "%s%s%s", k, eq, v); + tor_free(tmp); + } + + char *result = smartlist_join_strings(elements, " ", 0, NULL); + + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + + return result; +} + +/** + * Decode a <b>line</b> containing a series of space-separated 'Key=Value' + * pairs, using the provided <b>flags</b> to decode it. Return a newly + * allocated list of pairs on success, or NULL on failure. + * + * If KV_QUOTED is set in <b>flags</b>, then (double-)quoted values are + * allowed. Otherwise, such values are not allowed. + * + * If KV_OMIT_KEYS is set in <b>flags</b>, then values without keys are + * allowed. Otherwise, such values are not allowed. + */ +config_line_t * +kvline_parse(const char *line, unsigned flags) +{ + const char *cp = line, *cplast = NULL; + bool omit_keys = (flags & KV_OMIT_KEYS) != 0; + bool quoted = (flags & KV_QUOTED) != 0; + + config_line_t *result = NULL; + config_line_t **next_line = &result; + + char *key = NULL; + char *val = NULL; + + while (*cp) { + key = val = NULL; + { + size_t idx = strspn(cp, " \t\r\v\n"); + cp += idx; + } + if (BUG(cp == cplast)) { + /* If we didn't parse anything, this code is broken. */ + goto err; // LCOV_EXCL_LINE + } + cplast = cp; + if (! *cp) + break; /* End of string; we're done. */ + + /* Possible formats are K=V, K="V", V, and "V", depending on flags. */ + + /* Find the key. */ + if (*cp != '\"') { + size_t idx = strcspn(cp, " \t\r\v\n="); + + if (cp[idx] == '=') { + key = tor_memdup_nulterm(cp, idx); + cp += idx + 1; + } else { + if (!omit_keys) + goto err; + } + } + + if (*cp == '\"') { + /* The type is "V". */ + if (!quoted) + goto err; + size_t len=0; + cp = unescape_string(cp, &val, &len); + if (cp == NULL || len != strlen(val)) { + // The string contains a NUL or is badly coded. + goto err; + } + } else { + size_t idx = strcspn(cp, " \t\r\v\n"); + val = tor_memdup_nulterm(cp, idx); + cp += idx; + } + + if (key && strlen(key) == 0) { + /* We don't allow empty keys. */ + goto err; + } + + *next_line = tor_malloc_zero(sizeof(config_line_t)); + (*next_line)->key = key ? key : tor_strdup(""); + (*next_line)->value = val; + next_line = &(*next_line)->next; + key = val = NULL; + } + + if (!kvline_can_encode_lines(result, flags)) { + goto err; + } + return result; + + err: + tor_free(key); + tor_free(val); + config_free_lines(result); + return NULL; +} diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h new file mode 100644 index 0000000000..4eed30a223 --- /dev/null +++ b/src/lib/encoding/kvline.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.h + * + * \brief Header for kvline.c + **/ + +#ifndef TOR_KVLINE_H +#define TOR_KVLINE_H + +struct config_line_t; + +#define KV_QUOTED (1u<<0) +#define KV_OMIT_KEYS (1u<<1) + +struct config_line_t *kvline_parse(const char *line, unsigned flags); +char *kvline_encode(const struct config_line_t *line, unsigned flags); + +#endif /* !defined(TOR_KVLINE_H) */ diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include index 48cc0ef088..daa1b6e4ca 100644 --- a/src/lib/err/.may_include +++ b/src/lib/err/.may_include @@ -1,3 +1,5 @@ orconfig.h lib/cc/*.h lib/err/*.h +lib/subsys/*.h +lib/version/*.h
\ No newline at end of file diff --git a/src/lib/err/include.am b/src/lib/err/include.am index f2a409c51e..43adcd2694 100644 --- a/src/lib/err/include.am +++ b/src/lib/err/include.am @@ -6,8 +6,9 @@ noinst_LIBRARIES += src/lib/libtor-err-testing.a endif src_lib_libtor_err_a_SOURCES = \ - src/lib/err/backtrace.c \ - src/lib/err/torerr.c + src/lib/err/backtrace.c \ + src/lib/err/torerr.c \ + src/lib/err/torerr_sys.c src_lib_libtor_err_testing_a_SOURCES = \ $(src_lib_libtor_err_a_SOURCES) @@ -16,4 +17,5 @@ src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/err/backtrace.h \ - src/lib/err/torerr.h + src/lib/err/torerr.h \ + src/lib/err/torerr_sys.h diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c index 6b5224273a..1a246e995c 100644 --- a/src/lib/err/torerr.c +++ b/src/lib/err/torerr.c @@ -123,6 +123,16 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n) } /** + * Reset the list of emergency error fds to its default. + */ +void +tor_log_reset_sigsafe_err_fds(void) +{ + int fds[] = { STDERR_FILENO }; + tor_log_set_sigsafe_err_fds(fds, 1); +} + +/** * Set the granularity (in ms) to use when reporting fatal errors outside * the logging system. */ diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h index 6ae91fbe85..0badaf7c6d 100644 --- a/src/lib/err/torerr.h +++ b/src/lib/err/torerr.h @@ -39,6 +39,7 @@ void tor_raw_assertion_failed_msg_(const char *file, int line, void tor_log_err_sigsafe(const char *m, ...); int tor_log_get_sigsafe_err_fds(const int **out); void tor_log_set_sigsafe_err_fds(const int *fds, int n); +void tor_log_reset_sigsafe_err_fds(void); void tor_log_sigsafe_err_set_granularity(int ms); int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len); diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c new file mode 100644 index 0000000000..3ab1b3c4e1 --- /dev/null +++ b/src/lib/err/torerr_sys.c @@ -0,0 +1,40 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.c + * \brief Subsystem object for the error handling subsystem. + **/ + +#include "orconfig.h" +#include "lib/err/backtrace.h" +#include "lib/err/torerr.h" +#include "lib/err/torerr_sys.h" +#include "lib/subsys/subsys.h" +#include "lib/version/torversion.h" + +#include <stddef.h> + +static int +subsys_torerr_initialize(void) +{ + if (configure_backtrace_handler(get_version()) < 0) + return -1; + tor_log_reset_sigsafe_err_fds(); + + return 0; +} +static void +subsys_torerr_shutdown(void) +{ + tor_log_reset_sigsafe_err_fds(); + clean_up_backtrace_handler(); +} + +const subsys_fns_t sys_torerr = { + .name = "err", + .level = -100, + .supported = true, + .initialize = subsys_torerr_initialize, + .shutdown = subsys_torerr_shutdown +}; diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h new file mode 100644 index 0000000000..c947695689 --- /dev/null +++ b/src/lib/err/torerr_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.h + * \brief Declare subsystem object for torerr.c + **/ + +#ifndef TOR_TORERR_SYS_H +#define TOR_TORERR_SYS_H + +extern const struct subsys_fns_t sys_torerr; + +#endif /* !defined(TOR_TORERR_SYS_H) */ diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include index 30af508914..273de7bb94 100644 --- a/src/lib/evloop/.may_include +++ b/src/lib/evloop/.may_include @@ -12,5 +12,5 @@ lib/testsupport/*.h lib/thread/*.h lib/time/*.h -src/ext/timeouts/timeout.c -tor_queue.h
\ No newline at end of file +ext/timeouts/timeout.c +ext/tor_queue.h
\ No newline at end of file diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c index e46d2635a8..4b2a96ef7d 100644 --- a/src/lib/evloop/timers.c +++ b/src/lib/evloop/timers.c @@ -80,7 +80,8 @@ struct timeout_cb { * use 32-bit math. */ #define WHEEL_BIT 5 #endif -#include "src/ext/timeouts/timeout.c" + +#include "ext/timeouts/timeout.c" static struct timeouts *global_timeouts = NULL; static struct mainloop_event_t *global_timer_event = NULL; diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 931f65e710..b36a02da5e 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -15,7 +15,7 @@ * * The main thread informs the worker threads of pending work by using a * condition variable. The workers inform the main process of completed work - * by using an alert_sockets_t object, as implemented in compat_threads.c. + * by using an alert_sockets_t object, as implemented in net/alertsock.c. * * The main thread can also queue an "update" that will be handled by all the * workers. This is useful for updating state that all the workers share. @@ -36,7 +36,7 @@ #include "lib/net/socket.h" #include "lib/thread/threads.h" -#include "tor_queue.h" +#include "ext/tor_queue.h" #include <event2/event.h> #include <string.h> @@ -622,8 +622,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg) tp->reply_cb(tp); } -/** Register the threadpool <b>tp</b>'s reply queue with the libevent - * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after +/** Register the threadpool <b>tp</b>'s reply queue with Tor's global + * libevent mainloop. If <b>cb</b> is provided, it is run after * each time there is work to process from the reply queue. Return 0 on * success, -1 on failure. */ diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h index 333a3f6dde..d0ee8f2be2 100644 --- a/src/lib/evloop/workqueue.h +++ b/src/lib/evloop/workqueue.h @@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp); replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); void replyqueue_process(replyqueue_t *queue); -struct event_base; int threadpool_register_reply_event(threadpool_t *tp, void (*cb)(threadpool_t *tp)); diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c index 6c87af791d..078af6a9ba 100644 --- a/src/lib/fdio/fdio.c +++ b/src/lib/fdio/fdio.c @@ -17,12 +17,16 @@ #ifdef _WIN32 #include <windows.h> #endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif #include "lib/fdio/fdio.h" #include "lib/cc/torint.h" #include "lib/err/torerr.h" #include <stdlib.h> +#include <stdio.h> /** @{ */ /** Some old versions of Unix didn't define constants for these values, diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include index b1e49fc891..c192e6181c 100644 --- a/src/lib/fs/.may_include +++ b/src/lib/fs/.may_include @@ -13,4 +13,4 @@ lib/malloc/*.h lib/memarea/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h index d0b0e8b954..67a738861b 100644 --- a/src/lib/intmath/cmp.h +++ b/src/lib/intmath/cmp.h @@ -36,4 +36,7 @@ ((v) > (max)) ? (max) : \ (v) ) +/** Give the absolute value of <b>x</b>, independent of its type. */ +#define ABS(x) ( ((x)<0) ? -(x) : (x) ) + #endif /* !defined(TOR_INTMATH_CMP_H) */ diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include index 852173aab3..11c87f0a0d 100644 --- a/src/lib/log/.may_include +++ b/src/lib/log/.may_include @@ -9,7 +9,7 @@ lib/lock/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h +lib/version/*.h lib/wallclock/*.h - -micro-revision.i
\ No newline at end of file diff --git a/src/lib/log/include.am b/src/lib/log/include.am index 4a6c9b3686..9d3dbe3104 100644 --- a/src/lib/log/include.am +++ b/src/lib/log/include.am @@ -7,9 +7,9 @@ endif src_lib_libtor_log_a_SOURCES = \ src/lib/log/escape.c \ - src/lib/log/git_revision.c \ src/lib/log/ratelim.c \ src/lib/log/log.c \ + src/lib/log/log_sys.c \ src/lib/log/util_bug.c if WIN32 @@ -21,16 +21,10 @@ src_lib_libtor_log_testing_a_SOURCES = \ src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) -# Declare that these object files depend on micro-revision.i. Without this -# rule, we could try to build them before micro-revision.i was created. -src/lib/log/git_revision.$(OBJEXT) \ - src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \ - micro-revision.i - noinst_HEADERS += \ src/lib/log/escape.h \ - src/lib/log/git_revision.h \ src/lib/log/ratelim.h \ src/lib/log/log.h \ + src/lib/log/log_sys.h \ src/lib/log/util_bug.h \ src/lib/log/win32err.h diff --git a/src/lib/log/log.c b/src/lib/log/log.c index a9ad38fb25..d21d8d1d41 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -32,7 +32,8 @@ #define LOG_PRIVATE #include "lib/log/log.h" -#include "lib/log/git_revision.h" +#include "lib/log/log_sys.h" +#include "lib/version/git_revision.h" #include "lib/log/ratelim.h" #include "lib/lock/compat_mutex.h" #include "lib/smartlist_core/smartlist_core.h" @@ -1267,7 +1268,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", NULL }; /** Return a bitmask for the log domain for which <b>domain</b> is the name, diff --git a/src/lib/log/log.h b/src/lib/log/log.h index d7a5070610..dbc1c47021 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -107,8 +107,14 @@ #define LD_CONSDIFF (1u<<24) /** Denial of Service mitigation. */ #define LD_DOS (1u<<25) +/** Processes */ +#define LD_PROCESS (1u<<26) +/** Pluggable Transports. */ +#define LD_PT (1u<<27) +/** Bootstrap tracker. */ +#define LD_BTRACK (1u<<28) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 26 +#define N_LOGGING_DOMAINS 29 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c new file mode 100644 index 0000000000..d1080f2264 --- /dev/null +++ b/src/lib/log/log_sys.c @@ -0,0 +1,35 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.c + * \brief Setup and tear down the logging module. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/log_sys.h" + +static int +subsys_logging_initialize(void) +{ + init_logging(0); + return 0; +} + +static void +subsys_logging_shutdown(void) +{ + logs_free_all(); + escaped(NULL); +} + +const subsys_fns_t sys_logging = { + .name = "log", + .supported = true, + .level = -90, + .initialize = subsys_logging_initialize, + .shutdown = subsys_logging_shutdown, +}; diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h new file mode 100644 index 0000000000..7043253066 --- /dev/null +++ b/src/lib/log/log_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.h + * \brief Declare subsystem object for the logging module. + **/ + +#ifndef TOR_LOG_SYS_H +#define TOR_LOG_SYS_H + +extern const struct subsys_fns_t sys_logging; + +#endif /* !defined(TOR_LOG_SYS_H) */ diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include index cc62bb1013..7686bf862a 100644 --- a/src/lib/malloc/.may_include +++ b/src/lib/malloc/.may_include @@ -3,4 +3,4 @@ orconfig.h lib/cc/*.h lib/err/*.h lib/malloc/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am index 502cc1c6b7..95d96168e1 100644 --- a/src/lib/malloc/include.am +++ b/src/lib/malloc/include.am @@ -6,7 +6,8 @@ noinst_LIBRARIES += src/lib/libtor-malloc-testing.a endif src_lib_libtor_malloc_a_SOURCES = \ - src/lib/malloc/malloc.c + src/lib/malloc/malloc.c \ + src/lib/malloc/map_anon.c if USE_OPENBSD_MALLOC src_lib_libtor_malloc_a_SOURCES += src/ext/OpenBSD_malloc_Linux.c @@ -18,4 +19,5 @@ src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ - src/lib/malloc/malloc.h + src/lib/malloc/malloc.h \ + src/lib/malloc/map_anon.h diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c new file mode 100644 index 0000000000..2430f7ad11 --- /dev/null +++ b/src/lib/malloc/map_anon.c @@ -0,0 +1,217 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file map_anon.c + * \brief Manage anonymous mappings. + **/ + +#include "orconfig.h" +#include "lib/malloc/map_anon.h" +#include "lib/malloc/malloc.h" +#include "lib/err/torerr.h" + +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_MACH_VM_INHERIT_H +#include <mach/vm_inherit.h> +#endif + +#ifdef _WIN32 +#include <windows.h> +#endif + +/** + * Macro to get the high bytes of a size_t, if there are high bytes. + * Windows needs this; other operating systems define a size_t that does + * what it should. + */ +#if SIZEOF_SIZE_T > 4 +#define HIGH_SIZE_T_BYTES(sz) ((sz) >> 32) +#else +#define HIGH_SIZE_T_BYTES(sz) (0) +#endif + +/* Here we define a MINHERIT macro that is minherit() or madvise(), depending + * on what we actually want. + * + * If there's a flag that sets pages to zero after fork, we define FLAG_ZERO + * to be that flag. If there's a flag unmaps pages after fork, we define + * FLAG_NOINHERIT to be that flag. + */ +#if defined(HAVE_MINHERIT) +#define MINHERIT minherit + +#ifdef INHERIT_ZERO +#define FLAG_ZERO INHERIT_ZERO +#elif defined(MAP_INHERIT_ZERO) +#define FLAG_ZERO MAP_INHERIT_ZERO +#endif +#ifdef INHERIT_NONE +#define FLAG_NOINHERIT INHERIT_NONE +#elif defined(VM_INHERIT_NONE) +#define FLAG_NOINHERIT VM_INHERIT_NONE +#elif defined(MAP_INHERIT_NONE) +#define FLAG_NOINHERIT MAP_INHERIT_NONE +#endif + +#elif defined(HAVE_MADVISE) + +#define MINHERIT madvise + +#ifdef MADV_WIPEONFORK +#define FLAG_ZERO MADV_WIPEONFORK +#endif +#ifdef MADV_DONTFORK +#define FLAG_NOINHERIT MADV_DONTFORK +#endif + +#endif + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being swapped + * to disk. Return 0 on success or if the facility is not available on this + * OS; return -1 on failure. + */ +static int +lock_mem(void *mem, size_t sz) +{ +#ifdef _WIN32 + return VirtualLock(mem, sz) ? 0 : -1; +#elif defined(HAVE_MLOCK) + return mlock(mem, sz); +#else + (void) mem; + (void) sz; + + return 0; +#endif +} + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from appearing in + * a core dump. Return 0 on success or if the facility is not available on + * this OS; return -1 on failure. + */ +static int +nodump_mem(void *mem, size_t sz) +{ +#if defined(MADV_DONTDUMP) + return madvise(mem, sz, MADV_DONTDUMP); +#else + (void) mem; + (void) sz; + return 0; +#endif +} + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being + * accessible in child processes -- ideally by having them set to 0 after a + * fork, and if that doesn't work, by having them unmapped after a fork. + * Return 0 on success or if the facility is not available on this OS; return + * -1 on failure. + */ +static int +noinherit_mem(void *mem, size_t sz) +{ +#ifdef FLAG_ZERO + int r = MINHERIT(mem, sz, FLAG_ZERO); + if (r == 0) + return 0; +#endif +#ifdef FLAG_NOINHERIT + return MINHERIT(mem, sz, FLAG_NOINHERIT); +#else + (void)mem; + (void)sz; + return 0; +#endif +} + +/** + * Return a new anonymous memory mapping that holds <b>sz</b> bytes. + * + * Memory mappings are unlike the results from malloc() in that they are + * handled separately by the operating system, and as such can have different + * kernel-level flags set on them. + * + * The "flags" argument may be zero or more of ANONMAP_PRIVATE and + * ANONMAP_NOINHERIT. + * + * Memory returned from this function must be released with + * tor_munmap_anonymous(). + * + * [Note: OS people use the word "anonymous" here to mean that the memory + * isn't associated with any file. This has *nothing* to do with the kind of + * anonymity that Tor is trying to provide.] + */ +void * +tor_mmap_anonymous(size_t sz, unsigned flags) +{ + void *ptr; +#if defined(_WIN32) + HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE, + NULL, /*attributes*/ + PAGE_READWRITE, + HIGH_SIZE_T_BYTES(sz), + sz & 0xffffffff, + NULL /* name */); + raw_assert(mapping != NULL); + ptr = MapViewOfFile(mapping, FILE_MAP_WRITE, + 0, 0, /* Offset */ + 0 /* Extend to end of mapping */); + raw_assert(ptr); + CloseHandle(mapping); /* mapped view holds a reference */ +#elif defined(HAVE_SYS_MMAN_H) + ptr = mmap(NULL, sz, + PROT_READ|PROT_WRITE, + MAP_ANON|MAP_PRIVATE, + -1, 0); + raw_assert(ptr != MAP_FAILED); + raw_assert(ptr != NULL); +#else + ptr = tor_malloc_zero(sz); +#endif + + if (flags & ANONMAP_PRIVATE) { + int lock_result = lock_mem(ptr, sz); + raw_assert(lock_result == 0); + int nodump_result = nodump_mem(ptr, sz); + raw_assert(nodump_result == 0); + } + + if (flags & ANONMAP_NOINHERIT) { + int noinherit_result = noinherit_mem(ptr, sz); + raw_assert(noinherit_result == 0); + } + + return ptr; +} + +/** + * Release <b>sz</b> bytes of memory that were previously mapped at + * <b>mapping</b> by tor_mmap_anonymous(). + **/ +void +tor_munmap_anonymous(void *mapping, size_t sz) +{ + if (!mapping) + return; + +#if defined(_WIN32) + (void)sz; + UnmapViewOfFile(mapping); +#elif defined(HAVE_SYS_MMAN_H) + munmap(mapping, sz); +#else + (void)sz; + tor_free(mapping); +#endif +} diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h new file mode 100644 index 0000000000..cc5797e4ec --- /dev/null +++ b/src/lib/malloc/map_anon.h @@ -0,0 +1,37 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file map_anon.h + * \brief Headers for map_anon.c + **/ + +#ifndef TOR_MAP_ANON_H +#define TOR_MAP_ANON_H + +#include "lib/malloc/malloc.h" +#include <stddef.h> + +/** + * When this flag is specified, try to prevent the mapping from being + * swapped or dumped. + * + * In some operating systems, this flag is not implemented. + */ +#define ANONMAP_PRIVATE (1u<<0) +/** + * When this flag is specified, try to prevent the mapping from being + * inherited after a fork(). In some operating systems, trying to access it + * afterwards will cause its contents to be zero. In others, trying to access + * it afterwards will cause a crash. + * + * In some operating systems, this flag is not implemented at all. + */ +#define ANONMAP_NOINHERIT (1u<<1) + +void *tor_mmap_anonymous(size_t sz, unsigned flags); +void tor_munmap_anonymous(void *mapping, size_t sz); + +#endif /* !defined(TOR_MAP_ANON_H) */ diff --git a/src/lib/math/.may_include b/src/lib/math/.may_include index 1fd26864dc..f8bc264a5f 100644 --- a/src/lib/math/.may_include +++ b/src/lib/math/.may_include @@ -3,3 +3,5 @@ orconfig.h lib/cc/*.h lib/log/*.h lib/math/*.h +lib/testsupport/*.h +lib/crypt_ops/*.h diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c index eafad358c3..616e4f15c0 100644 --- a/src/lib/math/fp.c +++ b/src/lib/math/fp.c @@ -121,3 +121,23 @@ ENABLE_GCC_WARNING(double-promotion) ENABLE_GCC_WARNING(float-conversion) #endif } + +/* isinf() wrapper for tor */ +int +tor_isinf(double x) +{ + /* Same as above, work around the "double promotion" warnings */ +#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING +DISABLE_GCC_WARNING(float-conversion) +#endif +#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING +DISABLE_GCC_WARNING(double-promotion) +#endif + return isinf(x); +#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING +ENABLE_GCC_WARNING(double-promotion) +#endif +#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING +ENABLE_GCC_WARNING(float-conversion) +#endif +} diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h index 6f07152e92..cb24649e6c 100644 --- a/src/lib/math/fp.h +++ b/src/lib/math/fp.h @@ -19,5 +19,6 @@ double tor_mathlog(double d) ATTR_CONST; long tor_lround(double d) ATTR_CONST; int64_t tor_llround(double d) ATTR_CONST; int64_t clamp_double_to_int64(double number); +int tor_isinf(double x); #endif diff --git a/src/lib/math/include.am b/src/lib/math/include.am index b088b3f3cc..6d65ce90a7 100644 --- a/src/lib/math/include.am +++ b/src/lib/math/include.am @@ -7,7 +7,8 @@ endif src_lib_libtor_math_a_SOURCES = \ src/lib/math/fp.c \ - src/lib/math/laplace.c + src/lib/math/laplace.c \ + src/lib/math/prob_distr.c src_lib_libtor_math_testing_a_SOURCES = \ @@ -17,4 +18,5 @@ src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/math/fp.h \ - src/lib/math/laplace.h + src/lib/math/laplace.h \ + src/lib/math/prob_distr.h diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c new file mode 100644 index 0000000000..c952dadc06 --- /dev/null +++ b/src/lib/math/prob_distr.c @@ -0,0 +1,1717 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file prob_distr.c + * + * \brief + * Implements various probability distributions. + * Almost all code is courtesy of Riastradh. + * + * \details + * Here are some details that might help you understand this file: + * + * - Throughout this file, `eps' means the largest relative error of a + * correctly rounded floating-point operation, which in binary64 + * floating-point arithmetic is 2^-53. Here the relative error of a + * true value x from a computed value y is |x - y|/|x|. This + * definition of epsilon is conventional for numerical analysts when + * writing error analyses. (If your libm doesn't provide correctly + * rounded exp and log, their relative error is usually below 2*2^-53 + * and probably closer to 1.1*2^-53 instead.) + * + * The C constant DBL_EPSILON is actually twice this, and should + * perhaps rather be named ulp(1) -- that is, it is the distance from + * 1 to the next greater floating-point number, which is usually of + * more interest to programmers and hardware engineers. + * + * Since this file is concerned mainly with error bounds rather than + * with low-level bit-hacking of floating-point numbers, we adopt the + * numerical analysts' definition in the comments, though we do use + * DBL_EPSILON in a handful of places where it is convenient to use + * some function of eps = DBL_EPSILON/2 in a case analysis. + * + * - In various functions (e.g. sample_log_logistic()) we jump through hoops so + * that we can use reals closer to 0 than closer to 1, since we achieve much + * greater accuracy for floating point numbers near 0. In particular, we can + * represent differences as small as 10^-300 for numbers near 0, but of no + * less than 10^-16 for numbers near 1. + **/ + +#define PROB_DISTR_PRIVATE + +#include "orconfig.h" + +#include "lib/math/prob_distr.h" + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/cc/ctassert.h" + +#include <float.h> +#include <math.h> +#include <stddef.h> + +/** Validators for downcasting macros below */ +#define validate_container_of(PTR, TYPE, FIELD) \ + (0 * sizeof((PTR) - &((TYPE *)(((char *)(PTR)) - \ + offsetof(TYPE, FIELD)))->FIELD)) +#define validate_const_container_of(PTR, TYPE, FIELD) \ + (0 * sizeof((PTR) - &((const TYPE *)(((const char *)(PTR)) - \ + offsetof(TYPE, FIELD)))->FIELD)) +/** Downcasting macro */ +#define container_of(PTR, TYPE, FIELD) \ + ((TYPE *)(((char *)(PTR)) - offsetof(TYPE, FIELD)) \ + + validate_container_of(PTR, TYPE, FIELD)) +/** Constified downcasting macro */ +#define const_container_of(PTR, TYPE, FIELD) \ + ((const TYPE *)(((const char *)(PTR)) - offsetof(TYPE, FIELD)) \ + + validate_const_container_of(PTR, TYPE, FIELD)) + +/** + * Count number of one bits in 32-bit word. + */ +static unsigned +bitcount32(uint32_t x) +{ + + /* Count two-bit groups. */ + x -= (x >> 1) & UINT32_C(0x55555555); + + /* Count four-bit groups. */ + x = ((x >> 2) & UINT32_C(0x33333333)) + (x & UINT32_C(0x33333333)); + + /* Count eight-bit groups. */ + x = (x + (x >> 4)) & UINT32_C(0x0f0f0f0f); + + /* Sum all eight-bit groups, and extract the sum. */ + return (x * UINT32_C(0x01010101)) >> 24; +} + +/** + * Count leading zeros in 32-bit word. + */ +static unsigned +clz32(uint32_t x) +{ + + /* Round up to a power of two. */ + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + + /* Subtract count of one bits from 32. */ + return (32 - bitcount32(x)); +} + +/* + * Some lemmas that will be used throughout this file to prove various error + * bounds: + * + * Lemma 1. If |d| <= 1/2, then 1/(1 + d) <= 2. + * + * Proof. If 0 <= d <= 1/2, then 1 + d >= 1, so that 1/(1 + d) <= 1. + * If -1/2 <= d <= 0, then 1 + d >= 1/2, so that 1/(1 + d) <= 2. QED. + * + * Lemma 2. If b = a*(1 + d)/(1 + d') for |d'| < 1/2 and nonzero a, b, + * then b = a*(1 + e) for |e| <= 2|d' - d|. + * + * Proof. |a - b|/|a| + * = |a - a*(1 + d)/(1 + d')|/|a| + * = |1 - (1 + d)/(1 + d')| + * = |(1 + d' - 1 - d)/(1 + d')| + * = |(d' - d)/(1 + d')| + * <= 2|d' - d|, by Lemma 1, + * + * QED. + * + * Lemma 3. For |d|, |d'| < 1/4, + * + * |log((1 + d)/(1 + d'))| <= 4|d - d'|. + * + * Proof. Write + * + * log((1 + d)/(1 + d')) + * = log(1 + (1 + d)/(1 + d') - 1) + * = log(1 + (1 + d - 1 - d')/(1 + d') + * = log(1 + (d - d')/(1 + d')). + * + * By Lemma 1, |(d - d')/(1 + d')| < 2|d' - d| < 1, so the Taylor + * series of log(1 + x) converges absolutely for (d - d')/(1 + d'), + * and thus we have + * + * |log(1 + (d - d')/(1 + d'))| + * = |\sum_{n=1}^\infty ((d - d')/(1 + d'))^n/n| + * <= \sum_{n=1}^\infty |(d - d')/(1 + d')|^n/n + * <= \sum_{n=1}^\infty |2(d' - d)|^n/n + * <= \sum_{n=1}^\infty |2(d' - d)|^n + * = 1/(1 - |2(d' - d)|) + * <= 4|d' - d|, + * + * QED. + * + * Lemma 4. If 1/e <= 1 + x <= e, then + * + * log(1 + (1 + d) x) = (1 + d') log(1 + x) + * + * for |d'| < 8|d|. + * + * Proof. Write + * + * log(1 + (1 + d) x) + * = log(1 + x + x*d) + * = log((1 + x) (1 + x + x*d)/(1 + x)) + * = log(1 + x) + log((1 + x + x*d)/(1 + x)) + * = log(1 + x) (1 + log((1 + x + x*d)/(1 + x))/log(1 + x)). + * + * The relative error is bounded by + * + * |log((1 + x + x*d)/(1 + x))/log(1 + x)| + * <= 4|x + x*d - x|/|log(1 + x)|, by Lemma 3, + * = 4|x*d|/|log(1 + x)| + * < 8|d|, + * + * since in this range 0 < 1 - 1/e < x/log(1 + x) <= e - 1 < 2. QED. + */ + +/** + * Compute the logistic function: f(x) = 1/(1 + e^{-x}) = e^x/(1 + e^x). + * Maps a log-odds-space probability in [-\infty, +\infty] into a direct-space + * probability in [0,1]. Inverse of logit. + * + * Ill-conditioned for large x; the identity logistic(-x) = 1 - + * logistic(x) and the function logistichalf(x) = logistic(x) - 1/2 may + * help to rearrange a computation. + * + * This implementation gives relative error bounded by 7 eps. + */ +STATIC double +logistic(double x) +{ + if (x <= log(DBL_EPSILON/2)) { + /* + * If x <= log(DBL_EPSILON/2) = log(eps), then e^x <= eps. In this case + * we will approximate the logistic() function with e^x because the + * relative error is less than eps. Here is a calculation of the + * relative error between the logistic() function and e^x and a proof + * that it's less than eps: + * + * |e^x - e^x/(1 + e^x)|/|e^x/(1 + e^x)| + * <= |1 - 1/(1 + e^x)|*|1 + e^x| + * = |e^x/(1 + e^x)|*|1 + e^x| + * = |e^x| + * <= eps. + */ + return exp(x); /* return e^x */ + } else if (x <= -log(DBL_EPSILON/2)) { + /* + * e^{-x} > 0, so 1 + e^{-x} > 1, and 0 < 1/(1 + + * e^{-x}) < 1; further, since e^{-x} < 1 + e^{-x}, we + * also have 0 < 1/(1 + e^{-x}) < 1. Thus, if exp has + * relative error d0, + has relative error d1, and / + * has relative error d2, then we get + * + * (1 + d2)/[(1 + (1 + d0) e^{-x})(1 + d1)] + * = (1 + d0)/[1 + e^{-x} + d0 e^{-x} + * + d1 + d1 e^{-x} + d0 d1 e^{-x}] + * = (1 + d0)/[(1 + e^{-x}) + * * (1 + d0 e^{-x}/(1 + e^{-x}) + * + d1/(1 + e^{-x}) + * + d0 d1 e^{-x}/(1 + e^{-x}))]. + * = (1 + d0)/[(1 + e^{-x})(1 + d')] + * = [1/(1 + e^{-x})] (1 + d0)/(1 + d') + * + * where + * + * d' = d0 e^{-x}/(1 + e^{-x}) + * + d1/(1 + e^{-x}) + * + d0 d1 e^{-x}/(1 + e^{-x}). + * + * By Lemma 2 this relative error is bounded by + * + * 2|d0 - d'| + * = 2|d0 - d0 e^{-x}/(1 + e^{-x}) + * - d1/(1 + e^{-x}) + * - d0 d1 e^{-x}/(1 + e^{-x})| + * <= 2|d0| + 2|d0 e^{-x}/(1 + e^{-x})| + * + 2|d1/(1 + e^{-x})| + * + 2|d0 d1 e^{-x}/(1 + e^{-x})| + * <= 2|d0| + 2|d0| + 2|d1| + 2|d0 d1| + * <= 4|d0| + 2|d1| + 2|d0 d1| + * <= 6 eps + 2 eps^2. + */ + return 1/(1 + exp(-x)); + } else { + /* + * e^{-x} <= eps, so the relative error of 1 from 1/(1 + * + e^{-x}) is + * + * |1/(1 + e^{-x}) - 1|/|1/(1 + e^{-x})| + * = |e^{-x}/(1 + e^{-x})|/|1/(1 + e^{-x})| + * = |e^{-x}| + * <= eps. + * + * This computation avoids an intermediate overflow + * exception, although the effect on the result is + * harmless. + * + * XXX Should maybe raise inexact here. + */ + return 1; + } +} + +/** + * Compute the logit function: log p/(1 - p). Defined on [0,1]. Maps + * a direct-space probability in [0,1] to a log-odds-space probability + * in [-\infty, +\infty]. Inverse of logistic. + * + * Ill-conditioned near 1/2 and 1; the identity logit(1 - p) = + * -logit(p) and the function logithalf(p0) = logit(1/2 + p0) may help + * to rearrange a computation for p in [1/(1 + e), 1 - 1/(1 + e)]. + * + * This implementation gives relative error bounded by 10 eps. + */ +STATIC double +logit(double p) +{ + + /* logistic(-1) <= p <= logistic(+1) */ + if (1/(1 + exp(1)) <= p && p <= 1/(1 + exp(-1))) { + /* + * For inputs near 1/2, we want to compute log1p(near + * 0) rather than log(near 1), so write this as: + * + * log(p/(1 - p)) = -log((1 - p)/p) + * = -log(1 + (1 - p)/p - 1) + * = -log(1 + (1 - p - p)/p) + * = -log(1 + (1 - 2p)/p). + * + * Since p = 2p/2 <= 1 <= 2*2p = 4p, the floating-point + * evaluation of 1 - 2p is exact; the only error arises + * from division and log1p. First, note that if + * logistic(-1) <= p <= logistic(+1), (1 - 2p)/p lies + * in the bounds of Lemma 4. + * + * If division has relative error d0 and log1p has + * relative error d1, the outcome is + * + * -(1 + d1) log(1 + (1 - 2p) (1 + d0)/p) + * = -(1 + d1) (1 + d') log(1 + (1 - 2p)/p) + * = -(1 + d1 + d' + d1 d') log(1 + (1 - 2p)/p). + * + * where |d'| < 8|d0| by Lemma 4. The relative error + * is then bounded by + * + * |d1 + d' + d1 d'| + * <= |d1| + 8|d0| + 8|d1 d0| + * <= 9 eps + 8 eps^2. + */ + return -log1p((1 - 2*p)/p); + } else { + /* + * For inputs near 0, although 1 - p may be rounded to + * 1, it doesn't matter much because the magnitude of + * the result is so much larger. For inputs near 1, we + * can compute 1 - p exactly, although the precision on + * the input is limited so we won't ever get more than + * about 700 for the output. + * + * If - has relative error d0, / has relative error d1, + * and log has relative error d2, then + * + * (1 + d2) log((1 + d0) p/[(1 - p)(1 + d1)]) + * = (1 + d2) [log(p/(1 - p)) + log((1 + d0)/(1 + d1))] + * = log(p/(1 - p)) + d2 log(p/(1 - p)) + * + (1 + d2) log((1 + d0)/(1 + d1)) + * = log(p/(1 - p))*[1 + d2 + + * + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))] + * + * Since 0 <= p < logistic(-1) or logistic(+1) < p <= + * 1, we have |log(p/(1 - p))| > 1. Hence this error + * is bounded by + * + * |d2 + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))| + * <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1)) + * / log(p/(1 - p))| + * <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))| + * <= |d2| + 4|(1 + d2) (d0 - d1)|, by Lemma 3, + * <= |d2| + 4|d0 - d1 + d2 d0 - d1 d0| + * <= |d2| + 4|d0| + 4|d1| + 4|d2 d0| + 4|d1 d0| + * <= 9 eps + 8 eps^2. + */ + return log(p/(1 - p)); + } +} + +/** + * Compute the logit function, translated in input by 1/2: logithalf(p) + * = logit(1/2 + p). Defined on [-1/2, 1/2]. Inverse of logistichalf. + * + * Ill-conditioned near +/-1/2. If |p0| > 1/2 - 1/(1 + e), it may be + * better to compute 1/2 + p0 or -1/2 - p0 and to use logit instead. + * This implementation gives relative error bounded by 34 eps. + */ +STATIC double +logithalf(double p0) +{ + + if (fabs(p0) <= 0.5 - 1/(1 + exp(1))) { + /* + * logit(1/2 + p0) + * = log((1/2 + p0)/(1 - (1/2 + p0))) + * = log((1/2 + p0)/(1/2 - p0)) + * = log(1 + (1/2 + p0)/(1/2 - p0) - 1) + * = log(1 + (1/2 + p0 - (1/2 - p0))/(1/2 - p0)) + * = log(1 + (1/2 + p0 - 1/2 + p0)/(1/2 - p0)) + * = log(1 + 2 p0/(1/2 - p0)) + * + * If the error of subtraction is d0, the error of + * division is d1, and the error of log1p is d2, then + * what we compute is + * + * (1 + d2) log(1 + (1 + d1) 2 p0/[(1 + d0) (1/2 - p0)]) + * = (1 + d2) log(1 + (1 + d') 2 p0/(1/2 - p0)) + * = (1 + d2) (1 + d'') log(1 + 2 p0/(1/2 - p0)) + * = (1 + d2 + d'' + d2 d'') log(1 + 2 p0/(1/2 - p0)), + * + * where |d'| < 2|d0 - d1| <= 4 eps by Lemma 2, and + * |d''| < 8|d'| < 32 eps by Lemma 4 since + * + * 1/e <= 1 + 2*p0/(1/2 - p0) <= e + * + * when |p0| <= 1/2 - 1/(1 + e). Hence the relative + * error is bounded by + * + * |d2 + d'' + d2 d''| + * <= |d2| + |d''| + |d2 d''| + * <= |d1| + 32 |d0| + 32 |d1 d0| + * <= 33 eps + 32 eps^2. + */ + return log1p(2*p0/(0.5 - p0)); + } else { + /* + * We have a choice of computing logit(1/2 + p0) or + * -logit(1 - (1/2 + p0)) = -logit(1/2 - p0). It + * doesn't matter which way we do this: either way, + * since 1/2 p0 <= 1/2 <= 2 p0, the sum and difference + * are computed exactly. So let's do the one that + * skips the final negation. + * + * The result is + * + * (1 + d1) log((1 + d0) (1/2 + p0)/[(1 + d2) (1/2 - p0)]) + * = (1 + d1) (1 + log((1 + d0)/(1 + d2)) + * / log((1/2 + p0)/(1/2 - p0))) + * * log((1/2 + p0)/(1/2 - p0)) + * = (1 + d') log((1/2 + p0)/(1/2 - p0)) + * = (1 + d') logit(1/2 + p0) + * + * where + * + * d' = d1 + log((1 + d0)/(1 + d2))/logit(1/2 + p0) + * + d1 log((1 + d0)/(1 + d2))/logit(1/2 + p0). + * + * For |p| > 1/2 - 1/(1 + e), logit(1/2 + p0) > 1. + * Provided |d0|, |d2| < 1/4, by Lemma 3 we have + * + * |log((1 + d0)/(1 + d2))| <= 4|d0 - d2|. + * + * Hence the relative error is bounded by + * + * |d'| <= |d1| + 4|d0 - d2| + 4|d1| |d0 - d2| + * <= |d1| + 4|d0| + 4|d2| + 4|d1 d0| + 4|d1 d2| + * <= 9 eps + 8 eps^2. + */ + return log((0.5 + p0)/(0.5 - p0)); + } +} + +/* + * The following random_uniform_01 is tailored for IEEE 754 binary64 + * floating-point or smaller. It can be adapted to larger + * floating-point formats like i387 80-bit or IEEE 754 binary128, but + * it may require sampling more bits. + */ +CTASSERT(FLT_RADIX == 2); +CTASSERT(-DBL_MIN_EXP <= 1021); +CTASSERT(DBL_MANT_DIG <= 53); + +/** + * Draw a floating-point number in [0, 1] with uniform distribution. + * + * Note that the probability of returning 0 is less than 2^-1074, so + * callers need not check for it. However, callers that cannot handle + * rounding to 1 must deal with that, because it occurs with + * probability 2^-54, which is small but nonnegligible. + */ +STATIC double +random_uniform_01(void) +{ + uint32_t z, x, hi, lo; + double s; + + /* + * Draw an exponent, geometrically distributed, but give up if + * we get a run of more than 1088 zeros, which really means the + * system is broken. + */ + z = 0; + while ((x = crypto_rand_u32()) == 0) { + if (z >= 1088) + /* Your bit sampler is broken. Go home. */ + return 0; + z += 32; + } + z += clz32(x); + + /* + * Pick 32-bit halves of an odd normalized significand. + * Picking it odd breaks ties in the subsequent rounding, which + * occur only with measure zero in the uniform distribution on + * [0, 1]. + */ + hi = crypto_rand_u32() | UINT32_C(0x80000000); + lo = crypto_rand_u32() | UINT32_C(0x00000001); + + /* Round to nearest scaled significand in [2^63, 2^64]. */ + s = hi*(double)4294967296 + lo; + + /* Rescale into [1/2, 1] and apply exponent in one swell foop. */ + return s * ldexp(1, -(64 + z)); +} + +/*******************************************************************/ + +/* Functions for specific probability distributions start here: */ + +/* + * Logistic(mu, sigma) distribution, supported on (-\infty,+\infty) + * + * This is the uniform distribution on [0,1] mapped into log-odds + * space, scaled by sigma and translated by mu. + * + * pdf(x) = e^{-(x - mu)/sigma} sigma (1 + e^{-(x - mu)/sigma})^2 + * cdf(x) = 1/(1 + e^{-(x - mu)/sigma}) = logistic((x - mu)/sigma) + * sf(x) = 1 - cdf(x) = 1 - logistic((x - mu)/sigma = logistic(-(x - mu)/sigma) + * icdf(p) = mu + sigma log p/(1 - p) = mu + sigma logit(p) + * isf(p) = mu + sigma log (1 - p)/p = mu - sigma logit(p) + */ + +/** + * Compute the CDF of the Logistic(mu, sigma) distribution: the + * logistic function. Well-conditioned for negative inputs and small + * positive inputs; ill-conditioned for large positive inputs. + */ +STATIC double +cdf_logistic(double x, double mu, double sigma) +{ + return logistic((x - mu)/sigma); +} + +/** + * Compute the SF of the Logistic(mu, sigma) distribution: the logistic + * function reflected over the y axis. Well-conditioned for positive + * inputs and small negative inputs; ill-conditioned for large negative + * inputs. + */ +STATIC double +sf_logistic(double x, double mu, double sigma) +{ + return logistic(-(x - mu)/sigma); +} + +/** + * Compute the inverse of the CDF of the Logistic(mu, sigma) + * distribution: the logit function. Well-conditioned near 0; + * ill-conditioned near 1/2 and 1. + */ +STATIC double +icdf_logistic(double p, double mu, double sigma) +{ + return mu + sigma*logit(p); +} + +/** + * Compute the inverse of the SF of the Logistic(mu, sigma) + * distribution: the -logit function. Well-conditioned near 0; + * ill-conditioned near 1/2 and 1. + */ +STATIC double +isf_logistic(double p, double mu, double sigma) +{ + return mu - sigma*logit(p); +} + +/* + * LogLogistic(alpha, beta) distribution, supported on (0, +\infty). + * + * This is the uniform distribution on [0,1] mapped into odds space, + * scaled by positive alpha and shaped by positive beta. + * + * Equivalent to computing exp of a Logistic(log alpha, 1/beta) sample. + * (Name arises because the pdf has LogLogistic(x; alpha, beta) = + * Logistic(log x; log alpha, 1/beta) and mathematicians got their + * covariance contravariant.) + * + * pdf(x) = (beta/alpha) (x/alpha)^{beta - 1}/(1 + (x/alpha)^beta)^2 + * = (1/e^mu sigma) (x/e^mu)^{1/sigma - 1} / + * (1 + (x/e^mu)^{1/sigma})^2 + * cdf(x) = 1/(1 + (x/alpha)^-beta) = 1/(1 + (x/e^mu)^{-1/sigma}) + * = 1/(1 + (e^{log x}/e^mu)^{-1/sigma}) + * = 1/(1 + (e^{log x - mu})^{-1/sigma}) + * = 1/(1 + e^{-(log x - mu)/sigma}) + * = logistic((log x - mu)/sigma) + * = logistic((log x - log alpha)/(1/beta)) + * sf(x) = 1 - 1/(1 + (x/alpha)^-beta) + * = (x/alpha)^-beta/(1 + (x/alpha)^-beta) + * = 1/((x/alpha)^beta + 1) + * = 1/(1 + (x/alpha)^beta) + * icdf(p) = alpha (p/(1 - p))^{1/beta} + * = alpha e^{logit(p)/beta} + * = e^{mu + sigma logit(p)} + * isf(p) = alpha ((1 - p)/p)^{1/beta} + * = alpha e^{-logit(p)/beta} + * = e^{mu - sigma logit(p)} + */ + +/** + * Compute the CDF of the LogLogistic(alpha, beta) distribution. + * Well-conditioned for all x and alpha, and the condition number + * + * -beta/[1 + (x/alpha)^{-beta}] + * + * grows linearly with beta. + * + * Loosely, the relative error of this implementation is bounded by + * + * 4 eps + 2 eps^2 + O(beta eps), + * + * so don't bother trying this for beta anywhere near as large as + * 1/eps, around which point it levels off at 1. + */ +STATIC double +cdf_log_logistic(double x, double alpha, double beta) +{ + /* + * Let d0 be the error of x/alpha; d1, of pow; d2, of +; and + * d3, of the final quotient. The exponentiation gives + * + * ((1 + d0) x/alpha)^{-beta} + * = (x/alpha)^{-beta} (1 + d0)^{-beta} + * = (x/alpha)^{-beta} (1 + (1 + d0)^{-beta} - 1) + * = (x/alpha)^{-beta} (1 + d') + * + * where d' = (1 + d0)^{-beta} - 1. If y = (x/alpha)^{-beta}, + * the denominator is + * + * (1 + d2) (1 + (1 + d1) (1 + d') y) + * = (1 + d2) (1 + y + (d1 + d' + d1 d') y) + * = 1 + y + (1 + d2) (d1 + d' + d1 d') y + * = (1 + y) (1 + (1 + d2) (d1 + d' + d1 d') y/(1 + y)) + * = (1 + y) (1 + d''), + * + * where d'' = (1 + d2) (d1 + d' + d1 d') y/(1 + y). The + * final result is + * + * (1 + d3) / [(1 + d2) (1 + d'') (1 + y)] + * = (1 + d''') / (1 + y) + * + * for |d'''| <= 2|d3 - d''| by Lemma 2 as long as |d''| < 1/2 + * (which may not be the case for very large beta). This + * relative error is therefore bounded by + * + * |d'''| + * <= 2|d3 - d''| + * <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d') y/(1 + y)| + * <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d')| + * = 2|d3| + 2|d1 + d' + d1 d' + d2 d1 + d2 d' + d2 d1 d'| + * <= 2|d3| + 2|d1| + 2|d'| + 2|d1 d'| + 2|d2 d1| + 2|d2 d'| + * + 2|d2 d1 d'| + * <= 4 eps + 2 eps^2 + (2 + 2 eps + 2 eps^2) |d'|. + * + * Roughly, |d'| = |(1 + d0)^{-beta} - 1| grows like beta eps, + * until it levels off at 1. + */ + return 1/(1 + pow(x/alpha, -beta)); +} + +/** + * Compute the SF of the LogLogistic(alpha, beta) distribution. + * Well-conditioned for all x and alpha, and the condition number + * + * beta/[1 + (x/alpha)^beta] + * + * grows linearly with beta. + * + * Loosely, the relative error of this implementation is bounded by + * + * 4 eps + 2 eps^2 + O(beta eps) + * + * so don't bother trying this for beta anywhere near as large as + * 1/eps, beyond which point it grows unbounded. + */ +STATIC double +sf_log_logistic(double x, double alpha, double beta) +{ + /* + * The error analysis here is essentially the same as in + * cdf_log_logistic, except that rather than levelling off at + * 1, |(1 + d0)^beta - 1| grows unbounded. + */ + return 1/(1 + pow(x/alpha, beta)); +} + +/** + * Compute the inverse of the CDF of the LogLogistic(alpha, beta) + * distribution. Ill-conditioned for p near 1 and beta near 0 with + * condition number 1/[beta (1 - p)]. + */ +STATIC double +icdf_log_logistic(double p, double alpha, double beta) +{ + return alpha*pow(p/(1 - p), 1/beta); +} + +/** + * Compute the inverse of the SF of the LogLogistic(alpha, beta) + * distribution. Ill-conditioned for p near 1 and for large beta, with + * condition number -1/[beta (1 - p)]. + */ +STATIC double +isf_log_logistic(double p, double alpha, double beta) +{ + return alpha*pow((1 - p)/p, 1/beta); +} + +/* + * Weibull(lambda, k) distribution, supported on (0, +\infty). + * + * pdf(x) = (k/lambda) (x/lambda)^{k - 1} e^{-(x/lambda)^k} + * cdf(x) = 1 - e^{-(x/lambda)^k} + * icdf(p) = lambda * (-log (1 - p))^{1/k} + * sf(x) = e^{-(x/lambda)^k} + * isf(p) = lambda * (-log p)^{1/k} + */ + +/** + * Compute the CDF of the Weibull(lambda, k) distribution. + * Well-conditioned for small x and k, and for large lambda -- + * condition number + * + * -k (x/lambda)^k exp(-(x/lambda)^k)/[exp(-(x/lambda)^k) - 1] + * + * grows linearly with k, x^k, and lambda^{-k}. + */ +STATIC double +cdf_weibull(double x, double lambda, double k) +{ + return -expm1(-pow(x/lambda, k)); +} + +/** + * Compute the SF of the Weibull(lambda, k) distribution. + * Well-conditioned for small x and k, and for large lambda -- + * condition number + * + * -k (x/lambda)^k + * + * grows linearly with k, x^k, and lambda^{-k}. + */ +STATIC double +sf_weibull(double x, double lambda, double k) +{ + return exp(-pow(x/lambda, k)); +} + +/** + * Compute the inverse of the CDF of the Weibull(lambda, k) + * distribution. Ill-conditioned for p near 1, and for k near 0; + * condition number is + * + * (p/(1 - p))/(k log(1 - p)). + */ +STATIC double +icdf_weibull(double p, double lambda, double k) +{ + return lambda*pow(-log1p(-p), 1/k); +} + +/** + * Compute the inverse of the SF of the Weibull(lambda, k) + * distribution. Ill-conditioned for p near 0, and for k near 0; + * condition number is + * + * 1/(k log(p)). + */ +STATIC double +isf_weibull(double p, double lambda, double k) +{ + return lambda*pow(-log(p), 1/k); +} + +/* + * GeneralizedPareto(mu, sigma, xi), supported on (mu, +\infty) for + * nonnegative xi, or (mu, mu - sigma/xi) for negative xi. + * + * Samples: + * = mu - sigma log U, if xi = 0; + * = mu + sigma (U^{-xi} - 1)/xi = mu + sigma*expm1(-xi log U)/xi, if xi =/= 0, + * where U is uniform on (0,1]. + * = mu + sigma (e^{xi X} - 1)/xi, + * where X has standard exponential distribution. + * + * pdf(x) = sigma^{-1} (1 + xi (x - mu)/sigma)^{-(1 + 1/xi)} + * cdf(x) = 1 - (1 + xi (x - mu)/sigma)^{-1/xi} + * = 1 - e^{-log(1 + xi (x - mu)/sigma)/xi} + * --> 1 - e^{-(x - mu)/sigma} as xi --> 0 + * sf(x) = (1 + xi (x - mu)/sigma)^{-1/xi} + * --> e^{-(x - mu)/sigma} as xi --> 0 + * icdf(p) = mu + sigma*(p^{-xi} - 1)/xi + * = mu + sigma*expm1(-xi log p)/xi + * --> mu + sigma*log p as xi --> 0 + * isf(p) = mu + sigma*((1 - p)^{xi} - 1)/xi + * = mu + sigma*expm1(-xi log1p(-p))/xi + * --> mu + sigma*log1p(-p) as xi --> 0 + */ + +/** + * Compute the CDF of the GeneralizedPareto(mu, sigma, xi) + * distribution. Well-conditioned everywhere. For standard + * distribution (mu=0, sigma=1), condition number + * + * (x/(1 + x xi)) / ((1 + x xi)^{1/xi} - 1) + * + * is bounded by 1, attained only at x = 0. + */ +STATIC double +cdf_genpareto(double x, double mu, double sigma, double xi) +{ + double x_0 = (x - mu)/sigma; + + /* + * log(1 + xi x_0)/xi + * = (-1/xi) \sum_{n=1}^\infty (-xi x_0)^n/n + * = (-1/xi) (-xi x_0 + \sum_{n=2}^\infty (-xi x_0)^n/n) + * = x_0 - (1/xi) \sum_{n=2}^\infty (-xi x_0)^n/n + * = x_0 - x_0 \sum_{n=2}^\infty (-xi x_0)^{n-1}/n + * = x_0 (1 - d), + * + * where d = \sum_{n=2}^\infty (-xi x_0)^{n-1}/n. If |xi| < + * eps/4|x_0|, then + * + * |d| <= \sum_{n=2}^\infty (eps/4)^{n-1}/n + * <= \sum_{n=2}^\infty (eps/4)^{n-1} + * = \sum_{n=1}^\infty (eps/4)^n + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2 + * + * for any 0 < eps < 2. Thus, the relative error of x_0 from + * log(1 + xi x_0)/xi is bounded by eps. + */ + if (fabs(xi) < 1e-17/x_0) + return -expm1(-x_0); + else + return -expm1(-log1p(xi*x_0)/xi); +} + +/** + * Compute the SF of the GeneralizedPareto(mu, sigma, xi) distribution. + * For standard distribution (mu=0, sigma=1), ill-conditioned for xi + * near 0; condition number + * + * -x (1 + x xi)^{(-1 - xi)/xi}/(1 + x xi)^{-1/xi} + * = -x (1 + x xi)^{-1/xi - 1}/(1 + x xi)^{-1/xi} + * = -(x/(1 + x xi)) (1 + x xi)^{-1/xi}/(1 + x xi)^{-1/xi} + * = -x/(1 + x xi) + * + * is bounded by 1/xi. + */ +STATIC double +sf_genpareto(double x, double mu, double sigma, double xi) +{ + double x_0 = (x - mu)/sigma; + + if (fabs(xi) < 1e-17/x_0) + return exp(-x_0); + else + return exp(-log1p(xi*x_0)/xi); +} + +/** + * Compute the inverse of the CDF of the GeneralizedPareto(mu, sigma, + * xi) distribution. Ill-conditioned for p near 1; condition number is + * + * xi (p/(1 - p))/(1 - (1 - p)^xi) + */ +STATIC double +icdf_genpareto(double p, double mu, double sigma, double xi) +{ + /* + * To compute f(xi) = (U^{-xi} - 1)/xi = (e^{-xi log U} - 1)/xi + * for xi near zero (note f(xi) --> -log U as xi --> 0), write + * the absolutely convergent Taylor expansion + * + * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^\infty (-xi log U)^n/n! + * = -log U + (1/xi)*\sum_{n=2}^\infty (-xi log U)^n/n! + * = -log U + \sum_{n=2}^\infty xi^{n-1} (-log U)^n/n! + * = -log U - log U \sum_{n=2}^\infty (-xi log U)^{n-1}/n! + * = -log U (1 + \sum_{n=2}^\infty (-xi log U)^{n-1}/n!). + * + * Let d = \sum_{n=2}^\infty (-xi log U)^{n-1}/n!. What do we + * lose if we discard it and use -log U as an approximation to + * f(xi)? If |xi| < eps/-4log U, then + * + * |d| <= \sum_{n=2}^\infty |xi log U|^{n-1}/n! + * <= \sum_{n=2}^\infty (eps/4)^{n-1}/n! + * <= \sum_{n=1}^\infty (eps/4)^n + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2, + * + * for any 0 < eps < 2. Hence, as long as |xi| < eps/-2log U, + * f(xi) = -log U (1 + d) for |d| <= eps/2. |d| is the + * relative error of f(xi) from -log U; from this bound, the + * relative error of -log U from f(xi) is at most (eps/2)/(1 - + * eps/2) = eps/2 + (eps/2)^2 + (eps/2)^3 + ... < eps for 0 < + * eps < 1. Since -log U < 1000 for all U in (0, 1] in + * binary64 floating-point, we can safely cut xi off at 1e-20 < + * eps/4000 and attain <1ulp error from series truncation. + */ + if (fabs(xi) <= 1e-20) + return mu - sigma*log1p(-p); + else + return mu + sigma*expm1(-xi*log1p(-p))/xi; +} + +/** + * Compute the inverse of the SF of the GeneralizedPareto(mu, sigma, + * xi) distribution. Ill-conditioned for p near 1; conditon number is + * + * -xi/(1 - p^{-xi}) + */ +STATIC double +isf_genpareto(double p, double mu, double sigma, double xi) +{ + if (fabs(xi) <= 1e-20) + return mu - sigma*log(p); + else + return mu + sigma*expm1(-xi*log(p))/xi; +} + +/*******************************************************************/ + +/** + * Deterministic samplers, parametrized by uniform integer and (0,1] + * samples. No guarantees are made about _which_ mapping from the + * integer and (0,1] samples these use; all that is guaranteed is the + * distribution of the outputs conditioned on a uniform distribution on + * the inputs. The automatic tests in test_prob_distr.c double-check + * the particular mappings we use. + * + * Beware: Unlike random_uniform_01(), these are not guaranteed to be + * supported on all possible outputs. See Ilya Mironov, `On the + * Significance of the Least Significant Bits for Differential + * Privacy', for an example of what can go wrong if you try to use + * these to conceal information from an adversary but you expose the + * specific full-precision floating-point values. + * + * Note: None of these samplers use rejection sampling; they are all + * essentially inverse-CDF transforms with tweaks. If you were to add, + * say, a Gamma sampler with the Marsaglia-Tsang method, you would have + * to parametrize it by a potentially infinite stream of uniform (and + * perhaps normal) samples rather than a fixed number, which doesn't + * make for quite as nice automatic testing as for these. + */ + +/** + * Deterministically sample from the interval [a, b], indexed by a + * uniform random floating-point number p0 in (0, 1]. + * + * Note that even if p0 is nonzero, the result may be equal to a, if + * ulp(a)/2 is nonnegligible, e.g. if a = 1. For maximum resolution, + * arrange |a| <= |b|. + */ +STATIC double +sample_uniform_interval(double p0, double a, double b) +{ + /* + * XXX Prove that the distribution is, in fact, uniform on + * [a,b], particularly around p0 = 1, or at least has very + * small deviation from uniform, quantified appropriately + * (e.g., like in Monahan 1984, or by KL divergence). It + * almost certainly does but it would be nice to quantify the + * error. + */ + if ((a <= 0 && 0 <= b) || (b <= 0 && 0 <= a)) { + /* + * When ab < 0, (1 - t) a + t b is monotonic, since for + * a <= b it is a sum of nondecreasing functions of t, + * and for b <= a, of nonincreasing functions of t. + * Further, clearly at 0 and 1 it attains a and b, + * respectively. Hence it is bounded within [a, b]. + */ + return (1 - p0)*a + p0*b; + } else { + /* + * a + (b - a) t is monotonic -- it is obviously a + * nondecreasing function of t for a <= b. Further, it + * attains a at 0, and while it may overshoot b at 1, + * we have a + * + * Theorem. If 0 <= t < 1, then the floating-point + * evaluation of a + (b - a) t is bounded in [a, b]. + * + * Lemma 1. If 0 <= t < 1 is a floating-point number, + * then for any normal floating-point number x except + * the smallest in magnitude, |round(x*t)| < |x|. + * + * Proof. WLOG, assume x >= 0. Since the rounding + * function and t |---> x*t are nondecreasing, their + * composition t |---> round(x*t) is also + * nondecreasing, so it suffices to consider the + * largest floating-point number below 1, in particular + * t = 1 - ulp(1)/2. + * + * Case I: If x is a power of two, then the next + * floating-point number below x is x - ulp(x)/2 = x - + * x*ulp(1)/2 = x*(1 - ulp(1)/2) = x*t, so, since x*t + * is a floating-point number, multiplication is exact, + * and thus round(x*t) = x*t < x. + * + * Case II: If x is not a power of two, then the + * greatest lower bound of real numbers rounded to x is + * x - ulp(x)/2 = x - ulp(T(x))/2 = x - T(x)*ulp(1)/2, + * where T(X) is the largest power of two below x. + * Anything below this bound is rounded to a + * floating-point number smaller than x, and x*t = x*(1 + * - ulp(1)/2) = x - x*ulp(1)/2 < x - T(x)*ulp(1)/2 + * since T(x) < x, so round(x*t) < x*t < x. QED. + * + * Lemma 2. If x and y are subnormal, then round(x + + * y) = x + y. + * + * Proof. It is a matter of adding the significands, + * since if we treat subnormals as having an implicit + * zero bit before the `binary' point, their exponents + * are all the same. There is at most one carry/borrow + * bit, which can always be acommodated either in a + * subnormal, or, at largest, in the implicit one bit + * of a normal. + * + * Lemma 3. Let x and y be floating-point numbers. If + * round(x - y) is subnormal or zero, then it is equal + * to x - y. + * + * Proof. Case I (equal): round(x - y) = 0 iff x = y; + * hence if round(x - y) = 0, then round(x - y) = 0 = x + * - y. + * + * Case II (subnormal/subnormal): If x and y are both + * subnormal, this follows directly from Lemma 2. + * + * Case IIIa (normal/subnormal): If x is normal and y + * is subnormal, then x and y must share sign, or else + * x - y would be larger than x and thus rounded to + * normal. If s is the smallest normal positive + * floating-point number, |x| < 2s since by + * construction 2s - |y| is normal for all subnormal y. + * This means that x and y must have the same exponent, + * so the difference is the difference of significands, + * which is exact. + * + * Case IIIb (subnormal/normal): Same as case IIIa for + * -(y - x). + * + * Case IV (normal/normal): If x and y are both normal, + * then they must share sign, or else x - y would be + * larger than x and thus rounded to normal. Note that + * |y| < 2|x|, for if |y| >= 2|x|, then |x| - |y| <= + * -|x| but -|x| is normal like x. Also, |x|/2 < |y|: + * if |x|/2 is subnormal, it must hold because y is + * normal; if |x|/2 is normal, then |x|/2 >= s, so + * since |x| - |y| < s, + * + * |x|/2 = |x| - |x|/2 <= |x| - s <= |y|; + * + * that is, |x|/2 < |y| < 2|x|, so by the Sterbenz + * lemma, round(x - y) = x - y. QED. + * + * Proof of theorem. WLOG, assume 0 <= a <= b. Since + * round(a + round(round(b - a)*t) is nondecreasing in + * t and attains a at 0, the lower end of the bound is + * trivial; we must show the upper end of the bound + * strictly. It suffices to show this for the largest + * floating-point number below 1, namely 1 - ulp(1)/2. + * + * Case I: round(b - a) is normal. Then it is at most + * the smallest floating-point number above b - a. By + * Lemma 1, round(round(b - a)*t) < round(b - a). + * Since the inequality is strict, and since + * round(round(b - a)*t) is a floating-point number + * below round(b - a), and since there are no + * floating-point numbers between b - a and round(b - + * a), we must have round(round(b - a)*t) < b - a. + * Then since y |---> round(a + y) is nondecreasing, we + * must have + * + * round(a + round(round(b - a)*t)) + * <= round(a + (b - a)) + * = round(b) = b. + * + * Case II: round(b - a) is subnormal. In this case, + * Lemma 1 falls apart -- we are not guaranteed the + * strict inequality. However, by Lemma 3, the + * difference is exact: round(b - a) = b - a. Thus, + * + * round(a + round(round(b - a)*t)) + * <= round(a + round((b - a)*t)) + * <= round(a + (b - a)) + * = round(b) + * = b, + * + * QED. + */ + + /* p0 is restricted to [0,1], but we use >= to silence -Wfloat-equal. */ + if (p0 >= 1) + return b; + return a + (b - a)*p0; + } +} + +/** + * Deterministically sample from the standard logistic distribution, + * indexed by a uniform random 32-bit integer s and uniform random + * floating-point numbers t and p0 in (0, 1]. + */ +STATIC double +sample_logistic(uint32_t s, double t, double p0) +{ + double sign = (s & 1) ? -1 : +1; + double r; + + /* + * We carve up the interval (0, 1) into subregions to compute + * the inverse CDF precisely: + * + * A = (0, 1/(1 + e)] ---> (-\infty, -1] + * B = [1/(1 + e), 1/2] ---> [-1, 0] + * C = [1/2, 1 - 1/(1 + e)] ---> [0, 1] + * D = [1 - 1/(1 + e), 1) ---> [1, +\infty) + * + * Cases D and C are mirror images of cases A and B, + * respectively, so we choose between them by the sign chosen + * by a fair coin toss. We choose between cases A and B by a + * coin toss weighted by + * + * 2/(1 + e) = 1 - [1/2 - 1/(1 + e)]/(1/2): + * + * if it comes up heads, scale p0 into a uniform (0, 1/(1 + e)] + * sample p; if it comes up tails, scale p0 into a uniform (0, + * 1/2 - 1/(1 + e)] sample and compute the inverse CDF of p = + * 1/2 - p0. + */ + if (t <= 2/(1 + exp(1))) { + /* p uniform in (0, 1/(1 + e)], represented by p. */ + p0 /= 1 + exp(1); + r = logit(p0); + } else { + /* + * p uniform in [1/(1 + e), 1/2), actually represented + * by p0 = 1/2 - p uniform in (0, 1/2 - 1/(1 + e)], so + * that p = 1/2 - p. + */ + p0 *= 0.5 - 1/(1 + exp(1)); + r = logithalf(p0); + } + + /* + * We have chosen from the negative half of the standard + * logistic distribution, which is symmetric with the positive + * half. Now use the sign to choose uniformly between them. + */ + return sign*r; +} + +/** + * Deterministically sample from the logistic distribution scaled by + * sigma and translated by mu. + */ +static double +sample_logistic_locscale(uint32_t s, double t, double p0, double mu, + double sigma) +{ + + return mu + sigma*sample_logistic(s, t, p0); +} + +/** + * Deterministically sample from the standard log-logistic + * distribution, indexed by a uniform random 32-bit integer s and a + * uniform random floating-point number p0 in (0, 1]. + */ +STATIC double +sample_log_logistic(uint32_t s, double p0) +{ + + /* + * Carve up the interval (0, 1) into (0, 1/2] and [1/2, 1); the + * condition numbers of the icdf and the isf coincide at 1/2. + */ + p0 *= 0.5; + if ((s & 1) == 0) { + /* p = p0 in (0, 1/2] */ + return p0/(1 - p0); + } else { + /* p = 1 - p0 in [1/2, 1) */ + return (1 - p0)/p0; + } +} + +/** + * Deterministically sample from the log-logistic distribution with + * scale alpha and shape beta. + */ +static double +sample_log_logistic_scaleshape(uint32_t s, double p0, double alpha, + double beta) +{ + double x = sample_log_logistic(s, p0); + + return alpha*pow(x, 1/beta); +} + +/** + * Deterministically sample from the standard exponential distribution, + * indexed by a uniform random 32-bit integer s and a uniform random + * floating-point number p0 in (0, 1]. + */ +static double +sample_exponential(uint32_t s, double p0) +{ + /* + * We would like to evaluate log(p) for p near 0, and log1p(-p) + * for p near 1. Simply carve the interval into (0, 1/2] and + * [1/2, 1) by a fair coin toss. + */ + p0 *= 0.5; + if ((s & 1) == 0) + /* p = p0 in (0, 1/2] */ + return -log(p0); + else + /* p = 1 - p0 in [1/2, 1) */ + return -log1p(-p0); +} + +/** + * Deterministically sample from a Weibull distribution with scale + * lambda and shape k -- just an exponential with a shape parameter in + * addition to a scale parameter. (Yes, lambda really is the scale, + * _not_ the rate.) + */ +STATIC double +sample_weibull(uint32_t s, double p0, double lambda, double k) +{ + + return lambda*pow(sample_exponential(s, p0), 1/k); +} + +/** + * Deterministically sample from the generalized Pareto distribution + * with shape xi, indexed by a uniform random 32-bit integer s and a + * uniform random floating-point number p0 in (0, 1]. + */ +STATIC double +sample_genpareto(uint32_t s, double p0, double xi) +{ + double x = sample_exponential(s, p0); + + /* + * Write f(xi) = (e^{xi x} - 1)/xi for xi near zero as the + * absolutely convergent Taylor series + * + * f(x) = (1/xi) (xi x + \sum_{n=2}^\infty (xi x)^n/n!) + * = x + (1/xi) \sum_{n=2}^\inty (xi x)^n/n! + * = x + \sum_{n=2}^\infty xi^{n-1} x^n/n! + * = x + x \sum_{n=2}^\infty (xi x)^{n-1}/n! + * = x (1 + \sum_{n=2}^\infty (xi x)^{n-1}/n!). + * + * d = \sum_{n=2}^\infty (xi x)^{n-1}/n! is the relative error + * of f(x) from x. If |xi| < eps/4x, then + * + * |d| <= \sum_{n=2}^\infty |xi x|^{n-1}/n! + * <= \sum_{n=2}^\infty (eps/4)^{n-1}/n! + * <= \sum_{n=1}^\infty (eps/4) + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2, + * + * for any 0 < eps < 2. Hence, as long as |xi| < eps/2x, f(xi) + * = x (1 + d) for |d| <= eps/2, so x = f(xi) (1 + d') for |d'| + * <= eps. What bound should we use for x? + * + * - If x is exponentially distributed, x > 200 with + * probability below e^{-200} << 2^{-256}, i.e. never. + * + * - If x is computed by -log(U) for U in (0, 1], x is + * guaranteed to be below 1000 in IEEE 754 binary64 + * floating-point. + * + * We can safely cut xi off at 1e-20 < eps/4000 and attain an + * error bounded by 0.5 ulp for this expression. + */ + return (fabs(xi) < 1e-20 ? x : expm1(xi*x)/xi); +} + +/** + * Deterministically sample from a generalized Pareto distribution with + * shape xi, scaled by sigma and translated by mu. + */ +static double +sample_genpareto_locscale(uint32_t s, double p0, double mu, double sigma, + double xi) +{ + + return mu + sigma*sample_genpareto(s, p0, xi); +} + +/** + * Deterministically sample from the geometric distribution with + * per-trial success probability p. + * + * XXX Quantify the error (KL divergence?) of this + * ceiling-of-exponential sampler from a true geometric distribution, + * which we could get by rejection sampling. Relevant papers: + * + * John F. Monahan, `Accuracy in Random Number Generation', + * Mathematics of Computation 45(172), October 1984, pp. 559--568. +*https://pdfs.semanticscholar.org/aca6/74b96da1df77b2224e8cfc5dd6d61a471632.pdf + * + * Karl Bringmann and Tobias Friedrich, `Exact and Efficient + * Generation of Geometric Random Variates and Random Graphs', in + * Proceedings of the 40th International Colloaquium on Automata, + * Languages, and Programming -- ICALP 2013, Springer LNCS 7965, + * pp.267--278. + * https://doi.org/10.1007/978-3-642-39206-1_23 + * https://people.mpi-inf.mpg.de/~kbringma/paper/2013ICALP-1.pdf + */ +static double +sample_geometric(uint32_t s, double p0, double p) +{ + double x = sample_exponential(s, p0); + + /* This is actually a check against 1, but we do >= so that the compiler + does not raise a -Wfloat-equal */ + if (p >= 1) + return 1; + + return ceil(-x/log1p(-p)); +} + +/*******************************************************************/ + +/** Public API for probability distributions: + * + * For each probability distribution we define each public functions + * (sample/cdf/sf/icdf/isf) as part of its dist_ops structure. + */ + +const char * +dist_name(const struct dist *dist) +{ + return dist->ops->name; +} + +double +dist_sample(const struct dist *dist) +{ + return dist->ops->sample(dist); +} + +double +dist_cdf(const struct dist *dist, double x) +{ + return dist->ops->cdf(dist, x); +} + +double +dist_sf(const struct dist *dist, double x) +{ + return dist->ops->sf(dist, x); +} + +double +dist_icdf(const struct dist *dist, double p) +{ + return dist->ops->icdf(dist, p); +} + +double +dist_isf(const struct dist *dist, double p) +{ + return dist->ops->isf(dist, p); +} + +/** Functions for uniform distribution */ + +static double +uniform_sample(const struct dist *dist) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + double p0 = random_uniform_01(); + + return sample_uniform_interval(p0, U->a, U->b); +} + +static double +uniform_cdf(const struct dist *dist, double x) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + + if (x < U->a) + return 0; + else if (x < U->b) + return (x - U->a)/(U->b - U->a); + else + return 1; +} + +static double +uniform_sf(const struct dist *dist, double x) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + + if (x > U->b) + return 0; + else if (x > U->a) + return (U->b - x)/(U->b - U->a); + else + return 1; +} + +static double +uniform_icdf(const struct dist *dist, double p) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + double w = U->b - U->a; + + return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p))); +} + +static double +uniform_isf(const struct dist *dist, double p) +{ + const struct uniform *U = const_container_of(dist, struct uniform, + base); + double w = U->b - U->a; + + return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p))); +} + +const struct dist_ops uniform_ops = { + .name = "uniform", + .sample = uniform_sample, + .cdf = uniform_cdf, + .sf = uniform_sf, + .icdf = uniform_icdf, + .isf = uniform_isf, +}; + +/** Functions for logistic distribution: */ + +static double +logistic_sample(const struct dist *dist) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + uint32_t s = crypto_rand_u32(); + double t = random_uniform_01(); + double p0 = random_uniform_01(); + + return sample_logistic_locscale(s, t, p0, L->mu, L->sigma); +} + +static double +logistic_cdf(const struct dist *dist, double x) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return cdf_logistic(x, L->mu, L->sigma); +} + +static double +logistic_sf(const struct dist *dist, double x) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return sf_logistic(x, L->mu, L->sigma); +} + +static double +logistic_icdf(const struct dist *dist, double p) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return icdf_logistic(p, L->mu, L->sigma); +} + +static double +logistic_isf(const struct dist *dist, double p) +{ + const struct logistic *L = const_container_of(dist, struct logistic, + base); + + return isf_logistic(p, L->mu, L->sigma); +} + +const struct dist_ops logistic_ops = { + .name = "logistic", + .sample = logistic_sample, + .cdf = logistic_cdf, + .sf = logistic_sf, + .icdf = logistic_icdf, + .isf = logistic_isf, +}; + +/** Functions for log-logistic distribution: */ + +static double +log_logistic_sample(const struct dist *dist) +{ + const struct log_logistic *LL = const_container_of(dist, struct + log_logistic, base); + uint32_t s = crypto_rand_u32(); + double p0 = random_uniform_01(); + + return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta); +} + +static double +log_logistic_cdf(const struct dist *dist, double x) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return cdf_log_logistic(x, LL->alpha, LL->beta); +} + +static double +log_logistic_sf(const struct dist *dist, double x) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return sf_log_logistic(x, LL->alpha, LL->beta); +} + +static double +log_logistic_icdf(const struct dist *dist, double p) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return icdf_log_logistic(p, LL->alpha, LL->beta); +} + +static double +log_logistic_isf(const struct dist *dist, double p) +{ + const struct log_logistic *LL = const_container_of(dist, + struct log_logistic, base); + + return isf_log_logistic(p, LL->alpha, LL->beta); +} + +const struct dist_ops log_logistic_ops = { + .name = "log logistic", + .sample = log_logistic_sample, + .cdf = log_logistic_cdf, + .sf = log_logistic_sf, + .icdf = log_logistic_icdf, + .isf = log_logistic_isf, +}; + +/** Functions for Weibull distribution */ + +static double +weibull_sample(const struct dist *dist) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + uint32_t s = crypto_rand_u32(); + double p0 = random_uniform_01(); + + return sample_weibull(s, p0, W->lambda, W->k); +} + +static double +weibull_cdf(const struct dist *dist, double x) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return cdf_weibull(x, W->lambda, W->k); +} + +static double +weibull_sf(const struct dist *dist, double x) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return sf_weibull(x, W->lambda, W->k); +} + +static double +weibull_icdf(const struct dist *dist, double p) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return icdf_weibull(p, W->lambda, W->k); +} + +static double +weibull_isf(const struct dist *dist, double p) +{ + const struct weibull *W = const_container_of(dist, struct weibull, + base); + + return isf_weibull(p, W->lambda, W->k); +} + +const struct dist_ops weibull_ops = { + .name = "Weibull", + .sample = weibull_sample, + .cdf = weibull_cdf, + .sf = weibull_sf, + .icdf = weibull_icdf, + .isf = weibull_isf, +}; + +/** Functions for generalized Pareto distributions */ + +static double +genpareto_sample(const struct dist *dist) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + uint32_t s = crypto_rand_u32(); + double p0 = random_uniform_01(); + + return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_cdf(const struct dist *dist, double x) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_sf(const struct dist *dist, double x) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return sf_genpareto(x, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_icdf(const struct dist *dist, double p) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_isf(const struct dist *dist, double p) +{ + const struct genpareto *GP = const_container_of(dist, struct genpareto, + base); + + return isf_genpareto(p, GP->mu, GP->sigma, GP->xi); +} + +const struct dist_ops genpareto_ops = { + .name = "generalized Pareto", + .sample = genpareto_sample, + .cdf = genpareto_cdf, + .sf = genpareto_sf, + .icdf = genpareto_icdf, + .isf = genpareto_isf, +}; + +/** Functions for geometric distribution on number of trials before success */ + +static double +geometric_sample(const struct dist *dist) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + uint32_t s = crypto_rand_u32(); + double p0 = random_uniform_01(); + + return sample_geometric(s, p0, G->p); +} + +static double +geometric_cdf(const struct dist *dist, double x) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + if (x < 1) + return 0; + /* 1 - (1 - p)^floor(x) = 1 - e^{floor(x) log(1 - p)} */ + return -expm1(floor(x)*log1p(-G->p)); +} + +static double +geometric_sf(const struct dist *dist, double x) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + if (x < 1) + return 0; + /* (1 - p)^floor(x) = e^{ceil(x) log(1 - p)} */ + return exp(floor(x)*log1p(-G->p)); +} + +static double +geometric_icdf(const struct dist *dist, double p) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + return log1p(-p)/log1p(-G->p); +} + +static double +geometric_isf(const struct dist *dist, double p) +{ + const struct geometric *G = const_container_of(dist, struct geometric, base); + + return log(p)/log1p(-G->p); +} + +const struct dist_ops geometric_ops = { + .name = "geometric (1-based)", + .sample = geometric_sample, + .cdf = geometric_cdf, + .sf = geometric_sf, + .icdf = geometric_icdf, + .isf = geometric_isf, +}; diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h new file mode 100644 index 0000000000..66acb796fd --- /dev/null +++ b/src/lib/math/prob_distr.h @@ -0,0 +1,158 @@ + +/** + * \file prob_distr.h + * + * \brief Header for prob_distr.c + **/ + +#ifndef TOR_PROB_DISTR_H +#define TOR_PROB_DISTR_H + +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" +#include "lib/testsupport/testsupport.h" + +/** + * Container for distribution parameters for sampling, CDF, &c. + */ +struct dist { + const struct dist_ops *ops; +}; + +#define DIST_BASE(OPS) { .ops = (OPS) } +#define DIST_BASE_TYPED(OPS, OBJ, TYPE) \ + DIST_BASE((OPS) + 0*sizeof(&(OBJ) - (const TYPE *)&(OBJ))) + +const char *dist_name(const struct dist *); +double dist_sample(const struct dist *); +double dist_cdf(const struct dist *, double x); +double dist_sf(const struct dist *, double x); +double dist_icdf(const struct dist *, double p); +double dist_isf(const struct dist *, double p); + +struct dist_ops { + const char *name; + double (*sample)(const struct dist *); + double (*cdf)(const struct dist *, double x); + double (*sf)(const struct dist *, double x); + double (*icdf)(const struct dist *, double p); + double (*isf)(const struct dist *, double p); +}; + +/* Geometric distribution on positive number of trials before first success */ + +struct geometric { + struct dist base; + double p; /* success probability */ +}; + +extern const struct dist_ops geometric_ops; + +#define GEOMETRIC(OBJ) \ + DIST_BASE_TYPED(&geometric_ops, OBJ, struct geometric) + +/* Pareto distribution */ + +struct genpareto { + struct dist base; + double mu; + double sigma; + double xi; +}; + +extern const struct dist_ops genpareto_ops; + +#define GENPARETO(OBJ) \ + DIST_BASE_TYPED(&genpareto_ops, OBJ, struct genpareto) + +/* Weibull distribution */ + +struct weibull { + struct dist base; + double lambda; + double k; +}; + +extern const struct dist_ops weibull_ops; + +#define WEIBULL(OBJ) \ + DIST_BASE_TYPED(&weibull_ops, OBJ, struct weibull) + +/* Log-logistic distribution */ + +struct log_logistic { + struct dist base; + double alpha; + double beta; +}; + +extern const struct dist_ops log_logistic_ops; + +#define LOG_LOGISTIC(OBJ) \ + DIST_BASE_TYPED(&log_logistic_ops, OBJ, struct log_logistic) + +/* Logistic distribution */ + +struct logistic { + struct dist base; + double mu; + double sigma; +}; + +extern const struct dist_ops logistic_ops; + +#define LOGISTIC(OBJ) \ + DIST_BASE_TYPED(&logistic_ops, OBJ, struct logistic) + +/* Uniform distribution */ + +struct uniform { + struct dist base; + double a; + double b; +}; + +extern const struct dist_ops uniform_ops; + +#define UNIFORM(OBJ) \ + DIST_BASE_TYPED(&uniform_ops, OBJ, struct uniform) + +/** Only by unittests */ + +#ifdef PROB_DISTR_PRIVATE + +STATIC double logithalf(double p0); +STATIC double logit(double p); + +STATIC double random_uniform_01(void); + +STATIC double logistic(double x); +STATIC double cdf_logistic(double x, double mu, double sigma); +STATIC double sf_logistic(double x, double mu, double sigma); +STATIC double icdf_logistic(double p, double mu, double sigma); +STATIC double isf_logistic(double p, double mu, double sigma); +STATIC double sample_logistic(uint32_t s, double t, double p0); + +STATIC double cdf_log_logistic(double x, double alpha, double beta); +STATIC double sf_log_logistic(double x, double alpha, double beta); +STATIC double icdf_log_logistic(double p, double alpha, double beta); +STATIC double isf_log_logistic(double p, double alpha, double beta); +STATIC double sample_log_logistic(uint32_t s, double p0); + +STATIC double cdf_weibull(double x, double lambda, double k); +STATIC double sf_weibull(double x, double lambda, double k); +STATIC double icdf_weibull(double p, double lambda, double k); +STATIC double isf_weibull(double p, double lambda, double k); +STATIC double sample_weibull(uint32_t s, double p0, double lambda, double k); + +STATIC double sample_uniform_interval(double p0, double a, double b); + +STATIC double cdf_genpareto(double x, double mu, double sigma, double xi); +STATIC double sf_genpareto(double x, double mu, double sigma, double xi); +STATIC double icdf_genpareto(double p, double mu, double sigma, double xi); +STATIC double isf_genpareto(double p, double mu, double sigma, double xi); +STATIC double sample_genpareto(uint32_t s, double p0, double xi); + +#endif + +#endif diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include index 814652a93c..a1edaf2231 100644 --- a/src/lib/memarea/.may_include +++ b/src/lib/memarea/.may_include @@ -1,7 +1,7 @@ orconfig.h lib/arch/*.h lib/cc/*.h -lib/container/*.h lib/log/*.h lib/malloc/*.h lib/memarea/*.h +lib/smartlist_core/*.h
\ No newline at end of file diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index 486673116c..84c73b0b95 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -16,7 +16,8 @@ #include "lib/arch/bytes.h" #include "lib/cc/torint.h" -#include "lib/container/smartlist.h" +#include "lib/smartlist_core/smartlist_core.h" +#include "lib/smartlist_core/smartlist_foreach.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index 13b209bbed..e4368f799b 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -1,8 +1,9 @@ orconfig.h -siphash.h -ht.h +ext/siphash.h +ext/ht.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h @@ -11,5 +12,6 @@ lib/lock/*.h lib/log/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/malloc/*.h
\ No newline at end of file diff --git a/src/lib/net/address.c b/src/lib/net/address.c index a2d234b742..214d8aa3eb 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -40,6 +40,7 @@ #include "lib/net/address.h" #include "lib/net/socket.h" +#include "lib/cc/ctassert.h" #include "lib/container/smartlist.h" #include "lib/ctime/di_ops.h" #include "lib/log/log.h" @@ -52,7 +53,7 @@ #include "lib/string/printf.h" #include "lib/string/util_string.h" -#include "siphash.h" +#include "ext/siphash.h" #ifdef HAVE_SYS_TIME_H #include <sys/time.h> @@ -98,6 +99,7 @@ #if AF_UNSPEC != 0 #error We rely on AF_UNSPEC being 0. Let us know about your platform, please! #endif +CTASSERT(AF_UNSPEC == 0); /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a * sockaddr object in *<b>sa_out</b> of object size <b>len</b>. If not enough @@ -1198,14 +1200,22 @@ tor_addr_parse(tor_addr_t *addr, const char *src) int result; struct in_addr in_tmp; struct in6_addr in6_tmp; + int brackets_detected = 0; + tor_assert(addr && src); - if (src[0] == '[' && src[1]) + + size_t len = strlen(src); + + if (len && src[0] == '[' && src[len - 1] == ']') { + brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); + } if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { result = AF_INET6; tor_addr_from_in6(addr, &in6_tmp); - } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { + } else if (!brackets_detected && + tor_inet_pton(AF_INET, src, &in_tmp) > 0) { result = AF_INET; tor_addr_from_in(addr, &in_tmp); } else { diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 3eb0a033d5..cfe1a7dc26 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/net/buffers_net.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/net/nettypes.h" @@ -22,6 +22,10 @@ #include <stdlib.h> +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + #ifdef PARANOIA /** Helper: If PARANOIA is defined, assert that the buffer in local variable * <b>buf</b> is well-formed. */ @@ -30,27 +34,36 @@ #define check() STMT_NIL #endif /* defined(PARANOIA) */ -/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into +/** Read up to <b>at_most</b> bytes from the file descriptor <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set - * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, - * and the number of bytes read otherwise. */ + * *<b>reached_eof</b> to 1. Uses <b>tor_socket_recv()</b> iff <b>is_socket</b> + * is true, otherwise it uses <b>read()</b>. Return -1 on error (and sets + * *<b>error</b> to errno), 0 on eof or blocking, and the number of bytes read + * otherwise. */ static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, - int *reached_eof, int *socket_error) + int *reached_eof, int *error, bool is_socket) { ssize_t read_result; if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) at_most = CHUNK_REMAINING_CAPACITY(chunk); - read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (is_socket) + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + else + read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most); if (read_result < 0) { - int e = tor_socket_errno(fd); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) - log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); + log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?", + is_socket ? "recv" : "read"); #endif - *socket_error = e; + if (error) + *error = e; return -1; } return 0; /* would block. */ @@ -68,16 +81,17 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, } } -/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most - * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 - * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on - * error; else return the number of bytes read. +/** Read from file descriptor <b>fd</b>, writing onto end of <b>buf</b>. Read + * at most <b>at_most</b> bytes, growing the buffer as necessary. If recv() + * returns 0 (because of EOF), set *<b>reached_eof</b> to 1 and return 0. + * Return -1 on error; else return the number of bytes read. */ /* XXXX indicate "read blocked" somehow? */ -int -buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, - int *reached_eof, - int *socket_error) +static int +buf_read_from_fd(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error, + bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. @@ -87,7 +101,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, check(); tor_assert(reached_eof); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(buf->datalen >= INT_MAX)) return -1; @@ -108,7 +122,8 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + r = read_to_chunk(buf, chunk, fd, readlen, + reached_eof, socket_error, is_socket); check(); if (r < 0) return r; /* Error */ @@ -122,22 +137,27 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, } /** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk - * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. On success, deduct - * the bytes written from *<b>buf_flushlen</b>. Return the number of bytes - * written on success, 0 on blocking, -1 on failure. + * <b>chunk</b> of buffer <b>buf</b> onto file descriptor <b>fd</b>. On + * success, deduct the bytes written from *<b>buf_flushlen</b>. Return the + * number of bytes written on success, 0 on blocking, -1 on failure. */ static inline int -flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, - size_t *buf_flushlen) +flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen, bool is_socket) { ssize_t write_result; if (sz > chunk->datalen) sz = chunk->datalen; - write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (is_socket) + write_result = tor_socket_send(fd, chunk->data, sz, 0); + else + write_result = write(fd, chunk->data, sz); if (write_result < 0) { - int e = tor_socket_errno(s); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) @@ -155,15 +175,15 @@ flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, } } -/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most +/** Write data from <b>buf</b> to the file descriptor <b>fd</b>. Write at most * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by * the number of bytes actually written, and remove the written bytes * from the buffer. Return the number of bytes written on success, * -1 on failure. Return 0 if write() would block. */ -int -buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, - size_t *buf_flushlen) +static int +buf_flush_to_fd(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen, bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. @@ -171,7 +191,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, int r; size_t flushed = 0; tor_assert(buf_flushlen); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(*buf_flushlen > buf->datalen)) { *buf_flushlen = buf->datalen; } @@ -188,7 +208,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket); check(); if (r < 0) return r; @@ -200,3 +220,55 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, tor_assert(flushed < INT_MAX); return (int)flushed; } + +/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); +} + +/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); +} + +/** Write data from <b>buf</b> to the pipe <b>fd</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_pipe(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false); +} + +/** Read from pipe <b>fd</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If read() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_pipe(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false); +} diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h index 5f69bebedf..a3a90172a1 100644 --- a/src/lib/net/buffers_net.h +++ b/src/lib/net/buffers_net.h @@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most, int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz, size_t *buf_flushlen); +int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error); + +int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen); + #endif /* !defined(TOR_BUFFERS_H) */ diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c index 1a2406ce5f..d9ae7cd562 100644 --- a/src/lib/net/inaddr.c +++ b/src/lib/net/inaddr.c @@ -168,6 +168,13 @@ tor_inet_pton(int af, const char *src, void *dst) if (af == AF_INET) { return tor_inet_aton(src, dst); } else if (af == AF_INET6) { + ssize_t len = strlen(src); + + /* Reject if src has needless trailing ':'. */ + if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { + return 0; + } + struct in6_addr *out = dst; uint16_t words[8]; int gapPos = -1, i, setWords=0; @@ -207,7 +214,6 @@ tor_inet_pton(int af, const char *src, void *dst) return 0; if (TOR_ISXDIGIT(*src)) { char *next; - ssize_t len; long r = strtol(src, &next, 16); if (next == NULL || next == src) { /* The 'next == src' error case can happen on versions of openbsd diff --git a/src/lib/net/include.am b/src/lib/net/include.am index ff0967e786..8a88f0f2ae 100644 --- a/src/lib/net/include.am +++ b/src/lib/net/include.am @@ -11,6 +11,7 @@ src_lib_libtor_net_a_SOURCES = \ src/lib/net/buffers_net.c \ src/lib/net/gethostname.c \ src/lib/net/inaddr.c \ + src/lib/net/network_sys.c \ src/lib/net/resolve.c \ src/lib/net/socket.c \ src/lib/net/socketpair.c @@ -28,6 +29,7 @@ noinst_HEADERS += \ src/lib/net/inaddr.h \ src/lib/net/inaddr_st.h \ src/lib/net/nettypes.h \ + src/lib/net/network_sys.h \ src/lib/net/resolve.h \ src/lib/net/socket.h \ src/lib/net/socketpair.h \ diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c new file mode 100644 index 0000000000..9dfdb2b45a --- /dev/null +++ b/src/lib/net/network_sys.c @@ -0,0 +1,44 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file network_sys.c + * \brief Subsystem object for networking setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/net/network_sys.h" +#include "lib/net/resolve.h" +#include "lib/net/socket.h" + +#ifdef _WIN32 +#include <winsock2.h> +#include <windows.h> +#endif + +static int +subsys_network_initialize(void) +{ + if (network_init() < 0) + return -1; + + return 0; +} + +static void +subsys_network_shutdown(void) +{ +#ifdef _WIN32 + WSACleanup(); +#endif + tor_free_getaddrinfo_cache(); +} + +const subsys_fns_t sys_network = { + .name = "network", + .level = -90, + .supported = true, + .initialize = subsys_network_initialize, + .shutdown = subsys_network_shutdown, +}; diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h new file mode 100644 index 0000000000..43e62592ca --- /dev/null +++ b/src/lib/net/network_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_network.h + * \brief Declare subsystem object for the network module. + **/ + +#ifndef TOR_NETWORK_SYS_H +#define TOR_NETWORK_SYS_H + +extern const struct subsys_fns_t sys_network; + +#endif /* !defined(TOR_NETWORK_SYS_H) */ diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 8cee29df37..49c263faa2 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -16,8 +16,8 @@ #include "lib/string/parse_int.h" #include "lib/string/util_string.h" -#include "siphash.h" -#include "ht.h" +#include "ext/siphash.h" +#include "ext/ht.h" #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> @@ -421,4 +421,13 @@ tor_make_getaddrinfo_cache_active(void) { sandbox_getaddrinfo_is_active = 1; } +#else +void +sandbox_disable_getaddrinfo_cache(void) +{ +} +void +tor_make_getaddrinfo_cache_active(void) +{ +} #endif diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index 47a283c81c..0fb77f1661 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -42,7 +42,6 @@ int tor_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void tor_freeaddrinfo(struct addrinfo *addrinfo); void tor_free_getaddrinfo_cache(void); -void tor_make_getaddrinfo_cache_active(void); #else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ #define tor_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) @@ -54,5 +53,6 @@ void tor_make_getaddrinfo_cache_active(void); #endif /* defined(USE_SANDBOX_GETADDRINFO) */ void sandbox_disable_getaddrinfo_cache(void); +void tor_make_getaddrinfo_cache_active(void); #endif diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index fba90b7506..f978deeab8 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -31,6 +31,9 @@ #endif #include <stddef.h> #include <string.h> +#ifdef __FreeBSD__ +#include <sys/sysctl.h> +#endif /** Called before we make any calls to network-related functions. * (Some operating systems require their network libraries to be @@ -60,6 +63,32 @@ network_init(void) return 0; } +/** + * Warn the user if any system network parameters should be changed. + */ +void +check_network_configuration(bool server_mode) +{ +#ifdef __FreeBSD__ + if (server_mode) { + int random_id_state; + size_t state_size = sizeof(random_id_state); + + if (sysctlbyname("net.inet.ip.random_id", &random_id_state, + &state_size, NULL, 0)) { + log_warn(LD_CONFIG, + "Failed to figure out if IP ids are randomized."); + } else if (random_id_state == 0) { + log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " + "Please consider setting the net.inet.ip.random_id sysctl, " + "so your relay makes it harder to figure out how busy it is."); + } + } +#else + (void) server_mode; +#endif +} + /* When set_max_file_sockets() is called, update this with the max file * descriptor value so we can use it to check the limit when opening a new * socket. Default value is what Debian sets as the default hard limit. */ @@ -429,7 +458,9 @@ get_n_open_sockets(void) * localhost is inaccessible (for example, if the networking * stack is down). And even if it succeeds, the socket pair will not * be able to read while localhost is down later (the socket pair may - * even close, depending on OS-specific timeouts). + * even close, depending on OS-specific timeouts). The socket pair + * should work on IPv4-only, IPv6-only, and dual-stack systems, as long + * as they have the standard localhost addresses. * * Returns 0 on success and -errno on failure; do not rely on the value * of errno or WSAGetLastError(). diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 0909619510..86ae336dfb 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock); int set_socket_nonblocking(tor_socket_t socket); int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); int network_init(void); +void check_network_configuration(bool server_mode); int get_max_sockets(void); void set_max_sockets(int); diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c index 10eb749735..15c706bec7 100644 --- a/src/lib/net/socketpair.c +++ b/src/lib/net/socketpair.c @@ -105,7 +105,12 @@ sockaddr_eq(struct sockaddr *sa1, struct sockaddr *sa2) /** * Helper used to implement socketpair on systems that lack it, by * making a direct connection to localhost. - */ + * + * See tor_socketpair() for details. + * + * The direct connection defaults to IPv4, but falls back to IPv6 if + * IPv4 is not supported. + **/ int tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) { diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index 05414d2a96..ce1b6ecf59 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -1,17 +1,20 @@ orconfig.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h lib/err/*.h -lib/intmath/*.h +lib/evloop/*.h lib/fs/*.h +lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/net/*.h lib/process/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h -ht.h
\ No newline at end of file +ext/ht.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index c6cc3a6699..83b67bf029 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -9,10 +9,14 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/daemon.c \ src/lib/process/env.c \ src/lib/process/pidfile.c \ + src/lib/process/process.c \ + src/lib/process/process_sys.c \ + src/lib/process/process_unix.c \ + src/lib/process/process_win32.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ - src/lib/process/subprocess.c \ - src/lib/process/waitpid.c + src/lib/process/waitpid.c \ + src/lib/process/winprocess_sys.c src_lib_libtor_process_testing_a_SOURCES = \ $(src_lib_libtor_process_a_SOURCES) @@ -23,7 +27,11 @@ noinst_HEADERS += \ src/lib/process/daemon.h \ src/lib/process/env.h \ src/lib/process/pidfile.h \ + src/lib/process/process.h \ + src/lib/process/process_sys.h \ + src/lib/process/process_unix.h \ + src/lib/process/process_win32.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ - src/lib/process/subprocess.h \ - src/lib/process/waitpid.h + src/lib/process/waitpid.h \ + src/lib/process/winprocess_sys.h diff --git a/src/lib/process/process.c b/src/lib/process/process.c new file mode 100644 index 0000000000..422942dc83 --- /dev/null +++ b/src/lib/process/process.c @@ -0,0 +1,797 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.c + * \brief Module for working with other processes. + **/ + +#define PROCESS_PRIVATE +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/process_win32.h" +#include "lib/process/env.h" + +#ifdef HAVE_STDDEF_H +#include <stddef.h> +#endif + +/** A list of all <b>process_t</b> instances currently allocated. */ +static smartlist_t *processes; + +/** + * Boolean. If true, then Tor may call execve or CreateProcess via + * tor_spawn_background. + **/ +static int may_spawn_background_process = 1; + +/** Structure to represent a child process. */ +struct process_t { + /** Process status. */ + process_status_t status; + + /** Which protocol is the process using? */ + process_protocol_t protocol; + + /** Which function to call when we have data ready from stdout? */ + process_read_callback_t stdout_read_callback; + + /** Which function to call when we have data ready from stderr? */ + process_read_callback_t stderr_read_callback; + + /** Which function call when our process terminated? */ + process_exit_callback_t exit_callback; + + /** Our exit code when the process have terminated. */ + process_exit_code_t exit_code; + + /** Name of the command we want to execute (for example: /bin/ls). */ + char *command; + + /** The arguments used for the new process. The format here is one argument + * per element of the smartlist_t. On Windows these arguments are combined + * together using the <b>tor_join_win_cmdline</b> function. On Unix the + * process name (argv[0]) and the trailing NULL is added automatically before + * the process is executed. */ + smartlist_t *arguments; + + /** The environment used for the new process. */ + smartlist_t *environment; + + /** Buffer to store data from stdout when it is read. */ + buf_t *stdout_buffer; + + /** Buffer to store data from stderr when it is read. */ + buf_t *stderr_buffer; + + /** Buffer to store data to stdin before it is written. */ + buf_t *stdin_buffer; + + /** Do we need to store some custom data with the process? */ + void *data; + +#ifndef _WIN32 + /** Our Unix process handle. */ + process_unix_t *unix_process; +#else + /** Our Win32 process handle. */ + process_win32_t *win32_process; +#endif +}; + +/** Convert a given process status in <b>status</b> to its string + * representation. */ +const char * +process_status_to_string(process_status_t status) +{ + switch (status) { + case PROCESS_STATUS_NOT_RUNNING: + return "not running"; + case PROCESS_STATUS_RUNNING: + return "running"; + case PROCESS_STATUS_ERROR: + return "error"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Convert a given process protocol in <b>protocol</b> to its string + * representation. */ +const char * +process_protocol_to_string(process_protocol_t protocol) +{ + switch (protocol) { + case PROCESS_PROTOCOL_LINE: + return "Line"; + case PROCESS_PROTOCOL_RAW: + return "Raw"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** + * Turn off may_spawn_background_process, so that all future calls to + * tor_spawn_background are guaranteed to fail. + **/ +void +tor_disable_spawning_background_processes(void) +{ + may_spawn_background_process = 0; +} + +/** Initialize the Process subsystem. This function initializes the Process + * subsystem's global state. For cleaning up, <b>process_free_all()</b> should + * be called. */ +void +process_init(void) +{ + processes = smartlist_new(); + +#ifdef _WIN32 + process_win32_init(); +#endif +} + +/** Free up all resources that is handled by the Process subsystem. Note that + * this call does not terminate already running processes. */ +void +process_free_all(void) +{ +#ifdef _WIN32 + process_win32_deinit(); +#endif + + SMARTLIST_FOREACH(processes, process_t *, x, process_free(x)); + smartlist_free(processes); +} + +/** Get a list of all processes. This function returns a smartlist of + * <b>process_t</b> containing all the currently allocated processes. */ +const smartlist_t * +process_get_all_processes(void) +{ + return processes; +} + +/** Allocate and initialize a new process. This function returns a newly + * allocated and initialized process data, which can be used to configure and + * later run a subprocess of Tor. Use the various <b>process_set_*()</b> + * methods to configure it and run the process using <b>process_exec()</b>. Use + * <b>command</b> to specify the path to the command to run. You can either + * specify an absolute path to the command or relative where Tor will use the + * underlying operating system's functionality for finding the command to run. + * */ +process_t * +process_new(const char *command) +{ + tor_assert(command); + + process_t *process; + process = tor_malloc_zero(sizeof(process_t)); + + /* Set our command. */ + process->command = tor_strdup(command); + + /* By default we are not running. */ + process->status = PROCESS_STATUS_NOT_RUNNING; + + /* Prepare process environment. */ + process->arguments = smartlist_new(); + process->environment = smartlist_new(); + + /* Prepare the buffers. */ + process->stdout_buffer = buf_new(); + process->stderr_buffer = buf_new(); + process->stdin_buffer = buf_new(); + +#ifndef _WIN32 + /* Prepare our Unix process handle. */ + process->unix_process = process_unix_new(); +#else + /* Prepare our Win32 process handle. */ + process->win32_process = process_win32_new(); +#endif + + smartlist_add(processes, process); + + return process; +} + +/** Deallocate the given process in <b>process</b>. */ +void +process_free_(process_t *process) +{ + if (! process) + return; + + /* Cleanup parameters. */ + tor_free(process->command); + + /* Cleanup arguments and environment. */ + SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x)); + smartlist_free(process->arguments); + + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + + /* Cleanup the buffers. */ + buf_free(process->stdout_buffer); + buf_free(process->stderr_buffer); + buf_free(process->stdin_buffer); + +#ifndef _WIN32 + /* Cleanup our Unix process handle. */ + process_unix_free(process->unix_process); +#else + /* Cleanup our Win32 process handle. */ + process_win32_free(process->win32_process); +#endif + + smartlist_remove(processes, process); + + tor_free(process); +} + +/** Execute the given process. This function executes the given process as a + * subprocess of Tor. Returns <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_exec(process_t *process) +{ + tor_assert(process); + + if (BUG(may_spawn_background_process == 0)) + return PROCESS_STATUS_ERROR; + + process_status_t status = PROCESS_STATUS_NOT_RUNNING; + + log_info(LD_PROCESS, "Starting new process: %s", process->command); + +#ifndef _WIN32 + status = process_unix_exec(process); +#else + status = process_win32_exec(process); +#endif + + /* Update our state. */ + process_set_status(process, status); + + if (status != PROCESS_STATUS_RUNNING) { + log_warn(LD_PROCESS, "Failed to start process: %s", + process_get_command(process)); + } + + return status; +} + +/** Terminate the given process. Returns true on success, + * otherwise false. */ +bool +process_terminate(process_t *process) +{ + tor_assert(process); + + /* Terminating a non-running process isn't going to work. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + log_debug(LD_PROCESS, "Terminating process"); + +#ifndef _WIN32 + return process_unix_terminate(process); +#else + return process_win32_terminate(process); +#endif +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_get_pid(process_t *process) +{ + tor_assert(process); + +#ifndef _WIN32 + return process_unix_get_pid(process); +#else + return process_win32_get_pid(process); +#endif +} + +/** Set the callback function for output from the child process's standard out + * handle. This function sets the callback function which is called every time + * the child process have written output to its standard out file handle. + * + * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stdout_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stdout_read_callback = callback; +} + +/** Set the callback function for output from the child process's standard + * error handle. This function sets the callback function which is called + * every time the child process have written output to its standard error file + * handle. + * + * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stderr_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stderr_read_callback = callback; +} + +/** Set the callback function for process exit notification. The + * <b>callback</b> function will be called every time your child process have + * terminated. */ +void +process_set_exit_callback(process_t *process, + process_exit_callback_t callback) +{ + tor_assert(process); + process->exit_callback = callback; +} + +/** Get the current command of the given process. */ +const char * +process_get_command(const process_t *process) +{ + tor_assert(process); + return process->command; +} + +void +process_set_protocol(process_t *process, process_protocol_t protocol) +{ + tor_assert(process); + process->protocol = protocol; +} + +/** Get the currently used protocol of the given process. */ +process_protocol_t +process_get_protocol(const process_t *process) +{ + tor_assert(process); + return process->protocol; +} + +/** Set opague pointer to data. This function allows you to store a pointer to + * your own data in the given process. Use <b>process_get_data()</b> in the + * various callback functions to retrieve the data again. + * + * Note that the given process does NOT take ownership of the data and you are + * responsible for freeing up any resources allocated by the given data. + * */ +void +process_set_data(process_t *process, void *data) +{ + tor_assert(process); + process->data = data; +} + +/** Get the opaque pointer to callback data from the given process. This + * function allows you get the data you stored with <b>process_set_data()</b> + * in the different callback functions. */ +void * +process_get_data(const process_t *process) +{ + tor_assert(process); + return process->data; +} + +/** Set the status of a given process. */ +void +process_set_status(process_t *process, process_status_t status) +{ + tor_assert(process); + process->status = status; +} + +/** Get the status of the given process. */ +process_status_t +process_get_status(const process_t *process) +{ + tor_assert(process); + return process->status; +} + +/** Append an argument to the list of arguments in the given process. */ +void +process_append_argument(process_t *process, const char *argument) +{ + tor_assert(process); + tor_assert(argument); + + smartlist_add(process->arguments, tor_strdup(argument)); +} + +/** Returns a list of arguments (excluding the command itself) from the + * given process. */ +const smartlist_t * +process_get_arguments(const process_t *process) +{ + tor_assert(process); + return process->arguments; +} + +/** Returns a newly allocated Unix style argument vector. Use <b>tor_free()</b> + * to deallocate it after use. */ +char ** +process_get_argv(const process_t *process) +{ + tor_assert(process); + + /** Generate a Unix style process argument vector from our process's + * arguments smartlist_t. */ + char **argv = NULL; + + char *filename = process->command; + const smartlist_t *arguments = process->arguments; + const size_t size = smartlist_len(arguments); + + /* Make space for the process filename as argv[0] and a trailing NULL. */ + argv = tor_malloc_zero(sizeof(char *) * (size + 2)); + + /* Set our filename as first argument. */ + argv[0] = filename; + + /* Put in the rest of the values from arguments. */ + SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) { + tor_assert(arg_val != NULL); + + argv[arg_val_sl_idx + 1] = arg_val; + } SMARTLIST_FOREACH_END(arg_val); + + return argv; +} + +/** This function clears the internal environment and copies over every string + * from <b>env</b> as the new environment. */ +void +process_reset_environment(process_t *process, const smartlist_t *env) +{ + tor_assert(process); + tor_assert(env); + + /* Cleanup old environment. */ + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + process->environment = smartlist_new(); + + SMARTLIST_FOREACH(env, char *, x, + smartlist_add(process->environment, tor_strdup(x))); +} + +/** Set the given <b>key</b>/<b>value</b> pair as environment variable in the + * given process. */ +void +process_set_environment(process_t *process, + const char *key, + const char *value) +{ + tor_assert(process); + tor_assert(key); + tor_assert(value); + + smartlist_add_asprintf(process->environment, "%s=%s", key, value); +} + +/** Returns a newly allocated <b>process_environment_t</b> containing the + * environment variables for the given process. */ +process_environment_t * +process_get_environment(const process_t *process) +{ + tor_assert(process); + return process_environment_make(process->environment); +} + +#ifndef _WIN32 +/** Get the internal handle for the Unix backend. */ +process_unix_t * +process_get_unix_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->unix_process); + return process->unix_process; +} +#else +/** Get the internal handle for Windows backend. */ +process_win32_t * +process_get_win32_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->win32_process); + return process->win32_process; +} +#endif + +/** Write <b>size</b> bytes of <b>data</b> to the given process's standard + * input. */ +void +process_write(process_t *process, + const uint8_t *data, size_t size) +{ + tor_assert(process); + tor_assert(data); + + buf_add(process->stdin_buffer, (char *)data, size); + process_write_stdin(process, process->stdin_buffer); +} + +/** As tor_vsnprintf(), but write the data to the given process's standard + * input. */ +void +process_vprintf(process_t *process, + const char *format, va_list args) +{ + tor_assert(process); + tor_assert(format); + + int size; + char *data; + + size = tor_vasprintf(&data, format, args); + process_write(process, (uint8_t *)data, size); + tor_free(data); +} + +/** As tor_snprintf(), but write the data to the given process's standard + * input. */ +void +process_printf(process_t *process, + const char *format, ...) +{ + tor_assert(process); + tor_assert(format); + + va_list ap; + va_start(ap, format); + process_vprintf(process, format, ap); + va_end(ap); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard output + * handle. */ +void +process_notify_event_stdout(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stdout(process, process->stdout_buffer); + + if (ret > 0) + process_read_data(process, + process->stdout_buffer, + process->stdout_read_callback); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard error + * handle. */ +void +process_notify_event_stderr(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stderr(process, process->stderr_buffer); + + if (ret > 0) + process_read_data(process, + process->stderr_buffer, + process->stderr_read_callback); +} + +/** This function is called by the Process backend when a given process is + * allowed to begin writing data to the standard input of the child process. */ +void +process_notify_event_stdin(process_t *process) +{ + tor_assert(process); + + process_write_stdin(process, process->stdin_buffer); +} + +/** This function is called by the Process backend when a given process have + * terminated. The exit status code is passed in <b>exit_code</b>. We mark the + * process as no longer running and calls the <b>exit_callback</b> with + * information about the process termination. The given <b>process</b> is + * free'd iff the exit_callback returns true. */ +void +process_notify_event_exit(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_debug(LD_PROCESS, + "Process terminated with exit code: %"PRIu64, exit_code); + + /* Update our state. */ + process_set_status(process, PROCESS_STATUS_NOT_RUNNING); + process->exit_code = exit_code; + + /* Call our exit callback, if it exists. */ + bool free_process_handle = false; + + /* The exit callback will tell us if we should process_free() our handle. */ + if (process->exit_callback) + free_process_handle = process->exit_callback(process, exit_code); + + if (free_process_handle) + process_free(process); +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard out handle. Returns the number of + * bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + return process_unix_read_stdout(process, buffer); +#else + return process_win32_read_stdout(process, buffer); +#endif +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard error handle. Returns the number + * of bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + return process_unix_read_stderr(process, buffer); +#else + return process_win32_read_stderr(process, buffer); +#endif +} + +/** This function calls the backend function for the given process whenever + * there is data to be written to the backends' file handles. */ +MOCK_IMPL(STATIC void, process_write_stdin, + (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + process_unix_write(process, buffer); +#else + process_win32_write(process, buffer); +#endif +} + +/** This function calls the protocol handlers based on the value of + * <b>process_get_protocol(process)</b>. Currently we call + * <b>process_read_buffer()</b> for <b>PROCESS_PROTOCOL_RAW</b> and + * <b>process_read_lines()</b> for <b>PROCESS_PROTOCOL_LINE</b>. */ +STATIC void +process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + switch (process_get_protocol(process)) { + case PROCESS_PROTOCOL_RAW: + process_read_buffer(process, buffer, callback); + break; + case PROCESS_PROTOCOL_LINE: + process_read_lines(process, buffer, callback); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return; + /* LCOV_EXCL_STOP */ + } +} + +/** This function takes the content of the given <b>buffer</b> and passes it to + * the given <b>callback</b> function, but ensures that an additional zero byte + * is added to the end of the data such that the given callback implementation + * can threat the content as a ASCIIZ string. */ +STATIC void +process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer); + + /* We allocate an extra byte for the zero byte in the end. */ + char *data = tor_malloc_zero(size + 1); + + buf_get_bytes(buffer, data, size); + log_debug(LD_PROCESS, "Read data from process"); + + if (callback) + callback(process, data, size); + + tor_free(data); +} + +/** This function tries to extract complete lines from the given <b>buffer</b> + * and calls the given <b>callback</b> function whenever it has a complete + * line. Before calling <b>callback</b> we remove the trailing "\n" or "\r\n" + * from the line. If we are unable to extract a complete line we leave the data + * in the buffer for next call. */ +STATIC void +process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer) + 1; + size_t line_size = 0; + char *data = tor_malloc_zero(size); + int ret; + + while (true) { + line_size = size; + ret = buf_get_line(buffer, data, &line_size); + + /* A complete line should always be smaller than the size of our + * buffer. */ + tor_assert(ret != -1); + + /* Remove \n from the end of the line. */ + if (line_size >= 1 && data[line_size - 1] == '\n') { + data[line_size - 1] = '\0'; + --line_size; + } + + /* Remove \r from the end of the line. */ + if (line_size >= 1 && data[line_size - 1] == '\r') { + data[line_size - 1] = '\0'; + --line_size; + } + + if (ret == 1) { + log_debug(LD_PROCESS, "Read line from process: \"%s\"", data); + + if (callback) + callback(process, data, line_size); + + /* We have read a whole line, let's see if there is more lines to read. + * */ + continue; + } + + /* No complete line for us to read. We are done for now. */ + tor_assert_nonfatal(ret == 0); + break; + } + + tor_free(data); +} diff --git a/src/lib/process/process.h b/src/lib/process/process.h new file mode 100644 index 0000000000..14069923a0 --- /dev/null +++ b/src/lib/process/process.h @@ -0,0 +1,145 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.h + * \brief Header for process.c + **/ + +#ifndef TOR_PROCESS_H +#define TOR_PROCESS_H + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +/** Maximum number of bytes to write to a process' stdin. */ +#define PROCESS_MAX_WRITE (1024) + +/** Maximum number of bytes to read from a process' stdout/stderr. */ +#define PROCESS_MAX_READ (1024) + +typedef enum { + /** The process is not running. */ + PROCESS_STATUS_NOT_RUNNING, + + /** The process is running. */ + PROCESS_STATUS_RUNNING, + + /** The process is in an erroneous state. */ + PROCESS_STATUS_ERROR +} process_status_t; + +const char *process_status_to_string(process_status_t status); + +typedef enum { + /** Pass complete \n-terminated lines to the + * callback (with the \n or \r\n removed). */ + PROCESS_PROTOCOL_LINE, + + /** Pass the raw response from read() to the callback. */ + PROCESS_PROTOCOL_RAW +} process_protocol_t; + +const char *process_protocol_to_string(process_protocol_t protocol); + +void tor_disable_spawning_background_processes(void); + +struct smartlist_t; + +struct process_t; +typedef struct process_t process_t; + +typedef uint64_t process_exit_code_t; +typedef uint64_t process_pid_t; + +typedef void (*process_read_callback_t)(process_t *, + const char *, + size_t); +typedef bool +(*process_exit_callback_t)(process_t *, process_exit_code_t); + +void process_init(void); +void process_free_all(void); +const struct smartlist_t *process_get_all_processes(void); + +process_t *process_new(const char *command); +void process_free_(process_t *process); +#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s)) + +process_status_t process_exec(process_t *process); +bool process_terminate(process_t *process); + +process_pid_t process_get_pid(process_t *process); + +void process_set_stdout_read_callback(process_t *, + process_read_callback_t); +void process_set_stderr_read_callback(process_t *, + process_read_callback_t); +void process_set_exit_callback(process_t *, + process_exit_callback_t); + +const char *process_get_command(const process_t *process); + +void process_append_argument(process_t *process, const char *argument); +const struct smartlist_t *process_get_arguments(const process_t *process); +char **process_get_argv(const process_t *process); + +void process_reset_environment(process_t *process, + const struct smartlist_t *env); +void process_set_environment(process_t *process, + const char *key, + const char *value); + +struct process_environment_t; +struct process_environment_t *process_get_environment(const process_t *); + +void process_set_protocol(process_t *process, process_protocol_t protocol); +process_protocol_t process_get_protocol(const process_t *process); + +void process_set_data(process_t *process, void *data); +void *process_get_data(const process_t *process); + +void process_set_status(process_t *process, process_status_t status); +process_status_t process_get_status(const process_t *process); + +#ifndef _WIN32 +struct process_unix_t; +struct process_unix_t *process_get_unix_process(const process_t *process); +#else +struct process_win32_t; +struct process_win32_t *process_get_win32_process(const process_t *process); +#endif + +void process_write(process_t *process, + const uint8_t *data, size_t size); +void process_vprintf(process_t *process, + const char *format, va_list args) CHECK_PRINTF(2, 0); +void process_printf(process_t *process, + const char *format, ...) CHECK_PRINTF(2, 3); + +void process_notify_event_stdout(process_t *process); +void process_notify_event_stderr(process_t *process); +void process_notify_event_stdin(process_t *process); +void process_notify_event_exit(process_t *process, + process_exit_code_t); + +#ifdef PROCESS_PRIVATE +MOCK_DECL(STATIC int, process_read_stdout, (process_t *, buf_t *)); +MOCK_DECL(STATIC int, process_read_stderr, (process_t *, buf_t *)); +MOCK_DECL(STATIC void, process_write_stdin, (process_t *, buf_t *)); + +STATIC void process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +#endif /* defined(PROCESS_PRIVATE). */ + +#endif /* defined(TOR_PROCESS_H). */ diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c new file mode 100644 index 0000000000..3c809a00e8 --- /dev/null +++ b/src/lib/process/process_sys.c @@ -0,0 +1,33 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.c + * \brief Subsystem object for process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/process_sys.h" +#include "lib/process/process.h" + +static int +subsys_process_initialize(void) +{ + process_init(); + return 0; +} + +static void +subsys_process_shutdown(void) +{ + process_free_all(); +} + +const subsys_fns_t sys_process = { + .name = "process", + .level = -35, + .supported = true, + .initialize = subsys_process_initialize, + .shutdown = subsys_process_shutdown +}; diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h new file mode 100644 index 0000000000..b7a116d838 --- /dev/null +++ b/src/lib/process/process_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.h + * \brief Declare subsystem object for the process module. + **/ + +#ifndef TOR_PROCESS_SYS_H +#define TOR_PROCESS_SYS_H + +extern const struct subsys_fns_t sys_process; + +#endif /* !defined(TOR_PROCESS_SYS_H) */ diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c new file mode 100644 index 0000000000..b102afff44 --- /dev/null +++ b/src/lib/process/process_unix.c @@ -0,0 +1,698 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.c + * \brief Module for working with Unix processes. + **/ + +#define PROCESS_UNIX_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/waitpid.h" +#include "lib/process/env.h" + +#include <stdio.h> + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif + +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) +#include <sys/prctl.h> +#endif + +#if HAVE_SIGNAL_H +#include <signal.h> +#endif + +#ifndef _WIN32 + +/** Maximum number of file descriptors, if we cannot get it via sysconf() */ +#define DEFAULT_MAX_FD 256 + +/** Internal state for Unix handles. */ +struct process_unix_handle_t { + /** Unix File Descriptor. */ + int fd; + + /** Have we reached end of file? */ + bool reached_eof; + + /** Event structure for libevent. */ + struct event *event; + + /** Are we writing? */ + bool is_writing; +}; + +/** Internal state for our Unix process. */ +struct process_unix_t { + /** Standard in handle. */ + process_unix_handle_t stdin_handle; + + /** Standard out handle. */ + process_unix_handle_t stdout_handle; + + /** Standard error handle. */ + process_unix_handle_t stderr_handle; + + /** The process identifier of our process. */ + pid_t pid; + + /** Waitpid Callback structure. */ + waitpid_callback_t *waitpid; +}; + +/** Returns a newly allocated <b>process_unix_t</b>. */ +process_unix_t * +process_unix_new(void) +{ + process_unix_t *unix_process; + unix_process = tor_malloc_zero(sizeof(process_unix_t)); + + unix_process->stdin_handle.fd = -1; + unix_process->stderr_handle.fd = -1; + unix_process->stdout_handle.fd = -1; + + return unix_process; +} + +/** Deallocates the given <b>unix_process</b>. */ +void +process_unix_free_(process_unix_t *unix_process) +{ + if (! unix_process) + return; + + /* Clean up our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + + /* FIXME(ahf): Refactor waitpid code? */ + unix_process->waitpid = NULL; + + /* Close all our file descriptors. */ + process_unix_close_file_descriptors(unix_process); + + tor_event_free(unix_process->stdout_handle.event); + tor_event_free(unix_process->stderr_handle.event); + tor_event_free(unix_process->stdin_handle.event); + + tor_free(unix_process); +} + +/** Executes the given process as a child process of Tor. This function is + * responsible for setting up the child process and run it. This includes + * setting up pipes for interprocess communication, initialize the waitpid + * callbacks, and finally run fork() followed by execve(). Returns + * <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_unix_exec(process_t *process) +{ + static int max_fd = -1; + + process_unix_t *unix_process; + pid_t pid; + int stdin_pipe[2]; + int stdout_pipe[2]; + int stderr_pipe[2]; + int retval, fd; + + unix_process = process_get_unix_process(process); + + /* Create standard in pipe. */ + retval = pipe(stdin_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdin " + "communication with process: %s", + strerror(errno)); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard out pipe. */ + retval = pipe(stdout_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdout " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard error pipe. */ + retval = pipe(stderr_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stderr " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdout_pipe[0]); + close(stdout_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + +#ifdef _SC_OPEN_MAX + if (-1 == max_fd) { + max_fd = (int)sysconf(_SC_OPEN_MAX); + + if (max_fd == -1) { + max_fd = DEFAULT_MAX_FD; + log_warn(LD_PROCESS, + "Cannot find maximum file descriptor, assuming: %d", max_fd); + } + } +#else /* !(defined(_SC_OPEN_MAX)) */ + max_fd = DEFAULT_MAX_FD; +#endif /* defined(_SC_OPEN_MAX) */ + + pid = fork(); + + if (0 == pid) { + /* This code is running in the child process context. */ + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) + /* Attempt to have the kernel issue a SIGTERM if the parent + * goes away. Certain attributes of the binary being execve()ed + * will clear this during the execve() call, but it's better + * than nothing. + */ + prctl(PR_SET_PDEATHSIG, SIGTERM); +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ + + /* Link process stdout to the write end of the pipe. */ + retval = dup2(stdout_pipe[1], STDOUT_FILENO); + if (-1 == retval) + goto error; + + /* Link process stderr to the write end of the pipe. */ + retval = dup2(stderr_pipe[1], STDERR_FILENO); + if (-1 == retval) + goto error; + + /* Link process stdin to the read end of the pipe */ + retval = dup2(stdin_pipe[0], STDIN_FILENO); + if (-1 == retval) + goto error; + + /* Close our pipes now after they have been dup2()'ed. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + close(stdout_pipe[0]); + close(stdout_pipe[1]); + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /* Close all other fds, including the read end of the pipe. XXX: We should + * now be doing enough FD_CLOEXEC setting to make this needless. + */ + for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) + close(fd); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Call the requested program. */ + execve(argv[0], argv, env->unixoid_environment_block); + + /* If we made it here it is because execve failed :-( */ + tor_free(argv); + process_environment_free(env); + + error: + fprintf(stderr, "Error from child process: %s", strerror(errno)); + _exit(1); + } + + /* We are in the parent process. */ + if (-1 == pid) { + log_warn(LD_PROCESS, + "Failed to create child process: %s", strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdout_pipe[0]); + close(stdout_pipe[1]); + + /** Cleanup standard error pipe. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Register our PID. */ + unix_process->pid = pid; + + /* Setup waitpid callbacks. */ + unix_process->waitpid = set_waitpid_callback(pid, + process_unix_waitpid_callback, + process); + + /* Handle standard out. */ + unix_process->stdout_handle.fd = stdout_pipe[0]; + retval = close(stdout_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s", + strerror(errno)); + } + + /* Handle standard error. */ + unix_process->stderr_handle.fd = stderr_pipe[0]; + retval = close(stderr_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Failed to close write end of standard error pipe: %s", + strerror(errno)); + } + + /* Handle standard in. */ + unix_process->stdin_handle.fd = stdin_pipe[1]; + retval = close(stdin_pipe[0]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s", + strerror(errno)); + } + + /* Setup our handles. */ + process_unix_setup_handle(process, + &unix_process->stdout_handle, + EV_READ|EV_PERSIST, + stdout_read_callback); + + process_unix_setup_handle(process, + &unix_process->stderr_handle, + EV_READ|EV_PERSIST, + stderr_read_callback); + + process_unix_setup_handle(process, + &unix_process->stdin_handle, + EV_WRITE|EV_PERSIST, + stdin_write_callback); + + /* Start reading from standard out and standard error. */ + process_unix_start_reading(&unix_process->stdout_handle); + process_unix_start_reading(&unix_process->stderr_handle); + + return PROCESS_STATUS_RUNNING; +} + +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_unix_terminate(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + + /* All running processes should have a waitpid. */ + if (BUG(unix_process->waitpid == NULL)) + return false; + + bool success = true; + + /* Send a SIGTERM to our child process. */ + int ret; + + ret = kill(unix_process->pid, SIGTERM); + + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to terminate process: %s", + strerror(errno)); + success = false; + } + + /* Close all our FD's. */ + if (! process_unix_close_file_descriptors(unix_process)) + success = false; + + return success; +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_unix_get_pid(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + return (process_pid_t)unix_process->pid; +} + +/** Write the given <b>buffer</b> as input to the given <b>process</b>'s + * standard input. Returns the number of bytes written. */ +int +process_unix_write(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + size_t buffer_flush_len = buf_datalen(buffer); + const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len); + + /* If we have data to write (when buffer_flush_len > 0) and we are not + * currently getting file descriptor events from the kernel, we tell the + * kernel to start notifying us about when we can write to our file + * descriptor and return. */ + if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) { + process_unix_start_writing(&unix_process->stdin_handle); + return 0; + } + + /* We don't have any data to write, but the kernel is currently notifying us + * about whether we are able to write or not. Tell the kernel to stop + * notifying us until we have data to write. */ + if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) { + process_unix_stop_writing(&unix_process->stdin_handle); + return 0; + } + + /* We have data to write and the kernel have told us to write it. */ + return buf_flush_to_pipe(buffer, + process_get_unix_process(process)->stdin_handle.fd, + max_to_write, &buffer_flush_len); +} + +/** Read data from the given process's standard output and put it into + * <b>buffer</b>. Returns the number of bytes read. */ +int +process_unix_read_stdout(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stdout_handle, + buffer); +} + +/** Read data from the given process's standard error and put it into + * <b>buffer</b>. Returns the number of bytes read. */ +int +process_unix_read_stderr(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stderr_handle, + buffer); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard output. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stdout_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdout(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard error. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stderr_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stderr(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * written the child process's standard input. We notify the Process subsystem, + * which is then responsible for calling back to us for doing the actual write + * of the data. */ +STATIC void +stdin_write_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdin(process); +} + +/** This function tells libevent that we are interested in receiving read + * events from the given <b>handle</b>. */ +STATIC void +process_unix_start_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); +} + +/** This function tells libevent that we are no longer interested in receiving + * read events from the given <b>handle</b>. */ +STATIC void +process_unix_stop_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); +} + +/** This function tells libevent that we are interested in receiving write + * events from the given <b>handle</b>. */ +STATIC void +process_unix_start_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); + + handle->is_writing = true; +} + +/** This function tells libevent that we are no longer interested in receiving + * write events from the given <b>handle</b>. */ +STATIC void +process_unix_stop_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); + + handle->is_writing = false; +} + +/** This function is called when the waitpid system have detected that our + * process have terminated. We disable the waitpid system and notify the + * Process subsystem that we have terminated. */ +STATIC void +process_unix_waitpid_callback(int status, void *data) +{ + tor_assert(data); + + process_t *process = data; + process_unix_t *unix_process = process_get_unix_process(process); + + /* Remove our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + unix_process->waitpid = NULL; + + /* Notify our process. */ + process_notify_event_exit(process, status); + + /* Make sure you don't modify the process after we have called + * process_notify_event_exit() on it, to allow users to process_free() it in + * the exit callback. */ +} + +/** This function sets the file descriptor in the <b>handle</b> as non-blocking + * and configures the libevent event structure based on the given <b>flags</b> + * to ensure that <b>callback</b> is called whenever we have events on the + * given <b>handle</b>. */ +STATIC void +process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(callback); + + /* Put our file descriptor into non-blocking mode. */ + if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) { + log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s", + strerror(errno)); + } + + /* Setup libevent event. */ + handle->event = tor_event_new(tor_libevent_get_base(), + handle->fd, + flags, + callback, + process); +} + +/** This function reads data from the given <b>handle</b> and puts it into + * <b>buffer</b>. Returns the number of bytes read this way. */ +STATIC int +process_unix_read_handle(process_t *process, + process_unix_handle_t *handle, + buf_t *buffer) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(buffer); + + int ret = 0; + int eof = 0; + int error = 0; + + ret = buf_read_from_pipe(buffer, + handle->fd, + PROCESS_MAX_READ, + &eof, + &error); + + if (error) + log_warn(LD_PROCESS, + "Unable to read data: %s", strerror(error)); + + if (eof) { + handle->reached_eof = true; + process_unix_stop_reading(handle); + } + + return ret; +} + +/** Close the standard in, out, and error handles of the given + * <b>unix_process</b>. */ +STATIC bool +process_unix_close_file_descriptors(process_unix_t *unix_process) +{ + tor_assert(unix_process); + + int ret; + bool success = true; + + /* Stop reading and writing before we close() our + * file descriptors. */ + if (! unix_process->stdout_handle.reached_eof) + process_unix_stop_reading(&unix_process->stdout_handle); + + if (! unix_process->stderr_handle.reached_eof) + process_unix_stop_reading(&unix_process->stderr_handle); + + if (unix_process->stdin_handle.is_writing) + process_unix_stop_writing(&unix_process->stdin_handle); + + if (unix_process->stdin_handle.fd != -1) { + ret = close(unix_process->stdin_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard in"); + success = false; + } + + unix_process->stdin_handle.fd = -1; + } + + if (unix_process->stdout_handle.fd != -1) { + ret = close(unix_process->stdout_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard out"); + success = false; + } + + unix_process->stdout_handle.fd = -1; + } + + if (unix_process->stderr_handle.fd != -1) { + ret = close(unix_process->stderr_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard error"); + success = false; + } + + unix_process->stderr_handle.fd = -1; + } + + return success; +} + +#endif /* defined(_WIN32). */ diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h new file mode 100644 index 0000000000..a1d8f72993 --- /dev/null +++ b/src/lib/process/process_unix.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.h + * \brief Header for process_unix.c + **/ + +#ifndef TOR_PROCESS_UNIX_H +#define TOR_PROCESS_UNIX_H + +#ifndef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" + +#include <event2/event.h> + +struct process_t; + +struct process_unix_t; +typedef struct process_unix_t process_unix_t; + +process_unix_t *process_unix_new(void); +void process_unix_free_(process_unix_t *unix_process); +#define process_unix_free(s) \ + FREE_AND_NULL(process_unix_t, process_unix_free_, (s)) + +process_status_t process_unix_exec(struct process_t *process); +bool process_unix_terminate(struct process_t *process); + +process_pid_t process_unix_get_pid(struct process_t *process); + +int process_unix_write(struct process_t *process, buf_t *buffer); +int process_unix_read_stdout(struct process_t *process, buf_t *buffer); +int process_unix_read_stderr(struct process_t *process, buf_t *buffer); + +#ifdef PROCESS_UNIX_PRIVATE +struct process_unix_handle_t; +typedef struct process_unix_handle_t process_unix_handle_t; + +STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data); + +STATIC void process_unix_start_reading(process_unix_handle_t *); +STATIC void process_unix_stop_reading(process_unix_handle_t *); + +STATIC void process_unix_start_writing(process_unix_handle_t *); +STATIC void process_unix_stop_writing(process_unix_handle_t *); + +STATIC void process_unix_waitpid_callback(int status, void *data); + +STATIC void process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback); +STATIC int process_unix_read_handle(process_t *, + process_unix_handle_t *, + buf_t *); +STATIC bool process_unix_close_file_descriptors(process_unix_t *); +#endif /* defined(PROCESS_UNIX_PRIVATE). */ + +#endif /* defined(_WIN32). */ + +#endif /* defined(TOR_PROCESS_UNIX_H). */ diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c new file mode 100644 index 0000000000..1b1de6ad55 --- /dev/null +++ b/src/lib/process/process_win32.c @@ -0,0 +1,1105 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.c + * \brief Module for working with Windows processes. + **/ + +#define PROCESS_WIN32_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/log/win32err.h" +#include "lib/process/process.h" +#include "lib/process/process_win32.h" +#include "lib/process/env.h" + +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef _WIN32 + +/** The size of our intermediate buffers. */ +#define BUFFER_SIZE (1024) + +/** Timer that ticks once a second and calls the process_win32_timer_callback() + * function. */ +static periodic_timer_t *periodic_timer; + +/** Structure to represent the state around the pipe HANDLE. + * + * This structure is used to store state about a given HANDLE, including + * whether we have reached end of file, its intermediate buffers, and how much + * data that is available in the intermediate buffer. */ +struct process_win32_handle_t { + /** Standard out pipe handle. */ + HANDLE pipe; + + /** True iff we have reached EOF from the pipe. */ + bool reached_eof; + + /** How much data is available in buffer. */ + size_t data_available; + + /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */ + char buffer[BUFFER_SIZE]; + + /** Overlapped structure for ReadFileEx() and WriteFileEx(). */ + OVERLAPPED overlapped; + + /** Are we waiting for another I/O operation to complete? */ + bool busy; +}; + +/** Structure to represent the Windows specific implementation details of this + * Process backend. + * + * This structure is attached to <b>process_t</b> (see process.h) and is + * reachable from <b>process_t</b> via the <b>process_get_win32_process()</b> + * method. */ +struct process_win32_t { + /** Standard in state. */ + process_win32_handle_t stdin_handle; + + /** Standard out state. */ + process_win32_handle_t stdout_handle; + + /** Standard error state. */ + process_win32_handle_t stderr_handle; + + /** Process Information. */ + PROCESS_INFORMATION process_information; +}; + +/** Create a new <b>process_win32_t</b>. + * + * This function constructs a new <b>process_win32_t</b> and initializes the + * default values. */ +process_win32_t * +process_win32_new(void) +{ + process_win32_t *win32_process; + win32_process = tor_malloc_zero(sizeof(process_win32_t)); + + win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE; + + return win32_process; +} + +/** Free a given <b>process_win32_t</b>. + * + * This function deinitializes and frees up the resources allocated for the + * given <b>process_win32_t</b>. */ +void +process_win32_free_(process_win32_t *win32_process) +{ + if (! win32_process) + return; + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + tor_free(win32_process); +} + +/** Initialize the Windows backend of the Process subsystem. */ +void +process_win32_init(void) +{ + /* We don't start the periodic timer here because it makes no sense to have + * the timer running until we have some processes that benefits from the + * timer timer ticks. */ +} + +/** Deinitialize the Windows backend of the Process subsystem. */ +void +process_win32_deinit(void) +{ + /* Stop our timer, but only if it's running. */ + if (process_win32_timer_running()) + process_win32_timer_stop(); +} + +/** Execute the given process. This function is responsible for setting up + * named pipes for I/O between the child process and the Tor process. Returns + * <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_win32_exec(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + HANDLE stdout_pipe_read = NULL; + HANDLE stdout_pipe_write = NULL; + HANDLE stderr_pipe_read = NULL; + HANDLE stderr_pipe_write = NULL; + HANDLE stdin_pipe_read = NULL; + HANDLE stdin_pipe_write = NULL; + BOOL ret = FALSE; + + /* Setup our security attributes. */ + SECURITY_ATTRIBUTES security_attributes; + security_attributes.nLength = sizeof(security_attributes); + security_attributes.bInheritHandle = TRUE; + /* FIXME: should we set explicit security attributes? + * (See Ticket #2046, comment 5) */ + security_attributes.lpSecurityDescriptor = NULL; + + /* Create our standard out pipe. */ + if (! process_win32_create_pipe(&stdout_pipe_read, + &stdout_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create our standard error pipe. */ + if (! process_win32_create_pipe(&stderr_pipe_read, + &stderr_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create out standard in pipe. */ + if (! process_win32_create_pipe(&stdin_pipe_read, + &stdin_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_WRITER)) { + return PROCESS_STATUS_ERROR; + } + + /* Configure startup info for our child process. */ + STARTUPINFOA startup_info; + + memset(&startup_info, 0, sizeof(startup_info)); + startup_info.cb = sizeof(startup_info); + startup_info.hStdError = stderr_pipe_write; + startup_info.hStdOutput = stdout_pipe_write; + startup_info.hStdInput = stdin_pipe_read; + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Windows expects argv to be a whitespace delimited string, so join argv up + */ + char *joined_argv = tor_join_win_cmdline((const char **)argv); + + /* Create the child process */ + ret = CreateProcessA(NULL, + joined_argv, + NULL, + NULL, + TRUE, + CREATE_NO_WINDOW, + env->windows_environment_block[0] == '\0' ? + NULL : env->windows_environment_block, + NULL, + &startup_info, + &win32_process->process_information); + + tor_free(argv); + tor_free(joined_argv); + process_environment_free(env); + + if (! ret) { + log_warn(LD_PROCESS, "CreateProcessA() failed: %s", + format_win32_error(GetLastError())); + + /* Cleanup our handles. */ + CloseHandle(stdout_pipe_read); + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_read); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + CloseHandle(stdin_pipe_write); + + /* In the Unix backend, we do not get an error in the Tor process when a + * child process fails to spawn its target executable since we need to + * first do the fork() call in the Tor process and then the child process + * is responsible for doing the call to execve(). + * + * This means that the user of the process_exec() API must check for + * whether it returns PROCESS_STATUS_ERROR, which will rarely happen on + * Unix, but will happen for error cases on Windows where it does not + * happen on Unix. For example: when the target executable does not exist + * on the file system. + * + * To have somewhat feature compatibility between the Unix and the Windows + * backend, we here notify the process_t owner that the process have exited + * (even though it never managed to run) to ensure that the exit callback + * is executed. + */ + process_notify_event_exit(process, 0); + + return PROCESS_STATUS_ERROR; + } + + /* TODO: Should we close hProcess and hThread in + * process_handle->process_information? */ + win32_process->stdout_handle.pipe = stdout_pipe_read; + win32_process->stderr_handle.pipe = stderr_pipe_read; + win32_process->stdin_handle.pipe = stdin_pipe_write; + + /* Close our ends of the pipes that is now owned by the child process. */ + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + + /* Used by the callback functions from ReadFileEx() and WriteFileEx() such + * that we can figure out which process_t that was responsible for the event. + * + * Warning, here be dragons: + * + * MSDN says that the hEvent member of the overlapped structure is unused + * for ReadFileEx() and WriteFileEx, which allows us to store a pointer to + * our process state there. + */ + win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process; + + /* Start our timer if it is not already running. */ + if (! process_win32_timer_running()) + process_win32_timer_start(); + + /* We use Windows Extended I/O functions, so our completion callbacks are + * called automatically for us when there is data to read. Because of this + * we start the read of standard out and error right away. */ + process_notify_event_stdout(process); + process_notify_event_stderr(process); + + return PROCESS_STATUS_RUNNING; +} + +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_win32_terminate(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + /* Terminate our process. */ + BOOL ret; + + ret = TerminateProcess(win32_process->process_information.hProcess, 0); + + if (! ret) { + log_warn(LD_PROCESS, "TerminateProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + return true; +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_win32_get_pid(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + return (process_pid_t)win32_process->process_information.dwProcessId; +} + +/** Schedule an async write of the data found in <b>buffer</b> for the given + * process. This function runs an async write operation of the content of + * buffer, if we are not already waiting for a pending I/O request. Returns the + * number of bytes that Windows will hopefully write for us in the background. + * */ +int +process_win32_write(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD error_code = 0; + const size_t buffer_size = buf_datalen(buffer); + + /* Windows is still writing our buffer. */ + if (win32_process->stdin_handle.busy) + return 0; + + /* Nothing for us to do right now. */ + if (buffer_size == 0) + return 0; + + /* We have reached end of file already? */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return 0; + + /* Figure out how much data we should read. */ + const size_t write_size = MIN(buffer_size, + sizeof(win32_process->stdin_handle.buffer)); + + /* Read data from the process_t buffer into our intermediate buffer. */ + buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size); + + /* Because of the slightly weird API for WriteFileEx() we must set this to 0 + * before we call WriteFileEx() because WriteFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + + /* Schedule our write. */ + ret = WriteFileEx(win32_process->stdin_handle.pipe, + win32_process->stdin_handle.buffer, + write_size, + &win32_process->stdin_handle.overlapped, + process_win32_stdin_write_done); + + if (! ret) { + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "WriteFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "WriteFileEx() failed: %s", + format_win32_error(error_code)); + } + + win32_process->stdin_handle.reached_eof = true; + return 0; + } + + /* Here be dragons: According to MSDN's documentation for WriteFileEx() we + * should check GetLastError() after a call to WriteFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "WriteFileEx() failed after returning success: %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + return 0; + /* LCOV_EXCL_STOP */ + } + + /* This cast should be safe since our buffer can maximum be BUFFER_SIZE + * large. */ + return (int)write_size; +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard output of the child process + * and asks Windows to call process_win32_stdout_read_done() when that + * operation is finished. Returns the number of bytes moved into <b>buffer</b>. + * */ +int +process_win32_read_stdout(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stdout_handle, + buffer, + process_win32_stdout_read_done); +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard error of the child process + * and asks Windows to call process_win32_stderr_read_done() when that + * operation is finished. Returns the number of bytes moved into <b>buffer</b>. + * */ +int +process_win32_read_stderr(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stderr_handle, + buffer, + process_win32_stderr_read_done); +} + +/** This function is responsible for moving the Tor process into what Microsoft + * calls an "alertable" state. Once the process is in an alertable state the + * Windows kernel will notify us when our background I/O requests have finished + * and the callbacks will be executed. */ +void +process_win32_trigger_completion_callbacks(void) +{ + DWORD ret; + + /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep + * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause + * the process to enter alertable state, where the Windows kernel will notify + * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which + * will cause our completion callbacks to be executed. + * + * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION + * if one or more I/O callbacks were executed. */ + ret = SleepEx(0, TRUE); + + /* Warn us if the function returned something we did not anticipate. */ + if (ret != 0 && ret != WAIT_IO_COMPLETION) { + log_warn(LD_PROCESS, "SleepEx() returned %lu", ret); + } +} + +/** Start the periodic timer which is reponsible for checking whether processes + * are still alive and to make sure that the Tor process is periodically being + * moved into an alertable state. */ +void +process_win32_timer_start(void) +{ + /* Make sure we never start our timer if it's already running. */ + if (BUG(process_win32_timer_running())) + return; + + /* Wake up once a second. */ + static const struct timeval interval = {1, 0}; + + log_info(LD_PROCESS, "Starting Windows Process I/O timer"); + periodic_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + process_win32_timer_callback, + NULL); +} + +/** Stops the periodic timer. */ +void +process_win32_timer_stop(void) +{ + if (BUG(periodic_timer == NULL)) + return; + + log_info(LD_PROCESS, "Stopping Windows Process I/O timer"); + periodic_timer_free(periodic_timer); +} + +/** Returns true iff the periodic timer is running. */ +bool +process_win32_timer_running(void) +{ + return periodic_timer != NULL; +} + +/** This function is called whenever the periodic_timer ticks. The function is + * responsible for moving the Tor process into an alertable state once a second + * and checking for whether our child processes have terminated since the last + * tick. */ +STATIC void +process_win32_timer_callback(periodic_timer_t *timer, void *data) +{ + tor_assert(timer == periodic_timer); + tor_assert(data == NULL); + + /* Move the process into an alertable state. */ + process_win32_trigger_completion_callbacks(); + + /* Check if our processes are still alive. */ + + /* Since the call to process_win32_timer_test_process() might call + * process_notify_event_exit() which again might call process_free() which + * updates the list of processes returned by process_get_all_processes() it + * is important here that we make sure to not touch the list of processes if + * the call to process_win32_timer_test_process() returns true. */ + bool done; + + do { + const smartlist_t *processes = process_get_all_processes(); + done = true; + + SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) { + /* If process_win32_timer_test_process() returns true, it means that + * smartlist_remove() might have been called on the list returned by + * process_get_all_processes(). We start the loop over again until we + * have a succesful run over the entire list where the list was not + * modified. */ + if (process_win32_timer_test_process(process)) { + done = false; + break; + } + } SMARTLIST_FOREACH_END(process); + } while (! done); +} + +/** Test whether a given process is still alive. Notify the Process subsystem + * if our process have died. Returns true iff the given process have + * terminated. */ +STATIC bool +process_win32_timer_test_process(process_t *process) +{ + tor_assert(process); + + /* No need to look at processes that don't claim they are running. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD exit_code = 0; + + /* Sometimes the Windows kernel wont give us the EOF/Broken Pipe error + * message until some time after the process have actually terminated. We + * make sure that our ReadFileEx() calls for the process have *all* returned + * and both standard out and error have been marked as EOF before we try to + * see if the process terminated. + * + * This ensures that we *never* call the exit callback of the `process_t`, + * which potentially ends up calling `process_free()` on our `process_t`, + * before all data have been received from the process. + * + * We do NOT have a check here for whether standard in reached EOF since + * standard in's WriteFileEx() function is only called on-demand when we have + * something to write and is thus usually not awaiting to finish any + * operations. If we WriteFileEx() to a file that has terminated we'll simply + * get an error from ReadFileEx() or its completion routine and move on with + * life. */ + if (! win32_process->stdout_handle.reached_eof) + return false; + + if (! win32_process->stderr_handle.reached_eof) + return false; + + /* We start by testing whether our process is still running. */ + ret = GetExitCodeProcess(win32_process->process_information.hProcess, + &exit_code); + + if (! ret) { + log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Notify our process_t that our process have terminated. Since our + * exit_callback might decide to process_free() our process handle it is very + * important that we do not touch the process_t after the call to + * process_notify_event_exit(). */ + if (exit_code != STILL_ACTIVE) { + process_notify_event_exit(process, exit_code); + return true; + } + + return false; +} + +/** Create a new overlapped named pipe. This function creates a new connected, + * named, pipe in <b>*read_pipe</b> and <b>*write_pipe</b> if the function is + * succesful. Returns true on sucess, false on failure. */ +STATIC bool +process_win32_create_pipe(HANDLE *read_pipe, + HANDLE *write_pipe, + SECURITY_ATTRIBUTES *attributes, + process_win32_pipe_type_t pipe_type) +{ + tor_assert(read_pipe); + tor_assert(write_pipe); + tor_assert(attributes); + + BOOL ret = FALSE; + + /* Buffer size. */ + const size_t size = 4096; + + /* Our additional read/write modes that depends on which pipe type we are + * creating. */ + DWORD read_mode = 0; + DWORD write_mode = 0; + + /* Generate the unique pipe name. */ + char pipe_name[MAX_PATH]; + static DWORD process_id = 0; + static DWORD counter = 0; + + if (process_id == 0) + process_id = GetCurrentProcessId(); + + tor_snprintf(pipe_name, sizeof(pipe_name), + "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu", + process_id, counter++); + + /* Only one of our handles can be overlapped. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + read_mode = FILE_FLAG_OVERLAPPED; + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + write_mode = FILE_FLAG_OVERLAPPED; + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + /* Setup our read and write handles. */ + HANDLE read_handle; + HANDLE write_handle; + + /* Create our named pipe. */ + read_handle = CreateNamedPipeA(pipe_name, + (PIPE_ACCESS_INBOUND|read_mode), + (PIPE_TYPE_BYTE|PIPE_WAIT), + 1, + size, + size, + 1000, + attributes); + + if (read_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Create our file in the pipe namespace. */ + write_handle = CreateFileA(pipe_name, + GENERIC_WRITE, + 0, + attributes, + OPEN_EXISTING, + (FILE_ATTRIBUTE_NORMAL|write_mode), + NULL); + + if (write_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateFileA() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + + return false; + } + + /* Set the inherit flag for our pipe. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0); + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + if (! ret) { + log_warn(LD_PROCESS, "SetHandleInformation() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + CloseHandle(write_handle); + + return false; + } + + /* Everything is good. */ + *read_pipe = read_handle; + *write_pipe = write_handle; + + return true; +} + +/** Cleanup a given <b>handle</b>. */ +STATIC void +process_win32_cleanup_handle(process_win32_handle_t *handle) +{ + tor_assert(handle); + +#if 0 + BOOL ret; + DWORD error_code; + + /* Cancel any pending I/O requests: This means that instead of getting + * ERROR_BROKEN_PIPE we get ERROR_OPERATION_ABORTED, but it doesn't seem + * like this is needed. */ + ret = CancelIo(handle->pipe); + + if (! ret) { + error_code = GetLastError(); + + /* There was no pending I/O requests for our handle. */ + if (error_code != ERROR_NOT_FOUND) { + log_warn(LD_PROCESS, "CancelIo() failed: %s", + format_win32_error(error_code)); + } + } +#endif + + /* Close our handle. */ + if (handle->pipe != INVALID_HANDLE_VALUE) { + CloseHandle(handle->pipe); + handle->pipe = INVALID_HANDLE_VALUE; + handle->reached_eof = true; + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard output. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stdout_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stdout_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stdout(process); + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard error. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stderr_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stderr_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stderr(process); + } +} + +/** This function is called when WriteFileEx() completes its background write + * to the child process's standard input. We notify the Process subsystem that + * it can write data to us again. */ +STATIC VOID WINAPI +process_win32_stdin_write_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + (void)byte_count; + + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + /* Mark our handle as not having any outstanding I/O requests. */ + win32_process->stdin_handle.busy = false; + + /* Check if we have been asked to write to the handle that have been marked + * as having reached EOF. */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return; + + if (error_code == 0) { + /** Our data have been succesfully written. Clear our state and schedule + * the next write. */ + win32_process->stdin_handle.data_available = 0; + memset(win32_process->stdin_handle.buffer, 0, + sizeof(win32_process->stdin_handle.buffer)); + + /* Schedule the next write. */ + process_notify_event_stdin(process); + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our WriteFileEx() call was succesful, but we reached the end of our + * file. We mark our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + win32_process->stdin_handle.reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from WriteFileEx(): %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + } +} + +/** This function reads data from the given <b>handle</b>'s internal buffer and + * moves it into the given <b>buffer</b>. Additionally, we start the next + * ReadFileEx() background operation with the given <b>callback</b> as + * completion callback. Returns the number of bytes written to the buffer. */ +STATIC int +process_win32_read_from_handle(process_win32_handle_t *handle, + buf_t *buffer, + LPOVERLAPPED_COMPLETION_ROUTINE callback) +{ + tor_assert(handle); + tor_assert(buffer); + tor_assert(callback); + + BOOL ret = FALSE; + int bytes_available = 0; + DWORD error_code = 0; + + /* We already have a request to read data that isn't complete yet. */ + if (BUG(handle->busy)) + return 0; + + /* Check if we have been asked to read from a handle that have already told + * us that we have reached the end of the file. */ + if (BUG(handle->reached_eof)) + return 0; + + /* This cast should be safe since our buffer can be at maximum up to + * BUFFER_SIZE in size. */ + bytes_available = (int)handle->data_available; + + if (handle->data_available > 0) { + /* Read data from our intermediate buffer into the process_t buffer. */ + buf_add(buffer, handle->buffer, handle->data_available); + + /* Reset our read state. */ + handle->data_available = 0; + memset(handle->buffer, 0, sizeof(handle->buffer)); + } + + /* Because of the slightly weird API for ReadFileEx() we must set this to 0 + * before we call ReadFileEx() because ReadFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + + /* Ask the Windows kernel to read data from our pipe into our buffer and call + * the callback function when it is done. */ + ret = ReadFileEx(handle->pipe, + handle->buffer, + sizeof(handle->buffer), + &handle->overlapped, + callback); + + if (! ret) { + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "ReadFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "ReadFileEx() failed: %s", + format_win32_error(error_code)); + } + + handle->reached_eof = true; + return bytes_available; + } + + /* Here be dragons: According to MSDN's documentation for ReadFileEx() we + * should check GetLastError() after a call to ReadFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "ReadFileEx() failed after returning success: %s", + format_win32_error(error_code)); + handle->reached_eof = true; + return bytes_available; + /* LCOV_EXCL_STOP */ + } + + /* We mark our handle as having a pending I/O request. */ + handle->busy = true; + + return bytes_available; +} + +/** This function checks the callback values from ReadFileEx() in + * <b>error_code</b> and <b>byte_count</b> if we have read data. Returns true + * iff our caller should request more data from ReadFileEx(). */ +STATIC bool +process_win32_handle_read_completion(process_win32_handle_t *handle, + DWORD error_code, + DWORD byte_count) +{ + tor_assert(handle); + + /* Mark our handle as not having any outstanding I/O requests. */ + handle->busy = false; + + if (error_code == 0) { + /* Our ReadFileEx() call was succesful and there is data for us. */ + + /* This cast should be safe since byte_count should never be larger than + * BUFFER_SIZE. */ + tor_assert(byte_count <= BUFFER_SIZE); + handle->data_available = (size_t)byte_count; + + /* Tell our caller to schedule the next read. */ + return true; + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our ReadFileEx() finished, but we reached the end of our file. We mark + * our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + handle->reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from ReadFileEx(): %s", + format_win32_error(error_code)); + + handle->reached_eof = true; + } + + /* Our caller should NOT schedule the next read. */ + return false; +} + +/** Format a single argument for being put on a Windows command line. + * Returns a newly allocated string */ +STATIC char * +format_win_cmdline_argument(const char *arg) +{ + char *formatted_arg; + char need_quotes; + const char *c; + int i; + int bs_counter = 0; + /* Backslash we can point to when one is inserted into the string */ + const char backslash = '\\'; + + /* Smartlist of *char */ + smartlist_t *arg_chars; + arg_chars = smartlist_new(); + + /* Quote string if it contains whitespace or is empty */ + need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); + + /* Build up smartlist of *chars */ + for (c=arg; *c != '\0'; c++) { + if ('"' == *c) { + /* Double up backslashes preceding a quote */ + for (i=0; i<(bs_counter*2); i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + /* Escape the quote */ + smartlist_add(arg_chars, (void*)&backslash); + smartlist_add(arg_chars, (void*)c); + } else if ('\\' == *c) { + /* Count backslashes until we know whether to double up */ + bs_counter++; + } else { + /* Don't double up slashes preceding a non-quote */ + for (i=0; i<bs_counter; i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + smartlist_add(arg_chars, (void*)c); + } + } + /* Don't double up trailing backslashes */ + for (i=0; i<bs_counter; i++) + smartlist_add(arg_chars, (void*)&backslash); + + /* Allocate space for argument, quotes (if needed), and terminator */ + const size_t formatted_arg_len = smartlist_len(arg_chars) + + (need_quotes ? 2 : 0) + 1; + formatted_arg = tor_malloc_zero(formatted_arg_len); + + /* Add leading quote */ + i=0; + if (need_quotes) + formatted_arg[i++] = '"'; + + /* Add characters */ + SMARTLIST_FOREACH(arg_chars, char*, ch, + { + formatted_arg[i++] = *ch; + }); + + /* Add trailing quote */ + if (need_quotes) + formatted_arg[i++] = '"'; + formatted_arg[i] = '\0'; + + smartlist_free(arg_chars); + return formatted_arg; +} + +/** Format a command line for use on Windows, which takes the command as a + * string rather than string array. Follows the rules from "Parsing C++ + * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the + * Python subprocess module. Returns a newly allocated string */ +STATIC char * +tor_join_win_cmdline(const char *argv[]) +{ + smartlist_t *argv_list; + char *joined_argv; + int i; + + /* Format each argument and put the result in a smartlist */ + argv_list = smartlist_new(); + for (i=0; argv[i] != NULL; i++) { + smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i])); + } + + /* Join the arguments with whitespace */ + joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL); + + /* Free the newly allocated arguments, and the smartlist */ + SMARTLIST_FOREACH(argv_list, char *, arg, + { + tor_free(arg); + }); + smartlist_free(argv_list); + + return joined_argv; +} + +#endif /* ! defined(_WIN32). */ diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h new file mode 100644 index 0000000000..d79dde157e --- /dev/null +++ b/src/lib/process/process_win32.h @@ -0,0 +1,97 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.h + * \brief Header for process_win32.c + **/ + +#ifndef TOR_PROCESS_WIN32_H +#define TOR_PROCESS_WIN32_H + +#ifdef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/evloop/compat_libevent.h" + +#include <windows.h> + +struct process_t; + +struct process_win32_t; +typedef struct process_win32_t process_win32_t; + +process_win32_t *process_win32_new(void); +void process_win32_free_(process_win32_t *win32_process); +#define process_win32_free(s) \ + FREE_AND_NULL(process_win32_t, process_win32_free_, (s)) + +void process_win32_init(void); +void process_win32_deinit(void); + +process_status_t process_win32_exec(struct process_t *process); +bool process_win32_terminate(struct process_t *process); + +process_pid_t process_win32_get_pid(struct process_t *process); + +int process_win32_write(struct process_t *process, buf_t *buffer); +int process_win32_read_stdout(struct process_t *process, buf_t *buffer); +int process_win32_read_stderr(struct process_t *process, buf_t *buffer); + +void process_win32_trigger_completion_callbacks(void); + +/* Timer handling. */ +void process_win32_timer_start(void); +void process_win32_timer_stop(void); +bool process_win32_timer_running(void); + +#ifdef PROCESS_WIN32_PRIVATE +STATIC void process_win32_timer_callback(periodic_timer_t *, void *); +STATIC bool process_win32_timer_test_process(process_t *); + +/* I/O pipe handling. */ +struct process_win32_handle_t; +typedef struct process_win32_handle_t process_win32_handle_t; + +typedef enum process_win32_pipe_type_t { + /** This pipe is used for reading. */ + PROCESS_WIN32_PIPE_TYPE_READER, + + /** This pipe is used for writing. */ + PROCESS_WIN32_PIPE_TYPE_WRITER +} process_win32_pipe_type_t; + +STATIC bool process_win32_create_pipe(HANDLE *, + HANDLE *, + SECURITY_ATTRIBUTES *, + process_win32_pipe_type_t); + +STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle); + +STATIC VOID WINAPI process_win32_stdout_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stderr_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stdin_write_done(DWORD, + DWORD, + LPOVERLAPPED); + +STATIC int process_win32_read_from_handle(process_win32_handle_t *, + buf_t *, + LPOVERLAPPED_COMPLETION_ROUTINE); +STATIC bool process_win32_handle_read_completion(process_win32_handle_t *, + DWORD, + DWORD); + +STATIC char *format_win_cmdline_argument(const char *arg); +STATIC char *tor_join_win_cmdline(const char *argv[]); +#endif /* defined(PROCESS_WIN32_PRIVATE). */ + +#endif /* ! defined(_WIN32). */ + +#endif /* defined(TOR_PROCESS_WIN32_H). */ diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c deleted file mode 100644 index f4429d7f76..0000000000 --- a/src/lib/process/subprocess.c +++ /dev/null @@ -1,1236 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file subprocess.c - * \brief Launch and monitor other processes. - **/ - -#define SUBPROCESS_PRIVATE -#include "lib/process/subprocess.h" - -#include "lib/container/smartlist.h" -#include "lib/err/torerr.h" -#include "lib/log/log.h" -#include "lib/log/util_bug.h" -#include "lib/log/win32err.h" -#include "lib/malloc/malloc.h" -#include "lib/process/env.h" -#include "lib/process/waitpid.h" -#include "lib/string/compat_ctype.h" - -#ifdef HAVE_SYS_TYPES_H -#include <sys/types.h> -#endif -#ifdef HAVE_SYS_PRCTL_H -#include <sys/prctl.h> -#endif -#ifdef HAVE_UNISTD_H -#include <unistd.h> -#endif -#ifdef HAVE_SIGNAL_H -#include <signal.h> -#endif -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif -#ifdef HAVE_SYS_WAIT_H -#include <sys/wait.h> -#endif -#include <errno.h> -#include <string.h> - -/** Format a single argument for being put on a Windows command line. - * Returns a newly allocated string */ -static char * -format_win_cmdline_argument(const char *arg) -{ - char *formatted_arg; - char need_quotes; - const char *c; - int i; - int bs_counter = 0; - /* Backslash we can point to when one is inserted into the string */ - const char backslash = '\\'; - - /* Smartlist of *char */ - smartlist_t *arg_chars; - arg_chars = smartlist_new(); - - /* Quote string if it contains whitespace or is empty */ - need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); - - /* Build up smartlist of *chars */ - for (c=arg; *c != '\0'; c++) { - if ('"' == *c) { - /* Double up backslashes preceding a quote */ - for (i=0; i<(bs_counter*2); i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - /* Escape the quote */ - smartlist_add(arg_chars, (void*)&backslash); - smartlist_add(arg_chars, (void*)c); - } else if ('\\' == *c) { - /* Count backslashes until we know whether to double up */ - bs_counter++; - } else { - /* Don't double up slashes preceding a non-quote */ - for (i=0; i<bs_counter; i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - smartlist_add(arg_chars, (void*)c); - } - } - /* Don't double up trailing backslashes */ - for (i=0; i<bs_counter; i++) - smartlist_add(arg_chars, (void*)&backslash); - - /* Allocate space for argument, quotes (if needed), and terminator */ - const size_t formatted_arg_len = smartlist_len(arg_chars) + - (need_quotes ? 2 : 0) + 1; - formatted_arg = tor_malloc_zero(formatted_arg_len); - - /* Add leading quote */ - i=0; - if (need_quotes) - formatted_arg[i++] = '"'; - - /* Add characters */ - SMARTLIST_FOREACH(arg_chars, char*, ch, - { - formatted_arg[i++] = *ch; - }); - - /* Add trailing quote */ - if (need_quotes) - formatted_arg[i++] = '"'; - formatted_arg[i] = '\0'; - - smartlist_free(arg_chars); - return formatted_arg; -} - -/** Format a command line for use on Windows, which takes the command as a - * string rather than string array. Follows the rules from "Parsing C++ - * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the - * Python subprocess module. Returns a newly allocated string */ -char * -tor_join_win_cmdline(const char *argv[]) -{ - smartlist_t *argv_list; - char *joined_argv; - int i; - - /* Format each argument and put the result in a smartlist */ - argv_list = smartlist_new(); - for (i=0; argv[i] != NULL; i++) { - smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i])); - } - - /* Join the arguments with whitespace */ - joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL); - - /* Free the newly allocated arguments, and the smartlist */ - SMARTLIST_FOREACH(argv_list, char *, arg, - { - tor_free(arg); - }); - smartlist_free(argv_list); - - return joined_argv; -} - -#ifndef _WIN32 -/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in - * <b>hex_errno</b>. Called between fork and _exit, so must be signal-handler - * safe. - * - * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available. - * - * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded - * with spaces. CHILD_STATE indicates where - * in the process of starting the child process did the failure occur (see - * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of - * errno when the failure occurred. - * - * On success return the number of characters added to hex_errno, not counting - * the terminating NUL; return -1 on error. - */ -STATIC int -format_helper_exit_status(unsigned char child_state, int saved_errno, - char *hex_errno) -{ - unsigned int unsigned_errno; - int written, left; - char *cur; - size_t i; - int res = -1; - - /* Fill hex_errno with spaces, and a trailing newline (memset may - not be signal handler safe, so we can't use it) */ - for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++) - hex_errno[i] = ' '; - hex_errno[HEX_ERRNO_SIZE - 1] = '\n'; - - /* Convert errno to be unsigned for hex conversion */ - if (saved_errno < 0) { - // Avoid overflow on the cast to unsigned int when result is INT_MIN - // by adding 1 to the signed int negative value, - // then, after it has been negated and cast to unsigned, - // adding the original 1 back (the double-addition is intentional). - // Otherwise, the cast to signed could cause a temporary int - // to equal INT_MAX + 1, which is undefined. - unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1; - } else { - unsigned_errno = (unsigned int) saved_errno; - } - - /* - * Count how many chars of space we have left, and keep a pointer into the - * current point in the buffer. - */ - left = HEX_ERRNO_SIZE+1; - cur = hex_errno; - - /* Emit child_state */ - written = format_hex_number_sigsafe(child_state, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - if (left <= 0) - goto err; - - /* Now the '/' */ - *cur = '/'; - - /* Adjust left and cur */ - ++cur; - --left; - if (left <= 0) - goto err; - - /* Need minus? */ - if (saved_errno < 0) { - *cur = '-'; - ++cur; - --left; - if (left <= 0) - goto err; - } - - /* Emit unsigned_errno */ - written = format_hex_number_sigsafe(unsigned_errno, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - - /* Check that we have enough space left for a newline and a NUL */ - if (left <= 1) - goto err; - - /* Emit the newline and NUL */ - *cur++ = '\n'; - *cur++ = '\0'; - - res = (int)(cur - hex_errno - 1); - - goto done; - - err: - /* - * In error exit, just write a '\0' in the first char so whatever called - * this at least won't fall off the end. - */ - *hex_errno = '\0'; - - done: - return res; -} -#endif /* !defined(_WIN32) */ - -/* Maximum number of file descriptors, if we cannot get it via sysconf() */ -#define DEFAULT_MAX_FD 256 - -/** Terminate the process of <b>process_handle</b>, if that process has not - * already exited. - * - * Return 0 if we succeeded in terminating the process (or if the process - * already exited), and -1 if we tried to kill the process but failed. - * - * Based on code originally borrowed from Python's os.kill. */ -int -tor_terminate_process(process_handle_t *process_handle) -{ -#ifdef _WIN32 - if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) { - HANDLE handle = process_handle->pid.hProcess; - - if (!TerminateProcess(handle, 0)) - return -1; - else - return 0; - } -#else /* !(defined(_WIN32)) */ - if (process_handle->waitpid_cb) { - /* We haven't got a waitpid yet, so we can just kill off the process. */ - return kill(process_handle->pid, SIGTERM); - } -#endif /* defined(_WIN32) */ - - return 0; /* We didn't need to kill the process, so report success */ -} - -/** Return the Process ID of <b>process_handle</b>. */ -int -tor_process_get_pid(process_handle_t *process_handle) -{ -#ifdef _WIN32 - return (int) process_handle->pid.dwProcessId; -#else - return (int) process_handle->pid; -#endif -} - -#ifdef _WIN32 -HANDLE -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#else /* !(defined(_WIN32)) */ -/* DOCDOC tor_process_get_stdout_pipe */ -int -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#endif /* defined(_WIN32) */ - -/* DOCDOC process_handle_new */ -static process_handle_t * -process_handle_new(void) -{ - process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t)); - -#ifdef _WIN32 - out->stdin_pipe = INVALID_HANDLE_VALUE; - out->stdout_pipe = INVALID_HANDLE_VALUE; - out->stderr_pipe = INVALID_HANDLE_VALUE; -#else - out->stdin_pipe = -1; - out->stdout_pipe = -1; - out->stderr_pipe = -1; -#endif /* defined(_WIN32) */ - - return out; -} - -#ifndef _WIN32 -/** Invoked when a process that we've launched via tor_spawn_background() has - * been found to have terminated. - */ -static void -process_handle_waitpid_cb(int status, void *arg) -{ - process_handle_t *process_handle = arg; - - process_handle->waitpid_exit_status = status; - clear_waitpid_callback(process_handle->waitpid_cb); - if (process_handle->status == PROCESS_STATUS_RUNNING) - process_handle->status = PROCESS_STATUS_NOTRUNNING; - process_handle->waitpid_cb = 0; -} -#endif /* !defined(_WIN32) */ - -/** - * @name child-process states - * - * Each of these values represents a possible state that a child process can - * be in. They're used to determine what to say when telling the parent how - * far along we were before failure. - * - * @{ - */ -#define CHILD_STATE_INIT 0 -#define CHILD_STATE_PIPE 1 -#define CHILD_STATE_MAXFD 2 -#define CHILD_STATE_FORK 3 -#define CHILD_STATE_DUPOUT 4 -#define CHILD_STATE_DUPERR 5 -#define CHILD_STATE_DUPIN 6 -#define CHILD_STATE_CLOSEFD 7 -#define CHILD_STATE_EXEC 8 -#define CHILD_STATE_FAILEXEC 9 -/** @} */ -/** - * Boolean. If true, then Tor may call execve or CreateProcess via - * tor_spawn_background. - **/ -static int may_spawn_background_process = 1; -/** - * Turn off may_spawn_background_process, so that all future calls to - * tor_spawn_background are guaranteed to fail. - **/ -void -tor_disable_spawning_background_processes(void) -{ - may_spawn_background_process = 0; -} -/** Start a program in the background. If <b>filename</b> contains a '/', then - * it will be treated as an absolute or relative path. Otherwise, on - * non-Windows systems, the system path will be searched for <b>filename</b>. - * On Windows, only the current directory will be searched. Here, to search the - * system path (as well as the application directory, current working - * directory, and system directories), set filename to NULL. - * - * The strings in <b>argv</b> will be passed as the command line arguments of - * the child program (following convention, argv[0] should normally be the - * filename of the executable, and this must be the case if <b>filename</b> is - * NULL). The last element of argv must be NULL. A handle to the child process - * will be returned in process_handle (which must be non-NULL). Read - * process_handle.status to find out if the process was successfully launched. - * For convenience, process_handle.status is returned by this function. - * - * Some parts of this code are based on the POSIX subprocess module from - * Python, and example code from - * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx. - */ -int -tor_spawn_background(const char *const filename, const char **argv, - process_environment_t *env, - process_handle_t **process_handle_out) -{ - if (BUG(may_spawn_background_process == 0)) { - /* We should never reach this point if we're forbidden to spawn - * processes. Instead we should have caught the attempt earlier. */ - return PROCESS_STATUS_ERROR; - } - -#ifdef _WIN32 - HANDLE stdout_pipe_read = NULL; - HANDLE stdout_pipe_write = NULL; - HANDLE stderr_pipe_read = NULL; - HANDLE stderr_pipe_write = NULL; - HANDLE stdin_pipe_read = NULL; - HANDLE stdin_pipe_write = NULL; - process_handle_t *process_handle; - int status; - - STARTUPINFOA siStartInfo; - BOOL retval = FALSE; - - SECURITY_ATTRIBUTES saAttr; - char *joined_argv; - - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - saAttr.bInheritHandle = TRUE; - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - saAttr.lpSecurityDescriptor = NULL; - - /* Assume failure to start process */ - status = PROCESS_STATUS_ERROR; - - /* Set up pipe for stdout */ - if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdout communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdout communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stderr */ - if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stderr communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stderr communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stdin */ - if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdin communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdin communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Create the child process */ - - /* Windows expects argv to be a whitespace delimited string, so join argv up - */ - joined_argv = tor_join_win_cmdline(argv); - - process_handle = process_handle_new(); - process_handle->status = status; - - ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION)); - ZeroMemory(&siStartInfo, sizeof(STARTUPINFO)); - siStartInfo.cb = sizeof(STARTUPINFO); - siStartInfo.hStdError = stderr_pipe_write; - siStartInfo.hStdOutput = stdout_pipe_write; - siStartInfo.hStdInput = stdin_pipe_read; - siStartInfo.dwFlags |= STARTF_USESTDHANDLES; - - /* Create the child process */ - - retval = CreateProcessA(filename, // module name - joined_argv, // command line - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - NULL, // process security attributes - NULL, // primary thread security attributes - TRUE, // handles are inherited - /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess() - * work?) */ - CREATE_NO_WINDOW, // creation flags - (env==NULL) ? NULL : env->windows_environment_block, - NULL, // use parent's current directory - &siStartInfo, // STARTUPINFO pointer - &(process_handle->pid)); // receives PROCESS_INFORMATION - - tor_free(joined_argv); - - if (!retval) { - log_warn(LD_GENERAL, - "Failed to create child process %s: %s", filename?filename:argv[0], - format_win32_error(GetLastError())); - tor_free(process_handle); - } else { - /* TODO: Close hProcess and hThread in process_handle->pid? */ - process_handle->stdout_pipe = stdout_pipe_read; - process_handle->stderr_pipe = stderr_pipe_read; - process_handle->stdin_pipe = stdin_pipe_write; - status = process_handle->status = PROCESS_STATUS_RUNNING; - } - - /* TODO: Close pipes on exit */ - *process_handle_out = process_handle; - return status; -#else /* !(defined(_WIN32)) */ - pid_t pid; - int stdout_pipe[2]; - int stderr_pipe[2]; - int stdin_pipe[2]; - int fd, retval; - process_handle_t *process_handle; - int status; - - const char *error_message = SPAWN_ERROR_MESSAGE; - size_t error_message_length; - - /* Represents where in the process of spawning the program is; - this is used for printing out the error message */ - unsigned char child_state = CHILD_STATE_INIT; - - char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */ - - static int max_fd = -1; - - status = PROCESS_STATUS_ERROR; - - /* We do the strlen here because strlen() is not signal handler safe, - and we are not allowed to use unsafe functions between fork and exec */ - error_message_length = strlen(error_message); - - // child_state = CHILD_STATE_PIPE; - - /* Set up pipe for redirecting stdout, stderr, and stdin of child */ - retval = pipe(stdout_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdout communication with child process: %s", - strerror(errno)); - return status; - } - - retval = pipe(stderr_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stderr communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - - return status; - } - - retval = pipe(stdin_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdin communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - - return status; - } - - // child_state = CHILD_STATE_MAXFD; - -#ifdef _SC_OPEN_MAX - if (-1 == max_fd) { - max_fd = (int) sysconf(_SC_OPEN_MAX); - if (max_fd == -1) { - max_fd = DEFAULT_MAX_FD; - log_warn(LD_GENERAL, - "Cannot find maximum file descriptor, assuming %d", max_fd); - } - } -#else /* !(defined(_SC_OPEN_MAX)) */ - max_fd = DEFAULT_MAX_FD; -#endif /* defined(_SC_OPEN_MAX) */ - - // child_state = CHILD_STATE_FORK; - - pid = fork(); - if (0 == pid) { - /* In child */ - -#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) - /* Attempt to have the kernel issue a SIGTERM if the parent - * goes away. Certain attributes of the binary being execve()ed - * will clear this during the execve() call, but it's better - * than nothing. - */ - prctl(PR_SET_PDEATHSIG, SIGTERM); -#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ - - child_state = CHILD_STATE_DUPOUT; - - /* Link child stdout to the write end of the pipe */ - retval = dup2(stdout_pipe[1], STDOUT_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPERR; - - /* Link child stderr to the write end of the pipe */ - retval = dup2(stderr_pipe[1], STDERR_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPIN; - - /* Link child stdin to the read end of the pipe */ - retval = dup2(stdin_pipe[0], STDIN_FILENO); - if (-1 == retval) - goto error; - - // child_state = CHILD_STATE_CLOSEFD; - - close(stderr_pipe[0]); - close(stderr_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - - /* Close all other fds, including the read end of the pipe */ - /* XXX: We should now be doing enough FD_CLOEXEC setting to make - * this needless. */ - for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) { - close(fd); - } - - // child_state = CHILD_STATE_EXEC; - - /* Call the requested program. We need the cast because - execvp doesn't define argv as const, even though it - does not modify the arguments */ - if (env) - execve(filename, (char *const *) argv, env->unixoid_environment_block); - else { - static char *new_env[] = { NULL }; - execve(filename, (char *const *) argv, new_env); - } - - /* If we got here, the exec or open(/dev/null) failed */ - - child_state = CHILD_STATE_FAILEXEC; - - error: - { - /* XXX: are we leaking fds from the pipe? */ - int n, err=0; - ssize_t nbytes; - - n = format_helper_exit_status(child_state, errno, hex_errno); - - if (n >= 0) { - /* Write the error message. GCC requires that we check the return - value, but there is nothing we can do if it fails */ - /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */ - nbytes = write(STDOUT_FILENO, error_message, error_message_length); - err = (nbytes < 0); - nbytes = write(STDOUT_FILENO, hex_errno, n); - err += (nbytes < 0); - } - - _exit(err?254:255); // exit ok: in child. - } - - /* Never reached, but avoids compiler warning */ - return status; // LCOV_EXCL_LINE - } - - /* In parent */ - - if (-1 == pid) { - log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno)); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - return status; - } - - process_handle = process_handle_new(); - process_handle->status = status; - process_handle->pid = pid; - - /* TODO: If the child process forked but failed to exec, waitpid it */ - - /* Return read end of the pipes to caller, and close write end */ - process_handle->stdout_pipe = stdout_pipe[0]; - retval = close(stdout_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stdout pipe in parent process: %s", - strerror(errno)); - } - - process_handle->waitpid_cb = set_waitpid_callback(pid, - process_handle_waitpid_cb, - process_handle); - - process_handle->stderr_pipe = stderr_pipe[0]; - retval = close(stderr_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stderr pipe in parent process: %s", - strerror(errno)); - } - - /* Return write end of the stdin pipe to caller, and close the read end */ - process_handle->stdin_pipe = stdin_pipe[1]; - retval = close(stdin_pipe[0]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close read end of stdin pipe in parent process: %s", - strerror(errno)); - } - - status = process_handle->status = PROCESS_STATUS_RUNNING; - /* Set stdin/stdout/stderr pipes to be non-blocking */ - if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) { - log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " - "nonblocking in parent process: %s", strerror(errno)); - } - - *process_handle_out = process_handle; - return status; -#endif /* defined(_WIN32) */ -} - -/** Destroy all resources allocated by the process handle in - * <b>process_handle</b>. - * If <b>also_terminate_process</b> is true, also terminate the - * process of the process handle. */ -MOCK_IMPL(void, -tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)) -{ - if (!process_handle) - return; - - if (also_terminate_process) { - if (tor_terminate_process(process_handle) < 0) { - const char *errstr = -#ifdef _WIN32 - format_win32_error(GetLastError()); -#else - strerror(errno); -#endif - log_notice(LD_GENERAL, "Failed to terminate process with " - "PID '%d' ('%s').", tor_process_get_pid(process_handle), - errstr); - } else { - log_info(LD_GENERAL, "Terminated process with PID '%d'.", - tor_process_get_pid(process_handle)); - } - } - - process_handle->status = PROCESS_STATUS_NOTRUNNING; - -#ifdef _WIN32 - if (process_handle->stdout_pipe) - CloseHandle(process_handle->stdout_pipe); - - if (process_handle->stderr_pipe) - CloseHandle(process_handle->stderr_pipe); - - if (process_handle->stdin_pipe) - CloseHandle(process_handle->stdin_pipe); -#else /* !(defined(_WIN32)) */ - close(process_handle->stdout_pipe); - close(process_handle->stderr_pipe); - close(process_handle->stdin_pipe); - - clear_waitpid_callback(process_handle->waitpid_cb); -#endif /* defined(_WIN32) */ - - memset(process_handle, 0x0f, sizeof(process_handle_t)); - tor_free(process_handle); -} - -/** Get the exit code of a process specified by <b>process_handle</b> and store - * it in <b>exit_code</b>, if set to a non-NULL value. If <b>block</b> is set - * to true, the call will block until the process has exited. Otherwise if - * the process is still running, the function will return - * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns - * PROCESS_EXIT_EXITED if the process did exit. If there is a failure, - * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if - * non-NULL) will be undefined. N.B. Under *nix operating systems, this will - * probably not work in Tor, because waitpid() is called in main.c to reap any - * terminated child processes.*/ -int -tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code) -{ -#ifdef _WIN32 - DWORD retval; - BOOL success; - - if (block) { - /* Wait for the process to exit */ - retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE); - if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } else { - retval = WaitForSingleObject(process_handle->pid.hProcess, 0); - if (WAIT_TIMEOUT == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } - - if (exit_code != NULL) { - success = GetExitCodeProcess(process_handle->pid.hProcess, - (PDWORD)exit_code); - if (!success) { - log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s", - format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } -#else /* !(defined(_WIN32)) */ - int stat_loc; - int retval; - - if (process_handle->waitpid_cb) { - /* We haven't processed a SIGCHLD yet. */ - retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG); - if (retval == process_handle->pid) { - clear_waitpid_callback(process_handle->waitpid_cb); - process_handle->waitpid_cb = NULL; - process_handle->waitpid_exit_status = stat_loc; - } - } else { - /* We already got a SIGCHLD for this process, and handled it. */ - retval = process_handle->pid; - stat_loc = process_handle->waitpid_exit_status; - } - - if (!block && 0 == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != process_handle->pid) { - log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s", - (int)process_handle->pid, strerror(errno)); - return PROCESS_EXIT_ERROR; - } - - if (!WIFEXITED(stat_loc)) { - log_warn(LD_GENERAL, "Process %d did not exit normally", - (int)process_handle->pid); - return PROCESS_EXIT_ERROR; - } - - if (exit_code != NULL) - *exit_code = WEXITSTATUS(stat_loc); -#endif /* defined(_WIN32) */ - - return PROCESS_EXIT_EXITED; -} - -#ifdef _WIN32 -/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes. If - * <b>hProcess</b> is NULL, the function will return immediately if there is - * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle - * to the process owning the <b>h</b>. In this case, the function will exit - * only once the process has exited, or <b>count</b> bytes are read. Returns - * the number of bytes read, or -1 on error. */ -ssize_t -tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process) -{ - size_t numread = 0; - BOOL retval; - DWORD byte_count; - BOOL process_exited = FALSE; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - /* Check if there is anything to read */ - retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); - if (!retval) { - log_warn(LD_GENERAL, - "Failed to peek from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* Nothing available: process exited or it is busy */ - - /* Exit if we don't know whether the process is running */ - if (NULL == process) - break; - - /* The process exited and there's nothing left to read from it */ - if (process_exited) - break; - - /* If process is not running, check for output one more time in case - it wrote something after the peek was performed. Otherwise keep on - waiting for output */ - tor_assert(process != NULL); - byte_count = WaitForSingleObject(process->pid.hProcess, 0); - if (WAIT_TIMEOUT != byte_count) - process_exited = TRUE; - - continue; - } - - /* There is data to read; read it */ - retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL); - tor_assert(byte_count + numread <= count); - if (!retval) { - log_warn(LD_GENERAL, "Failed to read from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* End of file */ - break; - } - numread += byte_count; - } - return (ssize_t)numread; -} -#else /* !(defined(_WIN32)) */ -/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes. If - * <b>process</b> is NULL, the function will return immediately if there is - * nothing more to read. Otherwise data will be read until end of file, or - * <b>count</b> bytes are read. Returns the number of bytes read, or -1 on - * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the - * file has been reached. */ -ssize_t -tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof) -{ - size_t numread = 0; - ssize_t result; - - if (eof) - *eof = 0; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - result = read(fd, buf+numread, count-numread); - - if (result == 0) { - log_debug(LD_GENERAL, "read() reached end of file"); - if (eof) - *eof = 1; - break; - } else if (result < 0 && errno == EAGAIN) { - if (process) - continue; - else - break; - } else if (result < 0) { - log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); - return -1; - } - - numread += result; - } - - log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); - return (ssize_t)numread; -} -#endif /* defined(_WIN32) */ - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stdout(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stderr(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Return a string corresponding to <b>stream_status</b>. */ -const char * -stream_status_to_string(enum stream_status stream_status) -{ - switch (stream_status) { - case IO_STREAM_OKAY: - return "okay"; - case IO_STREAM_EAGAIN: - return "temporarily unavailable"; - case IO_STREAM_TERM: - return "terminated"; - case IO_STREAM_CLOSED: - return "closed"; - default: - tor_fragile_assert(); - return "unknown"; - } -} - -/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be - * modified. The resulting smartlist will consist of pointers to buf, so there - * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated - * string. <b>len</b> should be set to the length of the buffer excluding the - * NUL. Non-printable characters (including NUL) will be replaced with "." */ -int -tor_split_lines(smartlist_t *sl, char *buf, int len) -{ - /* Index in buf of the start of the current line */ - int start = 0; - /* Index in buf of the current character being processed */ - int cur = 0; - /* Are we currently in a line */ - char in_line = 0; - - /* Loop over string */ - while (cur < len) { - /* Loop until end of line or end of string */ - for (; cur < len; cur++) { - if (in_line) { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* End of line */ - buf[cur] = '\0'; - /* Point cur to the next line */ - cur++; - /* Line starts at start and ends with a nul */ - break; - } else { - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } else { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* Skip leading vertical space */ - ; - } else { - in_line = 1; - start = cur; - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } - } - /* We are at the end of the line or end of string. If in_line is true there - * is a line which starts at buf+start and ends at a NUL. cur points to - * the character after the NUL. */ - if (in_line) - smartlist_add(sl, (void *)(buf+start)); - in_line = 0; - } - return smartlist_len(sl); -} - -#ifdef _WIN32 - -/** Return a smartlist containing lines outputted from - * <b>handle</b>. Return NULL on error, and set - * <b>stream_status_out</b> appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (HANDLE *handle, - enum stream_status *stream_status_out)) -{ - int pos; - char stdout_buf[600] = {0}; - smartlist_t *lines = NULL; - - tor_assert(stream_status_out); - - *stream_status_out = IO_STREAM_TERM; - - pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL); - if (pos < 0) { - *stream_status_out = IO_STREAM_TERM; - return NULL; - } - if (pos == 0) { - *stream_status_out = IO_STREAM_EAGAIN; - return NULL; - } - - /* End with a null even if there isn't a \r\n at the end */ - /* TODO: What if this is a partial line? */ - stdout_buf[pos] = '\0'; - - /* Split up the buffer */ - lines = smartlist_new(); - tor_split_lines(lines, stdout_buf, pos); - - /* Currently 'lines' is populated with strings residing on the - stack. Replace them with their exact copies on the heap: */ - SMARTLIST_FOREACH(lines, char *, line, - SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line))); - - *stream_status_out = IO_STREAM_OKAY; - - return lines; -} - -#else /* !(defined(_WIN32)) */ - -/** Return a smartlist containing lines outputted from - * <b>fd</b>. Return NULL on error, and set - * <b>stream_status_out</b> appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) -{ - enum stream_status stream_status; - char stdout_buf[400]; - smartlist_t *lines = NULL; - - while (1) { - memset(stdout_buf, 0, sizeof(stdout_buf)); - - stream_status = get_string_from_pipe(fd, - stdout_buf, sizeof(stdout_buf) - 1); - if (stream_status != IO_STREAM_OKAY) - goto done; - - if (!lines) lines = smartlist_new(); - smartlist_split_string(lines, stdout_buf, "\n", 0, 0); - } - - done: - *stream_status_out = stream_status; - return lines; -} - -#endif /* defined(_WIN32) */ - -/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making - * sure it's below <b>count</b> bytes. - * If the string has a trailing newline, we strip it off. - * - * This function is specifically created to handle input from managed - * proxies, according to the pluggable transports spec. Make sure it - * fits your needs before using it. - * - * Returns: - * IO_STREAM_CLOSED: If the stream is closed. - * IO_STREAM_EAGAIN: If there is nothing to read and we should check back - * later. - * IO_STREAM_TERM: If something is wrong with the stream. - * IO_STREAM_OKAY: If everything went okay and we got a string - * in <b>buf_out</b>. */ -enum stream_status -get_string_from_pipe(int fd, char *buf_out, size_t count) -{ - ssize_t ret; - - tor_assert(count <= INT_MAX); - - ret = read(fd, buf_out, count); - - if (ret == 0) - return IO_STREAM_CLOSED; - else if (ret < 0 && errno == EAGAIN) - return IO_STREAM_EAGAIN; - else if (ret < 0) - return IO_STREAM_TERM; - - if (buf_out[ret - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[ret - 1] = '\0'; - } else - buf_out[ret] = '\0'; - - return IO_STREAM_OKAY; -} diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h deleted file mode 100644 index aa3127d62d..0000000000 --- a/src/lib/process/subprocess.h +++ /dev/null @@ -1,134 +0,0 @@ -/* Copyright (c) 2003-2004, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file subprocess.h - * \brief Header for subprocess.c - **/ - -#ifndef TOR_SUBPROCESS_H -#define TOR_SUBPROCESS_H - -#include "lib/cc/torint.h" -#include "lib/testsupport/testsupport.h" -#include <stddef.h> -#ifdef _WIN32 -#include <windows.h> -#endif - -struct smartlist_t; - -void tor_disable_spawning_background_processes(void); - -typedef struct process_handle_t process_handle_t; -struct process_environment_t; -int tor_spawn_background(const char *const filename, const char **argv, - struct process_environment_t *env, - process_handle_t **process_handle_out); - -#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code " - -/** Status of an I/O stream. */ -enum stream_status { - IO_STREAM_OKAY, - IO_STREAM_EAGAIN, - IO_STREAM_TERM, - IO_STREAM_CLOSED -}; - -const char *stream_status_to_string(enum stream_status stream_status); - -enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); - -/* Values of process_handle_t.status. */ -#define PROCESS_STATUS_NOTRUNNING 0 -#define PROCESS_STATUS_RUNNING 1 -#define PROCESS_STATUS_ERROR -1 - -#ifdef SUBPROCESS_PRIVATE -struct waitpid_callback_t; - -/** Structure to represent the state of a process with which Tor is - * communicating. The contents of this structure are private to util.c */ -struct process_handle_t { - /** One of the PROCESS_STATUS_* values */ - int status; -#ifdef _WIN32 - HANDLE stdin_pipe; - HANDLE stdout_pipe; - HANDLE stderr_pipe; - PROCESS_INFORMATION pid; -#else /* !(defined(_WIN32)) */ - int stdin_pipe; - int stdout_pipe; - int stderr_pipe; - pid_t pid; - /** If the process has not given us a SIGCHLD yet, this has the - * waitpid_callback_t that gets invoked once it has. Otherwise this - * contains NULL. */ - struct waitpid_callback_t *waitpid_cb; - /** The exit status reported by waitpid. */ - int waitpid_exit_status; -#endif /* defined(_WIN32) */ -}; -#endif /* defined(SUBPROCESS_PRIVATE) */ - -/* Return values of tor_get_exit_code() */ -#define PROCESS_EXIT_RUNNING 1 -#define PROCESS_EXIT_EXITED 0 -#define PROCESS_EXIT_ERROR -1 -int tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code); -int tor_split_lines(struct smartlist_t *sl, char *buf, int len); -#ifdef _WIN32 -ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process); -#else -ssize_t tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof); -#endif /* defined(_WIN32) */ -ssize_t tor_read_all_from_process_stdout( - const process_handle_t *process_handle, char *buf, size_t count); -ssize_t tor_read_all_from_process_stderr( - const process_handle_t *process_handle, char *buf, size_t count); -char *tor_join_win_cmdline(const char *argv[]); - -int tor_process_get_pid(process_handle_t *process_handle); -#ifdef _WIN32 -HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); -#else -int tor_process_get_stdout_pipe(process_handle_t *process_handle); -#endif - -#ifdef _WIN32 -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle, - enum stream_status *stream_status)); -#else -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd, - enum stream_status *stream_status)); -#endif /* defined(_WIN32) */ - -int tor_terminate_process(process_handle_t *process_handle); - -MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)); - -#ifdef SUBPROCESS_PRIVATE -/* Prototypes for private functions only used by util.c (and unit tests) */ - -#ifndef _WIN32 -STATIC int format_helper_exit_status(unsigned char child_state, - int saved_errno, char *hex_errno); - -/* Space for hex values of child state, a slash, saved_errno (with - leading minus) and newline (no null) */ -#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \ - 1 + sizeof(int) * 2 + 1) -#endif /* !defined(_WIN32) */ - -#endif /* defined(SUBPROCESS_PRIVATE) */ - -#endif diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c index 9b626394d2..2b38481aeb 100644 --- a/src/lib/process/waitpid.c +++ b/src/lib/process/waitpid.c @@ -16,7 +16,7 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" -#include "ht.h" +#include "ext/ht.h" #ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c new file mode 100644 index 0000000000..1266babca8 --- /dev/null +++ b/src/lib/process/winprocess_sys.c @@ -0,0 +1,64 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.c + * \brief Subsystem object for windows process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/winprocess_sys.h" + +#include <stdbool.h> +#include <stddef.h> + +#ifdef _WIN32 +#include <windows.h> + +#define WINPROCESS_SYS_ENABLED true + +static int +subsys_winprocess_initialize(void) +{ +#ifndef HeapEnableTerminationOnCorruption +#define HeapEnableTerminationOnCorruption 1 +#endif + + /* On heap corruption, just give up; don't try to play along. */ + HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); + + /* SetProcessDEPPolicy is only supported on 32-bit Windows. + * (On 64-bit Windows it always fails, and some compilers don't like the + * PSETDEP cast.) + * 32-bit Windows defines _WIN32. + * 64-bit Windows defines _WIN32 and _WIN64. */ +#ifndef _WIN64 + /* Call SetProcessDEPPolicy to permanently enable DEP. + The function will not resolve on earlier versions of Windows, + and failure is not dangerous. */ + HMODULE hMod = GetModuleHandleA("Kernel32.dll"); + if (hMod) { + typedef BOOL (WINAPI *PSETDEP)(DWORD); + PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, + "SetProcessDEPPolicy"); + if (setdeppolicy) { + /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ + setdeppolicy(3); + } + } +#endif /* !defined(_WIN64) */ + + return 0; +} +#else /* !defined(_WIN32) */ +#define WINPROCESS_SYS_ENABLED false +#define subsys_winprocess_initialize NULL +#endif /* defined(_WIN32) */ + +const subsys_fns_t sys_winprocess = { + .name = "winprocess", + .level = -100, + .supported = WINPROCESS_SYS_ENABLED, + .initialize = subsys_winprocess_initialize, +}; diff --git a/src/lib/process/winprocess_sys.h b/src/lib/process/winprocess_sys.h new file mode 100644 index 0000000000..7ab2aa04a6 --- /dev/null +++ b/src/lib/process/winprocess_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.h + * \brief Declare subsystem object for winprocess.c + **/ + +#ifndef TOR_WINPROCESS_SYS_H +#define TOR_WINPROCESS_SYS_H + +extern const struct subsys_fns_t sys_winprocess; + +#endif /* !defined(TOR_WINPROCESS_SYS_H) */ diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include index 84906dfb3d..853dae7880 100644 --- a/src/lib/sandbox/.may_include +++ b/src/lib/sandbox/.may_include @@ -5,11 +5,10 @@ lib/container/*.h lib/err/*.h lib/log/*.h lib/malloc/*.h -lib/net/*.h lib/sandbox/*.h lib/sandbox/*.inc lib/string/*.h -ht.h -siphash.h -tor_queue.h +ext/ht.h +ext/siphash.h +ext/tor_queue.h diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index e2356a1720..b652397f5a 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -38,13 +38,12 @@ #include "lib/err/torerr.h" #include "lib/log/log.h" #include "lib/cc/torint.h" -#include "lib/net/resolve.h" #include "lib/malloc/malloc.h" #include "lib/string/scanf.h" -#include "tor_queue.h" -#include "ht.h" -#include "siphash.h" +#include "ext/tor_queue.h" +#include "ext/ht.h" +#include "ext/siphash.h" #define DEBUGGING_CLOSE @@ -833,6 +832,12 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) if (rc) return rc; + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), + SCMP_CMP(2, SCMP_CMP_EQ, SO_ACCEPTCONN)); + if (rc) + return rc; + #ifdef HAVE_SYSTEMD rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), @@ -1553,7 +1558,6 @@ install_syscall_filter(sandbox_cfg_t* cfg) // marking the sandbox as active sandbox_active = 1; - tor_make_getaddrinfo_cache_active(); end: seccomp_release(ctx); @@ -1800,9 +1804,4 @@ sandbox_is_active(void) return 0; } -void -sandbox_disable_getaddrinfo_cache(void) -{ -} - #endif /* !defined(USE_LIBSECCOMP) */ diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include index a8507761a4..2f0c8d341e 100644 --- a/src/lib/smartlist_core/.may_include +++ b/src/lib/smartlist_core/.may_include @@ -4,4 +4,4 @@ lib/malloc/*.h lib/err/*.h lib/string/*.h lib/smartlist_core/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h index 0f6fe30074..a1fbcd444c 100644 --- a/src/lib/smartlist_core/smartlist_foreach.h +++ b/src/lib/smartlist_core/smartlist_foreach.h @@ -83,6 +83,19 @@ ++var ## _sl_idx) { \ var = (sl)->list[var ## _sl_idx]; +/** Iterates over the items in smartlist <b>sl</b> in reverse order, similar to + * SMARTLIST_FOREACH_BEGIN + * + * NOTE: This macro is incompatible with SMARTLIST_DEL_CURRENT. + */ +#define SMARTLIST_FOREACH_REVERSE_BEGIN(sl, type, var) \ + STMT_BEGIN \ + int var ## _sl_idx, var ## _sl_len=(sl)->num_used; \ + type var; \ + for (var ## _sl_idx = var ## _sl_len-1; var ## _sl_idx >= 0; \ + --var ## _sl_idx) { \ + var = (sl)->list[var ## _sl_idx]; + #define SMARTLIST_FOREACH_END(var) \ var = NULL; \ (void) var ## _sl_idx; \ diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include index ec5c769831..1fb9127f19 100644 --- a/src/lib/string/.may_include +++ b/src/lib/string/.may_include @@ -6,5 +6,5 @@ lib/malloc/*.h lib/ctime/*.h lib/string/*.h -strlcat.c -strlcpy.c +ext/strlcat.c +ext/strlcpy.c diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c index e125c921a4..187f784be5 100644 --- a/src/lib/string/compat_string.c +++ b/src/lib/string/compat_string.c @@ -14,10 +14,10 @@ /* Inline the strl functions if the platform doesn't have them. */ #ifndef HAVE_STRLCPY -#include "strlcpy.c" +#include "ext/strlcpy.c" #endif #ifndef HAVE_STRLCAT -#include "strlcat.c" +#include "ext/strlcat.c" #endif #include <stdlib.h> diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index f934f66f02..0c4e399008 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -212,21 +212,6 @@ strcmpstart(const char *s1, const char *s2) return strncmp(s1, s2, n); } -/** Compare the s1_len-byte string <b>s1</b> with <b>s2</b>, - * without depending on a terminating nul in s1. Sorting order is first by - * length, then lexically; return values are as for strcmp. - */ -int -strcmp_len(const char *s1, const char *s2, size_t s1_len) -{ - size_t s2_len = strlen(s2); - if (s1_len < s2_len) - return -1; - if (s1_len > s2_len) - return 1; - return fast_memcmp(s1, s2, s2_len); -} - /** Compares the first strlen(s2) characters of s1 with s2. Returns as for * strcasecmp. */ @@ -541,3 +526,16 @@ string_is_utf8(const char *str, size_t len) } return true; } + +/** As string_is_utf8(), but returns false if the string begins with a UTF-8 + * byte order mark (BOM). + */ +int +string_is_utf8_no_bom(const char *str, size_t len) +{ + if (len >= 3 && (!strcmpstart(str, "\uFEFF") || + !strcmpstart(str, "\uFFFE"))) { + return false; + } + return string_is_utf8(str, len); +} diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index d9fbf8c61e..da4fab159c 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -33,7 +33,6 @@ int tor_strisnonupper(const char *s); int tor_strisspace(const char *s); int strcmp_opt(const char *s1, const char *s2); int strcmpstart(const char *s1, const char *s2); -int strcmp_len(const char *s1, const char *s2, size_t len); int strcasecmpstart(const char *s1, const char *s2); int strcmpend(const char *s1, const char *s2); int strcasecmpend(const char *s1, const char *s2); @@ -53,5 +52,6 @@ const char *find_str_at_start_of_line(const char *haystack, int string_is_C_identifier(const char *string); int string_is_utf8(const char *str, size_t len); +int string_is_utf8_no_bom(const char *str, size_t len); #endif /* !defined(TOR_UTIL_STRING_H) */ diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include new file mode 100644 index 0000000000..2b06e8519c --- /dev/null +++ b/src/lib/subsys/.may_include @@ -0,0 +1 @@ +orconfig.h diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am new file mode 100644 index 0000000000..4741126b14 --- /dev/null +++ b/src/lib/subsys/include.am @@ -0,0 +1,3 @@ + +noinst_HEADERS += \ + src/lib/subsys/subsys.h diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h new file mode 100644 index 0000000000..241ad7829c --- /dev/null +++ b/src/lib/subsys/subsys.h @@ -0,0 +1,95 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYS_T +#define TOR_SUBSYS_T + +#include <stdbool.h> + +struct dispatch_connector_t; + +/** + * A subsystem is a part of Tor that is initialized, shut down, configured, + * and connected to other parts of Tor. + * + * All callbacks are optional -- if a callback is set to NULL, the subsystem + * manager will treat it as a no-op. + * + * You should use c99 named-field initializers with this structure: we + * will be adding more fields, often in the middle of the structure. + **/ +typedef struct subsys_fns_t { + /** + * The name of this subsystem. It should be a programmer-readable + * identifier. + **/ + const char *name; + + /** + * Whether this subsystem is supported -- that is, whether it is compiled + * into Tor. For most subsystems, this should be true. + **/ + bool supported; + + /** + * The 'initialization level' for the subsystem. It should run from -100 + * through +100. The subsystems are initialized from lowest level to + * highest, and shut down from highest level to lowest. + **/ + int level; + + /** + * Initialize any global components of this subsystem. + * + * This function MAY rely on any lower-level subsystem being initialized. + * + * This function MUST NOT rely on any runtime configuration information; + * it is only for global state or pre-configuration state. + * + * (If you need to do any setup that depends on configuration, you'll need + * to declare a configuration callback. (Not yet designed)) + * + * This function MUST NOT have any parts that can fail. + **/ + int (*initialize)(void); + + /** + * Connect a subsystem to the message dispatch system. + **/ + int (*add_pubsub)(struct dispatch_connector_t *); + + /** + * Perform any necessary pre-fork cleanup. This function may not fail. + */ + void (*prefork)(void); + + /** + * Perform any necessary post-fork setup. This function may not fail. + */ + void (*postfork)(void); + + /** + * Free any thread-local resources held by this subsystem. Called before + * the thread exits. + */ + void (*thread_cleanup)(void); + + /** + * Free all resources held by this subsystem. + * + * This function is not allowed to fail. + **/ + void (*shutdown)(void); + +} subsys_fns_t; + +#define MIN_SUBSYS_LEVEL -100 +#define MAX_SUBSYS_LEVEL 100 + +/* All tor "libraries" (in src/libs) should have a subsystem level equal to or + * less than this value. */ +#define SUBSYS_LEVEL_LIBS -10 + +#endif diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include index c93a06e59e..306fa57b7a 100644 --- a/src/lib/term/.may_include +++ b/src/lib/term/.may_include @@ -5,5 +5,4 @@ lib/log/*.h lib/term/*.h lib/malloc/*.h -# From src/ext -tor_readpassphrase.h +ext/tor_readpassphrase.h diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c index c6459f250f..8741344acf 100644 --- a/src/lib/term/getpass.c +++ b/src/lib/term/getpass.c @@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #elif defined(HAVE_READPASSPHRASE_H) #include <readpassphrase.h> #else -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #endif /* defined(_WIN32) || ... */ #include <stdlib.h> diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include index fc56f46836..02711348c5 100644 --- a/src/lib/thread/.may_include +++ b/src/lib/thread/.may_include @@ -2,6 +2,7 @@ orconfig.h lib/cc/*.h lib/lock/*.h lib/log/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h lib/wallclock/*.h diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 94ab021c52..35cfeba64c 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -14,9 +14,11 @@ #include "orconfig.h" #include <stdlib.h> #include "lib/thread/threads.h" +#include "lib/thread/thread_sys.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" +#include "lib/subsys/subsys.h" #include <string.h> @@ -109,3 +111,17 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return oldval; } #endif /* !defined(HAVE_WORKING_STDATOMIC) */ + +static int +subsys_threads_initialize(void) +{ + tor_threads_init(); + return 0; +} + +const subsys_fns_t sys_threads = { + .name = "threads", + .supported = true, + .level = -95, + .initialize = subsys_threads_initialize, +}; diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am index 9ec23d166e..695795a2c8 100644 --- a/src/lib/thread/include.am +++ b/src/lib/thread/include.am @@ -23,5 +23,6 @@ src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ - src/lib/thread/threads.h \ - src/lib/thread/numcpus.h + src/lib/thread/numcpus.h \ + src/lib/thread/thread_sys.h \ + src/lib/thread/threads.h diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h new file mode 100644 index 0000000000..c0daf2b5e9 --- /dev/null +++ b/src/lib/thread/thread_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file threads_sys.h + * \brief Declare subsystem object for threads library + **/ + +#ifndef TOR_THREADS_SYS_H +#define TOR_THREADS_SYS_H + +extern const struct subsys_fns_t sys_threads; + +#endif /* !defined(TOR_THREADS_SYS_H) */ diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include index 2c7e37a836..ae01431b60 100644 --- a/src/lib/time/.may_include +++ b/src/lib/time/.may_include @@ -4,8 +4,10 @@ lib/cc/*.h lib/err/*.h lib/intmath/*.h lib/log/*.h +lib/subsys/*.h lib/time/*.h lib/wallclock/*.h +lib/defs/time.h # For load_windows_system_lib. lib/fs/winlib.h
\ No newline at end of file diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index 98854bad2c..a047d6c2cd 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -522,7 +522,9 @@ monotime_init_internal(void) GetTickCount64_fn = (GetTickCount64_fn_t) (void(*)(void)) GetProcAddress(h, "GetTickCount64"); } - // FreeLibrary(h) ? + // We can't call FreeLibrary(h) here, because freeing the handle may + // unload the library, and cause future calls to GetTickCount64_fn() + // to fail. See 29642 for details. } void @@ -787,8 +789,8 @@ monotime_absolute_nsec(void) return monotime_diff_nsec(&initialized_at, &now); } -uint64_t -monotime_absolute_usec(void) +MOCK_IMPL(uint64_t, +monotime_absolute_usec,(void)) { return monotime_absolute_nsec() / 1000; } diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 480d426ac7..2cd4b3bee3 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -15,6 +15,102 @@ * of tens of milliseconds. */ +/* Q: Should you use monotime or monotime_coarse as your source? + * + * A: Generally, you get better precision with monotime, but better + * performance with monotime_coarse. + * + * Q: Should you use monotime_t or monotime_coarse_t directly? Should you use + * usec? msec? "stamp units?" + * + * A: Using monotime_t and monotime_coarse_t directly is most time-efficient, + * since no conversion needs to happen. But they can potentially use more + * memory than you would need for a usec/msec/"stamp unit" count. + * + * Converting to usec or msec on some platforms, and working with them in + * general, creates a risk of doing a 64-bit division. 64-bit division is + * expensive on 32-bit platforms, which still do exist. + * + * The "stamp unit" type is designed to give a type that is cheap to convert + * from monotime_coarse, has resolution of about 1-2ms, and fits nicely in a + * 32-bit integer. Its downside is that it does not correspond directly + * to a natural unit of time. + * + * There is not much point in using "coarse usec" or "coarse nsec", since the + * current coarse monotime implementations give you on the order of + * milliseconds of precision. + * + * Q: So, what backends is monotime_coarse using? + * + * A: Generally speaking, it uses "whatever monotonic-ish time implemenation + * does not require a context switch." The various implementations provide + * this by having a view of the current time in a read-only memory page that + * is updated with a frequency corresponding to the kernel's tick count. + * + * On Windows, monotime_coarse uses GetCount64() [or GetTickCount() on + * obsolete systems]. MSDN claims that the resolution is "typically in the + * range of 10-16 msec", but it has said that for years. Storing + * monotime_coarse_t uses 8 bytes. + * + * On OSX/iOS, monotime_coarse uses uses mach_approximate_time() where + * available, and falls back to regular monotime. The precision is not + * documented, but the implementation is open-source: it reads from a page + * that the kernel updates. Storing monotime_coarse_t uses 8 bytes. + * + * On unixy systems, monotime_coarse uses clock_gettime() with + * CLOCK_MONOTONIC_COARSE where available, and falls back to CLOCK_MONOTONIC. + * It typically uses vdso tricks to read from a page that the kernel updates. + * Its precision fixed, but you can get it with clock_getres(): on my Linux + * desktop, it claims to be 1 msec, but it will depend on the system HZ + * setting. Storing monotime_coarse_t uses 16 bytes. + * + * [TODO: Try CLOCK_MONOTONIC_FAST on foobsd.] + * + * Q: What backends is regular monotonic time using? + * + * A: In general, regular monotime uses something that requires a system call. + * On platforms where system calls are cheap, you win! Otherwise, you lose. + * + * On Windows, monotonic time uses QuereyPerformanceCounter. Storing + * monotime_t costs 8 bytes. + * + * On OSX/Apple, monotonic time uses mach_absolute_time. Storing + * monotime_t costs 8 bytes. + * + * On unixy systems, monotonic time uses CLOCK_MONOTONIC. Storing + * monotime_t costs 16 bytes. + * + * Q: Tell me about the costs of converting to a 64-bit nsec, usec, or msec + * count. + * + * A: Windows, coarse: Cheap, since it's all multiplication. + * + * Windows, precise: Expensive on 32-bit: it needs 64-bit division. + * + * Apple, all: Expensive on 32-bit: it needs 64-bit division. + * + * Unixy, all: Fairly cheap, since the only division required is dividing + * tv_nsec 1000, and nanoseconds-per-second fits in a 32-bit value. + * + * All, "timestamp units": Cheap everywhere: it never divides. + * + * Q: This is only somewhat related, but how much precision could I hope for + * from a libevent time.? + * + * A: Actually, it's _very_ related if you're timing in order to have a + * timeout happen. + * + * On Windows, it uses select: you could in theory have a microsecond + * resolution, but it usually isn't that accurate. + * + * On OSX, iOS, and BSD, you have kqueue: You could in theory have a nanosecond + * resolution, but it usually isn't that accurate. + * + * On Linux, you have epoll: It has a millisecond resolution. Some recent + * Libevents can also use timerfd for higher resolution if + * EVENT_BASE_FLAG_PRECISE_TIMER is set: Tor doesn't set that flag. + */ + #ifndef TOR_COMPAT_TIME_H #define TOR_COMPAT_TIME_H @@ -103,7 +199,7 @@ uint64_t monotime_absolute_nsec(void); /** * Return the number of microseconds since the timer system was initialized. */ -uint64_t monotime_absolute_usec(void); +MOCK_DECL(uint64_t, monotime_absolute_usec,(void)); /** * Return the number of milliseconds since the timer system was initialized. */ diff --git a/src/lib/time/include.am b/src/lib/time/include.am index a3f93a3744..dae16f49ac 100644 --- a/src/lib/time/include.am +++ b/src/lib/time/include.am @@ -7,6 +7,7 @@ endif src_lib_libtor_time_a_SOURCES = \ src/lib/time/compat_time.c \ + src/lib/time/time_sys.c \ src/lib/time/tvdiff.c src_lib_libtor_time_testing_a_SOURCES = \ @@ -16,4 +17,5 @@ src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) noinst_HEADERS += \ src/lib/time/compat_time.h \ + src/lib/time/time_sys.h \ src/lib/time/tvdiff.h diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c new file mode 100644 index 0000000000..b3feb7b46a --- /dev/null +++ b/src/lib/time/time_sys.c @@ -0,0 +1,26 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.c + * \brief Subsystem object for monotime setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/time/time_sys.h" +#include "lib/time/compat_time.h" + +static int +subsys_time_initialize(void) +{ + monotime_init(); + return 0; +} + +const subsys_fns_t sys_time = { + .name = "time", + .level = -90, + .supported = true, + .initialize = subsys_time_initialize, +}; diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h new file mode 100644 index 0000000000..6a860ffd08 --- /dev/null +++ b/src/lib/time/time_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.h + * \brief Declare subsystem object for the time module. + **/ + +#ifndef TOR_TIME_SYS_H +#define TOR_TIME_SYS_H + +extern const struct subsys_fns_t sys_time; + +#endif /* !defined(TOR_TIME_SYS_H) */ diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c index a87d0d96dc..d7c245f57a 100644 --- a/src/lib/time/tvdiff.c +++ b/src/lib/time/tvdiff.c @@ -11,6 +11,7 @@ #include "lib/time/tvdiff.h" #include "lib/cc/compat_compiler.h" +#include "lib/defs/time.h" #include "lib/log/log.h" #ifdef _WIN32 @@ -20,8 +21,6 @@ #include <sys/time.h> #endif -#define TOR_USEC_PER_SEC 1000000 - /** Return the difference between start->tv_sec and end->tv_sec. * Returns INT64_MAX on overflow and underflow. */ diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 2840e590b8..c550bde024 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -1,6 +1,7 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/crypt_ops/*.h @@ -11,7 +12,7 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/subsys/*.h +lib/testsupport/*.h lib/tls/*.h - -ciphers.inc +lib/tls/*.inc diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c index e92cb9163f..bf03b61459 100644 --- a/src/lib/tls/buffers_tls.c +++ b/src/lib/tls/buffers_tls.c @@ -12,7 +12,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include <stddef.h> -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/cc/torint.h" #include "lib/log/log.h" diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am index a664b29fb2..1817739eef 100644 --- a/src/lib/tls/include.am +++ b/src/lib/tls/include.am @@ -36,5 +36,6 @@ noinst_HEADERS += \ src/lib/tls/tortls.h \ src/lib/tls/tortls_internal.h \ src/lib/tls/tortls_st.h \ + src/lib/tls/tortls_sys.h \ src/lib/tls/x509.h \ src/lib/tls/x509_internal.h diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c index 4ca7c7d9d3..1aff40c437 100644 --- a/src/lib/tls/tortls.c +++ b/src/lib/tls/tortls.c @@ -7,6 +7,7 @@ #define TOR_X509_PRIVATE #include "lib/tls/x509.h" #include "lib/tls/x509_internal.h" +#include "lib/tls/tortls_sys.h" #include "lib/tls/tortls.h" #include "lib/tls/tortls_st.h" #include "lib/tls/tortls_internal.h" @@ -15,6 +16,7 @@ #include "lib/crypt_ops/crypto_rsa.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/net/socket.h" +#include "lib/subsys/subsys.h" #ifdef _WIN32 #include <winsock2.h> @@ -440,3 +442,15 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity) return rv; } + +static void +subsys_tortls_shutdown(void) +{ + tor_tls_free_all(); +} + +const subsys_fns_t sys_tortls = { + .name = "tortls", + .level = -50, + .shutdown = subsys_tortls_shutdown +}; diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index 80b0df301f..b40f948a3b 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -464,7 +464,7 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /** List of ciphers that clients should advertise, omitting items that * our OpenSSL doesn't know about. */ static const char CLIENT_CIPHER_LIST[] = -#include "ciphers.inc" +#include "lib/tls/ciphers.inc" /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version * of any cipher we say. */ "!SSLv2" diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h new file mode 100644 index 0000000000..4b04f85f0c --- /dev/null +++ b/src/lib/tls/tortls_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tortls_sys.h + * \brief Declare subsystem object for the tortls module + **/ + +#ifndef TOR_TORTLS_SYS_H +#define TOR_TORTLS_SYS_H + +extern const struct subsys_fns_t sys_tortls; + +#endif /* !defined(TOR_TORTLS_SYS_H) */ diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include new file mode 100644 index 0000000000..d159ceb41f --- /dev/null +++ b/src/lib/version/.may_include @@ -0,0 +1,3 @@ +orconfig.h +micro-revision.i +lib/version/*.h
\ No newline at end of file diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c index 7d27549cad..900a1e12a0 100644 --- a/src/lib/log/git_revision.c +++ b/src/lib/version/git_revision.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "orconfig.h" -#include "lib/log/git_revision.h" +#include "lib/version/git_revision.h" /** String describing which Tor Git repository version the source was * built from. This string is generated by a bit of shell kludging in diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h index 79e3c6684b..79e3c6684b 100644 --- a/src/lib/log/git_revision.h +++ b/src/lib/version/git_revision.h diff --git a/src/lib/version/include.am b/src/lib/version/include.am new file mode 100644 index 0000000000..6944eb05e3 --- /dev/null +++ b/src/lib/version/include.am @@ -0,0 +1,25 @@ + +noinst_LIBRARIES += src/lib/libtor-version.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-version-testing.a +endif + +src_lib_libtor_version_a_SOURCES = \ + src/lib/version/git_revision.c \ + src/lib/version/version.c + +src_lib_libtor_version_testing_a_SOURCES = \ + $(src_lib_libtor_version_a_SOURCES) +src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# Declare that these object files depend on micro-revision.i. Without this +# rule, we could try to build them before micro-revision.i was created. +src/lib/version/git_revision.$(OBJEXT) \ + src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \ + micro-revision.i + +noinst_HEADERS += \ + src/lib/version/git_revision.h \ + src/lib/version/torversion.h diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h new file mode 100644 index 0000000000..7b0fb66ec0 --- /dev/null +++ b/src/lib/version/torversion.h @@ -0,0 +1,12 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_VERSION_H +#define TOR_VERSION_H + +const char *get_version(void); +const char *get_short_version(void); + +#endif /* !defined(TOR_VERSION_H) */ diff --git a/src/lib/version/version.c b/src/lib/version/version.c new file mode 100644 index 0000000000..434e6fb424 --- /dev/null +++ b/src/lib/version/version.c @@ -0,0 +1,50 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "lib/version/torversion.h" +#include "lib/version/git_revision.h" + +#include <stdio.h> +#include <string.h> + +/** A shorter version of this Tor process's version, for export in our router + * descriptor. (Does not include the git version, if any.) */ +static const char the_short_tor_version[] = + VERSION +#ifdef TOR_BUILD_TAG + " ("TOR_BUILD_TAG")" +#endif + ""; + +#define MAX_VERSION_LEN 128 + +/** The version of this Tor process, possibly including git version */ +static char the_tor_version[MAX_VERSION_LEN] = ""; + +/** Return the current Tor version. */ +const char * +get_version(void) +{ + if (the_tor_version[0] == 0) { + if (strlen(tor_git_revision)) { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s (git-%s)", the_short_tor_version, tor_git_revision); + } else { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s", the_short_tor_version); + } + the_tor_version[sizeof(the_tor_version)-1] = 0; + } + + return the_tor_version; +} + +/** Return the current Tor version, without any git tag. */ +const char * +get_short_version(void) +{ + return the_short_tor_version; +} diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include index dc010da063..ce7a26472b 100644 --- a/src/lib/wallclock/.may_include +++ b/src/lib/wallclock/.may_include @@ -3,4 +3,5 @@ lib/cc/*.h lib/err/*.h lib/wallclock/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index ee498702d5..7b32804026 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -9,7 +9,9 @@ **/ #include "orconfig.h" +#include "lib/subsys/subsys.h" #include "lib/wallclock/approx_time.h" +#include "lib/wallclock/wallclock_sys.h" /* ===== * Cached time @@ -41,3 +43,17 @@ update_approx_time(time_t now) cached_approx_time = now; } #endif /* !defined(TIME_IS_FAST) */ + +static int +subsys_wallclock_initialize(void) +{ + update_approx_time(time(NULL)); + return 0; +} + +const subsys_fns_t sys_wallclock = { + .name = "wallclock", + .supported = true, + .level = -99, + .initialize = subsys_wallclock_initialize, +}; diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am index 1961639bd7..2351252e0c 100644 --- a/src/lib/wallclock/include.am +++ b/src/lib/wallclock/include.am @@ -19,4 +19,5 @@ noinst_HEADERS += \ src/lib/wallclock/approx_time.h \ src/lib/wallclock/timeval.h \ src/lib/wallclock/time_to_tm.h \ - src/lib/wallclock/tor_gettimeofday.h + src/lib/wallclock/tor_gettimeofday.h \ + src/lib/wallclock/wallclock_sys.h diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h new file mode 100644 index 0000000000..a30912b8fb --- /dev/null +++ b/src/lib/wallclock/wallclock_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file wallclock_sys.h + * \brief Declare subsystem object for the wallclock module. + **/ + +#ifndef TOR_WALLCLOCK_SYS_H +#define TOR_WALLCLOCK_SYS_H + +extern const struct subsys_fns_t sys_wallclock; + +#endif /* !defined(TOR_WALLCLOCK_SYS_H) */ diff --git a/src/rust/build.rs b/src/rust/build.rs index 123d5c0682..5626b35f75 100644 --- a/src/rust/build.rs +++ b/src/rust/build.rs @@ -149,8 +149,9 @@ pub fn main() { cfg.component("tor-sandbox-testing"); cfg.component("tor-encoding-testing"); cfg.component("tor-fs-testing"); - cfg.component("tor-time-testing"); cfg.component("tor-net-testing"); + cfg.component("tor-buf-testing"); + cfg.component("tor-time-testing"); cfg.component("tor-thread-testing"); cfg.component("tor-memarea-testing"); cfg.component("tor-log-testing"); @@ -162,6 +163,7 @@ pub fn main() { cfg.component("tor-malloc"); cfg.component("tor-wallclock"); cfg.component("tor-err-testing"); + cfg.component("tor-version-testing"); cfg.component("tor-intmath-testing"); cfg.component("tor-ctime-testing"); cfg.component("curve25519_donna"); diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs index 6ee63adb10..066b08eddb 100644 --- a/src/rust/protover/ffi.rs +++ b/src/rust/protover/ffi.rs @@ -30,6 +30,7 @@ fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> { 7 => Ok(Protocol::Desc), 8 => Ok(Protocol::Microdesc), 9 => Ok(Protocol::Cons), + 10 => Ok(Protocol::Padding), _ => Err(ProtoverError::UnknownProtocol), } } diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs index 2661d811c4..74158d9f6d 100644 --- a/src/rust/protover/protover.rs +++ b/src/rust/protover/protover.rs @@ -46,6 +46,7 @@ pub enum Protocol { LinkAuth, Microdesc, Relay, + Padding, } impl fmt::Display for Protocol { @@ -73,6 +74,7 @@ impl FromStr for Protocol { "LinkAuth" => Ok(Protocol::LinkAuth), "Microdesc" => Ok(Protocol::Microdesc), "Relay" => Ok(Protocol::Relay), + "Padding" => Ok(Protocol::Padding), _ => Err(ProtoverError::UnknownProtocol), } } @@ -163,7 +165,8 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { Link=1-5 \ LinkAuth=3 \ Microdesc=1-2 \ - Relay=1-2" + Relay=1-2 \ + Padding=1" ) } else { cstr!( @@ -176,7 +179,8 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { Link=1-5 \ LinkAuth=1,3 \ Microdesc=1-2 \ - Relay=1-2" + Relay=1-2 \ + Padding=1" ) } } diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs index 2a19458f46..ede42c6ea8 100644 --- a/src/rust/tor_util/strings.rs +++ b/src/rust/tor_util/strings.rs @@ -105,11 +105,7 @@ macro_rules! cstr { ($($bytes:expr),*) => ( ::std::ffi::CStr::from_bytes_with_nul( concat!($($bytes),*, "\0").as_bytes() - ).unwrap_or( - unsafe{ - ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0") - } - ) + ).unwrap_or_default() ) } diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake index cfbe281b94..ca6a84cf8a 100644 --- a/src/test/Makefile.nmake +++ b/src/test/Makefile.nmake @@ -1,4 +1,4 @@ -all: test.exe test-child.exe bench.exe +all: test.exe bench.exe CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common /I ..\or \ /I ..\ext @@ -19,6 +19,7 @@ TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \ test_cell_formats.obj test_relay.obj test_replay.obj \ test_channelpadding.obj \ test_circuitstats.obj \ + test_circuitpadding.obj \ test_scheduler.obj test_introduce.obj test_hs.obj tinytest.obj tinytest.obj: ..\ext\tinytest.c @@ -30,8 +31,5 @@ test.exe: $(TEST_OBJECTS) bench.exe: bench.obj $(CC) $(CFLAGS) bench.obj $(LIBS) ..\common\*.lib /Fe$@ -test-child.exe: test-child.obj - $(CC) $(CFLAGS) test-child.obj /Fe$@ - clean: - del *.obj *.lib test.exe bench.exe test-child.exe + del *.obj *.lib test.exe bench.exe diff --git a/src/test/bench.c b/src/test/bench.c index 06c616c3b0..65fa617cbd 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -14,6 +14,8 @@ #include "core/crypto/onion_tap.h" #include "core/crypto/relay_crypto.h" +#include "lib/intmath/weakrng.h" + #ifdef ENABLE_OPENSSL #include <openssl/opensslv.h> #include <openssl/evp.h> @@ -24,6 +26,7 @@ #include "core/or/circuitlist.h" #include "app/config/config.h" +#include "app/main/subsysmgr.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_dh.h" #include "core/crypto/onion_ntor.h" @@ -38,6 +41,9 @@ #include "lib/crypt_ops/digestset.h" #include "lib/crypt_ops/crypto_init.h" +#include "feature/dirparse/microdesc_parse.h" +#include "feature/nodelist/microdesc.h" + #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) static uint64_t nanostart; static inline uint64_t @@ -332,6 +338,65 @@ bench_ed25519(void) } static void +bench_rand_len(int len) +{ + const int N = 100000; + int i; + char *buf = tor_malloc(len); + uint64_t start,end; + + start = perftime(); + for (i = 0; i < N; ++i) { + crypto_rand(buf, len); + } + end = perftime(); + printf("crypto_rand(%d): %f nsec.\n", len, NANOCOUNT(start,end,N)); + + crypto_fast_rng_t *fr = crypto_fast_rng_new(); + start = perftime(); + for (i = 0; i < N; ++i) { + crypto_fast_rng_getbytes(fr,(uint8_t*)buf,len); + } + end = perftime(); + printf("crypto_fast_rng_getbytes(%d): %f nsec.\n", len, + NANOCOUNT(start,end,N)); + crypto_fast_rng_free(fr); + + if (len <= 32) { + start = perftime(); + for (i = 0; i < N; ++i) { + crypto_strongest_rand((uint8_t*)buf, len); + } + end = perftime(); + printf("crypto_strongest_rand(%d): %f nsec.\n", len, + NANOCOUNT(start,end,N)); + } + + if (len == 4) { + tor_weak_rng_t weak; + tor_init_weak_random(&weak, 1337); + + start = perftime(); + uint32_t t=0; + for (i = 0; i < N; ++i) { + t += tor_weak_random(&weak); + } + end = perftime(); + printf("weak_rand(4): %f nsec.\n", NANOCOUNT(start,end,N)); + } + + tor_free(buf); +} + +static void +bench_rand(void) +{ + bench_rand_len(4); + bench_rand_len(16); + bench_rand_len(128); +} + +static void bench_cell_aes(void) { uint64_t start, end; @@ -638,6 +703,41 @@ bench_ecdh_p224(void) } #endif +static void +bench_md_parse(void) +{ + uint64_t start, end; + const int N = 100000; + // selected arbitrarily + const char md_text[] = + "@last-listed 2018-12-14 18:14:14\n" + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMHkZeXNDX/49JqM2BVLmh1Fnb5iMVnatvZZTLJyedqDLkbXZ1WKP5oh\n" + "7ec14dj/k3ntpwHD4s2o3Lb6nfagWbug4+F/rNJ7JuFru/PSyOvDyHGNAuegOXph\n" + "3gTGjdDpv/yPoiadGebbVe8E7n6hO+XxM2W/4dqheKimF0/s9B7HAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key QgF/EjqlNG1wRHLIop/nCekEH+ETGZSgYOhu26eiTF4=\n" + "family $00E9A86E7733240E60D8435A7BBD634A23894098 " + "$329BD7545DEEEBBDC8C4285F243916F248972102 " + "$69E06EBB2573A4F89330BDF8BC869794A3E10E4D " + "$DCA2A3FAE50B3729DAA15BC95FB21AF03389818B\n" + "p accept 53,80,443,5222-5223,25565\n" + "id ed25519 BzffzY99z6Q8KltcFlUTLWjNTBU7yKK+uQhyi1Ivb3A\n"; + + reset_perftime(); + start = perftime(); + for (int i = 0; i < N; ++i) { + smartlist_t *s = microdescs_parse_from_string(md_text, NULL, 1, + SAVED_IN_CACHE, NULL); + SMARTLIST_FOREACH(s, microdesc_t *, md, microdesc_free(md)); + smartlist_free(s); + } + + end = perftime(); + printf("Microdesc parse: %f nsec\n", NANOCOUNT(start, end, N)); +} + typedef void (*bench_fn)(void); typedef struct benchmark_t { @@ -656,6 +756,7 @@ static struct benchmark_t benchmarks[] = { ENT(onion_TAP), ENT(onion_ntor), ENT(ed25519), + ENT(rand), ENT(cell_aes), ENT(cell_ops), @@ -665,6 +766,8 @@ static struct benchmark_t benchmarks[] = { ENT(ecdh_p256), ENT(ecdh_p224), #endif + + ENT(md_parse), {NULL,NULL,0} }; @@ -690,9 +793,10 @@ main(int argc, const char **argv) char *errmsg; or_options_t *options; - tor_threads_init(); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); + flush_log_messages_from_startup(); + tor_compress_init(); - init_logging(1); if (argc == 4 && !strcmp(argv[1], "diff")) { const int N = 200; @@ -702,11 +806,13 @@ main(int argc, const char **argv) perror("X"); return 1; } + size_t f1len = strlen(f1); + size_t f2len = strlen(f2); for (i = 0; i < N; ++i) { - char *diff = consensus_diff_generate(f1, f2); + char *diff = consensus_diff_generate(f1, f1len, f2, f2len); tor_free(diff); } - char *diff = consensus_diff_generate(f1, f2); + char *diff = consensus_diff_generate(f1, f1len, f2, f2len); printf("%s", diff); tor_free(f1); tor_free(f2); @@ -737,7 +843,6 @@ main(int argc, const char **argv) init_protocol_warning_severity_level(); options = options_new(); - init_logging(1); options->command = CMD_RUN_UNITTESTS; options->DataDirectory = tor_strdup(""); options->KeyDirectory = tor_strdup(""); diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c index 5947a3f48c..656ef0bdb2 100644 --- a/src/test/fuzz/fuzz_consensus.c +++ b/src/test/fuzz/fuzz_consensus.c @@ -61,13 +61,13 @@ int fuzz_main(const uint8_t *data, size_t sz) { networkstatus_t *ns; - char *str = tor_memdup_nulterm(data, sz); const char *eos = NULL; networkstatus_type_t tp = NS_TYPE_CONSENSUS; if (tor_memstr(data, MIN(sz, 1024), "tus vote")) tp = NS_TYPE_VOTE; const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote"; - ns = networkstatus_parse_vote_from_string(str, + ns = networkstatus_parse_vote_from_string((const char *)data, + sz, &eos, tp); if (ns) { @@ -76,6 +76,6 @@ fuzz_main(const uint8_t *data, size_t sz) } else { log_debug(LD_GENERAL, "Parsing as %s failed", what); } - tor_free(str); + return 0; } diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c index 1bc60e50ee..a31445666c 100644 --- a/src/test/fuzz/fuzz_diff.c +++ b/src/test/fuzz/fuzz_diff.c @@ -10,9 +10,11 @@ #include "test/fuzz/fuzzing.h" static int -mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +mock_consensus_compute_digest_(const char *c, size_t len, + consensus_digest_t *d) { (void)c; + (void)len; memset(d->sha3_256, 3, sizeof(d->sha3_256)); return 0; } @@ -42,28 +44,34 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) if (! separator) return 0; size_t c1_len = separator - stdin_buf; - char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + const char *c1 = (const char *)stdin_buf; size_t c2_len = data_size - c1_len - SEPLEN; - char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + const char *c2 = (const char *)separator + SEPLEN; - char *c3 = consensus_diff_generate(c1, c2); + const char *cp = memchr(c1, 0, c1_len); + if (cp) + c1_len = cp - c1; + + cp = memchr(c2, 0, c2_len); + if (cp) + c2_len = cp - c2; + + char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len); if (c3) { - char *c4 = consensus_diff_apply(c1, c3); + char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3)); tor_assert(c4); - if (strcmp(c2, c4)) { - printf("%s\n", escaped(c1)); - printf("%s\n", escaped(c2)); + int equal = (c2_len == strlen(c4)) && fast_memeq(c2, c4, c2_len); + if (! equal) { + //printf("%s\n", escaped(c1)); + //printf("%s\n", escaped(c2)); printf("%s\n", escaped(c3)); printf("%s\n", escaped(c4)); } - tor_assert(! strcmp(c2, c4)); + tor_assert(equal); tor_free(c3); tor_free(c4); } - tor_free(c1); - tor_free(c2); return 0; } - diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c index 9bd3cb0bf8..d8a0f9e590 100644 --- a/src/test/fuzz/fuzz_diff_apply.c +++ b/src/test/fuzz/fuzz_diff_apply.c @@ -10,9 +10,11 @@ #include "test/fuzz/fuzzing.h" static int -mock_consensus_compute_digest_(const char *c, consensus_digest_t *d) +mock_consensus_compute_digest_(const char *c, size_t len, + consensus_digest_t *d) { (void)c; + (void)len; memset(d->sha3_256, 3, sizeof(d->sha3_256)); return 0; } @@ -50,16 +52,13 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size) if (! separator) return 0; size_t c1_len = separator - stdin_buf; - char *c1 = tor_memdup_nulterm(stdin_buf, c1_len); + const char *c1 = (const char *)stdin_buf; size_t c2_len = data_size - c1_len - SEPLEN; - char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len); + const char *c2 = (const char *)separator + SEPLEN; - char *c3 = consensus_diff_apply(c1, c2); + char *c3 = consensus_diff_apply(c1, c1_len, c2, c2_len); - tor_free(c1); - tor_free(c2); tor_free(c3); return 0; } - diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c index 2798c47d23..44393b3a10 100644 --- a/src/test/fuzz/fuzz_http.c +++ b/src/test/fuzz/fuzz_http.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "feature/dircache/dircache.h" diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c index a60fc36804..2a597cae74 100644 --- a/src/test/fuzz/fuzz_http_connect.c +++ b/src/test/fuzz/fuzz_http_connect.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/or/connection_edge.h" diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c index 06cb08391e..d6c416a0f9 100644 --- a/src/test/fuzz/fuzz_socks.c +++ b/src/test/fuzz/fuzz_socks.c @@ -6,7 +6,7 @@ #define BUFFERS_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/err/backtrace.h" #include "lib/log/log.h" #include "core/proto/proto_socks.h" diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c new file mode 100644 index 0000000000..64a6453050 --- /dev/null +++ b/src/test/fuzz/fuzz_strops.c @@ -0,0 +1,247 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file fuzz_strops.c + * \brief Fuzzers for various string encoding/decoding operations + **/ + +#include "orconfig.h" + +#include "lib/cc/torint.h" +#include "lib/ctime/di_ops.h" +#include "lib/encoding/binascii.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/encoding/confline.h" +#include "lib/malloc/malloc.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" +#include "lib/intmath/muldiv.h" + +#include "test/fuzz/fuzzing.h" + +#include <stdio.h> +#include <string.h> + +int +fuzz_init(void) +{ + return 0; +} + +int +fuzz_cleanup(void) +{ + return 0; +} + +typedef struct chunk_t { + uint8_t *buf; + size_t len; +} chunk_t; + +#define chunk_free(ch) \ + FREE_AND_NULL(chunk_t, chunk_free_, (ch)) + +static chunk_t * +chunk_new(size_t len) +{ + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + ch->buf = tor_malloc(len); + ch->len = len; + return ch; +} +static void +chunk_free_(chunk_t *ch) +{ + if (!ch) + return; + tor_free(ch->buf); + tor_free(ch); +} +static bool +chunk_eq(const chunk_t *a, const chunk_t *b) +{ + return a->len == b->len && fast_memeq(a->buf, b->buf, a->len); +} + +static chunk_t * +b16_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(CEIL_DIV(inp->len, 2)); + int r = base16_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b16_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len * 2 + 1); + base16_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len); + return ch; +} + +#if 0 +static chunk_t * +b32_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len);//XXXX + int r = base32_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; // XXXX we need some way to get the actual length of + // XXXX the output here. + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b32_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(base32_encoded_size(inp->len)); + base32_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len); + ch->len = strlen((char *) ch->buf); + return ch; +} +#endif + +static chunk_t * +b64_dec(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(inp->len);//XXXX This could be shorter. + int r = base64_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len); + if (r >= 0) { + ch->len = r; + } else { + chunk_free(ch); + } + return ch; +} +static chunk_t * +b64_enc(const chunk_t *inp) +{ + chunk_t *ch = chunk_new(BASE64_BUFSIZE(inp->len)); + base64_encode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len, 0); + ch->len = strlen((char *) ch->buf); + return ch; +} + +static chunk_t * +c_dec(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + char *r = NULL; + (void) unescape_string(s, &r, &ch->len); + tor_free(s); + ch->buf = (uint8_t*) r; + if (!ch->buf) { + tor_free(ch); + } + return ch; +} +static chunk_t * +c_enc(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + chunk_t *ch = tor_malloc(sizeof(chunk_t)); + ch->buf = (uint8_t*)esc_for_log(s); + tor_free(s); + ch->len = strlen((char*)ch->buf); + return ch; +} + +static int kv_flags = 0; +static config_line_t * +kv_dec(const chunk_t *inp) +{ + char *s = tor_memdup_nulterm(inp->buf, inp->len); + config_line_t *res = kvline_parse(s, kv_flags); + tor_free(s); + return res; +} +static chunk_t * +kv_enc(const config_line_t *inp) +{ + char *s = kvline_encode(inp, kv_flags); + if (!s) + return NULL; + chunk_t *res = tor_malloc(sizeof(chunk_t)); + res->buf = (uint8_t*)s; + res->len = strlen(s); + return res; +} + +/* Given an encoder function, a decoder function, and a function to free + * the decoded object, check whether any string that successfully decoded + * will then survive an encode-decode-encode round-trip unchanged. + */ +#define ENCODE_ROUNDTRIP(E,D,FREE) \ + STMT_BEGIN { \ + bool err = false; \ + a = D(&inp); \ + if (!a) \ + return 0; \ + b = E(a); \ + tor_assert(b); \ + c = D(b); \ + tor_assert(c); \ + d = E(c); \ + tor_assert(d); \ + if (!chunk_eq(b,d)) { \ + printf("Unequal chunks: %s\n", \ + hex_str((char*)b->buf, b->len)); \ + printf(" vs %s\n", \ + hex_str((char*)d->buf, d->len)); \ + err = true; \ + } \ + FREE(a); \ + chunk_free(b); \ + FREE(c); \ + chunk_free(d); \ + tor_assert(!err); \ + } STMT_END + +int +fuzz_main(const uint8_t *stdin_buf, size_t data_size) +{ + if (!data_size) + return 0; + + chunk_t inp = { (uint8_t*)stdin_buf, data_size }; + chunk_t *b=NULL,*d=NULL; + void *a=NULL,*c=NULL; + + switch (stdin_buf[0]) { + case 0: + ENCODE_ROUNDTRIP(b16_enc, b16_dec, chunk_free_); + break; + case 1: + /* + XXXX see notes above about our base-32 functions. + ENCODE_ROUNDTRIP(b32_enc, b32_dec, chunk_free_); + */ + break; + case 2: + ENCODE_ROUNDTRIP(b64_enc, b64_dec, chunk_free_); + break; + case 3: + ENCODE_ROUNDTRIP(c_enc, c_dec, chunk_free_); + break; + case 5: + kv_flags = KV_QUOTED|KV_OMIT_KEYS; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + case 6: + kv_flags = 0; + ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_); + break; + } + + return 0; +} diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c index 967397d1af..7b61b8df2d 100644 --- a/src/test/fuzz/fuzz_vrs.c +++ b/src/test/fuzz/fuzz_vrs.c @@ -3,6 +3,7 @@ #define NS_PARSE_PRIVATE #define NETWORKSTATUS_PRIVATE #include "core/or/or.h" +#include "feature/dirauth/dirvote.h" #include "feature/dirparse/ns_parse.h" #include "feature/dirparse/unparseable.h" #include "lib/memarea/memarea.h" @@ -35,9 +36,12 @@ fuzz_init(void) dummy_vote = tor_malloc_zero(sizeof(*dummy_vote)); dummy_vote->known_flags = smartlist_new(); smartlist_split_string(dummy_vote->known_flags, - "Authority BadExit Exit Fast Guard HSDir " - "NoEdConsensus Running Stable V2Dir Valid", + DIRVOTE_UNIVERSAL_FLAGS, " ", 0, 0); + smartlist_split_string(dummy_vote->known_flags, + DIRVOTE_OPTIONAL_FLAGS, + " ", 0, 0); + smartlist_sort_strings(dummy_vote->known_flags); return 0; } @@ -53,24 +57,24 @@ fuzz_cleanup(void) int fuzz_main(const uint8_t *data, size_t sz) { - char *str = tor_memdup_nulterm(data, sz); const char *s; routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL; vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs)); smartlist_t *tokens = smartlist_new(); + const char *eos = (const char *)data + sz; - s = str; - rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_ns = routerstatus_parse_entry_from_string(area, &s, eos, tokens, NULL, NULL, 26, FLAV_NS); tor_assert(smartlist_len(tokens) == 0); - s = str; - rs_md = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_md = routerstatus_parse_entry_from_string(area, &s, eos, tokens, NULL, NULL, 26, FLAV_MICRODESC); tor_assert(smartlist_len(tokens) == 0); - s = str; - rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens, + s = (const char *)data; + rs_vote = routerstatus_parse_entry_from_string(area, &s, eos, tokens, dummy_vote, vrs, 26, FLAV_NS); tor_assert(smartlist_len(tokens) == 0); @@ -82,6 +86,6 @@ fuzz_main(const uint8_t *data, size_t sz) vote_routerstatus_free(vrs); memarea_clear(area); smartlist_free(tokens); - tor_free(str); + return 0; } diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c index 8ea4898522..387c865a9b 100644 --- a/src/test/fuzz/fuzzing_common.c +++ b/src/test/fuzz/fuzzing_common.c @@ -3,12 +3,14 @@ #define CRYPTO_ED25519_PRIVATE #include "orconfig.h" #include "core/or/or.h" +#include "app/main/subsysmgr.h" #include "lib/err/backtrace.h" #include "app/config/config.h" #include "test/fuzz/fuzzing.h" #include "lib/compress/compress.h" #include "lib/crypt_ops/crypto_ed25519.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" static or_options_t *mock_options = NULL; static const or_options_t * @@ -94,12 +96,10 @@ disable_signature_checking(void) static void global_init(void) { - tor_threads_init(); - tor_compress_init(); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); + flush_log_messages_from_startup(); - /* Initialise logging first */ - init_logging(1); - configure_backtrace_handler(get_version()); + tor_compress_init(); if (crypto_global_init(0, NULL, NULL) < 0) abort(); diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 27eeced8c5..d0711f05d6 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -153,6 +153,16 @@ src_test_fuzz_fuzz_socks_LDADD = $(FUZZING_LIBS) endif if UNITTESTS_ENABLED +src_test_fuzz_fuzz_strops_SOURCES = \ + src/test/fuzz/fuzzing_common.c \ + src/test/fuzz/fuzz_strops.c +src_test_fuzz_fuzz_strops_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_strops_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_strops_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_strops_LDADD = $(FUZZING_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_fuzz_vrs_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_vrs.c @@ -176,6 +186,7 @@ FUZZERS = \ src/test/fuzz/fuzz-iptsv2 \ src/test/fuzz/fuzz-microdesc \ src/test/fuzz/fuzz-socks \ + src/test/fuzz/fuzz-strops \ src/test/fuzz/fuzz-vrs endif @@ -291,6 +302,15 @@ src_test_fuzz_lf_fuzz_socks_LDADD = $(LIBFUZZER_LIBS) endif if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_strops_SOURCES = \ + $(src_test_fuzz_fuzz_strops_SOURCES) +src_test_fuzz_lf_fuzz_strops_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_strops_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_strops_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_strops_LDADD = $(LIBFUZZER_LIBS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_vrs_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) @@ -312,6 +332,7 @@ LIBFUZZER_FUZZERS = \ src/test/fuzz/lf-fuzz-iptsv2 \ src/test/fuzz/lf-fuzz-microdesc \ src/test/fuzz/lf-fuzz-socks \ + src/test/fuzz/lf-fuzz-strops \ src/test/fuzz/lf-fuzz-vrs else @@ -406,6 +427,13 @@ src_test_fuzz_liboss_fuzz_socks_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) endif if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_strops_a_SOURCES = \ + $(src_test_fuzz_fuzz_strops_SOURCES) +src_test_fuzz_liboss_fuzz_strops_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_strops_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif + +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) @@ -425,6 +453,7 @@ OSS_FUZZ_FUZZERS = \ src/test/fuzz/liboss-fuzz-iptsv2.a \ src/test/fuzz/liboss-fuzz-microdesc.a \ src/test/fuzz/liboss-fuzz-socks.a \ + src/test/fuzz/liboss-fuzz-strops.a \ src/test/fuzz/liboss-fuzz-vrs.a else diff --git a/src/test/include.am b/src/test/include.am index ecb7689579..d585c2a38a 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -65,10 +65,11 @@ noinst_PROGRAMS+= \ src/test/test \ src/test/test-slow \ src/test/test-memwipe \ - src/test/test-child \ + src/test/test-process \ src/test/test_workqueue \ src/test/test-switch-id \ - src/test/test-timers + src/test/test-timers \ + src/test/test-rng endif src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ @@ -94,12 +95,14 @@ src_test_test_SOURCES += \ src/test/test_address.c \ src/test/test_address_set.c \ src/test/test_bridges.c \ + src/test/test_btrack.c \ src/test/test_buffers.c \ src/test/test_bwmgt.c \ src/test/test_cell_formats.c \ src/test/test_cell_queue.c \ src/test/test_channel.c \ src/test/test_channelpadding.c \ + src/test/test_circuitpadding.c \ src/test/test_channeltls.c \ src/test/test_checkdir.c \ src/test/test_circuitlist.c \ @@ -118,6 +121,7 @@ src_test_test_SOURCES += \ src/test/test_controller_events.c \ src/test/test_crypto.c \ src/test/test_crypto_ope.c \ + src/test/test_crypto_rng.c \ src/test/test_data.c \ src/test/test_dir.c \ src/test/test_dir_common.c \ @@ -146,6 +150,7 @@ src_test_test_SOURCES += \ src/test/test_logging.c \ src/test/test_mainloop.c \ src/test/test_microdesc.c \ + src/test/test_netinfo.c \ src/test/test_nodelist.c \ src/test/test_oom.c \ src/test/test_oos.c \ @@ -153,6 +158,8 @@ src_test_test_SOURCES += \ src/test/test_pem.c \ src/test/test_periodic_event.c \ src/test/test_policy.c \ + src/test/test_process.c \ + src/test/test_prob_distr.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ src/test/test_proto_misc.c \ @@ -177,10 +184,12 @@ src_test_test_SOURCES += \ src/test/test_util.c \ src/test/test_util_format.c \ src/test/test_util_process.c \ + src/test/test_voting_flags.c \ src/test/test_voting_schedule.c \ src/test/test_x509.c \ src/test/test_helpers.c \ src/test/test_dns.c \ + src/test/test_parsecommon.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c @@ -200,7 +209,8 @@ if UNITTESTS_ENABLED src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ - src/test/test_util_slow.c \ + src/test/test_process_slow.c \ + src/test/test_prob_distr.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c @@ -251,6 +261,12 @@ src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) src_test_test_slow_LDADD = $(src_test_test_LDADD) src_test_test_slow_LDFLAGS = $(src_test_test_LDFLAGS) +src_test_test_rng_CPPFLAGS = $(src_test_test_CPPFLAGS) +src_test_test_rng_CFLAGS = $(src_test_test_CFLAGS) +src_test_test_rng_SOURCES = src/test/test_rng.c +src_test_test_rng_LDFLAGS = $(src_test_test_LDFLAGS) +src_test_test_rng_LDADD = $(src_test_test_LDADD) + src_test_test_memwipe_CPPFLAGS = $(src_test_test_CPPFLAGS) # Don't use bugtrap cflags here: memwipe tests require memory violations. src_test_test_memwipe_CFLAGS = $(TEST_CFLAGS) diff --git a/src/test/prob_distr_mpfr_ref.c b/src/test/prob_distr_mpfr_ref.c new file mode 100644 index 0000000000..425733dc1b --- /dev/null +++ b/src/test/prob_distr_mpfr_ref.c @@ -0,0 +1,64 @@ +/* Copyright 2012-2019, The Tor Project, Inc + * See LICENSE for licensing information */ + +/** prob_distr_mpfr_ref.c + * + * Example reference file for GNU MPFR vectors tested in test_prob_distr.c . + * Code by Riastradh. + */ + +#include <complex.h> +#include <float.h> +#include <math.h> +#include <stdio.h> + +/* Must come after <stdio.h> so we get mpfr_printf. */ +#include <mpfr.h> + +/* gcc -o mpfr prob_distr_mpfr_ref.c -lmpfr -lm */ + +/* Computes logit(p) for p = .49999 */ +int +main(void) +{ + mpfr_t p, q, r; + mpfr_init(p); + mpfr_set_prec(p, 200); + mpfr_init(q); + mpfr_set_prec(q, 200); + mpfr_init(r); + mpfr_set_prec(r, 200); + mpfr_set_d(p, .49999, MPFR_RNDN); + mpfr_set_d(q, 1, MPFR_RNDN); + /* r := q - p = 1 - p */ + mpfr_sub(r, q, p, MPFR_RNDN); + /* q := p/r = p/(1 - p) */ + mpfr_div(q, p, r, MPFR_RNDN); + /* r := log(q) = log(p/(1 - p)) */ + mpfr_log(r, q, MPFR_RNDN); + mpfr_printf("mpfr 200-bit\t%.128Rg\n", r); + + /* + * Print a double approximation to logit three different ways. All + * three agree bit for bit on the libms I tried, with the nextafter + * adjustment (which is well within the 10 eps relative error bound + * advertised). Apparently I must have used the Goldberg expression + * for what I wrote down in the test case. + */ + printf("mpfr 53-bit\t%.17g\n", nextafter(mpfr_get_d(r, MPFR_RNDN), 0), 0); + volatile double p0 = .49999; + printf("log1p\t\t%.17g\n", nextafter(-log1p((1 - 2*p0)/p0), 0)); + volatile double x = (1 - 2*p0)/p0; + volatile double xp1 = x + 1; + printf("Goldberg\t%.17g\n", -x*log(xp1)/(xp1 - 1)); + + /* + * Print a bad approximation, using the naive expression, to see a + * lot of wrong digits, far beyond the 10 eps relative error attained + * by -log1p((1 - 2*p)/p). + */ + printf("naive\t\t%.17g\n", log(p0/(1 - p0))); + + fflush(stdout); + return ferror(stdout); +} diff --git a/src/test/test-child.c b/src/test/test-child.c deleted file mode 100644 index 11a1695cad..0000000000 --- a/src/test/test-child.c +++ /dev/null @@ -1,61 +0,0 @@ -/* Copyright (c) 2011-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#include "orconfig.h" -#include <stdio.h> -#ifdef _WIN32 -#define WINDOWS_LEAN_AND_MEAN -#include <windows.h> -#else -#include <unistd.h> -#endif /* defined(_WIN32) */ -#include <string.h> - -#ifdef _WIN32 -#define SLEEP(sec) Sleep((sec)*1000) -#else -#define SLEEP(sec) sleep(sec) -#endif - -/** Trivial test program which prints out its command line arguments so we can - * check if tor_spawn_background() works */ -int -main(int argc, char **argv) -{ - int i; - int delay = 1; - int fast = 0; - - if (argc > 1) { - if (!strcmp(argv[1], "--hang")) { - delay = 60; - } else if (!strcmp(argv[1], "--fast")) { - fast = 1; - delay = 0; - } - } - - fprintf(stdout, "OUT\n"); - fprintf(stderr, "ERR\n"); - for (i = 1; i < argc; i++) - fprintf(stdout, "%s\n", argv[i]); - if (!fast) - fprintf(stdout, "SLEEPING\n"); - /* We need to flush stdout so that test_util_spawn_background_partial_read() - succeed. Otherwise ReadFile() will get the entire output in one */ - // XXX: Can we make stdio flush on newline? - fflush(stdout); - if (!fast) - SLEEP(1); - fprintf(stdout, "DONE\n"); - fflush(stdout); - if (fast) - return 0; - - while (--delay) { - SLEEP(1); - } - - return 0; -} - diff --git a/src/test/test-process.c b/src/test/test-process.c new file mode 100644 index 0000000000..eb28ad90e9 --- /dev/null +++ b/src/test/test-process.c @@ -0,0 +1,85 @@ +/* Copyright (c) 2011-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include <stdio.h> +#ifdef _WIN32 +#define WINDOWS_LEAN_AND_MEAN +#include <windows.h> +#else +#include <unistd.h> +#endif /* defined(_WIN32) */ +#include <string.h> +#include <stdlib.h> + +#ifdef _WIN32 +#define SLEEP(sec) Sleep((sec)*1000) +#else +#define SLEEP(sec) sleep(sec) +#endif + +/* Trivial test program to test process_t. */ +int +main(int argc, char **argv) +{ + /* Does our process get the right arguments? */ + for (int i = 0; i < argc; ++i) { + fprintf(stdout, "argv[%d] = '%s'\n", i, argv[i]); + fflush(stdout); + } + + /* Make sure our process got our environment variable. */ + fprintf(stdout, "Environment variable TOR_TEST_ENV = '%s'\n", + getenv("TOR_TEST_ENV")); + fflush(stdout); + + /* Test line handling on stdout and stderr. */ + fprintf(stdout, "Output on stdout\nThis is a new line\n"); + fflush(stdout); + + fprintf(stderr, "Output on stderr\nThis is a new line\n"); + fflush(stderr); + + fprintf(stdout, "Partial line on stdout ..."); + fflush(stdout); + + fprintf(stderr, "Partial line on stderr ..."); + fflush(stderr); + + SLEEP(2); + + fprintf(stdout, "end of partial line on stdout\n"); + fflush(stdout); + fprintf(stderr, "end of partial line on stderr\n"); + fflush(stderr); + + /* Echo input from stdin. */ + char buffer[1024]; + + int count = 0; + + while (fgets(buffer, sizeof(buffer), stdin)) { + /* Strip the newline. */ + size_t size = strlen(buffer); + + if (size >= 1 && buffer[size - 1] == '\n') { + buffer[size - 1] = '\0'; + --size; + } + + if (size >= 1 && buffer[size - 1] == '\r') { + buffer[size - 1] = '\0'; + --size; + } + + fprintf(stdout, "Read line from stdin: '%s'\n", buffer); + fflush(stdout); + + if (++count == 3) + break; + } + + fprintf(stdout, "We are done for here, thank you!\n"); + + return 0; +} diff --git a/src/test/test.c b/src/test/test.c index 58b468775c..25e9da5591 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -37,7 +37,7 @@ #include "core/or/or.h" #include "lib/err/backtrace.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/circuitlist.h" #include "core/or/circuitstats.h" #include "lib/compress/compress.h" @@ -845,10 +845,11 @@ struct testgroup_t testgroups[] = { { "channeltls/", channeltls_tests }, { "checkdir/", checkdir_tests }, { "circuitbuild/", circuitbuild_tests }, + { "circuitpadding/", circuitpadding_tests }, { "circuitlist/", circuitlist_tests }, { "circuitmux/", circuitmux_tests }, - { "circuituse/", circuituse_tests }, { "circuitstats/", circuitstats_tests }, + { "circuituse/", circuituse_tests }, { "compat/libevent/", compat_libevent_tests }, { "config/", config_tests }, { "connection/", connection_tests }, @@ -857,6 +858,7 @@ struct testgroup_t testgroups[] = { { "consdiffmgr/", consdiffmgr_tests }, { "container/", container_tests }, { "control/", controller_tests }, + { "control/btrack/", btrack_tests }, { "control/event/", controller_event_tests }, { "crypto/", crypto_tests }, { "crypto/ope/", crypto_ope_tests }, @@ -864,38 +866,45 @@ struct testgroup_t testgroups[] = { { "crypto/openssl/", crypto_openssl_tests }, #endif { "crypto/pem/", pem_tests }, + { "crypto/rng/", crypto_rng_tests }, { "dir/", dir_tests }, - { "dir_handle_get/", dir_handle_get_tests }, { "dir/md/", microdesc_tests }, - { "dir/voting-schedule/", voting_schedule_tests }, + { "dir/voting/flags/", voting_flags_tests }, + { "dir/voting/schedule/", voting_schedule_tests }, + { "dir_handle_get/", dir_handle_get_tests }, + { "dns/", dns_tests }, { "dos/", dos_tests }, { "entryconn/", entryconn_tests }, { "entrynodes/", entrynodes_tests }, - { "guardfraction/", guardfraction_tests }, { "extorport/", extorport_tests }, { "geoip/", geoip_tests }, - { "legacy_hs/", hs_tests }, + { "guardfraction/", guardfraction_tests }, { "hs_cache/", hs_cache }, { "hs_cell/", hs_cell_tests }, + { "hs_client/", hs_client_tests }, { "hs_common/", hs_common_tests }, { "hs_config/", hs_config_tests }, { "hs_control/", hs_control_tests }, { "hs_descriptor/", hs_descriptor }, + { "hs_intropoint/", hs_intropoint_tests }, { "hs_ntor/", hs_ntor_tests }, { "hs_service/", hs_service_tests }, - { "hs_client/", hs_client_tests }, - { "hs_intropoint/", hs_intropoint_tests }, { "introduce/", introduce_tests }, { "keypin/", keypin_tests }, + { "legacy_hs/", hs_tests }, { "link-handshake/", link_handshake_tests }, { "mainloop/", mainloop_tests }, + { "netinfo/", netinfo_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, { "oos/", oos_tests }, { "options/", options_tests }, + { "parsecommon/", parsecommon_tests }, { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, + { "prob_distr/", prob_distr_tests }, { "procmon/", procmon_tests }, + { "process/", process_tests }, { "proto/http/", proto_http_tests }, { "proto/misc/", proto_misc_tests }, { "protover/", protover_tests }, @@ -910,8 +919,8 @@ struct testgroup_t testgroups[] = { { "routerlist/", routerlist_tests }, { "routerset/" , routerset_tests }, { "scheduler/", scheduler_tests }, - { "socks/", socks_tests }, { "shared-random/", sr_tests }, + { "socks/", socks_tests }, { "status/" , status_tests }, { "storagedir/", storagedir_tests }, { "tortls/", tortls_tests }, @@ -921,10 +930,9 @@ struct testgroup_t testgroups[] = { { "tortls/x509/", x509_tests }, { "util/", util_tests }, { "util/format/", util_format_tests }, + { "util/handle/", handle_tests }, { "util/logging/", logging_tests }, { "util/process/", util_process_tests }, { "util/thread/", thread_tests }, - { "util/handle/", handle_tests }, - { "dns/", dns_tests }, END_OF_GROUPS }; diff --git a/src/test/test.h b/src/test/test.h index aacc9dba87..2564432985 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -177,22 +177,24 @@ extern const struct testcase_setup_t ed25519_test_setup; extern struct testcase_t accounting_tests[]; extern struct testcase_t addr_tests[]; -extern struct testcase_t address_tests[]; extern struct testcase_t address_set_tests[]; +extern struct testcase_t address_tests[]; extern struct testcase_t bridges_tests[]; -extern struct testcase_t bwmgt_tests[]; +extern struct testcase_t btrack_tests[]; extern struct testcase_t buffer_tests[]; +extern struct testcase_t bwmgt_tests[]; extern struct testcase_t cell_format_tests[]; extern struct testcase_t cell_queue_tests[]; extern struct testcase_t channel_tests[]; extern struct testcase_t channelpadding_tests[]; +extern struct testcase_t circuitpadding_tests[]; extern struct testcase_t channeltls_tests[]; extern struct testcase_t checkdir_tests[]; extern struct testcase_t circuitbuild_tests[]; extern struct testcase_t circuitlist_tests[]; extern struct testcase_t circuitmux_tests[]; -extern struct testcase_t circuituse_tests[]; extern struct testcase_t circuitstats_tests[]; +extern struct testcase_t circuituse_tests[]; extern struct testcase_t compat_libevent_tests[]; extern struct testcase_t config_tests[]; extern struct testcase_t connection_tests[]; @@ -200,44 +202,52 @@ extern struct testcase_t conscache_tests[]; extern struct testcase_t consdiff_tests[]; extern struct testcase_t consdiffmgr_tests[]; extern struct testcase_t container_tests[]; -extern struct testcase_t controller_tests[]; extern struct testcase_t controller_event_tests[]; -extern struct testcase_t crypto_tests[]; +extern struct testcase_t controller_tests[]; extern struct testcase_t crypto_ope_tests[]; extern struct testcase_t crypto_openssl_tests[]; -extern struct testcase_t dir_tests[]; +extern struct testcase_t crypto_rng_tests[]; +extern struct testcase_t crypto_tests[]; extern struct testcase_t dir_handle_get_tests[]; +extern struct testcase_t dir_tests[]; +extern struct testcase_t dns_tests[]; extern struct testcase_t dos_tests[]; extern struct testcase_t entryconn_tests[]; extern struct testcase_t entrynodes_tests[]; -extern struct testcase_t guardfraction_tests[]; extern struct testcase_t extorport_tests[]; extern struct testcase_t geoip_tests[]; -extern struct testcase_t hs_tests[]; +extern struct testcase_t guardfraction_tests[]; +extern struct testcase_t handle_tests[]; extern struct testcase_t hs_cache[]; extern struct testcase_t hs_cell_tests[]; +extern struct testcase_t hs_client_tests[]; extern struct testcase_t hs_common_tests[]; extern struct testcase_t hs_config_tests[]; extern struct testcase_t hs_control_tests[]; extern struct testcase_t hs_descriptor[]; +extern struct testcase_t hs_intropoint_tests[]; extern struct testcase_t hs_ntor_tests[]; extern struct testcase_t hs_service_tests[]; -extern struct testcase_t hs_client_tests[]; -extern struct testcase_t hs_intropoint_tests[]; +extern struct testcase_t hs_tests[]; extern struct testcase_t introduce_tests[]; extern struct testcase_t keypin_tests[]; extern struct testcase_t link_handshake_tests[]; extern struct testcase_t logging_tests[]; extern struct testcase_t mainloop_tests[]; extern struct testcase_t microdesc_tests[]; +extern struct testcase_t netinfo_tests[]; extern struct testcase_t nodelist_tests[]; extern struct testcase_t oom_tests[]; extern struct testcase_t oos_tests[]; extern struct testcase_t options_tests[]; +extern struct testcase_t parsecommon_tests[]; extern struct testcase_t pem_tests[]; extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; +extern struct testcase_t prob_distr_tests[]; +extern struct testcase_t slow_stochastic_prob_distr_tests[]; extern struct testcase_t procmon_tests[]; +extern struct testcase_t process_tests[]; extern struct testcase_t proto_http_tests[]; extern struct testcase_t proto_misc_tests[]; extern struct testcase_t protover_tests[]; @@ -252,23 +262,22 @@ extern struct testcase_t routerkeys_tests[]; extern struct testcase_t routerlist_tests[]; extern struct testcase_t routerset_tests[]; extern struct testcase_t scheduler_tests[]; -extern struct testcase_t storagedir_tests[]; extern struct testcase_t socks_tests[]; +extern struct testcase_t sr_tests[]; extern struct testcase_t status_tests[]; +extern struct testcase_t storagedir_tests[]; extern struct testcase_t thread_tests[]; -extern struct testcase_t tortls_tests[]; extern struct testcase_t tortls_openssl_tests[]; -extern struct testcase_t util_tests[]; +extern struct testcase_t tortls_tests[]; extern struct testcase_t util_format_tests[]; extern struct testcase_t util_process_tests[]; +extern struct testcase_t util_tests[]; +extern struct testcase_t voting_flags_tests[]; extern struct testcase_t voting_schedule_tests[]; -extern struct testcase_t dns_tests[]; -extern struct testcase_t handle_tests[]; -extern struct testcase_t sr_tests[]; extern struct testcase_t x509_tests[]; extern struct testcase_t slow_crypto_tests[]; -extern struct testcase_t slow_util_tests[]; +extern struct testcase_t slow_process_tests[]; extern struct testgroup_t testgroups[]; diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 8868edce25..fb8df5f0fb 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -723,7 +723,7 @@ test_addr_ip6_helpers(void *arg) ; } -/** Test tor_addr_port_parse(). */ +/** Test tor_addr_parse() and tor_addr_port_parse(). */ static void test_addr_parse(void *arg) { @@ -734,6 +734,60 @@ test_addr_parse(void *arg) /* Correct call. */ (void)arg; + r= tor_addr_parse(&addr, "192.0.2.1"); + tt_int_op(r,OP_EQ, AF_INET); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "192.0.2.1"); + + r= tor_addr_parse(&addr, "11:22::33:44"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44"); + + r= tor_addr_parse(&addr, "[11:22::33:44]"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44"); + + r= tor_addr_parse(&addr, "11:22:33:44:55:66:1.2.3.4"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22:33:44:55:66:102:304"); + + r= tor_addr_parse(&addr, "11:22::33:44:1.2.3.4"); + tt_int_op(r,OP_EQ, AF_INET6); + tor_addr_to_str(buf, &addr, sizeof(buf), 0); + tt_str_op(buf,OP_EQ, "11:22::33:44:102:304"); + + /* Empty string. */ + r= tor_addr_parse(&addr, ""); + tt_int_op(r,OP_EQ, -1); + + /* Square brackets around IPv4 address. */ + r= tor_addr_parse(&addr, "[192.0.2.1]"); + tt_int_op(r,OP_EQ, -1); + + /* Only left square bracket. */ + r= tor_addr_parse(&addr, "[11:22::33:44"); + tt_int_op(r,OP_EQ, -1); + + /* Only right square bracket. */ + r= tor_addr_parse(&addr, "11:22::33:44]"); + tt_int_op(r,OP_EQ, -1); + + /* Leading colon. */ + r= tor_addr_parse(&addr, ":11:22::33:44"); + tt_int_op(r,OP_EQ, -1); + + /* Trailing colon. */ + r= tor_addr_parse(&addr, "11:22::33:44:"); + tt_int_op(r,OP_EQ, -1); + + /* Too many hex words in IPv4-mapped IPv6 address. */ + r= tor_addr_parse(&addr, "11:22:33:44:55:66:77:88:1.2.3.4"); + tt_int_op(r,OP_EQ, -1); + + /* Correct call. */ r= tor_addr_port_parse(LOG_DEBUG, "192.0.2.1:1234", &addr, &port, -1); diff --git a/src/test/test_address.c b/src/test/test_address.c index c33c30aee5..bf9ca047dc 100644 --- a/src/test/test_address.c +++ b/src/test/test_address.c @@ -24,6 +24,8 @@ #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ #include "core/or/or.h" +#include "feature/nodelist/routerinfo_st.h" +#include "feature/nodelist/node_st.h" #include "feature/nodelist/nodelist.h" #include "lib/net/address.h" #include "test/test.h" @@ -1170,6 +1172,78 @@ test_address_tor_addr_in_same_network_family(void *ignored) return; } +static node_t * +helper_create_mock_node(char id_char) +{ + node_t *node = tor_malloc_zero(sizeof(node_t)); + routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t)); + tor_addr_make_null(&ri->ipv6_addr, AF_INET6); + node->ri = ri; + memset(node->identity, id_char, sizeof(node->identity)); + return node; +} + +static void +helper_free_mock_node(node_t *node) +{ + if (!node) + return; + tor_free(node->ri); + tor_free(node); +} + +#define NODE_SET_IPV4(node, ipv4_addr, ipv4_port) { \ + tor_addr_t addr; \ + tor_addr_parse(&addr, ipv4_addr); \ + node->ri->addr = tor_addr_to_ipv4h(&addr); \ + node->ri->or_port = ipv4_port; \ + } + +#define NODE_CLEAR_IPV4(node) { \ + node->ri->addr = 0; \ + node->ri->or_port = 0; \ + } + +#define NODE_SET_IPV6(node, ipv6_addr_str, ipv6_port) { \ + tor_addr_parse(&node->ri->ipv6_addr, ipv6_addr_str); \ + node->ri->ipv6_orport = ipv6_port; \ + } + +static void +test_address_tor_node_in_same_network_family(void *ignored) +{ + (void)ignored; + node_t *node_a = helper_create_mock_node('a'); + node_t *node_b = helper_create_mock_node('b'); + + NODE_SET_IPV4(node_a, "8.8.8.8", 1); + NODE_SET_IPV4(node_b, "8.8.4.4", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + + NODE_SET_IPV4(node_a, "8.8.8.8", 1); + NODE_SET_IPV4(node_b, "1.1.1.1", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_CLEAR_IPV4(node_a); + NODE_SET_IPV6(node_a, "2001:470:20::2", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_CLEAR_IPV4(node_b); + NODE_SET_IPV6(node_b, "2606:4700:4700::1111", 1); + + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0); + + NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1); + tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1); + + done: + helper_free_mock_node(node_a); + helper_free_mock_node(node_b); +} + #define ADDRESS_TEST(name, flags) \ { #name, test_address_ ## name, flags, NULL, NULL } @@ -1202,5 +1276,6 @@ struct testcase_t address_tests[] = { ADDRESS_TEST(tor_addr_to_mapped_ipv4h, 0), ADDRESS_TEST(tor_addr_eq_ipv4h, 0), ADDRESS_TEST(tor_addr_in_same_network_family, 0), + ADDRESS_TEST(tor_node_in_same_network_family, 0), END_OF_TESTCASES }; diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c new file mode 100644 index 0000000000..48486fb5a1 --- /dev/null +++ b/src/test/test_btrack.c @@ -0,0 +1,100 @@ +/* Copyright (c) 2013-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" + +#include "test/test.h" +#include "test/log_test_helpers.h" + +#define OCIRC_EVENT_PRIVATE +#define ORCONN_EVENT_PRIVATE +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" + +static void +test_btrack_launch(void *arg) +{ + orconn_event_msg_t conn; + ocirc_event_msg_t circ; + + (void)arg; + conn.type = ORCONN_MSGTYPE_STATE; + conn.u.state.gid = 1; + conn.u.state.chan = 1; + conn.u.state.proxy_type = PROXY_NONE; + conn.u.state.state = OR_CONN_STATE_CONNECTING; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0 state=1"); + expect_no_log_msg_containing("ORCONN BEST_"); + teardown_capture_of_logs(); + + circ.type = OCIRC_MSGTYPE_CHAN; + circ.u.chan.chan = 1; + circ.u.chan.onehop = true; + + setup_full_capture_of_logs(LOG_DEBUG); + ocirc_event_publish(&circ); + expect_log_msg_containing("ORCONN LAUNCH chan=1 onehop=1"); + expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1"); + teardown_capture_of_logs(); + + conn.u.state.gid = 2; + conn.u.state.chan = 2; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=2 chan=2 proxy_type=0 state=1"); + expect_no_log_msg_containing("ORCONN BEST_"); + teardown_capture_of_logs(); + + circ.u.chan.chan = 2; + circ.u.chan.onehop = false; + + setup_full_capture_of_logs(LOG_DEBUG); + ocirc_event_publish(&circ); + expect_log_msg_containing("ORCONN LAUNCH chan=2 onehop=0"); + expect_log_msg_containing("ORCONN BEST_AP state -1->1 gid=2"); + teardown_capture_of_logs(); + + done: + ; +} + +static void +test_btrack_delete(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + conn.type = ORCONN_MSGTYPE_STATE; + conn.u.state.gid = 1; + conn.u.state.chan = 1; + conn.u.state.proxy_type = PROXY_NONE; + conn.u.state.state = OR_CONN_STATE_CONNECTING; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0"); + teardown_capture_of_logs(); + + conn.type = ORCONN_MSGTYPE_STATUS; + conn.u.status.gid = 1; + conn.u.status.status = OR_CONN_EVENT_CLOSED; + conn.u.status.reason = 0; + + setup_full_capture_of_logs(LOG_DEBUG); + orconn_event_publish(&conn); + expect_log_msg_containing("ORCONN DELETE gid=1 status=3 reason=0"); + teardown_capture_of_logs(); + + done: + ; +} + +struct testcase_t btrack_tests[] = { + { "launch", test_btrack_launch, TT_FORK, 0, NULL }, + { "delete", test_btrack_delete, TT_FORK, 0, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index 2f9ad1fbe3..97311c85cc 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -6,7 +6,7 @@ #define BUFFERS_PRIVATE #define PROTO_HTTP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/tls/tortls.h" #include "lib/compress/compress.h" diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c index 5da2d81377..5d012e462b 100644 --- a/src/test/test_channelpadding.c +++ b/src/test/test_channelpadding.c @@ -21,7 +21,7 @@ #include "test/log_test_helpers.h" #include "lib/tls/tortls.h" #include "lib/evloop/timers.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/cell_st.h" #include "feature/nodelist/networkstatus_st.h" diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index 10513e451d..054d3910e4 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -8,7 +8,7 @@ #define TOR_CHANNEL_INTERNAL_ #include "core/or/or.h" #include "lib/net/address.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c index 538f20781f..ccbaa02552 100644 --- a/src/test/test_circuitbuild.c +++ b/src/test/test_circuitbuild.c @@ -27,11 +27,11 @@ static smartlist_t dummy_nodes; static extend_info_t dummy_ei; static int -mock_count_acceptable_nodes(smartlist_t *nodes) +mock_count_acceptable_nodes(smartlist_t *nodes, int direct) { (void)nodes; - return DEFAULT_ROUTE_LEN + 1; + return direct ? 1 : DEFAULT_ROUTE_LEN + 1; } /* Test route lengths when the caller of new_route_len() doesn't diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c new file mode 100644 index 0000000000..09a4c9a0ca --- /dev/null +++ b/src/test/test_circuitpadding.c @@ -0,0 +1,2128 @@ +#define TOR_CHANNEL_INTERNAL_ +#define TOR_TIMERS_PRIVATE +#define CIRCUITPADDING_PRIVATE +#define NETWORKSTATUS_PRIVATE + +#include "core/or/or.h" +#include "test.h" +#include "lib/testsupport/testsupport.h" +#include "core/or/connection_or.h" +#include "core/or/channel.h" +#include "core/or/channeltls.h" +#include <event.h> +#include "lib/evloop/compat_libevent.h" +#include "lib/time/compat_time.h" +#include "lib/defs/time.h" +#include "core/or/relay.h" +#include "core/or/circuitlist.h" +#include "core/or/circuitbuild.h" +#include "core/or/circuitpadding.h" +#include "core/crypto/relay_crypto.h" +#include "core/or/protover.h" +#include "feature/nodelist/nodelist.h" +#include "lib/evloop/compat_libevent.h" +#include "app/config/config.h" + +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "core/or/cell_st.h" +#include "core/or/crypt_path_st.h" +#include "core/or/or_circuit_st.h" +#include "core/or/origin_circuit_st.h" + +/* Start our monotime mocking at 1 second past whatever monotime_init() + * thought the actual wall clock time was, for platforms with bad resolution + * and weird timevalues during monotime_init() before mocking. */ +#define MONOTIME_MOCK_START (monotime_absolute_nsec()+\ + TOR_NSEC_PER_USEC*TOR_USEC_PER_SEC) + +extern smartlist_t *connection_array; + +circid_t get_unique_circ_id_by_chan(channel_t *chan); +void helper_create_basic_machine(void); +static void helper_create_conditional_machines(void); + +static or_circuit_t * new_fake_orcirc(channel_t *nchan, channel_t *pchan); +channel_t *new_fake_channel(void); +void test_circuitpadding_negotiation(void *arg); +void test_circuitpadding_wronghop(void *arg); +void test_circuitpadding_conditions(void *arg); + +void test_circuitpadding_serialize(void *arg); +void test_circuitpadding_rtt(void *arg); +void test_circuitpadding_tokens(void *arg); + +static void +simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, + int padding); +void free_fake_orcirc(circuit_t *circ); +void free_fake_origin_circuit(origin_circuit_t *circ); + +static int deliver_negotiated = 1; +static int64_t curr_mocked_time; + +static node_t padding_node; +static node_t non_padding_node; + +static channel_t dummy_channel; +static circpad_machine_spec_t circ_client_machine; + +static void +timers_advance_and_run(int64_t msec_update) +{ + curr_mocked_time += msec_update*TOR_NSEC_PER_MSEC; + monotime_coarse_set_mock_time_nsec(curr_mocked_time); + monotime_set_mock_time_nsec(curr_mocked_time); + timers_run_pending(); +} + +static void +nodes_init(void) +{ + padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t)); + padding_node.rs->pv.supports_padding = 1; + + non_padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t)); + non_padding_node.rs->pv.supports_padding = 0; +} + +static void +nodes_free(void) +{ + tor_free(padding_node.rs); + + tor_free(non_padding_node.rs); +} + +static const node_t * +node_get_by_id_mock(const char *identity_digest) +{ + if (identity_digest[0] == 1) { + return &padding_node; + } else if (identity_digest[0] == 0) { + return &non_padding_node; + } + + return NULL; +} + +static or_circuit_t * +new_fake_orcirc(channel_t *nchan, channel_t *pchan) +{ + or_circuit_t *orcirc = NULL; + circuit_t *circ = NULL; + crypt_path_t tmp_cpath; + char whatevs_key[CPATH_KEY_MATERIAL_LEN]; + + orcirc = tor_malloc_zero(sizeof(*orcirc)); + circ = &(orcirc->base_); + circ->magic = OR_CIRCUIT_MAGIC; + + //circ->n_chan = nchan; + circ->n_circ_id = get_unique_circ_id_by_chan(nchan); + circ->n_mux = NULL; /* ?? */ + cell_queue_init(&(circ->n_chan_cells)); + circ->n_hop = NULL; + circ->streams_blocked_on_n_chan = 0; + circ->streams_blocked_on_p_chan = 0; + circ->n_delete_pending = 0; + circ->p_delete_pending = 0; + circ->received_destroy = 0; + circ->state = CIRCUIT_STATE_OPEN; + circ->purpose = CIRCUIT_PURPOSE_OR; + circ->package_window = CIRCWINDOW_START_MAX; + circ->deliver_window = CIRCWINDOW_START_MAX; + circ->n_chan_create_cell = NULL; + + //orcirc->p_chan = pchan; + orcirc->p_circ_id = get_unique_circ_id_by_chan(pchan); + cell_queue_init(&(orcirc->p_chan_cells)); + + circuit_set_p_circid_chan(orcirc, orcirc->p_circ_id, pchan); + circuit_set_n_circid_chan(circ, circ->n_circ_id, nchan); + + memset(&tmp_cpath, 0, sizeof(tmp_cpath)); + if (circuit_init_cpath_crypto(&tmp_cpath, whatevs_key, + sizeof(whatevs_key), 0, 0)<0) { + log_warn(LD_BUG,"Circuit initialization failed"); + return NULL; + } + orcirc->crypto = tmp_cpath.crypto; + + return orcirc; +} + +void +free_fake_orcirc(circuit_t *circ) +{ + or_circuit_t *orcirc = TO_OR_CIRCUIT(circ); + + relay_crypto_clear(&orcirc->crypto); + + circpad_circuit_free_all_machineinfos(circ); + tor_free(circ); +} + +void +free_fake_origin_circuit(origin_circuit_t *circ) +{ + circpad_circuit_free_all_machineinfos(TO_CIRCUIT(circ)); + circuit_clear_cpath(circ); + tor_free(circ); +} + +void dummy_nop_timer(void); + +//static int dont_stop_libevent = 0; + +static circuit_t *client_side; +static circuit_t *relay_side; + +static int n_client_cells = 0; +static int n_relay_cells = 0; + +static int +circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction, + crypt_path_t *layer_hint, streamid_t on_stream, + const char *filename, int lineno); + +static void +circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ, + cell_direction_t direction); + +static void +circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ, + cell_direction_t direction) +{ + (void)cmux; + (void)circ; + (void)direction; + + return; +} + +static int +circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ, + cell_direction_t cell_direction, + crypt_path_t *layer_hint, streamid_t on_stream, + const char *filename, int lineno) +{ + (void)cell; (void)on_stream; (void)filename; (void)lineno; + + if (circ == client_side) { + if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) { + // Deliver to relay + circpad_handle_padding_negotiate(relay_side, cell); + } else { + + int is_target_hop = circpad_padding_is_from_expected_hop(circ, + layer_hint); + tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_OUT); + tt_int_op(is_target_hop, OP_EQ, 1); + + // No need to pretend a padding cell was sent: This event is + // now emitted internally when the circuitpadding code sends them. + //circpad_cell_event_padding_sent(client_side); + + // Receive padding cell at middle + circpad_deliver_recognized_relay_cell_events(relay_side, + cell->payload[0], NULL); + } + n_client_cells++; + } else if (circ == relay_side) { + tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_IN); + + if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATED) { + // XXX: blah need right layer_hint.. + if (deliver_negotiated) + circpad_handle_padding_negotiated(client_side, cell, + TO_ORIGIN_CIRCUIT(client_side) + ->cpath->next); + } else if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) { + circpad_handle_padding_negotiate(client_side, cell); + } else { + // No need to pretend a padding cell was sent: This event is + // now emitted internally when the circuitpadding code sends them. + //circpad_cell_event_padding_sent(relay_side); + + // Receive padding cell at client + circpad_deliver_recognized_relay_cell_events(client_side, + cell->payload[0], + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + } + + n_relay_cells++; + } + + done: + timers_advance_and_run(1); + return 0; +} + +// Test reading and writing padding to strings (or options_t + consensus) +void +test_circuitpadding_serialize(void *arg) +{ + (void)arg; +} + +static signed_error_t +circpad_send_command_to_hop_mock(origin_circuit_t *circ, uint8_t hopnum, + uint8_t relay_command, const uint8_t *payload, + ssize_t payload_len) +{ + (void) circ; + (void) hopnum; + (void) relay_command; + (void) payload; + (void) payload_len; + return 0; +} + +void +test_circuitpadding_rtt(void *arg) +{ + /* Test Plan: + * + * 1. Test RTT measurement server side + * a. test usage of measured RTT + * 2. Test termination of RTT measurement + * a. test non-update of RTT + * 3. Test client side circuit and non-application of RTT.. + */ + circpad_delay_t rtt_estimate; + int64_t actual_mocked_monotime_start; + (void)arg; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock); + + dummy_channel.cmux = circuitmux_alloc(); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + circpad_machines_init(); + helper_create_basic_machine(); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side, + 0); + + relay_side->padding_machine[0] = &circ_client_machine; + relay_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,0); + + /* Test 1: Test measuring RTT */ + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); + + timers_advance_and_run(20); + + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 19000); + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 30000); + tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0), + OP_EQ, + relay_side->padding_info[0]->rtt_estimate_usec+ + circpad_machine_current_state( + relay_side->padding_info[0])->start_usec); + + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0); + timers_advance_and_run(20); + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 20000); + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 21000); + tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0), + OP_EQ, + relay_side->padding_info[0]->rtt_estimate_usec+ + circpad_machine_current_state( + relay_side->padding_info[0])->start_usec); + + /* Test 2: Termination of RTT measurement (from the previous test) */ + tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1); + rtt_estimate = relay_side->padding_info[0]->rtt_estimate_usec; + + circpad_cell_event_nonpadding_received((circuit_t*)relay_side); + timers_advance_and_run(4); + circpad_cell_event_nonpadding_sent((circuit_t*)relay_side); + + tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_EQ, + rtt_estimate); + tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1); + tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0), + OP_EQ, + relay_side->padding_info[0]->rtt_estimate_usec+ + circpad_machine_current_state( + relay_side->padding_info[0])->start_usec); + + /* Test 3: Make sure client side machine properly ignores RTT */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + timers_advance_and_run(20); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0); + + tt_int_op(client_side->padding_info[0]->rtt_estimate_usec, OP_EQ, 0); + tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0), + OP_NE, client_side->padding_info[0]->rtt_estimate_usec); + tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0), + OP_EQ, + circpad_machine_current_state( + client_side->padding_info[0])->start_usec); + done: + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + timers_shutdown(); + monotime_disable_test_mocking(); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + tor_free(circ_client_machine.states); + + return; +} + +void +helper_create_basic_machine(void) +{ + /* Start, burst */ + circpad_machine_states_init(&circ_client_machine, 2); + + circ_client_machine.states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + circ_client_machine.states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + circ_client_machine.states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + circ_client_machine.states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL; + + // FIXME: Is this what we want? + circ_client_machine.states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_HIGHER; + + // FIXME: Tune this histogram + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_len = 5; + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 500; + circ_client_machine.states[CIRCPAD_STATE_BURST].range_usec = 1000000; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[0] = 1; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[1] = 0; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[2] = 2; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[3] = 2; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[4] = 2; + circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_total_tokens = 7; + circ_client_machine.states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1; + + return; +} + +#define BIG_HISTOGRAM_LEN 10 + +/** Setup a machine with a big histogram */ +static void +helper_create_machine_with_big_histogram(circpad_removal_t removal_strategy) +{ + const int tokens_per_bin = 2; + + /* Start, burst */ + circpad_machine_states_init(&circ_client_machine, 2); + + circpad_state_t *burst_state = + &circ_client_machine.states[CIRCPAD_STATE_BURST]; + + circ_client_machine.states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST; + + burst_state->next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST; + burst_state->next_state[CIRCPAD_EVENT_NONPADDING_RECV] =CIRCPAD_STATE_BURST; + + burst_state->next_state[CIRCPAD_EVENT_NONPADDING_SENT] =CIRCPAD_STATE_CANCEL; + + burst_state->token_removal = CIRCPAD_TOKEN_REMOVAL_HIGHER; + + burst_state->histogram_len = BIG_HISTOGRAM_LEN; + burst_state->start_usec = 0; + burst_state->range_usec = 1000; + + int n_tokens = 0; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + burst_state->histogram[i] = tokens_per_bin; + n_tokens += tokens_per_bin; + } + + burst_state->histogram_total_tokens = n_tokens; + burst_state->length_dist.type = CIRCPAD_DIST_UNIFORM; + burst_state->length_dist.param1 = n_tokens; + burst_state->length_dist.param2 = n_tokens; + burst_state->max_length = n_tokens; + burst_state->length_includes_nonpadding = 1; + burst_state->use_rtt_estimate = 0; + burst_state->token_removal = removal_strategy; +} + +static circpad_decision_t +circpad_machine_schedule_padding_mock(circpad_machine_state_t *mi) +{ + (void)mi; + return 0; +} + +static uint64_t +mock_monotime_absolute_usec(void) +{ + return 100; +} + +/** Test higher token removal strategy by bin */ +static void +test_circuitpadding_token_removal_higher(void *arg) +{ + circpad_machine_state_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_HIGHER); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 5, 6, 7, 8}; + unsigned i; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all lower bins are not touched */ + for (i=0; i < 4 ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test below the lowest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 1; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[0], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test lower token removal strategy by bin */ +static void +test_circuitpadding_token_removal_lower(void *arg) +{ + circpad_machine_state_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_LOWER); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 3, 2, 1, 0}; + unsigned i; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all higher bins are untouched */ + for (i = 5; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test above the highest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 29202; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test closest token removal strategy by bin */ +static void +test_circuitpadding_closest_token_removal(void *arg) +{ + circpad_machine_state_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 3, 5, 2, 6, 1, 7, 0, 8, 9}; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all bins have been refilled */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test below the lowest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 102; + mi->histogram[0] = 0; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[1], OP_EQ, 1); + + /* Test above the highest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 29202; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test closest token removal strategy with usec */ +static void +test_circuitpadding_closest_token_removal_usec(void *arg) +{ + circpad_machine_state_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /*************************************************************************/ + + uint64_t current_time = monotime_absolute_usec(); + + /* Test left boundaries of each histogram bin: */ + const circpad_delay_t bin_left_bounds[] = + {0, 1, 7, 15, 31, 62, 125, 250, 500, CIRCPAD_DELAY_INFINITE}; + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_uint_op(bin_left_bounds[i], OP_EQ, + circpad_histogram_bin_to_usec(mi, i)); + } + + /* XXX we want to test remove_token_exact and + circpad_machine_remove_closest_token() with usec */ + + /* Check that all bins have two tokens right now */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* This is the right order to remove tokens from this histogram. That is, we + * first remove tokens from the 4th bin since 57 usec is nearest to the 4th + * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for + * the same reason, then from the 5th, etc. */ + const int bin_removal_order[] = {4, 3, 5, 2, 1, 0, 6, 7, 8, 9}; + + /* Remove all tokens from all bins apart from the infinity bin */ + for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) { + int bin_to_remove = bin_removal_order[i]; + log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin", + i, bin_to_remove); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1); + + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + + /* Test that we cleaned out this bin. Don't do this in the case of the last + bin since the tokens will get refilled */ + if (i != BIG_HISTOGRAM_LEN - 2) { + tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0); + } + } + + /* Check that all bins have been refilled */ + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + + /* Test below the lowest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 102; + mi->histogram[0] = 0; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[1], OP_EQ, 1); + + /* Test above the highest bin, for coverage */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + circ_client_machine.states[CIRCPAD_STATE_BURST].start_usec = 100; + mi->padding_scheduled_at_usec = current_time - 29202; + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +/** Test closest token removal strategy with usec */ +static void +test_circuitpadding_token_removal_exact(void *arg) +{ + circpad_machine_state_t *mi; + (void)arg; + + /* Mock it up */ + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + /* Setup test environment (time etc.) */ + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + monotime_enable_test_mocking(); + + /* Create test machine */ + helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_EXACT); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + + /* move the machine to the right state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + /* Get the machine and setup tokens */ + mi = client_side->padding_info[0]; + tt_assert(mi); + + /**********************************************************************/ + uint64_t current_time = monotime_absolute_usec(); + + /* Ensure that we will clear out bin #4 with this usec */ + mi->padding_scheduled_at_usec = current_time - 57; + tt_int_op(mi->histogram[4], OP_EQ, 2); + circpad_machine_remove_token(mi); + mi->padding_scheduled_at_usec = current_time - 57; + tt_int_op(mi->histogram[4], OP_EQ, 1); + circpad_machine_remove_token(mi); + tt_int_op(mi->histogram[4], OP_EQ, 0); + + /* Ensure that we will not remove any other tokens even tho we try to, since + * this is what the exact strategy dictates */ + mi->padding_scheduled_at_usec = current_time - 57; + circpad_machine_remove_token(mi); + for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) { + if (i != 4) { + tt_int_op(mi->histogram[i], OP_EQ, 2); + } + } + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +#undef BIG_HISTOGRAM_LEN + +void +test_circuitpadding_tokens(void *arg) +{ + const circpad_state_t *state; + circpad_machine_state_t *mi; + int64_t actual_mocked_monotime_start; + (void)arg; + + /** Test plan: + * + * 1. Test symmetry between bin_to_usec and usec_to_bin + * a. Test conversion + * b. Test edge transitions (lower, upper) + * 2. Test remove higher on an empty bin + * a. Normal bin + * b. Infinity bin + * c. Bin 0 + * d. No higher + * 3. Test remove lower + * a. Normal bin + * b. Bin 0 + * c. No lower + * 4. Test remove closest + * a. Closest lower + * b. Closest higher + * c. Closest 0 + * d. Closest Infinity + */ + client_side = TO_CIRCUIT(origin_circuit_new()); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + + helper_create_basic_machine(); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side, + 0); + + mi = client_side->padding_info[0]; + + // Pretend a non-padding cell was sent + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + /* We have to save the infinity bin because one inf delay + * could have been chosen when we transition to burst */ + circpad_hist_token_t inf_bin = mi->histogram[4]; + + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, + CIRCPAD_STATE_BURST); + + state = circpad_machine_current_state(client_side->padding_info[0]); + + // Test 0: convert bin->usec->bin + // Bin 0+1 have different semantics + for (int bin = 0; bin < 2; bin++) { + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + } + for (int bin = 2; bin < state->histogram_len-1; bin++) { + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + /* Verify we round down */ + bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec+3); + tt_int_op(bin, OP_EQ, bin2); + + bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec-1); + tt_int_op(bin, OP_EQ, bin2+1); + } + + // Test 1: converting usec->bin->usec->bin + // Bin 0+1 have different semantics. + for (circpad_delay_t i = 0; i <= state->start_usec+1; i++) { + int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0], + i); + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + tt_int_op(i, OP_LE, usec); + } + for (circpad_delay_t i = state->start_usec+1; + i <= state->start_usec + state->range_usec; i++) { + int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0], + i); + circpad_delay_t usec = + circpad_histogram_bin_to_usec(client_side->padding_info[0], bin); + int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0], + usec); + tt_int_op(bin, OP_EQ, bin2); + tt_int_op(i, OP_GE, usec); + } + + /* 2.a. Normal higher bin */ + { + tt_int_op(mi->histogram[2], OP_EQ, 2); + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[3], OP_EQ, 2); + tt_int_op(mi->histogram[2], OP_EQ, 1); + + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[2], OP_EQ, 0); + + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + } + + /* 2.b. Higher Infinity bin */ + { + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1); + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + + /* Test past the infinity bin */ + circpad_machine_remove_higher_token(mi, + circpad_histogram_bin_to_usec(mi, 5)+1000000); + + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + } + + /* 2.c. Bin 0 */ + { + tt_int_op(mi->histogram[0], OP_EQ, 0); + mi->histogram[0] = 1; + circpad_machine_remove_higher_token(mi, + state->start_usec/2); + tt_int_op(mi->histogram[0], OP_EQ, 0); + } + + /* Drain the infinity bin and cause a refill */ + while (inf_bin != 0) { + tt_int_op(mi->histogram[4], OP_EQ, inf_bin); + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + inf_bin--; + } + + circpad_cell_event_nonpadding_sent((circuit_t*)client_side); + + // We should have refilled here. + tt_int_op(mi->histogram[4], OP_EQ, 2); + + /* 3.a. Bin 0 */ + { + tt_int_op(mi->histogram[0], OP_EQ, 1); + circpad_machine_remove_higher_token(mi, + state->start_usec/2); + tt_int_op(mi->histogram[0], OP_EQ, 0); + } + + /* 3.b. Test remove lower normal bin */ + { + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + tt_int_op(mi->histogram[2], OP_EQ, 2); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + /* 3.c. No lower */ + circpad_machine_remove_lower_token(mi, + circpad_histogram_bin_to_usec(mi, 3)+1); + tt_int_op(mi->histogram[2], OP_EQ, 0); + } + + /* 4. Test remove closest + * a. Closest lower + * b. Closest higher + * c. Closest 0 + * d. Closest Infinity + */ + circpad_machine_setup_tokens(mi); + tt_int_op(mi->histogram[2], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[2], OP_EQ, 0); + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[3], OP_EQ, 0); + tt_int_op(mi->histogram[0], OP_EQ, 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[0], OP_EQ, 0); + tt_int_op(mi->histogram[4], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 2)+1, 0); + tt_int_op(mi->histogram[4], OP_EQ, 2); + + /* 5. Test remove closest usec + * a. Closest 0 + * b. Closest lower (below midpoint) + * c. Closest higher (above midpoint) + * d. Closest Infinity + */ + circpad_machine_setup_tokens(mi); + + tt_int_op(mi->histogram[0], OP_EQ, 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 0)/3, 1); + tt_int_op(mi->histogram[0], OP_EQ, 0); + tt_int_op(mi->histogram[2], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 0)/3, 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 0)/3, 1); + tt_int_op(mi->histogram[2], OP_EQ, 0); + tt_int_op(mi->histogram[3], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + tt_int_op(mi->histogram[3], OP_EQ, 0); + tt_int_op(mi->histogram[4], OP_EQ, 2); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + circpad_machine_remove_closest_token(mi, + circpad_histogram_bin_to_usec(mi, 4), 1); + tt_int_op(mi->histogram[4], OP_EQ, 2); + + // XXX: Need more coverage of the actual usec branches + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + monotime_disable_test_mocking(); + tor_free(circ_client_machine.states); +} + +void +test_circuitpadding_wronghop(void *arg) +{ + /** + * Test plan: + * 1. Padding sent from hop 1 and 3 to client + * 2. Send negotiated from hop 1 and 3 to client + * 3. Garbled negotiated cell + * 4. Padding negotiate sent to client + * 5. Send negotiate stop command for unknown machine + * 6. Send negotiated to relay + * 7. Garbled padding negotiate cell + */ + (void)arg; + uint32_t read_bw = 0, overhead_bw = 0; + cell_t cell; + signed_error_t ret; + origin_circuit_t *orig_client; + int64_t actual_mocked_monotime_start; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + /* Mock this function so that our cell counting tests don't get confused by + * padding that gets sent by scheduled timers. */ + MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock); + + client_side = (circuit_t *)origin_circuit_new(); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, + &dummy_channel); + orig_client = TO_ORIGIN_CIRCUIT(client_side); + + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + nodes_init(); + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + circpad_machines_init(); + + MOCK(node_get_by_id, + node_get_by_id_mock); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + + /* Build three hops */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* verify padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + + /* verify echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + read_bw = orig_client->n_delivered_read_circ_bw; + overhead_bw = orig_client->n_overhead_read_circ_bw; + + /* 1. Test padding from first and third hop */ + circpad_deliver_recognized_relay_cell_events(client_side, + RELAY_COMMAND_DROP, + TO_ORIGIN_CIRCUIT(client_side)->cpath); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_EQ, + orig_client->n_overhead_read_circ_bw); + + circpad_deliver_recognized_relay_cell_events(client_side, + RELAY_COMMAND_DROP, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_EQ, + orig_client->n_overhead_read_circ_bw); + + circpad_deliver_recognized_relay_cell_events(client_side, + RELAY_COMMAND_DROP, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_LT, + orig_client->n_overhead_read_circ_bw); + + /* 2. Test padding negotiated not handled from hops 1,3 */ + ret = circpad_handle_padding_negotiated(client_side, &cell, + TO_ORIGIN_CIRCUIT(client_side)->cpath); + tt_int_op(ret, OP_EQ, -1); + + ret = circpad_handle_padding_negotiated(client_side, &cell, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next); + tt_int_op(ret, OP_EQ, -1); + + /* 3. Garbled negotiated cell */ + memset(&cell, 255, sizeof(cell)); + ret = circpad_handle_padding_negotiated(client_side, &cell, + TO_ORIGIN_CIRCUIT(client_side)->cpath->next); + tt_int_op(ret, OP_EQ, -1); + + /* 4. Test that negotiate is dropped at origin */ + read_bw = orig_client->n_delivered_read_circ_bw; + overhead_bw = orig_client->n_overhead_read_circ_bw; + relay_send_command_from_edge(0, relay_side, + RELAY_COMMAND_PADDING_NEGOTIATE, + (void*)cell.payload, + (size_t)3, NULL); + tt_int_op(read_bw, OP_EQ, + orig_client->n_delivered_read_circ_bw); + tt_int_op(overhead_bw, OP_EQ, + orig_client->n_overhead_read_circ_bw); + + tt_int_op(n_relay_cells, OP_EQ, 2); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* 5. Test that asking to stop the wrong machine does nothing */ + circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side), + 255, 2, CIRCPAD_COMMAND_STOP); + tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* 6. Sending negotiated command to relay does nothing */ + ret = circpad_handle_padding_negotiated(relay_side, &cell, NULL); + tt_int_op(ret, OP_EQ, -1); + + /* 7. Test garbled negotated cell (bad command 255) */ + memset(&cell, 0, sizeof(cell)); + ret = circpad_handle_padding_negotiate(relay_side, &cell); + tt_int_op(ret, OP_EQ, -1); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* Test 2: Test no padding */ + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + + client_side = (circuit_t *)origin_circuit_new(); + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, + &dummy_channel); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 0); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* verify no echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 3); + tt_int_op(n_client_cells, OP_EQ, 2); + + /* Finish circuit */ + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Spoof padding negotiated on circuit with no padding */ + circpad_padding_negotiated(relay_side, + CIRCPAD_MACHINE_CIRC_SETUP, + CIRCPAD_COMMAND_START, + CIRCPAD_RESPONSE_OK); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + circpad_padding_negotiated(relay_side, + CIRCPAD_MACHINE_CIRC_SETUP, + CIRCPAD_COMMAND_START, + CIRCPAD_RESPONSE_ERR); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + monotime_disable_test_mocking(); + UNMOCK(node_get_by_id); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + nodes_free(); +} + +void +test_circuitpadding_negotiation(void *arg) +{ + /** + * Test plan: + * 1. Test circuit where padding is supported by middle + * a. Make sure padding negotiation is sent + * b. Test padding negotiation delivery and parsing + * 2. Test circuit where padding is unsupported by middle + * a. Make sure padding negotiation is not sent + * 3. Test failure to negotiate a machine due to desync. + */ + int64_t actual_mocked_monotime_start; + (void)arg; + + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + client_side = TO_CIRCUIT(origin_circuit_new()); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + nodes_init(); + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + circpad_machines_init(); + + MOCK(node_get_by_id, + node_get_by_id_mock); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + + /* Build two hops */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* verify padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + + /* verify echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* Finish circuit */ + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Test 2: Test no padding */ + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 0); + + /* verify no padding was negotiated */ + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* verify no echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* Finish circuit */ + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Force negotiate padding. */ + circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side), + CIRCPAD_MACHINE_CIRC_SETUP, + 2, CIRCPAD_COMMAND_START); + + /* verify no padding was negotiated */ + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + /* verify no echo was sent */ + tt_int_op(n_relay_cells, OP_EQ, 1); + tt_int_op(n_client_cells, OP_EQ, 1); + + /* 3. Test failure to negotiate a machine due to desync */ + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + + client_side = TO_CIRCUIT(origin_circuit_new()); + relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel)); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + SMARTLIST_FOREACH(relay_padding_machines, + circpad_machine_spec_t *, + m, tor_free(m->states); tor_free(m)); + smartlist_free(relay_padding_machines); + relay_padding_machines = smartlist_new(); + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* verify echo was sent */ + tt_int_op(n_client_cells, OP_EQ, 2); + tt_int_op(n_relay_cells, OP_EQ, 2); + + /* verify no padding was negotiated */ + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + monotime_disable_test_mocking(); + UNMOCK(node_get_by_id); + UNMOCK(circuit_package_relay_cell); + UNMOCK(circuitmux_attach_circuit); + nodes_free(); +} + +static void +simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay, + int padding) +{ + char whatevs_key[CPATH_KEY_MATERIAL_LEN]; + char digest[DIGEST_LEN]; + tor_addr_t addr; + + // Pretend a non-padding cell was sent + circpad_cell_event_nonpadding_sent((circuit_t*)client); + + // Receive extend cell at middle + circpad_cell_event_nonpadding_received((circuit_t*)mid_relay); + + // Advance time a tiny bit so we can calculate an RTT + curr_mocked_time += 10 * TOR_NSEC_PER_MSEC; + monotime_coarse_set_mock_time_nsec(curr_mocked_time); + monotime_set_mock_time_nsec(curr_mocked_time); + + // Receive extended cell at middle + circpad_cell_event_nonpadding_sent((circuit_t*)mid_relay); + + // Receive extended cell at first hop + circpad_cell_event_nonpadding_received((circuit_t*)client); + + // Add a hop to cpath + crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t)); + onion_append_to_cpath(&TO_ORIGIN_CIRCUIT(client)->cpath, hop); + + hop->magic = CRYPT_PATH_MAGIC; + hop->state = CPATH_STATE_OPEN; + + // add an extend info to indicate if this node supports padding or not. + // (set the first byte of the digest for our mocked node_get_by_id) + digest[0] = padding; + + hop->extend_info = extend_info_new( + padding ? "padding" : "non-padding", + digest, NULL, NULL, NULL, + &addr, padding); + + circuit_init_cpath_crypto(hop, whatevs_key, sizeof(whatevs_key), 0, 0); + + hop->package_window = circuit_initial_package_window(); + hop->deliver_window = CIRCWINDOW_START; + + // Signal that the hop was added + circpad_machine_event_circ_added_hop(TO_ORIGIN_CIRCUIT(client)); +} + +static circpad_machine_spec_t * +helper_create_conditional_machine(void) +{ + circpad_machine_spec_t *ret = + tor_malloc_zero(sizeof(circpad_machine_spec_t)); + + /* Start, burst */ + circpad_machine_states_init(ret, 2); + + ret->states[CIRCPAD_STATE_START]. + next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST; + + ret->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST; + + ret->states[CIRCPAD_STATE_BURST]. + next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END; + + ret->states[CIRCPAD_STATE_BURST].token_removal = + CIRCPAD_TOKEN_REMOVAL_NONE; + + ret->states[CIRCPAD_STATE_BURST].histogram_len = 3; + ret->states[CIRCPAD_STATE_BURST].start_usec = 0; + ret->states[CIRCPAD_STATE_BURST].range_usec = 1000000; + ret->states[CIRCPAD_STATE_BURST].histogram[0] = 6; + ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0; + ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6; + ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0; + ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1; + + return ret; +} + +static void +helper_create_conditional_machines(void) +{ + circpad_machine_spec_t *add = helper_create_conditional_machine(); + origin_padding_machines = smartlist_new(); + relay_padding_machines = smartlist_new(); + + add->machine_num = 2; + add->is_origin_side = 1; + add->should_negotiate_end = 1; + add->target_hopnum = 2; + + /* Let's have this one end after 4 packets */ + add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM; + add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4; + add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4; + add->states[CIRCPAD_STATE_BURST].max_length = 4; + + add->conditions.requires_vanguards = 0; + add->conditions.min_hops = 2; + add->conditions.state_mask = CIRCPAD_CIRC_BUILDING| + CIRCPAD_CIRC_NO_STREAMS|CIRCPAD_CIRC_HAS_RELAY_EARLY; + add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; + + smartlist_add(origin_padding_machines, add); + + add = helper_create_conditional_machine(); + add->machine_num = 3; + add->is_origin_side = 1; + add->should_negotiate_end = 1; + add->target_hopnum = 2; + + /* Let's have this one end after 4 packets */ + add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM; + add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4; + add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4; + add->states[CIRCPAD_STATE_BURST].max_length = 4; + + add->conditions.requires_vanguards = 1; + add->conditions.min_hops = 3; + add->conditions.state_mask = CIRCPAD_CIRC_OPENED| + CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY; + add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL; + smartlist_add(origin_padding_machines, add); + + add = helper_create_conditional_machine(); + add->machine_num = 2; + smartlist_add(relay_padding_machines, add); + + add = helper_create_conditional_machine(); + add->machine_num = 3; + smartlist_add(relay_padding_machines, add); +} + +void +test_circuitpadding_conditions(void *arg) +{ + /** + * Test plan: + * 0. Make a few origin and client machines with diff conditions + * * vanguards, purposes, has_opened circs, no relay early + * * Client side should_negotiate_end + * * Length limits + * 1. Test STATE_END transitions + * 2. Test new machine after end with same conditions + * 3. Test new machine due to changed conditions + * * Esp: built event, no relay early, no streams + * XXX: Diff test: + * 1. Test STATE_END with pending timers + * 2. Test marking a circuit before padding callback fires + * 3. Test freeing a circuit before padding callback fires + */ + int64_t actual_mocked_monotime_start; + (void)arg; + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + + nodes_init(); + dummy_channel.cmux = circuitmux_alloc(); + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, + &dummy_channel); + client_side = (circuit_t *)origin_circuit_new(); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + + timers_initialize(); + helper_create_conditional_machines(); + + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + MOCK(node_get_by_id, + node_get_by_id_mock); + + /* Simulate extend. This should result in the original machine getting + * added, since the circuit is not built */ + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Verify that machine #2 is added */ + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + /* Deliver a padding cell to the client, to trigger burst state */ + circpad_cell_event_padding_sent(client_side); + + /* This should have trigger length shutdown condition on client.. */ + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL); + + /* Verify machine is gone from both sides */ + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + /* Send another event.. verify machine gets re-added properly + * (test race with shutdown) */ + simulate_single_hop_extend(client_side, relay_side, 1); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + TO_ORIGIN_CIRCUIT(client_side)->p_streams = 0; + circpad_machine_event_circ_has_no_streams(TO_ORIGIN_CIRCUIT(client_side)); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + /* Now make the circuit opened and send built event */ + TO_ORIGIN_CIRCUIT(client_side)->has_opened = 1; + circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side)); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + TO_ORIGIN_CIRCUIT(client_side)->remaining_relay_early_cells = 0; + circpad_machine_event_circ_has_no_relay_early( + TO_ORIGIN_CIRCUIT(client_side)); + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2); + + get_options_mutable()->HSLayer2Nodes = (void*)1; + TO_ORIGIN_CIRCUIT(client_side)->p_streams = (void*)1; + circpad_machine_event_circ_has_streams(TO_ORIGIN_CIRCUIT(client_side)); + + /* Verify different machine is added */ + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3); + + /* Hold off on negotiated */ + deliver_negotiated = 0; + + /* Deliver a padding cell to the client, to trigger burst state */ + circpad_cell_event_padding_sent(client_side); + + /* This should have trigger length shutdown condition on client + * but not the response for the padding machine */ + tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL); + + /* Verify machine is gone from the relay (but negotiated not back yet */ + tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL); + tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL); + + /* Add another hop and verify it's back */ + simulate_single_hop_extend(client_side, relay_side, 1); + + tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3); + tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3); + + tt_ptr_op(client_side->padding_info[0], OP_NE, NULL); + tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL); + + done: + /* XXX: Free everything */ + return; +} + +/** Helper function: Initializes a padding machine where every state uses the + * uniform probability distribution. */ +static void +helper_circpad_circ_distribution_machine_setup(int min, int max) +{ + circpad_machine_states_init(&circ_client_machine, 7); + + circpad_state_t *zero_st = &circ_client_machine.states[0]; + zero_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 1; + zero_st->iat_dist.type = CIRCPAD_DIST_UNIFORM; + zero_st->iat_dist.param1 = min; + zero_st->iat_dist.param2 = max; + zero_st->start_usec = min; + zero_st->range_usec = max; + + circpad_state_t *first_st = &circ_client_machine.states[1]; + first_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 2; + first_st->iat_dist.type = CIRCPAD_DIST_LOGISTIC; + first_st->iat_dist.param1 = min; + first_st->iat_dist.param2 = max; + first_st->start_usec = min; + first_st->range_usec = max; + + circpad_state_t *second_st = &circ_client_machine.states[2]; + second_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 3; + second_st->iat_dist.type = CIRCPAD_DIST_LOG_LOGISTIC; + second_st->iat_dist.param1 = min; + second_st->iat_dist.param2 = max; + second_st->start_usec = min; + second_st->range_usec = max; + + circpad_state_t *third_st = &circ_client_machine.states[3]; + third_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 4; + third_st->iat_dist.type = CIRCPAD_DIST_GEOMETRIC; + third_st->iat_dist.param1 = min; + third_st->iat_dist.param2 = max; + third_st->start_usec = min; + third_st->range_usec = max; + + circpad_state_t *fourth_st = &circ_client_machine.states[4]; + fourth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 5; + fourth_st->iat_dist.type = CIRCPAD_DIST_WEIBULL; + fourth_st->iat_dist.param1 = min; + fourth_st->iat_dist.param2 = max; + fourth_st->start_usec = min; + fourth_st->range_usec = max; + + circpad_state_t *fifth_st = &circ_client_machine.states[5]; + fifth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 6; + fifth_st->iat_dist.type = CIRCPAD_DIST_PARETO; + fifth_st->iat_dist.param1 = min; + fifth_st->iat_dist.param2 = max; + fifth_st->start_usec = min; + fifth_st->range_usec = max; +} + +/** Simple test that the padding delays sampled from a uniform distribution + * actually faill within the uniform distribution range. */ +/* TODO: Upgrade this test so that each state tests a different prob + * distribution */ +static void +test_circuitpadding_sample_distribution(void *arg) +{ + circpad_machine_state_t *mi; + int n_samples; + int n_states; + + (void) arg; + + /* mock this function so that we dont actually schedule any padding */ + MOCK(circpad_machine_schedule_padding, + circpad_machine_schedule_padding_mock); + + /* Initialize a machine with multiple probability distributions that should + * return values between 0 and 5 */ + circpad_machines_init(); + helper_circpad_circ_distribution_machine_setup(0, 10); + + /* Initialize machine and circuits */ + client_side = TO_CIRCUIT(origin_circuit_new()); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + mi = client_side->padding_info[0]; + + /* For every state, sample a bunch of values from the distribution and ensure + * they fall within range. */ + for (n_states = 0 ; n_states < 6; n_states++) { + /* Make sure we in the right state */ + tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, n_states); + + for (n_samples = 0; n_samples < 100; n_samples++) { + circpad_delay_t delay = circpad_machine_sample_delay(mi); + tt_int_op(delay, OP_GE, 0); + tt_int_op(delay, OP_LE, 10); + } + + /* send a non-padding cell to move to the next machine state */ + circpad_cell_event_nonpadding_received((circuit_t*)client_side); + } + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); + UNMOCK(circpad_machine_schedule_padding); +} + +static circpad_decision_t +circpad_machine_spec_transition_mock(circpad_machine_state_t *mi, + circpad_event_t event) +{ + (void) mi; + (void) event; + + return CIRCPAD_STATE_UNCHANGED; +} + +/* Test per-machine padding rate limits */ +static void +test_circuitpadding_machine_rate_limiting(void *arg) +{ + (void) arg; + bool retval; + circpad_machine_state_t *mi; + int i; + + /* Ignore machine transitions for the purposes of this function, we only + * really care about padding counts */ + MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock); + MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock); + + /* Setup machine and circuits */ + client_side = TO_CIRCUIT(origin_circuit_new()); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + helper_create_basic_machine(); + client_side->padding_machine[0] = &circ_client_machine; + client_side->padding_info[0] = + circpad_circuit_machineinfo_new(client_side, 0); + mi = client_side->padding_info[0]; + /* Set up the machine info so that we can get through the basic functions */ + mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE; + + /* First we are going to test the per-machine rate limits */ + circ_client_machine.max_padding_percent = 50; + circ_client_machine.allowed_padding_count = 100; + + /* Check padding limit, should be fine since we haven't sent anything yet. */ + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so + * the rate limit will not trigger */ + for (i=0;i<99;i++) { + circpad_send_padding_cell_for_callback(mi); + } + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Now send another padding cell to pass circpad_global_allowed_cells=100, + and see that the limit will trigger */ + circpad_send_padding_cell_for_callback(mi); + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 1); + + retval = circpad_machine_schedule_padding(mi); + tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED); + + /* Cover wrap */ + for (;i<UINT16_MAX;i++) { + circpad_send_padding_cell_for_callback(mi); + } + tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/2+1); + + tt_ptr_op(client_side->padding_info[0], OP_EQ, mi); + for (i=0;i<UINT16_MAX;i++) { + circpad_cell_event_nonpadding_sent(client_side); + } + + tt_int_op(mi->nonpadding_sent, OP_EQ, UINT16_MAX/2); + tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/4+1); + + done: + free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side)); +} + +/* Test global padding rate limits */ +static void +test_circuitpadding_global_rate_limiting(void *arg) +{ + (void) arg; + bool retval; + circpad_machine_state_t *mi; + int i; + int64_t actual_mocked_monotime_start; + + /* Ignore machine transitions for the purposes of this function, we only + * really care about padding counts */ + MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock); + MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock); + MOCK(circuit_package_relay_cell, + circuit_package_relay_cell_mock); + MOCK(monotime_absolute_usec, mock_monotime_absolute_usec); + + monotime_init(); + monotime_enable_test_mocking(); + actual_mocked_monotime_start = MONOTIME_MOCK_START; + monotime_set_mock_time_nsec(actual_mocked_monotime_start); + monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start); + curr_mocked_time = actual_mocked_monotime_start; + timers_initialize(); + + client_side = (circuit_t *)origin_circuit_new(); + client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL; + dummy_channel.cmux = circuitmux_alloc(); + + /* Setup machine and circuits */ + relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel, &dummy_channel); + relay_side->purpose = CIRCUIT_PURPOSE_OR; + helper_create_basic_machine(); + relay_side->padding_machine[0] = &circ_client_machine; + relay_side->padding_info[0] = + circpad_circuit_machineinfo_new(relay_side, 0); + mi = relay_side->padding_info[0]; + /* Set up the machine info so that we can get through the basic functions */ + mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE; + + simulate_single_hop_extend(client_side, relay_side, 1); + simulate_single_hop_extend(client_side, relay_side, 1); + + /* Now test the global limits by setting up the consensus */ + networkstatus_t vote1; + vote1.net_params = smartlist_new(); + smartlist_split_string(vote1.net_params, + "circpad_global_allowed_cells=100 circpad_global_max_padding_pct=50", + NULL, 0, 0); + /* Register global limits with the padding subsystem */ + circpad_new_consensus_params(&vote1); + + /* Check padding limit, should be fine since we haven't sent anything yet. */ + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so + * the rate limit will not trigger */ + for (i=0;i<99;i++) { + circpad_send_padding_cell_for_callback(mi); + } + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + /* Now send another padding cell to pass circpad_global_allowed_cells=100, + and see that the limit will trigger */ + circpad_send_padding_cell_for_callback(mi); + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 1); + + retval = circpad_machine_schedule_padding(mi); + tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED); + + /* Now send 92 non-padding cells to get near the + * circpad_global_max_padding_pct=50 limit; in particular with 96 non-padding + * cells, the padding traffic is still 51% of total traffic so limit should + * trigger */ + for (i=0;i<92;i++) { + circpad_cell_event_nonpadding_sent(relay_side); + } + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 1); + + /* Send another non-padding cell to bring the padding traffic to 50% of total + * traffic and get past the limit */ + circpad_cell_event_nonpadding_sent(relay_side); + retval = circpad_machine_reached_padding_limit(mi); + tt_int_op(retval, OP_EQ, 0); + + done: + free_fake_orcirc(relay_side); + circuitmux_detach_all_circuits(dummy_channel.cmux, NULL); + circuitmux_free(dummy_channel.cmux); + SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp)); + smartlist_free(vote1.net_params); +} + +#define TEST_CIRCUITPADDING(name, flags) \ + { #name, test_##name, (flags), NULL, NULL } + +struct testcase_t circuitpadding_tests[] = { + TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_conditions, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_rtt, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_machine_rate_limiting, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_global_rate_limiting, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_token_removal_lower, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_token_removal_higher, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_closest_token_removal, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_closest_token_removal_usec, TT_FORK), + TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK), + END_OF_TESTCASES +}; diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index 2f8646e897..5d625483da 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -187,4 +187,3 @@ struct testcase_t compat_libevent_tests[] = { TT_FORK, NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_config.c b/src/test/test_config.c index 8f011ce1f1..994016a710 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -54,6 +54,7 @@ #include "lib/meminfo/meminfo.h" #include "lib/net/gethostname.h" #include "lib/encoding/confline.h" +#include "lib/encoding/kvline.h" #ifdef HAVE_UNISTD_H #include <unistd.h> @@ -5847,6 +5848,7 @@ test_config_extended_fmt(void *arg) tt_str_op(lp->value, OP_EQ, "is back here"); tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL); lp = lp->next; + tt_assert(!lp); config_free_lines(lines); /* Try with the "extended" flag enabled. */ @@ -5873,11 +5875,88 @@ test_config_extended_fmt(void *arg) tt_str_op(lp->value, OP_EQ, ""); tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR); lp = lp->next; + tt_assert(!lp); done: config_free_lines(lines); } +static void +test_config_kvline_parse(void *arg) +{ + (void)arg; + + config_line_t *lines = NULL; + char *enc = NULL; + + lines = kvline_parse("A=B CD=EF", 0); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "A"); + tt_str_op(lines->value, OP_EQ, "B"); + tt_str_op(lines->next->key, OP_EQ, "CD"); + tt_str_op(lines->next->value, OP_EQ, "EF"); + enc = kvline_encode(lines, 0); + tt_str_op(enc, OP_EQ, "A=B CD=EF"); + tor_free(enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "A=B CD=EF"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB CDE=F", 0); + tt_assert(! lines); + + lines = kvline_parse("AB CDE=F", KV_OMIT_KEYS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, ""); + tt_str_op(lines->value, OP_EQ, "AB"); + tt_str_op(lines->next->key, OP_EQ, "CDE"); + tt_str_op(lines->next->value, OP_EQ, "F"); + tt_assert(lines); + enc = kvline_encode(lines, 0); + tt_assert(!enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "AB CDE=F"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("AB=C CDE=\"F G\"", 0); + tt_assert(!lines); + + lines = kvline_parse("AB=C CDE=\"F G\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, "C"); + tt_str_op(lines->next->key, OP_EQ, "CDE"); + tt_str_op(lines->next->value, OP_EQ, "F G"); + tt_str_op(lines->next->next->key, OP_EQ, ""); + tt_str_op(lines->next->next->value, OP_EQ, "GHI"); + enc = kvline_encode(lines, 0); + tt_assert(!enc); + enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS); + tt_str_op(enc, OP_EQ, "AB=C CDE=\"F G\" GHI"); + tor_free(enc); + config_free_lines(lines); + + lines = kvline_parse("A\"B=C CDE=\"F\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS); + tt_assert(! lines); + + lines = kvline_parse("AB=", KV_QUOTED); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + config_free_lines(lines); + + lines = kvline_parse("AB=", 0); + tt_assert(lines); + tt_str_op(lines->key, OP_EQ, "AB"); + tt_str_op(lines->value, OP_EQ, ""); + + done: + config_free_lines(lines); + tor_free(enc); +} + #define CONFIG_TEST(name, flags) \ { #name, test_config_ ## name, flags, NULL, NULL } @@ -5930,5 +6009,6 @@ struct testcase_t config_tests[] = { CONFIG_TEST(include_opened_file_list, 0), CONFIG_TEST(compute_max_mem_in_queues, 0), CONFIG_TEST(extended_fmt, 0), + CONFIG_TEST(kvline_parse, 0), END_OF_TESTCASES }; diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c index 682ba5b970..7c4c92ea42 100644 --- a/src/test/test_consdiff.c +++ b/src/test/test_consdiff.c @@ -14,6 +14,39 @@ #define tt_str_eq_line(a,b) \ tt_assert(line_str_eq((b),(a))) +static int +consensus_split_lines_(smartlist_t *out, const char *s, memarea_t *area) +{ + size_t len = strlen(s); + return consensus_split_lines(out, s, len, area); +} + +static int +consensus_compute_digest_(const char *cons, + consensus_digest_t *digest_out) +{ + size_t len = strlen(cons); + char *tmp = tor_memdup(cons, len); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + int r = consensus_compute_digest(tmp, len, digest_out); + tor_free(tmp); + return r; +} + +static int +consensus_compute_digest_as_signed_(const char *cons, + consensus_digest_t *digest_out) +{ + size_t len = strlen(cons); + char *tmp = tor_memdup(cons, len); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + int r = consensus_compute_digest_as_signed(tmp, len, digest_out); + tor_free(tmp); + return r; +} + static void test_consdiff_smartlist_slice(void *arg) { @@ -58,7 +91,7 @@ test_consdiff_smartlist_slice_string_pos(void *arg) /* Create a regular smartlist. */ (void)arg; - consensus_split_lines(sl, "a\nd\nc\na\nb\n", area); + consensus_split_lines_(sl, "a\nd\nc\na\nb\n", area); /* See that smartlist_slice_string_pos respects the bounds of the slice. */ sls = smartlist_slice(sl, 2, 5); @@ -87,8 +120,8 @@ test_consdiff_lcs_lengths(void *arg) int e_lengths2[] = { 0, 1, 1, 2, 3, 4 }; (void)arg; - consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area); - consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area); + consensus_split_lines_(sl1, "a\nb\nc\nd\ne\n", area); + consensus_split_lines_(sl2, "a\nc\nd\ni\ne\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); @@ -119,10 +152,10 @@ test_consdiff_trim_slices(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area); - consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area); - consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area); - consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area); + consensus_split_lines_(sl1, "a\nb\nb\nb\nd\n", area); + consensus_split_lines_(sl2, "a\nc\nc\nc\nd\n", area); + consensus_split_lines_(sl3, "a\nb\nb\nb\na\n", area); + consensus_split_lines_(sl4, "c\nb\nb\nb\nc\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); sls3 = smartlist_slice(sl3, 0, -1); @@ -165,8 +198,8 @@ test_consdiff_set_changed(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\nb\na\na\n", area); - consensus_split_lines(sl2, "a\na\na\na\n", area); + consensus_split_lines_(sl1, "a\nb\na\na\n", area); + consensus_split_lines_(sl2, "a\na\na\na\n", area); /* Length of sls1 is 0. */ sls1 = smartlist_slice(sl1, 0, 0); @@ -240,8 +273,8 @@ test_consdiff_calc_changes(void *arg) memarea_t *area = memarea_new(); (void)arg; - consensus_split_lines(sl1, "a\na\na\na\n", area); - consensus_split_lines(sl2, "a\na\na\na\n", area); + consensus_split_lines_(sl1, "a\na\na\na\n", area); + consensus_split_lines_(sl2, "a\na\na\na\n", area); sls1 = smartlist_slice(sl1, 0, -1); sls2 = smartlist_slice(sl2, 0, -1); @@ -259,7 +292,7 @@ test_consdiff_calc_changes(void *arg) tt_assert(!bitarray_is_set(changed2, 3)); smartlist_clear(sl2); - consensus_split_lines(sl2, "a\nb\na\nb\n", area); + consensus_split_lines_(sl2, "a\nb\na\nb\n", area); tor_free(sls1); tor_free(sls2); sls1 = smartlist_slice(sl1, 0, -1); @@ -282,7 +315,7 @@ test_consdiff_calc_changes(void *arg) bitarray_clear(changed1, 3); smartlist_clear(sl2); - consensus_split_lines(sl2, "b\nb\nb\nb\n", area); + consensus_split_lines_(sl2, "b\nb\nb\nb\n", area); tor_free(sls1); tor_free(sls2); sls1 = smartlist_slice(sl1, 0, -1); @@ -610,8 +643,8 @@ test_consdiff_gen_ed_diff(void *arg) /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */ smartlist_clear(cons1); smartlist_clear(cons2); - consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); - consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area); + consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines_(cons2, "A\nC\nO\nE\nU\n", area); diff = gen_ed_diff(cons1, cons2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(7, OP_EQ, smartlist_len(diff)); @@ -627,8 +660,8 @@ test_consdiff_gen_ed_diff(void *arg) smartlist_clear(cons1); smartlist_clear(cons2); - consensus_split_lines(cons1, "B\n", area); - consensus_split_lines(cons2, "A\nB\n", area); + consensus_split_lines_(cons1, "B\n", area); + consensus_split_lines_(cons2, "A\nB\n", area); diff = gen_ed_diff(cons1, cons2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(3, OP_EQ, smartlist_len(diff)); @@ -656,7 +689,7 @@ test_consdiff_apply_ed_diff(void *arg) diff = smartlist_new(); setup_capture_of_logs(LOG_WARN); - consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area); + consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area); /* Command without range. */ smartlist_add_linecpy(diff, area, "a"); @@ -829,7 +862,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_clear(diff); /* Test appending text, 'a'. */ - consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); + consensus_split_lines_(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(8, OP_EQ, smartlist_len(cons2)); @@ -846,7 +879,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test deleting text, 'd'. */ - consensus_split_lines(diff, "4d\n1,2d\n", area); + consensus_split_lines_(diff, "4d\n1,2d\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(2, OP_EQ, smartlist_len(cons2)); @@ -857,7 +890,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test changing text, 'c'. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); + consensus_split_lines_(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(5, OP_EQ, smartlist_len(cons2)); @@ -871,7 +904,7 @@ test_consdiff_apply_ed_diff(void *arg) smartlist_free(cons2); /* Test 'a', 'd' and 'c' together. */ - consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); + consensus_split_lines_(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area); cons2 = apply_ed_diff(cons1, diff, 0); tt_ptr_op(NULL, OP_NE, cons2); tt_int_op(6, OP_EQ, smartlist_len(cons2)); @@ -918,12 +951,12 @@ test_consdiff_gen_diff(void *arg) ); tt_int_op(0, OP_EQ, - consensus_compute_digest_as_signed(cons1_str, &digests1)); + consensus_compute_digest_as_signed_(cons1_str, &digests1)); tt_int_op(0, OP_EQ, - consensus_compute_digest(cons2_str, &digests2)); + consensus_compute_digest_(cons2_str, &digests2)); - consensus_split_lines(cons1, cons1_str, area); - consensus_split_lines(cons2, cons2_str, area); + consensus_split_lines_(cons1, cons1_str, area); + consensus_split_lines_(cons2, cons2_str, area); diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); tt_ptr_op(NULL, OP_EQ, diff); @@ -937,9 +970,9 @@ test_consdiff_gen_diff(void *arg) "directory-signature foo bar\nbar\n" ); tt_int_op(0, OP_EQ, - consensus_compute_digest_as_signed(cons1_str, &digests1)); + consensus_compute_digest_as_signed_(cons1_str, &digests1)); smartlist_clear(cons1); - consensus_split_lines(cons1, cons1_str, area); + consensus_split_lines_(cons1, cons1_str, area); diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area); tt_ptr_op(NULL, OP_NE, diff); tt_int_op(11, OP_EQ, smartlist_len(diff)); @@ -991,8 +1024,8 @@ test_consdiff_apply_diff(void *arg) "directory-signature foo bar\nbar\n" ); tt_int_op(0, OP_EQ, - consensus_compute_digest(cons1_str, &digests1)); - consensus_split_lines(cons1, cons1_str, area); + consensus_compute_digest_(cons1_str, &digests1)); + consensus_split_lines_(cons1, cons1_str, area); /* diff doesn't have enough lines. */ cons2 = consdiff_apply_diff(cons1, diff, &digests1); @@ -1182,4 +1215,3 @@ struct testcase_t consdiff_tests[] = { CONSDIFF_LEGACY(apply_diff), END_OF_TESTCASES }; - diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index 254a5ba5d0..74226b8c52 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -21,6 +21,23 @@ #include "test/test.h" #include "test/log_test_helpers.h" +#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm + +static char * +consensus_diff_apply_(const char *c, const char *d) +{ + size_t c_len = strlen(c); + size_t d_len = strlen(d); + // We use memdup here to ensure that the input is NOT nul-terminated. + // This makes it likelier for us to spot bugs. + char *c_tmp = tor_memdup(c, c_len); + char *d_tmp = tor_memdup(d, d_len); + char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len); + tor_free(c_tmp); + tor_free(d_tmp); + return result; +} + // ============================== Setup/teardown the consdiffmgr // These functions get run before/after each test in this module @@ -153,7 +170,8 @@ lookup_diff_from(consensus_cache_entry_t **out, const char *str1) { uint8_t digest[DIGEST256_LEN]; - if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) { + if (router_get_networkstatus_v3_sha3_as_signed(digest, + str1, strlen(str1))<0) { TT_FAIL(("Unable to compute sha3-as-signed")); return CONSDIFF_NOT_FOUND; } @@ -175,14 +193,15 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav, consensus_cache_entry_incref(ent); size_t size; - char *diff_string = NULL; - int r = uncompress_or_copy(&diff_string, &size, ent); + const char *diff_string = NULL; + char *diff_owned = NULL; + int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent); consensus_cache_entry_decref(ent); if (diff_string == NULL || r < 0) return -1; - char *applied = consensus_diff_apply(str1, diff_string); - tor_free(diff_string); + char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size); + tor_free(diff_owned); if (applied == NULL) return -1; @@ -282,7 +301,8 @@ test_consdiffmgr_add(void *arg) (void) arg; time_t now = approx_time(); - char *body = NULL; + const char *body = NULL; + char *body_owned = NULL; consensus_cache_entry_t *ent = NULL; networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now); @@ -324,7 +344,7 @@ test_consdiffmgr_add(void *arg) tt_assert(ent); consensus_cache_entry_incref(ent); size_t s; - r = uncompress_or_copy(&body, &s, ent); + r = uncompress_or_set_ptr(&body, &s, &body_owned, ent); tt_int_op(r, OP_EQ, 0); tt_int_op(s, OP_EQ, 4); tt_mem_op(body, OP_EQ, "quux", 4); @@ -337,7 +357,7 @@ test_consdiffmgr_add(void *arg) networkstatus_vote_free(ns_tmp); teardown_capture_of_logs(); consensus_cache_entry_decref(ent); - tor_free(body); + tor_free(body_owned); } static void @@ -370,7 +390,8 @@ test_consdiffmgr_make_diffs(void *arg) ns = fake_ns_new(FLAV_MICRODESC, now-3600); md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600); r = consdiffmgr_add_consensus(md_ns_body, ns); - router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body); + router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body, + strlen(md_ns_body)); networkstatus_vote_free(ns); tt_int_op(r, OP_EQ, 0); @@ -414,7 +435,7 @@ test_consdiffmgr_make_diffs(void *arg) r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size); tt_int_op(r, OP_EQ, 0); diff_text = tor_memdup_nulterm(diff_body, diff_size); - applied = consensus_diff_apply(md_ns_body, diff_text); + applied = consensus_diff_apply_(md_ns_body, diff_text); tt_assert(applied); tt_str_op(applied, OP_EQ, md_ns_body_2); diff --git a/src/test/test_containers.c b/src/test/test_containers.c index aedd2f7a89..a0832f868e 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -96,6 +96,30 @@ test_container_smartlist_basic(void *arg) tor_free(v555); } +/** Test SMARTLIST_FOREACH_REVERSE_BEGIN loop macro */ +static void +test_container_smartlist_foreach_reverse(void *arg) +{ + smartlist_t *sl = smartlist_new(); + int i; + + (void) arg; + + /* Add integers to smartlist in increasing order */ + for (i=0;i<100;i++) { + smartlist_add(sl, (void*)(uintptr_t)i); + } + + /* Pop them out in reverse and test their value */ + SMARTLIST_FOREACH_REVERSE_BEGIN(sl, void*, k) { + i--; + tt_ptr_op(k, OP_EQ, (void*)(uintptr_t)i); + } SMARTLIST_FOREACH_END(k); + + done: + smartlist_free(sl); +} + /** Run unit tests for smartlist-of-strings functionality. */ static void test_container_smartlist_strings(void *arg) @@ -1281,6 +1305,7 @@ test_container_smartlist_strings_eq(void *arg) struct testcase_t container_tests[] = { CONTAINER_LEGACY(smartlist_basic), CONTAINER_LEGACY(smartlist_strings), + CONTAINER_LEGACY(smartlist_foreach_reverse), CONTAINER_LEGACY(smartlist_overlap), CONTAINER_LEGACY(smartlist_digests), CONTAINER_LEGACY(smartlist_join), diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 70d36e53d4..647eac43c7 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -4,10 +4,14 @@ #define CONNECTION_PRIVATE #define TOR_CHANNEL_INTERNAL_ #define CONTROL_PRIVATE +#define OCIRC_EVENT_PRIVATE +#define ORCONN_EVENT_PRIVATE #include "core/or/or.h" #include "core/or/channel.h" #include "core/or/channeltls.h" #include "core/or/circuitlist.h" +#include "core/or/ocirc_event.h" +#include "core/or/orconn_event.h" #include "core/mainloop/connection.h" #include "feature/control/control.h" #include "test/test.h" @@ -351,10 +355,10 @@ test_cntev_dirboot_defer_desc(void *arg) /* This event should get deferred */ control_event_boot_dir(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0); assert_bootmsg("0 TAG=starting"); - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); - assert_bootmsg("5 TAG=conn_dir"); + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + assert_bootmsg("5 TAG=conn"); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - assert_bootmsg("10 TAG=handshake_dir"); + assert_bootmsg("14 TAG=handshake"); /* The deferred event should appear */ control_event_boot_first_orconn(); assert_bootmsg("45 TAG=requesting_descriptors"); @@ -374,21 +378,159 @@ test_cntev_dirboot_defer_orconn(void *arg) control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); assert_bootmsg("0 TAG=starting"); /* This event should get deferred */ - control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0); + control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0); assert_bootmsg("0 TAG=starting"); - control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0); - assert_bootmsg("5 TAG=conn_dir"); + control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0); + assert_bootmsg("5 TAG=conn"); control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0); - assert_bootmsg("10 TAG=handshake_dir"); + assert_bootmsg("14 TAG=handshake"); /* The deferred event should appear */ control_event_boot_first_orconn(); - assert_bootmsg("80 TAG=conn_or"); + assert_bootmsg("75 TAG=enough_dirinfo"); done: tor_free(saved_event_str); UNMOCK(queue_control_event_string); } -#define TEST(name, flags) \ +static void +setup_orconn_state(orconn_event_msg_t *msg, uint64_t gid, uint64_t chan, + int proxy_type) +{ + msg->type = ORCONN_MSGTYPE_STATE; + msg->u.state.gid = gid; + msg->u.state.chan = chan; + msg->u.state.proxy_type = proxy_type; +} + +static void +send_orconn_state(orconn_event_msg_t *msg, uint8_t state) +{ + msg->u.state.state = state; + orconn_event_publish(msg); +} + +static void +send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop) +{ + ocirc_event_msg_t msg; + + msg.type = OCIRC_MSGTYPE_CHAN; + msg.u.chan.gid = gid; + msg.u.chan.chan = chan; + msg.u.chan.onehop = onehop; + ocirc_event_publish(&msg); +} + +static void +test_cntev_orconn_state(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_NONE); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + send_ocirc_chan(1, 1, true); + assert_bootmsg("5 TAG=conn"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + /* It doesn't know it's an origin circuit yet */ + assert_bootmsg("15 TAG=handshake_done"); + send_ocirc_chan(2, 2, false); + assert_bootmsg("80 TAG=ap_conn"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("85 TAG=ap_conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("89 TAG=ap_handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("90 TAG=ap_handshake_done"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +static void +test_cntev_orconn_state_pt(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_PLUGGABLE); + send_ocirc_chan(1, 1, true); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("1 TAG=conn_pt"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("2 TAG=conn_done_pt"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + send_ocirc_chan(2, 2, false); + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("76 TAG=ap_conn_pt"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("77 TAG=ap_conn_done_pt"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +static void +test_cntev_orconn_state_proxy(void *arg) +{ + orconn_event_msg_t conn; + + (void)arg; + MOCK(queue_control_event_string, mock_queue_control_event_string); + control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT)); + setup_orconn_state(&conn, 1, 1, PROXY_CONNECT); + send_ocirc_chan(1, 1, true); + + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("3 TAG=conn_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("4 TAG=conn_done_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING); + assert_bootmsg("10 TAG=conn_done"); + send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3); + assert_bootmsg("14 TAG=handshake"); + send_orconn_state(&conn, OR_CONN_STATE_OPEN); + assert_bootmsg("15 TAG=handshake_done"); + + send_ocirc_chan(2, 2, false); + conn.u.state.gid = 2; + conn.u.state.chan = 2; + send_orconn_state(&conn, OR_CONN_STATE_CONNECTING); + assert_bootmsg("78 TAG=ap_conn_proxy"); + send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING); + assert_bootmsg("79 TAG=ap_conn_done_proxy"); + + done: + tor_free(saved_event_str); + UNMOCK(queue_control_event_string); +} + +#define TEST(name, flags) \ { #name, test_cntev_ ## name, flags, 0, NULL } struct testcase_t controller_event_tests[] = { @@ -398,5 +540,8 @@ struct testcase_t controller_event_tests[] = { TEST(event_mask, TT_FORK), TEST(dirboot_defer_desc, TT_FORK), TEST(dirboot_defer_orconn, TT_FORK), + TEST(orconn_state, TT_FORK), + TEST(orconn_state_pt, TT_FORK), + TEST(orconn_state_proxy, TT_FORK), END_OF_TESTCASES }; diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index fa79f4cc47..0b57448bcf 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -254,168 +254,6 @@ test_crypto_openssl_version(void *arg) ; } -/** Run unit tests for our random number generation function and its wrappers. - */ -static void -test_crypto_rng(void *arg) -{ - int i, j, allok; - char data1[100], data2[100]; - double d; - char *h=NULL; - - /* Try out RNG. */ - (void)arg; - tt_assert(! crypto_seed_rng()); - crypto_rand(data1, 100); - crypto_rand(data2, 100); - tt_mem_op(data1,OP_NE, data2,100); - allok = 1; - for (i = 0; i < 100; ++i) { - uint64_t big; - char *host; - j = crypto_rand_int(100); - if (j < 0 || j >= 100) - allok = 0; - big = crypto_rand_uint64(UINT64_C(1)<<40); - if (big >= (UINT64_C(1)<<40)) - allok = 0; - big = crypto_rand_uint64(UINT64_C(5)); - if (big >= 5) - allok = 0; - d = crypto_rand_double(); - tt_assert(d >= 0); - tt_assert(d < 1.0); - host = crypto_random_hostname(3,8,"www.",".onion"); - if (strcmpstart(host,"www.") || - strcmpend(host,".onion") || - strlen(host) < 13 || - strlen(host) > 18) - allok = 0; - tor_free(host); - } - - /* Make sure crypto_random_hostname clips its inputs properly. */ - h = crypto_random_hostname(20000, 9000, "www.", ".onion"); - tt_assert(! strcmpstart(h,"www.")); - tt_assert(! strcmpend(h,".onion")); - tt_int_op(63+4+6, OP_EQ, strlen(h)); - - tt_assert(allok); - done: - tor_free(h); -} - -static void -test_crypto_rng_range(void *arg) -{ - int got_smallest = 0, got_largest = 0; - int i; - - (void)arg; - for (i = 0; i < 1000; ++i) { - int x = crypto_rand_int_range(5,9); - tt_int_op(x, OP_GE, 5); - tt_int_op(x, OP_LT, 9); - if (x == 5) - got_smallest = 1; - if (x == 8) - got_largest = 1; - } - /* These fail with probability 1/10^603. */ - tt_assert(got_smallest); - tt_assert(got_largest); - - got_smallest = got_largest = 0; - const uint64_t ten_billion = 10 * ((uint64_t)1000000000000); - for (i = 0; i < 1000; ++i) { - uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10); - tt_u64_op(x, OP_GE, ten_billion); - tt_u64_op(x, OP_LT, ten_billion+10); - if (x == ten_billion) - got_smallest = 1; - if (x == ten_billion+9) - got_largest = 1; - } - - tt_assert(got_smallest); - tt_assert(got_largest); - - const time_t now = time(NULL); - for (i = 0; i < 2000; ++i) { - time_t x = crypto_rand_time_range(now, now+60); - tt_i64_op(x, OP_GE, now); - tt_i64_op(x, OP_LT, now+60); - if (x == now) - got_smallest = 1; - if (x == now+59) - got_largest = 1; - } - - tt_assert(got_smallest); - tt_assert(got_largest); - done: - ; -} - -static void -test_crypto_rng_strongest(void *arg) -{ - const char *how = arg; - int broken = 0; - - if (how == NULL) { - ; - } else if (!strcmp(how, "nosyscall")) { - break_strongest_rng_syscall = 1; - } else if (!strcmp(how, "nofallback")) { - break_strongest_rng_fallback = 1; - } else if (!strcmp(how, "broken")) { - broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1; - } - -#define N 128 - uint8_t combine_and[N]; - uint8_t combine_or[N]; - int i, j; - - memset(combine_and, 0xff, N); - memset(combine_or, 0, N); - - for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */ - uint8_t output[N]; - memset(output, 0, N); - if (how == NULL) { - /* this one can't fail. */ - crypto_strongest_rand(output, sizeof(output)); - } else { - int r = crypto_strongest_rand_raw(output, sizeof(output)); - if (r == -1) { - if (broken) { - goto done; /* we're fine. */ - } - /* This function is allowed to break, but only if it always breaks. */ - tt_int_op(i, OP_EQ, 0); - tt_skip(); - } else { - tt_assert(! broken); - } - } - for (j = 0; j < N; ++j) { - combine_and[j] &= output[j]; - combine_or[j] |= output[j]; - } - } - - for (j = 0; j < N; ++j) { - tt_int_op(combine_and[j], OP_EQ, 0); - tt_int_op(combine_or[j], OP_EQ, 0xff); - } - done: - ; -#undef N -} - /** Run unit tests for our AES128 functionality */ static void test_crypto_aes128(void *arg) @@ -3140,15 +2978,6 @@ test_crypto_failure_modes(void *arg) struct testcase_t crypto_tests[] = { CRYPTO_LEGACY(formats), - CRYPTO_LEGACY(rng), - { "rng_range", test_crypto_rng_range, 0, NULL, NULL }, - { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL }, - { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK, - &passthrough_setup, (void*)"nosyscall" }, - { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK, - &passthrough_setup, (void*)"nofallback" }, - { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK, - &passthrough_setup, (void*)"broken" }, { "openssl_version", test_crypto_openssl_version, TT_FORK, NULL, NULL }, { "aes_AES", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"aes" }, { "aes_EVP", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"evp" }, diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c new file mode 100644 index 0000000000..23b0c66514 --- /dev/null +++ b/src/test/test_crypto_rng.c @@ -0,0 +1,324 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#define CRYPTO_RAND_PRIVATE +#include "core/or/or.h" +#include "test/test.h" +#include "lib/crypt_ops/aes.h" +#include "lib/crypt_ops/crypto_format.h" +#include "lib/crypt_ops/crypto_rand.h" + +/** Run unit tests for our random number generation function and its wrappers. + */ +static void +test_crypto_rng(void *arg) +{ + int i, j, allok; + char data1[100], data2[100]; + double d; + char *h=NULL; + + /* Try out RNG. */ + (void)arg; + tt_assert(! crypto_seed_rng()); + crypto_rand(data1, 100); + crypto_rand(data2, 100); + tt_mem_op(data1,OP_NE, data2,100); + allok = 1; + for (i = 0; i < 100; ++i) { + uint64_t big; + char *host; + j = crypto_rand_int(100); + if (j < 0 || j >= 100) + allok = 0; + big = crypto_rand_uint64(UINT64_C(1)<<40); + if (big >= (UINT64_C(1)<<40)) + allok = 0; + big = crypto_rand_uint64(UINT64_C(5)); + if (big >= 5) + allok = 0; + d = crypto_rand_double(); + tt_assert(d >= 0); + tt_assert(d < 1.0); + host = crypto_random_hostname(3,8,"www.",".onion"); + if (strcmpstart(host,"www.") || + strcmpend(host,".onion") || + strlen(host) < 13 || + strlen(host) > 18) + allok = 0; + tor_free(host); + } + + /* Make sure crypto_random_hostname clips its inputs properly. */ + h = crypto_random_hostname(20000, 9000, "www.", ".onion"); + tt_assert(! strcmpstart(h,"www.")); + tt_assert(! strcmpend(h,".onion")); + tt_int_op(63+4+6, OP_EQ, strlen(h)); + + tt_assert(allok); + done: + tor_free(h); +} + +static void +test_crypto_rng_range(void *arg) +{ + int got_smallest = 0, got_largest = 0; + int i; + + (void)arg; + for (i = 0; i < 1000; ++i) { + int x = crypto_rand_int_range(5,9); + tt_int_op(x, OP_GE, 5); + tt_int_op(x, OP_LT, 9); + if (x == 5) + got_smallest = 1; + if (x == 8) + got_largest = 1; + } + /* These fail with probability 1/10^603. */ + tt_assert(got_smallest); + tt_assert(got_largest); + + got_smallest = got_largest = 0; + const uint64_t ten_billion = 10 * ((uint64_t)1000000000000); + for (i = 0; i < 1000; ++i) { + uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10); + tt_u64_op(x, OP_GE, ten_billion); + tt_u64_op(x, OP_LT, ten_billion+10); + if (x == ten_billion) + got_smallest = 1; + if (x == ten_billion+9) + got_largest = 1; + } + + tt_assert(got_smallest); + tt_assert(got_largest); + + const time_t now = time(NULL); + for (i = 0; i < 2000; ++i) { + time_t x = crypto_rand_time_range(now, now+60); + tt_i64_op(x, OP_GE, now); + tt_i64_op(x, OP_LT, now+60); + if (x == now) + got_smallest = 1; + if (x == now+59) + got_largest = 1; + } + + tt_assert(got_smallest); + tt_assert(got_largest); + done: + ; +} + +static void +test_crypto_rng_strongest(void *arg) +{ + const char *how = arg; + int broken = 0; + + if (how == NULL) { + ; + } else if (!strcmp(how, "nosyscall")) { + break_strongest_rng_syscall = 1; + } else if (!strcmp(how, "nofallback")) { + break_strongest_rng_fallback = 1; + } else if (!strcmp(how, "broken")) { + broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1; + } + +#define N 128 + uint8_t combine_and[N]; + uint8_t combine_or[N]; + int i, j; + + memset(combine_and, 0xff, N); + memset(combine_or, 0, N); + + for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */ + uint8_t output[N]; + memset(output, 0, N); + if (how == NULL) { + /* this one can't fail. */ + crypto_strongest_rand(output, sizeof(output)); + } else { + int r = crypto_strongest_rand_raw(output, sizeof(output)); + if (r == -1) { + if (broken) { + goto done; /* we're fine. */ + } + /* This function is allowed to break, but only if it always breaks. */ + tt_int_op(i, OP_EQ, 0); + tt_skip(); + } else { + tt_assert(! broken); + } + } + for (j = 0; j < N; ++j) { + combine_and[j] &= output[j]; + combine_or[j] |= output[j]; + } + } + + for (j = 0; j < N; ++j) { + tt_int_op(combine_and[j], OP_EQ, 0); + tt_int_op(combine_or[j], OP_EQ, 0xff); + } + done: + ; +#undef N +} + +static void +test_crypto_rng_fast(void *arg) +{ + (void)arg; + crypto_fast_rng_t *rng = crypto_fast_rng_new(); + tt_assert(rng); + + /* Rudimentary black-block test to make sure that our prng outputs + * have all bits sometimes on and all bits sometimes off. */ + uint64_t m1 = 0, m2 = ~(uint64_t)0; + const int N = 128; + + for (int i=0; i < N; ++i) { + uint64_t v; + crypto_fast_rng_getbytes(rng, (void*)&v, sizeof(v)); + m1 |= v; + m2 &= v; + } + + tt_u64_op(m1, OP_EQ, ~(uint64_t)0); + tt_u64_op(m2, OP_EQ, 0); + + /* Check range functions. */ + int counts[5]; + memset(counts, 0, sizeof(counts)); + for (int i=0; i < N; ++i) { + unsigned u = crypto_fast_rng_get_uint(rng, 5); + tt_int_op(u, OP_GE, 0); + tt_int_op(u, OP_LT, 5); + counts[u]++; + + uint64_t u64 = crypto_fast_rng_get_uint64(rng, UINT64_C(1)<<40); + tt_u64_op(u64, OP_GE, 0); + tt_u64_op(u64, OP_LT, UINT64_C(1)<<40); + + double d = crypto_fast_rng_get_double(rng); + tt_assert(d >= 0.0); + tt_assert(d < 1.0); + } + + /* All values should have come up once. */ + for (int i=0; i<5; ++i) { + tt_int_op(counts[i], OP_GT, 0); + } + + done: + crypto_fast_rng_free(rng); +} + +static void +test_crypto_rng_fast_whitebox(void *arg) +{ + (void)arg; + const size_t buflen = crypto_fast_rng_get_bytes_used_per_stream(); + char *buf = tor_malloc_zero(buflen); + char *buf2 = tor_malloc_zero(buflen); + char *buf3 = NULL, *buf4 = NULL; + + crypto_cipher_t *cipher = NULL, *cipher2 = NULL; + uint8_t seed[CRYPTO_FAST_RNG_SEED_LEN]; + memset(seed, 0, sizeof(seed)); + + /* Start with a prng with zero key and zero IV. */ + crypto_fast_rng_t *rng = crypto_fast_rng_new_from_seed(seed); + tt_assert(rng); + + /* We'll use a stream cipher to keep in sync */ + cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256); + + /* The first 48 bytes are used for the next seed -- let's make sure we have + * them. + */ + memset(seed, 0, sizeof(seed)); + crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed)); + + /* if we get 128 bytes, they should match the bytes from the aes256-counter + * stream, starting at position 48. + */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128); + memset(buf2, 0, 128); + crypto_cipher_crypt_inplace(cipher, buf2, 128); + tt_mem_op(buf, OP_EQ, buf2, 128); + + /* Try that again, with an odd number of bytes. */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 199); + memset(buf2, 0, 199); + crypto_cipher_crypt_inplace(cipher, buf2, 199); + tt_mem_op(buf, OP_EQ, buf2, 199); + + /* Make sure that refilling works as expected: skip all but the last 5 bytes + * of this steam. */ + size_t skip = buflen - (199+128) - 5; + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, skip); + crypto_cipher_crypt_inplace(cipher, buf2, skip); + + /* Now get the next 128 bytes. The first 5 will come from this stream, and + * the next 5 will come from the stream keyed by the new value of 'seed'. */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128); + memset(buf2, 0, 128); + crypto_cipher_crypt_inplace(cipher, buf2, 5); + crypto_cipher_free(cipher); + cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256); + memset(seed, 0, sizeof(seed)); + crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed)); + crypto_cipher_crypt_inplace(cipher, buf2+5, 128-5); + tt_mem_op(buf, OP_EQ, buf2, 128); + + /* And check the next 7 bytes to make sure we didn't discard anything. */ + crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 7); + memset(buf2, 0, 7); + crypto_cipher_crypt_inplace(cipher, buf2, 7); + tt_mem_op(buf, OP_EQ, buf2, 7); + + /* Now try the optimization for long outputs. */ + buf3 = tor_malloc(65536); + crypto_fast_rng_getbytes(rng, (uint8_t*)buf3, 65536); + + buf4 = tor_malloc_zero(65536); + uint8_t seed2[CRYPTO_FAST_RNG_SEED_LEN]; + memset(seed2, 0, sizeof(seed2)); + crypto_cipher_crypt_inplace(cipher, (char*)seed2, sizeof(seed2)); + cipher2 = crypto_cipher_new_with_iv_and_bits(seed2, seed2+32, 256); + crypto_cipher_crypt_inplace(cipher2, buf4, 65536); + tt_mem_op(buf3, OP_EQ, buf4, 65536); + + done: + crypto_fast_rng_free(rng); + crypto_cipher_free(cipher); + crypto_cipher_free(cipher2); + tor_free(buf); + tor_free(buf2); + tor_free(buf3); + tor_free(buf4); +} + +struct testcase_t crypto_rng_tests[] = { + { "rng", test_crypto_rng, 0, NULL, NULL }, + { "rng_range", test_crypto_rng_range, 0, NULL, NULL }, + { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL }, + { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"nosyscall" }, + { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"nofallback" }, + { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"broken" }, + { "fast", test_crypto_rng_fast, 0, NULL, NULL }, + { "fast_whitebox", test_crypto_rng_fast_whitebox, 0, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 0e44c47f3f..07a2641c9f 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -91,9 +91,29 @@ #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif #define NS_MODULE dir +static networkstatus_t * +networkstatus_parse_vote_from_string_(const char *s, + const char **eos_out, + enum networkstatus_type_t ns_type) +{ + size_t len = strlen(s); + // memdup so that it won't be nul-terminated. + char *tmp = tor_memdup(s, len); + networkstatus_t *result = + networkstatus_parse_vote_from_string(tmp, len, eos_out, ns_type); + if (eos_out && *eos_out) { + *eos_out = s + (*eos_out - tmp); + } + tor_free(tmp); + return result; +} + static void test_dir_nicknames(void *arg) { @@ -1767,7 +1787,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, "", 0); setup_capture_of_logs(LOG_WARN); tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); expect_log_msg("Empty bandwidth file\n"); teardown_capture_of_logs(); bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); @@ -1783,7 +1804,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op("", OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1794,7 +1817,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, header_lines_v100, 0); bw_file_headers = smartlist_new(); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1807,7 +1832,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1818,7 +1845,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) tor_asprintf(&content, "%s%s", header_lines_v100, relay_lines_v100); write_str_to_file(fname, content, 0); tor_free(content); - tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL)); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL, + NULL)); /* Test bandwidth file including v1.1.0 bandwidth headers and * v1.0.0 relay lines. bw_file_headers will contain the v1.1.0 headers. */ @@ -1828,7 +1856,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1844,7 +1874,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1861,7 +1893,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1872,7 +1906,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) bw_file_headers = smartlist_new(); write_str_to_file(fname, header_lines_v110_no_terminator, 0); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1883,7 +1919,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) bw_file_headers = smartlist_new(); write_str_to_file(fname, header_lines_v110, 0); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1898,7 +1936,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1913,7 +1953,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1929,7 +1971,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1946,7 +1990,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); tt_str_op(bw_file_headers_str_bad, OP_EQ, bw_file_headers_str); SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c)); @@ -1964,7 +2010,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ, smartlist_len(bw_file_headers)); bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL); @@ -1985,7 +2033,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) write_str_to_file(fname, content, 0); tor_free(content); tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, - bw_file_headers)); + bw_file_headers, + NULL)); + tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ, smartlist_len(bw_file_headers)); /* force bw_file_headers to be bigger than @@ -2014,7 +2064,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) /* Read the bandwidth file */ setup_full_capture_of_logs(LOG_DEBUG); - tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL)); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL, + NULL)); expect_log_msg_containing("Ignoring bandwidth file line"); teardown_capture_of_logs(); @@ -2032,11 +2083,13 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) /* Read the bandwidth file */ setup_full_capture_of_logs(LOG_DEBUG); - tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL)); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL, + NULL)); expect_log_msg_not_containing("Ignoring bandwidth file line"); teardown_capture_of_logs(); done: + unlink(fname); tor_free(fname); tor_free(header_lines_v100); tor_free(header_lines_v110_no_terminator); @@ -2863,11 +2916,17 @@ test_a_networkstatus( MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); /* Parse certificates and keys. */ - cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(cert1); - cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, + strlen(AUTHORITY_CERT_2), + NULL); tt_assert(cert2); - cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, + strlen(AUTHORITY_CERT_3), + NULL); tt_assert(cert3); sign_skey_1 = crypto_pk_new(); sign_skey_2 = crypto_pk_new(); @@ -2969,7 +3028,7 @@ test_a_networkstatus( sign_skey_leg1, FLAV_NS); tt_assert(consensus_text); - con = networkstatus_parse_vote_from_string(consensus_text, NULL, + con = networkstatus_parse_vote_from_string_(consensus_text, NULL, NS_TYPE_CONSENSUS); tt_assert(con); //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n", @@ -2981,7 +3040,7 @@ test_a_networkstatus( sign_skey_leg1, FLAV_MICRODESC); tt_assert(consensus_text_md); - con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + con_md = networkstatus_parse_vote_from_string_(consensus_text_md, NULL, NS_TYPE_CONSENSUS); tt_assert(con_md); tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC); @@ -3080,13 +3139,13 @@ test_a_networkstatus( tt_assert(consensus_text3); tt_assert(consensus_text_md2); tt_assert(consensus_text_md3); - con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL, + con2 = networkstatus_parse_vote_from_string_(consensus_text2, NULL, NS_TYPE_CONSENSUS); - con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL, + con3 = networkstatus_parse_vote_from_string_(consensus_text3, NULL, NS_TYPE_CONSENSUS); - con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL, + con_md2 = networkstatus_parse_vote_from_string_(consensus_text_md2, NULL, NS_TYPE_CONSENSUS); - con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL, + con_md3 = networkstatus_parse_vote_from_string_(consensus_text_md3, NULL, NS_TYPE_CONSENSUS); tt_assert(con2); tt_assert(con3); @@ -3864,6 +3923,62 @@ mock_get_options(void) return mock_options; } +/** + * Test dirauth_get_b64_digest_bw_file. + * This function should be near the other bwauth functions, but it needs + * mock_get_options, that is only defined here. + */ + +static void +test_dir_bwauth_bw_file_digest256(void *arg) +{ + (void)arg; + const char *content = + "1541171221\n" + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 " + "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ " + "bw=760 nick=Test time=2018-05-08T16:13:26\n"; + + char *fname = tor_strdup(get_fname("V3BandwidthsFile")); + /* Initialize to a wrong digest. */ + uint8_t digest[DIGEST256_LEN] = "01234567890123456789abcdefghijkl"; + + /* Digest of an empty string. Initialize to a wrong digest. */ + char digest_empty_str[DIGEST256_LEN] = "01234567890123456789abcdefghijkl"; + crypto_digest256(digest_empty_str, "", 0, DIGEST_SHA256); + + /* Digest of the content. Initialize to a wrong digest. */ + char digest_expected[DIGEST256_LEN] = "01234567890123456789abcdefghijkl"; + crypto_digest256(digest_expected, content, strlen(content), DIGEST_SHA256); + + /* When the bandwidth file can not be found. */ + tt_int_op(-1, OP_EQ, + dirserv_read_measured_bandwidths(fname, + NULL, NULL, digest)); + tt_mem_op(digest, OP_EQ, digest_empty_str, DIGEST256_LEN); + + /* When there is a timestamp but it is too old. */ + write_str_to_file(fname, content, 0); + tt_int_op(-1, OP_EQ, + dirserv_read_measured_bandwidths(fname, + NULL, NULL, digest)); + /* The digest will be correct. */ + tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN); + + update_approx_time(1541171221); + + /* When there is a bandwidth file and it can be read. */ + tt_int_op(0, OP_EQ, + dirserv_read_measured_bandwidths(fname, + NULL, NULL, digest)); + tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN); + + done: + unlink(fname); + tor_free(fname); + update_approx_time(time(NULL)); +} + static void reset_routerstatus(routerstatus_t *rs, const char *hex_identity_digest, @@ -6087,6 +6202,80 @@ test_dir_find_dl_min_delay(void* data) } static void +test_dir_matching_flags(void *arg) +{ + (void) arg; + routerstatus_t *rs_noflags = NULL; + routerstatus_t *rs = NULL; + char *s = NULL; + + smartlist_t *tokens = smartlist_new(); + memarea_t *area = memarea_new(); + + int expected_val_when_unused = 0; + + const char *ex_noflags = + "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " + "192.168.0.1 9001 0\n" + "m thisoneislongerbecauseitisa256bitmddigest33\n" + "s\n"; + const char *cp = ex_noflags; + rs_noflags = routerstatus_parse_entry_from_string( + area, &cp, + cp + strlen(cp), + tokens, NULL, NULL, + MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC); + tt_assert(rs_noflags); + +#define FLAG(string, field) STMT_BEGIN { \ + tor_asprintf(&s,\ + "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " \ + "192.168.0.1 9001 0\n" \ + "m thisoneislongerbecauseitisa256bitmddigest33\n" \ + "s %s\n", string); \ + cp = s; \ + rs = routerstatus_parse_entry_from_string( \ + area, &cp, \ + cp + strlen(cp), \ + tokens, NULL, NULL, \ + MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC); \ + /* the field should usually be 0 when no flags are listed */ \ + tt_int_op(rs_noflags->field, OP_EQ, expected_val_when_unused); \ + /* the field should be 1 when this flags islisted */ \ + tt_int_op(rs->field, OP_EQ, 1); \ + tor_free(s); \ + routerstatus_free(rs); \ +} STMT_END + + FLAG("Authority", is_authority); + FLAG("BadExit", is_bad_exit); + FLAG("Exit", is_exit); + FLAG("Fast", is_fast); + FLAG("Guard", is_possible_guard); + FLAG("HSDir", is_hs_dir); + FLAG("Stable", is_stable); + FLAG("StaleDesc", is_staledesc); + FLAG("V2Dir", is_v2_dir); + + // These flags are assumed to be set whether they're declared or not. + expected_val_when_unused = 1; + FLAG("Running", is_flagged_running); + FLAG("Valid", is_valid); + expected_val_when_unused = 0; + + // These flags are no longer used, but still parsed. + FLAG("Named", is_named); + FLAG("Unnamed", is_unnamed); + + done: + tor_free(s); + routerstatus_free(rs); + routerstatus_free(rs_noflags); + memarea_drop_all(area); + smartlist_free(tokens); +} + +static void test_dir_assumed_flags(void *arg) { (void)arg; @@ -6101,9 +6290,10 @@ test_dir_assumed_flags(void *arg) "192.168.0.1 9001 0\n" "m thisoneislongerbecauseitisa256bitmddigest33\n" "s Fast Guard Stable\n"; + const char *eos = str1 + strlen(str1); const char *cp = str1; - rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL, + rs = routerstatus_parse_entry_from_string(area, &cp, eos, tokens, NULL, NULL, 24, FLAV_MICRODESC); tt_assert(rs); tt_assert(rs->is_flagged_running); @@ -6370,6 +6560,7 @@ struct testcase_t dir_tests[] = { DIR_LEGACY(measured_bw_kb_line_is_after_headers), DIR_LEGACY(measured_bw_kb_cache), DIR_LEGACY(dirserv_read_measured_bandwidths), + DIR(bwauth_bw_file_digest256, 0), DIR_LEGACY(param_voting), DIR(param_voting_lookup, 0), DIR_LEGACY(v3_networkstatus), @@ -6410,6 +6601,7 @@ struct testcase_t dir_tests[] = { DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"), DIR_ARG(find_dl_min_delay, TT_FORK, "car"), DIR(assumed_flags, 0), + DIR(matching_flags, 0), DIR(networkstatus_compute_bw_weights_v10, 0), DIR(platform_str, 0), DIR(networkstatus_consensus_has_ipv6, TT_FORK), diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index 3723d6c31b..0b87e29873 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -42,14 +42,20 @@ dir_common_authority_pk_init(authority_cert_t **cert1, { /* Parse certificates and keys. */ authority_cert_t *cert; - cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(cert); tt_assert(cert->identity_key); *cert1 = cert; tt_assert(*cert1); - *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, + strlen(AUTHORITY_CERT_2), + NULL); tt_assert(*cert2); - *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, + strlen(AUTHORITY_CERT_3), + NULL); tt_assert(*cert3); *sign_skey_1 = crypto_pk_new(); *sign_skey_2 = crypto_pk_new(); @@ -266,7 +272,9 @@ dir_common_add_rs_and_parse(networkstatus_t *vote, networkstatus_t **vote_out, /* dump the vote and try to parse it. */ v_text = format_networkstatus_vote(sign_skey, vote); tt_assert(v_text); - *vote_out = networkstatus_parse_vote_from_string(v_text, NULL, NS_TYPE_VOTE); + *vote_out = networkstatus_parse_vote_from_string(v_text, + strlen(v_text), + NULL, NS_TYPE_VOTE); done: if (v_text) @@ -424,4 +432,3 @@ dir_common_construct_vote_3(networkstatus_t **vote, authority_cert_t *cert, return 0; } - diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 90691fff94..e57bd02584 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -72,6 +72,8 @@ ENABLE_GCC_WARNING(overlength-strings) #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \ "Consensus not signed by sufficient number of requested authorities\r\n\r\n" +#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm + static dir_connection_t * new_dir_conn(void) { @@ -1275,7 +1277,9 @@ test_dir_handle_get_server_keys_authority(void* data) size_t body_used = 0; (void) data; - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -1425,7 +1429,9 @@ test_dir_handle_get_server_keys_sk(void* data) size_t body_used = 0; (void) data; - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -2217,6 +2223,31 @@ test_dir_handle_get_status_vote_next_authority_not_found(void* data) tor_free(header); } +static void +test_dir_handle_get_status_vote_next_bandwidth_not_found(void* data) +{ + dir_connection_t *conn = NULL; + char *header = NULL; + (void) data; + + MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); + + conn = new_dir_conn(); + + tt_int_op(0, OP_EQ, directory_handle_command_get(conn, + GET("/tor/status-vote/next/bandwdith"), NULL, 0)); + + fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, + NULL, NULL, 1, 0); + tt_assert(header); + tt_str_op(NOT_FOUND, OP_EQ, header); + + done: + UNMOCK(connection_write_to_buf_impl_); + connection_free_minimal(TO_CONN(conn)); + tor_free(header); +} + NS_DECL(const char*, dirvote_get_pending_consensus, (consensus_flavor_t flav)); @@ -2393,7 +2424,9 @@ test_dir_handle_get_status_vote_next_authority(void* data) routerlist_free_all(); dirvote_free_all(); - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); /* create a trusted ds */ ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest, @@ -2455,6 +2488,85 @@ test_dir_handle_get_status_vote_next_authority(void* data) } static void +test_dir_handle_get_status_vote_next_bandwidth(void* data) +{ + dir_connection_t *conn = NULL; + char *header = NULL, *body = NULL; + size_t body_used = 0; + (void) data; + + const char *content = + "1541171221\n" + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 " + "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ " + "bw=760 nick=Test time=2018-05-08T16:13:26\n"; + + init_mock_options(); + MOCK(get_options, mock_get_options); + mock_options->V3BandwidthsFile = tor_strdup( + get_fname_rnd("V3BandwidthsFile") + ); + + write_str_to_file(mock_options->V3BandwidthsFile, content, 0); + + MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); + + conn = new_dir_conn(); + tt_int_op(0, OP_EQ, directory_handle_command_get(conn, + GET("/tor/status-vote/next/bandwidth"), NULL, 0)); + + fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, + &body, &body_used, strlen(content)+1, 0); + + tt_assert(header); + tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header); + tt_assert(strstr(header, "Content-Type: text/plain\r\n")); + tt_assert(strstr(header, "Content-Encoding: identity\r\n")); + tt_assert(strstr(header, "Content-Length: 167\r\n")); + + /* Check cache lifetime */ + char expbuf[RFC1123_TIME_LEN+1]; + time_t now = approx_time(); + /* BANDWIDTH_CACHE_LIFETIME is defined in dircache.c. */ + format_rfc1123_time(expbuf, (time_t)(now + 30*60)); + char *expires = NULL; + /* Change to 'Cache-control: max-age=%d' if using http/1.1. */ + tor_asprintf(&expires, "Expires: %s\r\n", expbuf); + tt_assert(strstr(header, expires)); + + tt_int_op(body_used, OP_EQ, strlen(body)); + tt_str_op(content, OP_EQ, body); + + tor_free(header); + tor_free(body); + + /* Request the file using compression, the result should be the same. */ + tt_int_op(0, OP_EQ, directory_handle_command_get(conn, + GET("/tor/status-vote/next/bandwidth.z"), NULL, 0)); + + fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, + &body, &body_used, strlen(content)+1, 0); + + tt_assert(header); + tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header); + tt_assert(strstr(header, "Content-Encoding: deflate\r\n")); + + /* Since using connection_write_to_buf_mock instead of mocking + * connection_buf_add_compress, the content is not actually compressed. + * If it would, the size and content would be different than the original. + */ + + done: + UNMOCK(get_options); + UNMOCK(connection_write_to_buf_impl_); + connection_free_minimal(TO_CONN(conn)); + tor_free(header); + tor_free(body); + tor_free(expires); + or_options_free(mock_options); +} + +static void test_dir_handle_get_status_vote_current_authority(void* data) { dir_connection_t *conn = NULL; @@ -2471,7 +2583,9 @@ test_dir_handle_get_status_vote_current_authority(void* data) routerlist_free_all(); dirvote_free_all(); - mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL); + mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, + strlen(TEST_CERTIFICATE), + NULL); /* create a trusted ds */ ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest, @@ -2627,6 +2741,8 @@ struct testcase_t dir_handle_get_tests[] = { DIR_HANDLE_CMD(status_vote_current_authority, 0), DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0), DIR_HANDLE_CMD(status_vote_next_authority, 0), + DIR_HANDLE_CMD(status_vote_next_bandwidth_not_found, 0), + DIR_HANDLE_CMD(status_vote_next_bandwidth, 0), DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK), DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK), DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK), diff --git a/src/test/test_dns.c b/src/test/test_dns.c index 41a56f65d8..231e6965f7 100644 --- a/src/test/test_dns.c +++ b/src/test/test_dns.c @@ -1,6 +1,7 @@ /* Copyright (c) 2015-2019, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#include "orconfig.h" #include "core/or/or.h" #include "test/test.h" @@ -13,9 +14,71 @@ #include "core/or/edge_connection_st.h" #include "core/or/or_circuit_st.h" +#include "app/config/or_options_st.h" +#include "app/config/config.h" + +#include <event2/event.h> +#include <event2/dns.h> #define NS_MODULE dns +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR +#define NS_SUBMODULE configure_nameservers_fallback + +static or_options_t options = { + .ORPort_set = 1, +}; + +static const or_options_t * +mock_get_options(void) +{ + return &options; +} + +static void +NS(test_main)(void *arg) +{ + (void)arg; + tor_addr_t *nameserver_addr = NULL; + + MOCK(get_options, mock_get_options); + + options.ServerDNSResolvConfFile = (char *)"no_such_file!!!"; + + dns_init(); // calls configure_nameservers() + + tt_int_op(number_of_configured_nameservers(), OP_EQ, 1); + + nameserver_addr = configured_nameserver_address(0); + + tt_assert(tor_addr_family(nameserver_addr) == AF_INET); + tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); + +#ifndef _WIN32 + tor_free(nameserver_addr); + + options.ServerDNSResolvConfFile = (char *)"/dev/null"; + + dns_init(); + + tt_int_op(number_of_configured_nameservers(), OP_EQ, 1); + + nameserver_addr = configured_nameserver_address(0); + + tt_assert(tor_addr_family(nameserver_addr) == AF_INET); + tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001)); +#endif + + UNMOCK(get_options); + + done: + tor_free(nameserver_addr); + return; +} + +#undef NS_SUBMODULE +#endif + #define NS_SUBMODULE clip_ttl static void @@ -736,6 +799,9 @@ NS(test_main)(void *arg) #undef NS_SUBMODULE struct testcase_t dns_tests[] = { +#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR + TEST_CASE(configure_nameservers_fallback), +#endif TEST_CASE(clip_ttl), TEST_CASE(resolve), TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve), diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index a486b13ae1..9e213393b0 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -127,6 +127,9 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr) return 1; /* NOP */ } +#define REASONABLY_FUTURE " reasonably-future" +#define REASONABLY_PAST " reasonably-past" + /* Unittest setup function: Setup a fake network. */ static void * big_fake_network_setup(const struct testcase_t *testcase) @@ -138,9 +141,10 @@ big_fake_network_setup(const struct testcase_t *testcase) const int N_NODES = 271; const char *argument = testcase->setup_data; - int reasonably_live_consensus = 0; + int reasonably_future_consensus = 0, reasonably_past_consensus = 0; if (argument) { - reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL; + reasonably_future_consensus = strstr(argument, REASONABLY_FUTURE) != NULL; + reasonably_past_consensus = strstr(argument, REASONABLY_PAST) != NULL; } big_fake_net_nodes = smartlist_new(); @@ -198,11 +202,15 @@ big_fake_network_setup(const struct testcase_t *testcase) dummy_state = tor_malloc_zero(sizeof(or_state_t)); dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t)); - if (reasonably_live_consensus) { - /* Make the dummy consensus valid from 4 hours ago, but expired an hour + if (reasonably_future_consensus) { + /* Make the dummy consensus valid in 6 hours, and expiring in 7 hours. */ + dummy_consensus->valid_after = approx_time() + 6*3600; + dummy_consensus->valid_until = approx_time() + 7*3600; + } else if (reasonably_past_consensus) { + /* Make the dummy consensus valid from 16 hours ago, but expired 12 hours * ago. */ - dummy_consensus->valid_after = approx_time() - 4*3600; - dummy_consensus->valid_until = approx_time() - 3600; + dummy_consensus->valid_after = approx_time() - 16*3600; + dummy_consensus->valid_until = approx_time() - 12*3600; } else { /* Make the dummy consensus valid for an hour either side of now. */ dummy_consensus->valid_after = approx_time() - 3600; @@ -3039,13 +3047,17 @@ static const struct testcase_setup_t upgrade_circuits = { #define BFN_TEST(name) \ EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \ - { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \ - &big_fake_network, (void*)("reasonably-live") } + { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \ + &big_fake_network, (void*)(REASONABLY_FUTURE) }, \ + { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \ + &big_fake_network, (void*)(REASONABLY_PAST) } #define UPGRADE_TEST(name, arg) \ EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \ - { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \ - &upgrade_circuits, (void*)(arg " reasonably-live") } + { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \ + &upgrade_circuits, (void*)(arg REASONABLY_FUTURE) }, \ + { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \ + &upgrade_circuits, (void*)(arg REASONABLY_PAST) } struct testcase_t entrynodes_tests[] = { NO_PREFIX_TEST(node_preferred_orport), diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index 0c34d37a71..aeb71ec583 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -5,7 +5,7 @@ #define EXT_ORPORT_PRIVATE #define MAINLOOP_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/mainloop/connection.h" #include "core/or/connection_or.h" #include "app/config/config.h" diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 802d0a9ebe..13de1e154b 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -14,7 +14,7 @@ #include "orconfig.h" #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "app/config/confparse.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index 2aff179687..eb7f3bfbb0 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -630,7 +630,7 @@ test_disaster_srv(void *arg) get_disaster_srv(1, srv_one); get_disaster_srv(2, srv_two); - /* Check that the cached ones where updated */ + /* Check that the cached ones were updated */ tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN); tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN); diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index 32b08ecf37..43bf894383 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -267,7 +267,7 @@ helper_clone_authorized_client(const hs_service_authorized_client_t *client) /* Helper: Return a newly allocated service object with the identity keypair * sets and the current descriptor. Then register it to the global map. - * Caller should us hs_free_all() to free this service or remove it from the + * Caller should use hs_free_all() to free this service or remove it from the * global map before freeing. */ static hs_service_t * helper_create_service(void) @@ -289,6 +289,20 @@ helper_create_service(void) return service; } +/* Helper: Deallocate a given service object, its child objects and + * remove it from onion service map. + * */ +static void +helper_destroy_service(hs_service_t *service) +{ + if (!service) + return; + + remove_service(get_hs_service_map(), service); + + hs_service_free(service); +} + /* Helper: Return a newly allocated service object with clients. */ static hs_service_t * helper_create_service_with_clients(int num_clients) @@ -1396,7 +1410,6 @@ static void test_build_update_descriptors(void *arg) { int ret; - time_t now = time(NULL); node_t *node; hs_service_t *service; hs_service_intro_point_t *ip_cur, *ip_next; @@ -1422,7 +1435,8 @@ test_build_update_descriptors(void *arg) voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); update_approx_time(mock_ns.valid_after+1); - now = mock_ns.valid_after+1; + + time_t now = mock_ns.valid_after+1; /* Create a service without a current descriptor to trigger a build. */ service = helper_create_service(); @@ -1626,6 +1640,7 @@ test_build_descriptors(void *arg) { int ret; time_t now = time(NULL); + hs_service_t *last_service = NULL; (void) arg; @@ -1650,19 +1665,27 @@ test_build_descriptors(void *arg) * is disabled. */ { hs_service_t *service = helper_create_service(); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; build_all_descriptors(now); + tt_assert(service->desc_current); + tt_assert(service->desc_current->desc); + hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16); + + helper_destroy_service(service); + last_service = NULL; } /* Generate a valid number of fake auth clients when the number of * clients is zero. */ { hs_service_t *service = helper_create_service_with_clients(0); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; @@ -1670,12 +1693,16 @@ test_build_descriptors(void *arg) hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16); + + helper_destroy_service(service); + last_service = NULL; } /* Generate a valid number of fake auth clients when the number of * clients is not a multiple of 16. */ { hs_service_t *service = helper_create_service_with_clients(20); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; @@ -1683,12 +1710,16 @@ test_build_descriptors(void *arg) hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32); + + helper_destroy_service(service); + last_service = NULL; } /* Do not generate any fake desc client when the number of clients is * a multiple of 16 but not zero. */ { hs_service_t *service = helper_create_service_with_clients(32); + last_service = service; service_descriptor_free(service->desc_current); service->desc_current = NULL; @@ -1696,9 +1727,13 @@ test_build_descriptors(void *arg) hs_desc_superencrypted_data_t *superencrypted; superencrypted = &service->desc_current->desc->superencrypted_data; tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32); + + helper_destroy_service(service); + last_service = NULL; } done: + helper_destroy_service(last_service); hs_free_all(); } diff --git a/src/test/test_logging.c b/src/test/test_logging.c index 95a2fce757..6416e98a4e 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -9,7 +9,6 @@ #include "lib/err/torerr.h" #include "lib/log/log.h" #include "test/test.h" -#include "lib/process/subprocess.h" #ifdef HAVE_UNISTD_H #include <unistd.h> @@ -117,22 +116,27 @@ test_sigsafe_err(void *arg) content = read_file_to_str(fn, 0, NULL); tt_ptr_op(content, OP_NE, NULL); - tor_split_lines(lines, content, (int)strlen(content)); + smartlist_split_string(lines, content, "\n", 0, 0); tt_int_op(smartlist_len(lines), OP_GE, 5); - if (strstr(smartlist_get(lines, 0), "opening new log file")) + if (strstr(smartlist_get(lines, 0), "opening new log file")) { + void *item = smartlist_get(lines, 0); smartlist_del_keeporder(lines, 0); + tor_free(item); + } + tt_assert(strstr(smartlist_get(lines, 0), "Say, this isn't too cool")); - /* Next line is blank. */ - tt_assert(!strcmpstart(smartlist_get(lines, 1), "==============")); - tt_assert(!strcmpstart(smartlist_get(lines, 2), "Minimal.")); - /* Next line is blank. */ - tt_assert(!strcmpstart(smartlist_get(lines, 3), "==============")); - tt_str_op(smartlist_get(lines, 4), OP_EQ, + tt_str_op(smartlist_get(lines, 1), OP_EQ, ""); + tt_assert(!strcmpstart(smartlist_get(lines, 2), "==============")); + tt_assert(!strcmpstart(smartlist_get(lines, 3), "Minimal.")); + tt_str_op(smartlist_get(lines, 4), OP_EQ, ""); + tt_assert(!strcmpstart(smartlist_get(lines, 5), "==============")); + tt_str_op(smartlist_get(lines, 6), OP_EQ, "Testing any attempt to manually log from a signal."); done: tor_free(content); + SMARTLIST_FOREACH(lines, char *, x, tor_free(x)); smartlist_free(lines); } diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c index 089ea812cf..ed6b8a9b66 100644 --- a/src/test/test_mainloop.c +++ b/src/test/test_mainloop.c @@ -6,11 +6,23 @@ * \brief Tests for functions closely related to the Tor main loop */ +#define CONFIG_PRIVATE +#define MAINLOOP_PRIVATE +#define STATEFILE_PRIVATE + #include "test/test.h" #include "test/log_test_helpers.h" #include "core/or/or.h" +#include "core/mainloop/connection.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" + +#include "feature/hs/hs_service.h" + +#include "app/config/config.h" +#include "app/config/statefile.h" +#include "app/config/or_state_st.h" static const uint64_t BILLION = 1000000000; @@ -131,12 +143,227 @@ test_mainloop_update_time_jumps(void *arg) monotime_disable_test_mocking(); } +static int schedule_rescan_called = 0; +static void +mock_schedule_rescan_periodic_events(void) +{ + ++schedule_rescan_called; +} + +static void +test_mainloop_user_activity(void *arg) +{ + (void)arg; + const time_t start = 1542658829; + update_approx_time(start); + + MOCK(schedule_rescan_periodic_events, mock_schedule_rescan_periodic_events); + + reset_user_activity(start); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + set_network_participation(false); + + // reset can move backwards and forwards, but does not change network + // participation. + reset_user_activity(start-10); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-10); + reset_user_activity(start+10); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10); + + tt_int_op(schedule_rescan_called, OP_EQ, 0); + tt_int_op(false, OP_EQ, is_participating_on_network()); + + // "note" can only move forward. Calling it from a non-participating + // state makes us rescan the periodic callbacks and set participation. + note_user_activity(start+20); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+20); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // Calling it again will move us forward, but not call rescan again. + note_user_activity(start+25); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // We won't move backwards. + note_user_activity(start+20); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25); + tt_int_op(true, OP_EQ, is_participating_on_network()); + tt_int_op(schedule_rescan_called, OP_EQ, 1); + + // We _will_ adjust if the clock jumps though. + netstatus_note_clock_jumped(500); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+525); + + netstatus_note_clock_jumped(-400); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+125); + + done: + UNMOCK(schedule_rescan_periodic_events); +} + +static unsigned int +mock_get_num_services(void) +{ + return 1; +} + +static connection_t * +mock_connection_gbtu(int type) +{ + (void) type; + return (void *)"hello fellow connections"; +} + +static void +test_mainloop_check_participation(void *arg) +{ + (void)arg; + or_options_t *options = options_new(); + const time_t start = 1542658829; + const time_t ONE_DAY = 24*60*60; + + // Suppose we've been idle for a day or two + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + check_network_participation_callback(start, options); + tt_int_op(is_participating_on_network(), OP_EQ, false); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + + // suppose we've been idle for 2 days... but we are a server. + reset_user_activity(start - 2*ONE_DAY); + options->ORPort_set = 1; + set_network_participation(true); + check_network_participation_callback(start+2, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+2); + options->ORPort_set = 0; + + // idle for 2 days, but we have a hidden service. + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + MOCK(hs_service_get_num_services, mock_get_num_services); + check_network_participation_callback(start+3, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+3); + UNMOCK(hs_service_get_num_services); + + // idle for 2 days but we have at least one user connection + MOCK(connection_get_by_type_nonlinked, mock_connection_gbtu); + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantTimeoutDisabledByIdleStreams = 1; + check_network_participation_callback(start+10, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10); + + // as above, but DormantTimeoutDisabledByIdleStreams is not set + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantTimeoutDisabledByIdleStreams = 0; + check_network_participation_callback(start+13, options); + tt_int_op(is_participating_on_network(), OP_EQ, false); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + UNMOCK(connection_get_by_type_nonlinked); + options->DormantTimeoutDisabledByIdleStreams = 1; + + // idle for 2 days but DormantClientTimeout is 3 days + reset_user_activity(start - 2*ONE_DAY); + set_network_participation(true); + options->DormantClientTimeout = ONE_DAY * 3; + check_network_participation_callback(start+30, options); + tt_int_op(is_participating_on_network(), OP_EQ, true); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY); + + done: + or_options_free(options); + UNMOCK(hs_service_get_num_services); + UNMOCK(connection_get_by_type_nonlinked); +} + +static void +test_mainloop_dormant_load_state(void *arg) +{ + (void)arg; + or_state_t *state = or_state_new(); + const time_t start = 1543956575; + + reset_user_activity(0); + set_network_participation(false); + + // When we construct a new state, it starts out in "auto" mode. + tt_int_op(state->Dormant, OP_EQ, -1); + + // Initializing from "auto" makes us start out (by default) non-Dormant, + // with activity right now. + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + // Initializing from dormant clears the last user activity time, and + // makes us dormant. + state->Dormant = 1; + netstatus_load_from_state(state, start); + tt_assert(! is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, 0); + + // Initializing from non-dormant sets the last user activity time, and + // makes us non-dormant. + state->Dormant = 0; + state->MinutesSinceUserActivity = 123; + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start - 123*60); + + // If we would start dormant, but DormantCanceledByStartup is set, then + // we start up non-dormant. + state->Dormant = 1; + get_options_mutable()->DormantCanceledByStartup = 1; + netstatus_load_from_state(state, start); + tt_assert(is_participating_on_network()); + tt_i64_op(get_last_user_activity_time(), OP_EQ, start); + + done: + or_state_free(state); +} + +static void +test_mainloop_dormant_save_state(void *arg) +{ + (void)arg; + or_state_t *state = or_state_new(); + const time_t start = 1543956575; + + // Can we save a non-dormant state correctly? + reset_user_activity(start - 1000); + set_network_participation(true); + netstatus_flush_to_state(state, start); + + tt_int_op(state->Dormant, OP_EQ, 0); + tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 1000 / 60); + + // Can we save a dormant state correctly? + set_network_participation(false); + netstatus_flush_to_state(state, start); + + tt_int_op(state->Dormant, OP_EQ, 1); + tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 0); + + done: + or_state_free(state); +} + #define MAINLOOP_TEST(name) \ { #name, test_mainloop_## name , TT_FORK, NULL, NULL } struct testcase_t mainloop_tests[] = { MAINLOOP_TEST(update_time_normal), MAINLOOP_TEST(update_time_jumps), + MAINLOOP_TEST(user_activity), + MAINLOOP_TEST(check_participation), + MAINLOOP_TEST(dormant_load_state), + MAINLOOP_TEST(dormant_save_state), END_OF_TESTCASES }; - diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 4c4317d81a..53ee799185 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -11,6 +11,7 @@ #include "feature/dirparse/routerparse.h" #include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/routerlist.h" #include "feature/nodelist/torcert.h" @@ -70,6 +71,7 @@ test_md_cache(void *data) const char *test_md3_noannotation = strchr(test_md3, '\n')+1; time_t time1, time2, time3; char *fn = NULL, *s = NULL; + char *encoded_family = NULL; (void)data; options = get_options_mutable(); @@ -172,8 +174,9 @@ test_md_cache(void *data) tt_ptr_op(md1->family, OP_EQ, NULL); tt_ptr_op(md3->family, OP_NE, NULL); - tt_int_op(smartlist_len(md3->family), OP_EQ, 3); - tt_str_op(smartlist_get(md3->family, 0), OP_EQ, "nodeX"); + + encoded_family = nodefamily_format(md3->family); + tt_str_op(encoded_family, OP_EQ, "nodex nodey nodez"); /* Now rebuild the cache! */ tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0); @@ -254,6 +257,7 @@ test_md_cache(void *data) smartlist_free(wanted); tor_free(s); tor_free(fn); + tor_free(encoded_family); } static const char truncated_md[] = @@ -417,6 +421,28 @@ static const char test_md2_21[] = "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; +static const char test_md2_withfamily_28[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" + "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" + "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" + "family OtherNode !Strange\n" + "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; + +static const char test_md2_withfamily_29[] = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" + "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" + "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n" + "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" + "family !Strange $B7E27F104213C36F13E7E9829182845E495997A0 othernode\n" + "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n"; + static void test_md_generate(void *arg) { @@ -447,6 +473,17 @@ test_md_generate(void *arg) tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey, &ri->cache_info.signing_key_cert->signing_key)); + // Try family encoding. + microdesc_free(md); + ri->declared_family = smartlist_new(); + smartlist_add_strdup(ri->declared_family, "OtherNode !Strange"); + md = dirvote_create_microdescriptor(ri, 28); + tt_str_op(md->body, OP_EQ, test_md2_withfamily_28); + + microdesc_free(md); + md = dirvote_create_microdescriptor(ri, 29); + tt_str_op(md->body, OP_EQ, test_md2_withfamily_29); + done: microdesc_free(md); routerinfo_free(ri); diff --git a/src/test/test_netinfo.c b/src/test/test_netinfo.c new file mode 100644 index 0000000000..27d276d42f --- /dev/null +++ b/src/test/test_netinfo.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "trunnel/netinfo.h" +#include "test/test.h" + +static void +test_netinfo_unsupported_addr(void *arg) +{ + const uint8_t wire_data[] = + { // TIME + 0x00, 0x00, 0x00, 0x01, + // OTHERADDR + 0x04, // ATYPE + 0x04, // ALEN + 0x08, 0x08, 0x08, 0x08, // AVAL + 0x01, // NMYADDR + 0x03, // ATYPE (unsupported) + 0x05, // ALEN + 'a', 'd', 'r', 'r', '!' // AVAL (unsupported) + }; + + (void)arg; + + netinfo_cell_t *parsed_cell = NULL; + + ssize_t parsed = netinfo_cell_parse(&parsed_cell, wire_data, + sizeof(wire_data)); + + tt_assert(parsed == sizeof(wire_data)); + + netinfo_addr_t *addr = netinfo_cell_get_my_addrs(parsed_cell, 0); + tt_assert(addr); + + tt_int_op(3, OP_EQ, netinfo_addr_get_addr_type(addr)); + tt_int_op(5, OP_EQ, netinfo_addr_get_len(addr)); + + done: + netinfo_cell_free(parsed_cell); +} + +struct testcase_t netinfo_tests[] = { + { "unsupported_addr", test_netinfo_unsupported_addr, 0, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 53eb0413e5..8d6d3cb974 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -6,15 +6,19 @@ * \brief Unit tests for nodelist related functions. **/ +#define NODELIST_PRIVATE + #include "core/or/or.h" #include "lib/crypt_ops/crypto_rand.h" #include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodefamily.h" #include "feature/nodelist/nodelist.h" #include "feature/nodelist/torcert.h" #include "feature/nodelist/microdesc_st.h" #include "feature/nodelist/networkstatus_st.h" #include "feature/nodelist/node_st.h" +#include "feature/nodelist/nodefamily_st.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerstatus_st.h" @@ -231,6 +235,411 @@ test_nodelist_ed_id(void *arg) #undef N_NODES } +static void +test_nodelist_nodefamily(void *arg) +{ + (void)arg; + /* hex ID digests */ + const char h1[] = "5B435D6869206861206C65207363617270652070"; + const char h2[] = "75C3B220616E6461726520696E206769726F2061"; + const char h3[] = "2074726F766172206461206D616E67696172652C"; + const char h4[] = "206D656E747265206E6F6E2076616C65206C2769"; + const char h5[] = "6E766572736F2E202D2D5072696D6F204C657669"; + + /* binary ID digests */ + uint8_t d1[DIGEST_LEN], d2[DIGEST_LEN], d3[DIGEST_LEN], d4[DIGEST_LEN], + d5[DIGEST_LEN]; + base16_decode((char*)d1, sizeof(d1), h1, strlen(h1)); + base16_decode((char*)d2, sizeof(d2), h2, strlen(h2)); + base16_decode((char*)d3, sizeof(d3), h3, strlen(h3)); + base16_decode((char*)d4, sizeof(d4), h4, strlen(h4)); + base16_decode((char*)d5, sizeof(d5), h5, strlen(h5)); + + char *enc=NULL, *enc2=NULL; + + nodefamily_t *nf1 = NULL; + nodefamily_t *nf2 = NULL; + nodefamily_t *nf3 = NULL; + + enc = nodefamily_format(NULL); + tt_str_op(enc, OP_EQ, ""); + tor_free(enc); + + /* Make sure that sorting and de-duplication work. */ + tor_asprintf(&enc, "$%s hello", h1); + nf1 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf1); + tor_free(enc); + + tor_asprintf(&enc, "hello hello $%s hello", h1); + nf2 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf2); + tt_ptr_op(nf1, OP_EQ, nf2); + tor_free(enc); + + tor_asprintf(&enc, "%s $%s hello", h1, h1); + nf3 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_EQ, nf3); + tor_free(enc); + + tt_assert(nodefamily_contains_rsa_id(nf1, d1)); + tt_assert(! nodefamily_contains_rsa_id(nf1, d2)); + tt_assert(nodefamily_contains_nickname(nf1, "hello")); + tt_assert(nodefamily_contains_nickname(nf1, "HELLO")); + tt_assert(! nodefamily_contains_nickname(nf1, "goodbye")); + + tt_int_op(nf1->refcnt, OP_EQ, 3); + nodefamily_free(nf3); + tt_int_op(nf1->refcnt, OP_EQ, 2); + + /* Try parsing with a provided self RSA digest. */ + nf3 = nodefamily_parse("hello ", d1, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_EQ, nf3); + + /* Do we get the expected result when we re-encode? */ + tor_asprintf(&enc, "$%s hello", h1); + enc2 = nodefamily_format(nf1); + tt_str_op(enc2, OP_EQ, enc); + tor_free(enc2); + tor_free(enc); + + /* Make sure that we get a different result if we give a different digest. */ + nodefamily_free(nf3); + tor_asprintf(&enc, "hello $%s hello", h3); + nf3 = nodefamily_parse(enc, NULL, 0); + tt_assert(nf3); + tt_ptr_op(nf1, OP_NE, nf3); + tor_free(enc); + + tt_assert(nodefamily_contains_rsa_id(nf3, d3)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d2)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d1)); + tt_assert(nodefamily_contains_nickname(nf3, "hello")); + tt_assert(! nodefamily_contains_nickname(nf3, "goodbye")); + + nodefamily_free(nf1); + nodefamily_free(nf2); + nodefamily_free(nf3); + + /* Try one with several digests, all with nicknames appended, in different + formats. */ + tor_asprintf(&enc, "%s $%s $%s=res $%s~ist", h1, h2, h3, h4); + nf1 = nodefamily_parse(enc, d5, 0); + tt_assert(nf1); + tt_assert(nodefamily_contains_rsa_id(nf1, d1)); + tt_assert(nodefamily_contains_rsa_id(nf1, d2)); + tt_assert(nodefamily_contains_rsa_id(nf1, d3)); + tt_assert(nodefamily_contains_rsa_id(nf1, d4)); + tt_assert(nodefamily_contains_rsa_id(nf1, d5)); + /* Nicknames aren't preserved when ids are present, since node naming is + * deprecated */ + tt_assert(! nodefamily_contains_nickname(nf3, "res")); + tor_free(enc); + tor_asprintf(&enc, "$%s $%s $%s $%s $%s", h4, h3, h1, h5, h2); + enc2 = nodefamily_format(nf1); + tt_str_op(enc, OP_EQ, enc2); + tor_free(enc); + tor_free(enc2); + + /* Try ones where we parse the empty string. */ + nf2 = nodefamily_parse("", NULL, 0); + nf3 = nodefamily_parse("", d4, 0); + tt_assert(nf2); + tt_assert(nf3); + tt_ptr_op(nf2, OP_NE, nf3); + + tt_assert(! nodefamily_contains_rsa_id(nf2, d4)); + tt_assert(nodefamily_contains_rsa_id(nf3, d4)); + tt_assert(! nodefamily_contains_rsa_id(nf2, d5)); + tt_assert(! nodefamily_contains_rsa_id(nf3, d5)); + tt_assert(! nodefamily_contains_nickname(nf2, "fred")); + tt_assert(! nodefamily_contains_nickname(nf3, "bosco")); + + /* The NULL family should contain nothing. */ + tt_assert(! nodefamily_contains_rsa_id(NULL, d4)); + tt_assert(! nodefamily_contains_rsa_id(NULL, d5)); + + done: + tor_free(enc); + tor_free(enc2); + nodefamily_free(nf1); + nodefamily_free(nf2); + nodefamily_free(nf3); + nodefamily_free_all(); +} + +static void +test_nodelist_nodefamily_parse_err(void *arg) +{ + (void)arg; + nodefamily_t *nf1 = NULL; + char *enc = NULL; + const char *semibogus = + "sdakljfdslkfjdsaklfjdkl9sdf " // too long for nickname + "$jkASDFLkjsadfjhkl " // not hex + "$7468696e67732d696e2d7468656d73656c766573 " // ok + "reticulatogranulate "// ok + "$73656d69616e7468726f706f6c6f676963616c6c79 " // too long for hex + "$616273656e746d696e6465646e6573736573" // too short for hex + ; + + setup_capture_of_logs(LOG_WARN); + + // We only get two items when we parse this. + for (int reject = 0; reject <= 1; ++reject) { + for (int log_at_warn = 0; log_at_warn <= 1; ++log_at_warn) { + unsigned flags = log_at_warn ? NF_WARN_MALFORMED : 0; + flags |= reject ? NF_REJECT_MALFORMED : 0; + nf1 = nodefamily_parse(semibogus, NULL, flags); + if (reject) { + tt_assert(nf1 == NULL); + } else { + tt_assert(nf1); + enc = nodefamily_format(nf1); + tt_str_op(enc, OP_EQ, + "$7468696E67732D696E2D7468656D73656C766573 " + "reticulatogranulate"); + tor_free(enc); + } + + if (log_at_warn) { + expect_log_msg_containing("$616273656e746d696e6465646e6573736573"); + expect_log_msg_containing("sdakljfdslkfjdsaklfjdkl9sdf"); + } else { + tt_int_op(mock_saved_log_n_entries(), OP_EQ, 0); + } + mock_clean_saved_logs(); + } + } + + done: + tor_free(enc); + nodefamily_free(nf1); + teardown_capture_of_logs(); +} + +static const node_t * +mock_node_get_by_id(const char *id) +{ + if (fast_memeq(id, "!!!!!!!!!!!!!!!!!!!!", DIGEST_LEN)) + return NULL; + + // use tor_free, not node_free. + node_t *fake_node = tor_malloc_zero(sizeof(node_t)); + memcpy(fake_node->identity, id, DIGEST_LEN); + return fake_node; +} + +static const node_t * +mock_node_get_by_nickname(const char *nn, unsigned flags) +{ + (void)flags; + if (!strcmp(nn, "nonesuch")) + return NULL; + + // use tor_free, not node_free. + node_t *fake_node = tor_malloc_zero(sizeof(node_t)); + strlcpy(fake_node->identity, nn, DIGEST_LEN); + return fake_node; +} + +static void +test_nodelist_nodefamily_lookup(void *arg) +{ + (void)arg; + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + MOCK(node_get_by_id, mock_node_get_by_id); + smartlist_t *sl = smartlist_new(); + nodefamily_t *nf1 = NULL; + char *mem_op_hex_tmp = NULL; + + // 'null' is allowed. + nodefamily_add_nodes_to_smartlist(NULL, sl); + tt_int_op(smartlist_len(sl), OP_EQ, 0); + + // Try a real family + nf1 = nodefamily_parse("$EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE " + "$2121212121212121212121212121212121212121 " + "$3333333333333333333333333333333333333333 " + "erewhon nonesuch", NULL, 0); + tt_assert(nf1); + nodefamily_add_nodes_to_smartlist(nf1, sl); + // There were 5 elements; 2 were dropped because the mocked lookup failed. + tt_int_op(smartlist_len(sl), OP_EQ, 3); + + const node_t *n = smartlist_get(sl, 0); + test_memeq_hex(n->identity, "3333333333333333333333333333333333333333"); + n = smartlist_get(sl, 1); + test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE"); + n = smartlist_get(sl, 2); + tt_str_op(n->identity, OP_EQ, "erewhon"); + + done: + UNMOCK(node_get_by_nickname); + UNMOCK(node_get_by_id); + SMARTLIST_FOREACH(sl, node_t *, fake_node, tor_free(fake_node)); + smartlist_free(sl); + nodefamily_free(nf1); + tor_free(mem_op_hex_tmp); +} + +static void +test_nodelist_nickname_matches(void *arg) +{ + (void)arg; + node_t mock_node; + routerstatus_t mock_rs; + memset(&mock_node, 0, sizeof(mock_node)); + memset(&mock_rs, 0, sizeof(mock_rs)); + + strlcpy(mock_rs.nickname, "evilgeniuses", sizeof(mock_rs.nickname)); + mock_node.rs = &mock_rs; + memcpy(mock_node.identity, ".forabettertomorrow.", DIGEST_LEN); + +#define match(x) tt_assert(node_nickname_matches(&mock_node, (x))) +#define no_match(x) tt_assert(! node_nickname_matches(&mock_node, (x))) + + match("evilgeniuses"); + match("EvilGeniuses"); + match("EvilGeniuses"); + match("2e666f7261626574746572746f6d6f72726f772e"); + match("2E666F7261626574746572746F6D6F72726F772E"); + match("$2e666f7261626574746572746f6d6f72726f772e"); + match("$2E666F7261626574746572746F6D6F72726F772E"); + match("$2E666F7261626574746572746F6D6F72726F772E~evilgeniuses"); + match("$2E666F7261626574746572746F6D6F72726F772E~EVILGENIUSES"); + + no_match("evilgenius"); + no_match("evilgeniuseses"); + no_match("evil.genius"); + no_match("$2E666F7261626574746572746F6D6F72726FFFFF"); + no_match("2E666F7261626574746572746F6D6F72726FFFFF"); + no_match("$2E666F7261626574746572746F6D6F72726F772E~fred"); + no_match("$2E666F7261626574746572746F6D6F72726F772E=EVILGENIUSES"); + done: + ; +} + +static void +test_nodelist_node_nodefamily(void *arg) +{ + (void)arg; + node_t mock_node1; + routerstatus_t mock_rs; + microdesc_t mock_md; + + node_t mock_node2; + routerinfo_t mock_ri; + + smartlist_t *nodes=smartlist_new(); + + memset(&mock_node1, 0, sizeof(mock_node1)); + memset(&mock_node2, 0, sizeof(mock_node2)); + memset(&mock_rs, 0, sizeof(mock_rs)); + memset(&mock_md, 0, sizeof(mock_md)); + memset(&mock_ri, 0, sizeof(mock_ri)); + + mock_node1.rs = &mock_rs; + mock_node1.md = &mock_md; + + mock_node2.ri = &mock_ri; + + strlcpy(mock_rs.nickname, "nodeone", sizeof(mock_rs.nickname)); + mock_ri.nickname = tor_strdup("nodetwo"); + + memcpy(mock_node1.identity, "NodeOneNode1NodeOne1", DIGEST_LEN); + memcpy(mock_node2.identity, "SecondNodeWe'reTestn", DIGEST_LEN); + + // empty families. + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(! node_family_contains(&mock_node2, &mock_node1)); + + // Families contain nodes, but not these nodes + mock_ri.declared_family = smartlist_new(); + smartlist_add(mock_ri.declared_family, (char*)"NodeThree"); + mock_md.family = nodefamily_parse("NodeFour", NULL, 0); + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(! node_family_contains(&mock_node2, &mock_node1)); + + // Families contain one another. + smartlist_add(mock_ri.declared_family, (char*) + "4e6f64654f6e654e6f6465314e6f64654f6e6531"); + tt_assert(! node_family_contains(&mock_node1, &mock_node2)); + tt_assert(node_family_contains(&mock_node2, &mock_node1)); + + nodefamily_free(mock_md.family); + mock_md.family = nodefamily_parse( + "NodeFour " + "5365636f6e644e6f64655765277265546573746e", NULL, 0); + tt_assert(node_family_contains(&mock_node1, &mock_node2)); + tt_assert(node_family_contains(&mock_node2, &mock_node1)); + + // Try looking up families now. + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + MOCK(node_get_by_id, mock_node_get_by_id); + + node_lookup_declared_family(nodes, &mock_node1); + tt_int_op(smartlist_len(nodes), OP_EQ, 2); + const node_t *n = smartlist_get(nodes, 0); + tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN); + n = smartlist_get(nodes, 1); + tt_str_op(n->identity, OP_EQ, "nodefour"); + + // free, try the other one. + SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); + smartlist_clear(nodes); + + node_lookup_declared_family(nodes, &mock_node2); + tt_int_op(smartlist_len(nodes), OP_EQ, 2); + n = smartlist_get(nodes, 0); + // This gets a truncated hex hex ID since it was looked up by name + tt_str_op(n->identity, OP_EQ, "NodeThree"); + n = smartlist_get(nodes, 1); + tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6"); + + done: + UNMOCK(node_get_by_nickname); + UNMOCK(node_get_by_id); + smartlist_free(mock_ri.declared_family); + nodefamily_free(mock_md.family); + tor_free(mock_ri.nickname); + // use tor_free, these aren't real nodes + SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x)); + smartlist_free(nodes); +} + +static void +test_nodelist_nodefamily_canonicalize(void *arg) +{ + (void)arg; + char *c = NULL; + + c = nodefamily_canonicalize("", NULL, 0); + tt_str_op(c, OP_EQ, ""); + tor_free(c); + + uint8_t own_id[20]; + memset(own_id, 0, sizeof(own_id)); + c = nodefamily_canonicalize( + "alice BOB caroL %potrzebie !!!@#@# " + "$bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=fred " + "ffffffffffffffffffffffffffffffffffffffff " + "$cccccccccccccccccccccccccccccccccccccccc ", own_id, 0); + tt_str_op(c, OP_EQ, + "!!!@#@# " + "$0000000000000000000000000000000000000000 " + "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB " + "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC " + "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF " + "%potrzebie " + "alice bob carol"); + + done: + tor_free(c); +} + #define NODE(name, flags) \ { #name, test_nodelist_##name, (flags), NULL, NULL } @@ -239,6 +648,11 @@ struct testcase_t nodelist_tests[] = { NODE(node_get_verbose_nickname_not_named, TT_FORK), NODE(node_is_dir, TT_FORK), NODE(ed_id, TT_FORK), + NODE(nodefamily, TT_FORK), + NODE(nodefamily_parse_err, TT_FORK), + NODE(nodefamily_lookup, TT_FORK), + NODE(nickname_matches, 0), + NODE(node_nodefamily, TT_FORK), + NODE(nodefamily_canonicalize, 0), END_OF_TESTCASES }; - diff --git a/src/test/test_oom.c b/src/test/test_oom.c index b813fa43a3..da6b2ee14d 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -8,7 +8,7 @@ #define CIRCUITLIST_PRIVATE #define CONNECTION_PRIVATE #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/circuitlist.h" #include "lib/evloop/compat_libevent.h" #include "core/mainloop/connection.h" diff --git a/src/test/test_options.c b/src/test/test_options.c index 66b0e7ef11..f12e6b6763 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -429,6 +429,7 @@ get_options_test_data(const char *conf) // with options_init(), but about a dozen tests break when I do that. // Being kinda lame and just fixing the immedate breakage for now.. result->opt->ConnectionPadding = -1; // default must be "auto" + result->opt->DormantClientTimeout = 1800; // must be over 600. rv = config_get_lines(conf, &cl, 1); tt_int_op(rv, OP_EQ, 0); diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c new file mode 100644 index 0000000000..0c8f467a45 --- /dev/null +++ b/src/test/test_parsecommon.c @@ -0,0 +1,594 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "core/or/or.h" +#include "test/test.h" +#include "lib/memarea/memarea.h" +#include "lib/encoding/binascii.h" +#include "feature/dirparse/parsecommon.h" +#include "test/log_test_helpers.h" + +static void +test_parsecommon_tokenize_string_null(void *arg) +{ + + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + const char *str_with_null = "a\0bccccccccc"; + + int retval = + tokenize_string(area, str_with_null, + str_with_null + 3, + tokens, NULL, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_multiple_lines(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, GE(1), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ), + END_OF_TABLE, + }; + + char *str = tor_strdup( + "hibernating 0\nuptime 1024\n" + "published 2018-10-15 10:00:00\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(smartlist_len(tokens), OP_EQ, 3); + directory_token_t *token = smartlist_get(tokens, 0); + + tt_int_op(token->tp, OP_EQ, K_HIBERNATING); + + token = smartlist_get(tokens, 1); + + tt_int_op(token->tp, OP_EQ, K_UPTIME); + + token = smartlist_get(tokens, 2); + + tt_int_op(token->tp, OP_EQ, K_PUBLISHED); + + tt_int_op(retval, OP_EQ, 0); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_min_cnt(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, EQ(2), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + END_OF_TABLE, + }; + + // Missing "uptime" + char *str = tor_strdup("uptime 1024\nhibernating 0\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_max_cnt(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ), + END_OF_TABLE, + }; + + // "uptime" expected once, but occurs twice in input. + char *str = tor_strdup( + "uptime 1024\nuptime 2048\nhibernating 0\n"); + + int retval = + tokenize_string(area, str, NULL, + tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_at_start(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ), + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + END_OF_TABLE, + }; + + // "client-name" is not the first line. + char *str = tor_strdup( + "uptime 1024\nclient-name Alice\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_at_end(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + T1_END("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ), + T01("uptime", K_UPTIME, EQ(1), NO_OBJ), + END_OF_TABLE, + }; + + // "client-name" is not the last line. + char *str = tor_strdup( + "client-name Alice\nuptime 1024\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_tokenize_string_no_annotations(void *arg) +{ + memarea_t *area = memarea_new(); + smartlist_t *tokens = smartlist_new(); + + (void)arg; + + token_rule_t table[] = { + A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ), + END_OF_TABLE, + }; + + char *str = tor_strdup("@last-listed 2018-09-21 15:30:03\n"); + + int retval = + tokenize_string(area, str, NULL, tokens, table, 0); + + tt_int_op(retval, OP_EQ, -1); + + done: + tor_free(str); + memarea_drop_all(area); + smartlist_free(tokens); + return; +} + +static void +test_parsecommon_get_next_token_success(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime 1024"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, GE(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, K_UPTIME); + tt_int_op(token->n_args, OP_EQ, 1); + tt_str_op(*(token->args), OP_EQ, "1024"); + tt_assert(!token->object_type); + tt_int_op(token->object_size, OP_EQ, 0); + tt_assert(!token->object_body); + + tt_ptr_op(*s, OP_EQ, end); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_concat_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "proto A=1 B=2"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T01("proto", K_PROTO, CONCAT_ARGS, NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, K_PROTO); + tt_int_op(token->n_args, OP_EQ, 1); + tt_str_op(*(token->args), OP_EQ, "A=1 B=2"); + + done: + memarea_drop_all(area); +} + +static void +test_parsecommon_get_next_token_parse_keys(void *arg) +{ + (void)arg; + + memarea_t *area = memarea_new(); + const char *str = + "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" + "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n" + "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n" + "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n" + "-----END RSA PUBLIC KEY-----\n"; + + const char *end = str + strlen(str); + const char **s = (const char **)&str; + directory_token_t *token = NULL; + directory_token_t *token2 = NULL; + + token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024); + + token = get_next_token(area, s, end, &rule); + tt_assert(token); + + tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY); + tt_int_op(token->n_args, OP_EQ, 0); + tt_str_op(token->object_type, OP_EQ, "RSA PUBLIC KEY"); + tt_int_op(token->object_size, OP_EQ, 140); + tt_assert(token->object_body); + tt_assert(token->key); + tt_assert(!token->error); + + const char *str2 = + "client-key\n" + "-----BEGIN RSA PRIVATE KEY-----\n" + "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n" + "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n" + "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n" + "AoGAbDg8CKkdQOnX9c7xFpCnsE8fKqz9eddgHHNwXw1NFTwOt+2gDWKSMZmv2X5S\n" + "CVZg3owZxf5W0nT0D6Ny2+6nliak7foYAvkD0BsCiBhgftwC0zAo6k5rIbUKB3PJ\n" + "QLFXgpJhqWuXkODyt/hS/GTernR437WVSEGp1bnALqiFabECQQDaqHOxzoWY/nvH\n" + "KrfUi8EhqCnqERlRHwrW0MQZ1RPvF16OPPma+xa+ht/amfh3vYN5tZY82Zm43gGl\n" + "XWL5cZhNAkEAzmdSootYVnqLLLRMfHKXnO1XbaEcA/08MDNKGlSclBJixFenE8jX\n" + "iQsUbHwMJuGONvzWpRGPBP2f8xBd28ZtxwJARY+LZshtpfNniz/ixYJESaHG28je\n" + "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n" + "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n" + "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n" + "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n" + "-----END RSA PRIVATE KEY-----\n"; + + const char *end2 = str2 + strlen(str2); + const char **s2 = (const char **)&str2; + + token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS, + NEED_SKEY_1024); + + token2 = get_next_token(area, s2, end2, &rule2); + tt_assert(token2); + + tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY); + tt_int_op(token2->n_args, OP_EQ, 0); + tt_str_op(token2->object_type, OP_EQ, "RSA PRIVATE KEY"); + tt_int_op(token2->object_size, OP_EQ, 608); + tt_assert(token2->object_body); + tt_assert(token2->key); + tt_assert(!token->error); + + done: + if (token) token_clear(token); + if (token2) token_clear(token2); + memarea_drop_all(area); +} + +static void +test_parsecommon_get_next_token_object(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, K_DIRECTORY_SIGNATURE); + tt_int_op(token->n_args, OP_EQ, 2); + tt_str_op(token->args[0], OP_EQ, + "0232AF901C31A04EE9848595AF9BB7620D4C5B2E"); + tt_str_op(token->args[1], OP_EQ, + "CD1FD971855430880D3C31E0331C5C55800C2F79"); + + tt_assert(!token->error); + + char decoded[256]; + const char *signature = + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n"; + tt_assert(signature); + size_t signature_len = strlen(signature); + base64_decode(decoded, sizeof(decoded), signature, signature_len); + + tt_str_op(token->object_type, OP_EQ, "SIGNATURE"); + tt_int_op(token->object_size, OP_EQ, 256); + tt_mem_op(token->object_body, OP_EQ, decoded, 256); + + tt_assert(!token->key); + + done: + memarea_drop_all(area); +} + +static void +test_parsecommon_get_next_token_err_too_many_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime 1024 1024 1024"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Too many arguments to uptime"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_too_few_args(void *arg) +{ + memarea_t *area = memarea_new(); + const char *str = "uptime"; + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &table); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Too few arguments to uptime"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_obj_missing_endline(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: missing object end line"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_bad_beginline(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-Z---\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: bad begin line"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_tag_mismatch(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n" + "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n" + "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n" + "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n" + "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n" + "Ua9DEZB9KbJHVX1rGShrLA==\n" + "-----END SOMETHINGELSE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, + "Malformed object: mismatched end tag SIGNATURE"); + + done: + memarea_drop_all(area); + return; +} + +static void +test_parsecommon_get_next_token_err_bad_base64(void *arg) +{ + memarea_t *area = memarea_new(); + + const char *str = + "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E " + "CD1FD971855430880D3C31E0331C5C55800C2F79\n" + "-----BEGIN SIGNATURE-----\n" + "%%@%%%%%%%!!!'\n" + "-----END SIGNATURE-----\n"; + + const char *end = str + strlen(str); + const char **s = &str; + token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE, + GE(2), NEED_OBJ); + (void)arg; + + directory_token_t *token = get_next_token(area, s, end, &rule); + + tt_int_op(token->tp, OP_EQ, ERR_); + tt_str_op(token->error, OP_EQ, "Malformed object: bad base64-encoded data"); + + done: + memarea_drop_all(area); + return; +} + +#define PARSECOMMON_TEST(name) \ + { #name, test_parsecommon_ ## name, 0, NULL, NULL } + +struct testcase_t parsecommon_tests[] = { + PARSECOMMON_TEST(tokenize_string_null), + PARSECOMMON_TEST(tokenize_string_multiple_lines), + PARSECOMMON_TEST(tokenize_string_min_cnt), + PARSECOMMON_TEST(tokenize_string_max_cnt), + PARSECOMMON_TEST(tokenize_string_at_start), + PARSECOMMON_TEST(tokenize_string_at_end), + PARSECOMMON_TEST(tokenize_string_no_annotations), + PARSECOMMON_TEST(get_next_token_success), + PARSECOMMON_TEST(get_next_token_concat_args), + PARSECOMMON_TEST(get_next_token_parse_keys), + PARSECOMMON_TEST(get_next_token_object), + PARSECOMMON_TEST(get_next_token_err_too_many_args), + PARSECOMMON_TEST(get_next_token_err_too_few_args), + PARSECOMMON_TEST(get_next_token_err_obj_missing_endline), + PARSECOMMON_TEST(get_next_token_err_bad_beginline), + PARSECOMMON_TEST(get_next_token_err_tag_mismatch), + PARSECOMMON_TEST(get_next_token_err_bad_base64), + END_OF_TESTCASES +}; diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c index 4a53639dad..ebac20838f 100644 --- a/src/test/test_periodic_event.c +++ b/src/test/test_periodic_event.c @@ -19,6 +19,7 @@ #include "feature/hibernate/hibernate.h" #include "feature/hs/hs_service.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/netstatus.h" #include "core/mainloop/periodic.h" /** Helper function: This is replaced in some tests for the event callbacks so @@ -50,6 +51,8 @@ test_pe_initialize(void *arg) * need to run the main loop and then wait for a second delaying the unit * tests. Instead, we'll test the callback work indepedently elsewhere. */ initialize_periodic_events(); + set_network_participation(false); + rescan_periodic_events(get_options()); /* Validate that all events have been set up. */ for (int i = 0; periodic_events[i].name; ++i) { @@ -59,7 +62,9 @@ test_pe_initialize(void *arg) tt_u64_op(item->last_action_time, OP_EQ, 0); /* Every event must have role(s) assign to it. This is done statically. */ tt_u64_op(item->roles, OP_NE, 0); - tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0); + int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && + !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); + tt_uint_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); } done: @@ -79,6 +84,8 @@ test_pe_launch(void *arg) * network gets enabled. */ consider_hibernation(time(NULL)); + set_network_participation(true); + /* Hack: We'll set a dumb fn() of each events so they don't get called when * dispatching them. We just want to test the state of the callbacks, not * the whole code path. */ @@ -90,6 +97,7 @@ test_pe_launch(void *arg) options = get_options_mutable(); options->SocksPort_set = 1; periodic_events_on_new_options(options); + #if 0 /* Lets make sure that before intialization, we can't scan the periodic * events list and launch them. Lets try by being a Client. */ @@ -106,13 +114,12 @@ test_pe_launch(void *arg) /* Now that we've initialized, rescan the list to launch. */ periodic_events_on_new_options(options); + int mask = PERIODIC_EVENT_ROLE_CLIENT|PERIODIC_EVENT_ROLE_ALL| + PERIODIC_EVENT_ROLE_NET_PARTICIPANT; for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) { - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); - } else { - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); - } + int should_be_enabled = !!(item->roles & mask); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); // enabled or not, the event has not yet been run. tt_u64_op(item->last_action_time, OP_EQ, 0); } @@ -124,7 +131,8 @@ test_pe_launch(void *arg) unsigned roles = get_my_roles(options); tt_uint_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER| + PERIODIC_EVENT_ROLE_ALL|PERIODIC_EVENT_ROLE_NET_PARTICIPANT); for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; @@ -144,17 +152,23 @@ test_pe_launch(void *arg) /* Disable everything and we'll enable them ALL. */ options->SocksPort_set = 0; options->ORPort_set = 0; + options->DisableNetwork = 1; + set_network_participation(false); periodic_events_on_new_options(options); for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) && + !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled); } /* Enable everything. */ options->SocksPort_set = 1; options->ORPort_set = 1; options->BridgeRelay = 1; options->AuthoritativeDir = 1; options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1; + options->DisableNetwork = 0; + set_network_participation(true); register_dummy_hidden_service(&service); periodic_events_on_new_options(options); /* Note down the reference because we need to remove this service from the @@ -165,7 +179,8 @@ test_pe_launch(void *arg) for (int i = 0; periodic_events[i].name; ++i) { periodic_event_item_t *item = &periodic_events[i]; - tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + tt_int_op(periodic_event_is_enabled(item), OP_EQ, + (item->roles != PERIODIC_EVENT_ROLE_CONTROLEV)); } done: @@ -187,42 +202,49 @@ test_pe_get_roles(void *arg) or_options_t *options = get_options_mutable(); tt_assert(options); + set_network_participation(true); + + const int ALL = PERIODIC_EVENT_ROLE_ALL | + PERIODIC_EVENT_ROLE_NET_PARTICIPANT; /* Nothing configured, should be no roles. */ + tt_assert(net_is_disabled()); roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, 0); + tt_int_op(roles, OP_EQ, ALL); /* Indicate we have a SocksPort, roles should be come Client. */ options->SocksPort_set = 1; roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT); + tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT|ALL); /* Now, we'll add a ORPort so should now be a Relay + Client. */ options->ORPort_set = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_DIRSERVER | ALL)); /* Now add a Bridge. */ options->BridgeRelay = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER); /* Unset client so we can solely test Router role. */ options->SocksPort_set = 0; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL); /* Reset options so we can test authorities. */ options->SocksPort_set = 0; options->ORPort_set = 0; options->BridgeRelay = 0; roles = get_my_roles(options); - tt_int_op(roles, OP_EQ, 0); + tt_int_op(roles, OP_EQ, ALL); /* Now upgrade to Dirauth. */ options->DirPort_set = 1; @@ -230,7 +252,7 @@ test_pe_get_roles(void *arg) options->V3AuthoritativeDir = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* Now Bridge Authority. */ @@ -238,7 +260,7 @@ test_pe_get_roles(void *arg) options->BridgeAuthoritativeDir = 1; roles = get_my_roles(options); tt_int_op(roles, OP_EQ, - PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* Move that bridge auth to become a relay. */ @@ -246,7 +268,7 @@ test_pe_get_roles(void *arg) roles = get_my_roles(options); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY - | PERIODIC_EVENT_ROLE_DIRSERVER)); + | PERIODIC_EVENT_ROLE_DIRSERVER|ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); /* And now an Hidden service. */ @@ -257,7 +279,8 @@ test_pe_get_roles(void *arg) remove_service(get_hs_service_map(), &service); tt_int_op(roles, OP_EQ, (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY | - PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER)); + PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER | + ALL)); tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); done: diff --git a/src/test/test_policy.c b/src/test/test_policy.c index 9c001c294a..46d4a1b94a 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -2024,6 +2024,20 @@ test_policies_fascist_firewall_allows_address(void *arg) expect_ap); \ STMT_END +/** Mock the preferred address function to return zero (prefer IPv4). */ +static int +mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4(void) +{ + return 0; +} + +/** Mock the preferred address function to return one (prefer IPv6). */ +static int +mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6(void) +{ + return 1; +} + /** Run unit tests for fascist_firewall_choose_address */ static void test_policies_fascist_firewall_choose_address(void *arg) @@ -2422,6 +2436,42 @@ test_policies_fascist_firewall_choose_address(void *arg) CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_DIR_CONNECTION, 1, 1, ipv4_dir_ap); + /* Test ClientAutoIPv6ORPort and pretend we prefer IPv4. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientAutoIPv6ORPort = 1; + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + MOCK(fascist_firewall_rand_prefer_ipv6_addr, + mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4); + /* Simulate the initialisation of fake_node.ipv6_preferred */ + fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport( + &mock_options); + + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1, + ipv4_or_ap); + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1, + ipv4_or_ap); + + UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + + /* Test ClientAutoIPv6ORPort and pretend we prefer IPv6. */ + memset(&mock_options, 0, sizeof(or_options_t)); + mock_options.ClientAutoIPv6ORPort = 1; + mock_options.ClientUseIPv4 = 1; + mock_options.ClientUseIPv6 = 1; + MOCK(fascist_firewall_rand_prefer_ipv6_addr, + mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6); + /* Simulate the initialisation of fake_node.ipv6_preferred */ + fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport( + &mock_options); + + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1, + ipv6_or_ap); + CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1, + ipv6_or_ap); + + UNMOCK(fascist_firewall_rand_prefer_ipv6_addr); + done: UNMOCK(get_options); } diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c new file mode 100644 index 0000000000..37cfdae7d9 --- /dev/null +++ b/src/test/test_prob_distr.c @@ -0,0 +1,1456 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_prob_distr.c + * \brief Test probability distributions. + * \detail + * + * For each probability distribution we do two kinds of tests: + * + * a) We do numerical deterministic testing of their cdf/icdf/sf/isf functions + * and the various relationships between them for each distribution. We also + * do deterministic tests on their sampling functions. Test vectors for + * these tests were computed from alternative implementations and were + * eyeballed to make sure they make sense + * (e.g. src/test/prob_distr_mpfr_ref.c computes logit(p) using GNU mpfr + * with 200-bit precision and is then tested in test_logit_logistic()). + * + * b) We do stochastic hypothesis testing (G-test) to ensure that sampling from + * the given distributions is distributed properly. The stochastic tests are + * slow and their false positive rate is not well suited for CI, so they are + * currently disabled-by-default and put into 'tests-slow'. + */ + +#define PROB_DISTR_PRIVATE + +#include "orconfig.h" + +#include "test/test.h" + +#include "core/or/or.h" + +#include "lib/math/prob_distr.h" +#include "lib/math/fp.h" +#include "lib/crypt_ops/crypto_rand.h" + +#include <float.h> +#include <math.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +/** + * Return floor(d) converted to size_t, as a workaround for complaints + * under -Wbad-function-cast for (size_t)floor(d). + */ +static size_t +floor_to_size_t(double d) +{ + double integral_d = floor(d); + return (size_t)integral_d; +} + +/** + * Return ceil(d) converted to size_t, as a workaround for complaints + * under -Wbad-function-cast for (size_t)ceil(d). + */ +static size_t +ceil_to_size_t(double d) +{ + double integral_d = ceil(d); + return (size_t)integral_d; +} + +/* + * Geometric(p) distribution, supported on {1, 2, 3, ...}. + * + * Compute the probability mass function Geom(n; p) of the number of + * trials before the first success when success has probability p. + */ +static double +logpmf_geometric(unsigned n, double p) +{ + /* This is actually a check against 1, but we do >= so that the compiler + does not raise a -Wfloat-equal */ + if (p >= 1) { + if (n == 1) + return 0; + else + return -HUGE_VAL; + } + return (n - 1)*log1p(-p) + log(p); +} + +/** + * Compute the logistic function, translated in output by 1/2: + * logistichalf(x) = logistic(x) - 1/2. Well-conditioned on the entire + * real plane, with maximum condition number 1 at 0. + * + * This implementation gives relative error bounded by 5 eps. + */ +static double +logistichalf(double x) +{ + /* + * Rewrite this with the identity + * + * 1/(1 + e^{-x}) - 1/2 + * = (1 - 1/2 - e^{-x}/2)/(1 + e^{-x}) + * = (1/2 - e^{-x}/2)/(1 + e^{-x}) + * = (1 - e^{-x})/[2 (1 + e^{-x})] + * = -(e^{-x} - 1)/[2 (1 + e^{-x})], + * + * which we can evaluate by -expm1(-x)/[2 (1 + exp(-x))]. + * + * Suppose exp has error d0, + has error d1, expm1 has error + * d2, and / has error d3, so we evaluate + * + * -(1 + d2) (1 + d3) (e^{-x} - 1) + * / [2 (1 + d1) (1 + (1 + d0) e^{-x})]. + * + * In the denominator, + * + * 1 + (1 + d0) e^{-x} + * = 1 + e^{-x} + d0 e^{-x} + * = (1 + e^{-x}) (1 + d0 e^{-x}/(1 + e^{-x})), + * + * so the relative error of the numerator is + * + * d' = d2 + d3 + d2 d3, + * and of the denominator, + * d'' = d1 + d0 e^{-x}/(1 + e^{-x}) + d0 d1 e^{-x}/(1 + e^{-x}) + * = d1 + d0 L(-x) + d0 d1 L(-x), + * + * where L(-x) is logistic(-x). By Lemma 1 the relative error + * of the quotient is bounded by + * + * 2|d2 + d3 + d2 d3 - d1 - d0 L(x) + d0 d1 L(x)|, + * + * Since 0 < L(x) < 1, this is bounded by + * + * 2|d2| + 2|d3| + 2|d2 d3| + 2|d1| + 2|d0| + 2|d0 d1| + * <= 4 eps + 2 eps^2. + */ + if (x < log(DBL_EPSILON/8)) { + /* + * Avoid overflow in e^{-x}. When x < log(eps/4), we + * we further have x < logit(eps/4), so that + * logistic(x) < eps/4. Hence the relative error of + * logistic(x) - 1/2 from -1/2 is bounded by eps/2, and + * so the relative error of -1/2 from logistic(x) - 1/2 + * is bounded by eps. + */ + return -0.5; + } else { + return -expm1(-x)/(2*(1 + exp(-x))); + } +} + +/** + * Compute the log of the sum of the exps. Caller should arrange the + * array in descending order to minimize error because I don't want to + * deal with using temporary space and the one caller in this file + * arranges that anyway. + * + * Warning: This implementation does not handle infinite or NaN inputs + * sensibly, because I don't need that here at the moment. (NaN, or + * -inf and +inf together, should yield NaN; +inf and finite should + * yield +inf; otherwise all -inf should be ignored because exp(-inf) = + * 0.) + */ +static double +logsumexp(double *A, size_t n) +{ + double maximum, sum; + size_t i; + + if (n == 0) + return log(0); + + maximum = A[0]; + for (i = 1; i < n; i++) { + if (A[i] > maximum) + maximum = A[i]; + } + + sum = 0; + for (i = n; i --> 0;) + sum += exp(A[i] - maximum); + + return log(sum) + maximum; +} + +/** + * Compute log(1 - e^x). Defined only for negative x so that e^x < 1. + * This is the complement of a probability in log space. + */ +static double +log1mexp(double x) +{ + + /* + * We want to compute log on [0, 1/2) but log1p on [1/2, +inf), + * so partition x at -log(2) = log(1/2). + */ + if (-log(2) < x) + return log(-expm1(x)); + else + return log1p(-exp(x)); +} + +/* + * Tests of numerical errors in computing logit, logistic, and the + * various cdfs, sfs, icdfs, and isfs. + */ + +#define arraycount(A) (sizeof(A)/sizeof(A[0])) + +/** Return relative error between <b>actual</b> and <b>expected</b>. + * Special cases: If <b>expected</b> is zero or infinite, return 1 if + * <b>actual</b> is equal to <b>expected</b> and 0 if not, since the + * usual notion of relative error is undefined but we only use this + * for testing relerr(e, a) <= bound. If either is NaN, return NaN, + * which has the property that NaN <= bound is false no matter what + * bound is. + * + * Beware: if you test !(relerr(e, a) > bound), then then the result + * is true when a is NaN because NaN > bound is false too. See + * CHECK_RELERR for correct use to decide when to report failure. + */ +static double +relerr(double expected, double actual) +{ + /* + * To silence -Wfloat-equal, we have to test for equality using + * inequalities: we have (fabs(expected) <= 0) iff (expected == 0), + * and (actual <= expected && actual >= expected) iff actual == + * expected whether expected is zero or infinite. + */ + if (fabs(expected) <= 0 || tor_isinf(expected)) { + if (actual <= expected && actual >= expected) + return 0; + else + return 1; + } else { + return fabs((expected - actual)/expected); + } +} + +/** Check that relative error of <b>expected</b> and <b>actual</b> is within + * <b>relerr_bound</b>. Caller must arrange to have i and relerr_bound in + * scope. */ +#define CHECK_RELERR(expected, actual) do { \ + double check_expected = (expected); \ + double check_actual = (actual); \ + const char *str_expected = #expected; \ + const char *str_actual = #actual; \ + double check_relerr = relerr(expected, actual); \ + if (!(relerr(check_expected, check_actual) <= relerr_bound)) { \ + log_warn(LD_GENERAL, "%s:%d: case %u: relerr(%s=%.17e, %s=%.17e)" \ + " = %.17e > %.17e\n", \ + __func__, __LINE__, (unsigned) i, \ + str_expected, check_expected, \ + str_actual, check_actual, \ + check_relerr, relerr_bound); \ + ok = false; \ + } \ +} while (0) + +/* Check that a <= b. + * Caller must arrange to have i in scope. */ +#define CHECK_LE(a, b) do { \ + double check_a = (a); \ + double check_b = (b); \ + const char *str_a = #a; \ + const char *str_b = #b; \ + if (!(check_a <= check_b)) { \ + log_warn(LD_GENERAL, "%s:%d: case %u: %s=%.17e > %s=%.17e\n", \ + __func__, __LINE__, (unsigned) i, \ + str_a, check_a, str_b, check_b); \ + ok = false; \ + } \ +} while (0) + +/** + * Test the logit and logistic functions. Confirm that they agree with + * the cdf, sf, icdf, and isf of the standard Logistic distribution. + * Confirm that the sampler for the standard logistic distribution maps + * [0, 1] into the right subinterval for the inverse transform, for + * this implementation. + */ +static void +test_logit_logistic(void *arg) +{ + (void) arg; + + static const struct { + double x; /* x = logit(p) */ + double p; /* p = logistic(x) */ + double phalf; /* p - 1/2 = logistic(x) - 1/2 */ + } cases[] = { + { -HUGE_VAL, 0, -0.5 }, + { -1000, 0, -0.5 }, + { -710, 4.47628622567513e-309, -0.5 }, + { -708, 3.307553003638408e-308, -0.5 }, + { -2, .11920292202211755, -.3807970779778824 }, + { -1.0000001, .2689414017088022, -.23105859829119776 }, + { -1, .2689414213699951, -.23105857863000487 }, + { -0.9999999, .26894144103118883, -.2310585589688111 }, + /* see src/test/prob_distr_mpfr_ref.c for computation */ + { -4.000000000537333e-5, .49999, -1.0000000000010001e-5 }, + { -4.000000000533334e-5, .49999, -.00001 }, + { -4.000000108916878e-9, .499999999, -1.0000000272292198e-9 }, + { -4e-9, .499999999, -1e-9 }, + { -4e-16, .5, -1e-16 }, + { -4e-300, .5, -1e-300 }, + { 0, .5, 0 }, + { 4e-300, .5, 1e-300 }, + { 4e-16, .5, 1e-16 }, + { 3.999999886872274e-9, .500000001, 9.999999717180685e-10 }, + { 4e-9, .500000001, 1e-9 }, + { 4.0000000005333336e-5, .50001, .00001 }, + { 8.000042667076272e-3, .502, .002 }, + { 0.9999999, .7310585589688111, .2310585589688111 }, + { 1, .7310585786300049, .23105857863000487 }, + { 1.0000001, .7310585982911977, .23105859829119774 }, + { 2, .8807970779778823, .3807970779778824 }, + { 708, 1, .5 }, + { 710, 1, .5 }, + { 1000, 1, .5 }, + { HUGE_VAL, 1, .5 }, + }; + double relerr_bound = 3e-15; /* >10eps */ + size_t i; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double x = cases[i].x; + double p = cases[i].p; + double phalf = cases[i].phalf; + + /* + * cdf is logistic, icdf is logit, and symmetry for + * sf/isf. + */ + CHECK_RELERR(logistic(x), cdf_logistic(x, 0, 1)); + CHECK_RELERR(logistic(-x), sf_logistic(x, 0, 1)); + CHECK_RELERR(logit(p), icdf_logistic(p, 0, 1)); + CHECK_RELERR(-logit(p), isf_logistic(p, 0, 1)); + + CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2, 0, 2)); + CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2, 0, 2)); + CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0, 2)/2); + CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, 2)/2); + + CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x/2, 0, .5)); + CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x/2, 0, .5)); + CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0,.5)*2); + CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, .5)*2); + + CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2 + 1, 1, 2)); + CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2 + 1, 1, 2)); + + /* + * For p near 0 and p near 1/2, the arithmetic of + * translating by 1 loses precision. + */ + if (fabs(p) > DBL_EPSILON && fabs(p) < 0.4) { + CHECK_RELERR(icdf_logistic(p, 0, 1), + (icdf_logistic(p, 1, 2) - 1)/2); + CHECK_RELERR(isf_logistic(p, 0, 1), + (isf_logistic(p, 1, 2) - 1)/2); + } + + CHECK_RELERR(p, logistic(x)); + CHECK_RELERR(phalf, logistichalf(x)); + + /* + * On the interior floating-point numbers, either logit or + * logithalf had better give the correct answer. + * + * For probabilities near 0, we can get much finer resolution with + * logit, and for probabilities near 1/2, we can get much finer + * resolution with logithalf by representing them using p - 1/2. + * + * E.g., we can write -.00001 for phalf, and .49999 for p, but the + * difference 1/2 - .00001 gives 1.0000000000010001e-5 in binary64 + * arithmetic. So test logit(.49999) which should give the same + * answer as logithalf(-1.0000000000010001e-5), namely + * -4.000000000537333e-5, and also test logithalf(-.00001) which + * gives -4.000000000533334e-5 instead -- but don't expect + * logit(.49999) to give -4.000000000533334e-5 even though it looks + * like 1/2 - .00001. + * + * A naive implementation of logit will just use log(p/(1 - p)) and + * give the answer -4.000000000551673e-05 for .49999, which is + * wrong in a lot of digits, which happens because log is + * ill-conditioned near 1 and thus amplifies whatever relative + * error we made in computing p/(1 - p). + */ + if ((0 < p && p < 1) || tor_isinf(x)) { + if (phalf >= p - 0.5 && phalf <= p - 0.5) + CHECK_RELERR(x, logit(p)); + if (p >= 0.5 + phalf && p <= 0.5 + phalf) + CHECK_RELERR(x, logithalf(phalf)); + } + + CHECK_RELERR(-phalf, logistichalf(-x)); + if (fabs(phalf) < 0.5 || tor_isinf(x)) + CHECK_RELERR(-x, logithalf(-phalf)); + if (p < 1 || tor_isinf(x)) { + CHECK_RELERR(1 - p, logistic(-x)); + if (p > .75 || tor_isinf(x)) + CHECK_RELERR(-x, logit(1 - p)); + } else { + CHECK_LE(logistic(-x), 1e-300); + } + } + + for (i = 0; i <= 100; i++) { + double p0 = (double)i/100; + + CHECK_RELERR(logit(p0/(1 + M_E)), sample_logistic(0, 0, p0)); + CHECK_RELERR(-logit(p0/(1 + M_E)), sample_logistic(1, 0, p0)); + CHECK_RELERR(logithalf(p0*(0.5 - 1/(1 + M_E))), + sample_logistic(0, 1, p0)); + CHECK_RELERR(-logithalf(p0*(0.5 - 1/(1 + M_E))), + sample_logistic(1, 1, p0)); + } + + if (!ok) + printf("fail logit/logistic / logistic cdf/sf\n"); + + tt_assert(ok); + + done: + ; +} + +/** + * Test the cdf, sf, icdf, and isf of the LogLogistic distribution. + */ +static void +test_log_logistic(void *arg) +{ + (void) arg; + + static const struct { + /* x is a point in the support of the LogLogistic distribution */ + double x; + /* 'p' is the probability that a random variable X for a given LogLogistic + * probability ditribution will take value less-or-equal to x */ + double p; + /* 'np' is the probability that a random variable X for a given LogLogistic + * probability distribution will take value greater-or-equal to x. */ + double np; + } cases[] = { + { 0, 0, 1 }, + { 1e-300, 1e-300, 1 }, + { 1e-17, 1e-17, 1 }, + { 1e-15, 1e-15, .999999999999999 }, + { .1, .09090909090909091, .90909090909090909 }, + { .25, .2, .8 }, + { .5, .33333333333333333, .66666666666666667 }, + { .75, .42857142857142855, .5714285714285714 }, + { .9999, .49997499874993756, .5000250012500626 }, + { .99999999, .49999999749999996, .5000000025 }, + { .999999999999999, .49999999999999994, .5000000000000002 }, + { 1, .5, .5 }, + }; + double relerr_bound = 3e-15; + size_t i; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double x = cases[i].x; + double p = cases[i].p; + double np = cases[i].np; + + CHECK_RELERR(p, cdf_log_logistic(x, 1, 1)); + CHECK_RELERR(p, cdf_log_logistic(x/2, .5, 1)); + CHECK_RELERR(p, cdf_log_logistic(x*2, 2, 1)); + CHECK_RELERR(p, cdf_log_logistic(sqrt(x), 1, 2)); + CHECK_RELERR(p, cdf_log_logistic(sqrt(x)/2, .5, 2)); + CHECK_RELERR(p, cdf_log_logistic(sqrt(x)*2, 2, 2)); + if (2*sqrt(DBL_MIN) < x) { + CHECK_RELERR(p, cdf_log_logistic(x*x, 1, .5)); + CHECK_RELERR(p, cdf_log_logistic(x*x/2, .5, .5)); + CHECK_RELERR(p, cdf_log_logistic(x*x*2, 2, .5)); + } + + CHECK_RELERR(np, sf_log_logistic(x, 1, 1)); + CHECK_RELERR(np, sf_log_logistic(x/2, .5, 1)); + CHECK_RELERR(np, sf_log_logistic(x*2, 2, 1)); + CHECK_RELERR(np, sf_log_logistic(sqrt(x), 1, 2)); + CHECK_RELERR(np, sf_log_logistic(sqrt(x)/2, .5, 2)); + CHECK_RELERR(np, sf_log_logistic(sqrt(x)*2, 2, 2)); + if (2*sqrt(DBL_MIN) < x) { + CHECK_RELERR(np, sf_log_logistic(x*x, 1, .5)); + CHECK_RELERR(np, sf_log_logistic(x*x/2, .5, .5)); + CHECK_RELERR(np, sf_log_logistic(x*x*2, 2, .5)); + } + + CHECK_RELERR(np, cdf_log_logistic(1/x, 1, 1)); + CHECK_RELERR(np, cdf_log_logistic(1/(2*x), .5, 1)); + CHECK_RELERR(np, cdf_log_logistic(2/x, 2, 1)); + CHECK_RELERR(np, cdf_log_logistic(1/sqrt(x), 1, 2)); + CHECK_RELERR(np, cdf_log_logistic(1/(2*sqrt(x)), .5, 2)); + CHECK_RELERR(np, cdf_log_logistic(2/sqrt(x), 2, 2)); + if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) { + CHECK_RELERR(np, cdf_log_logistic(1/(x*x), 1, .5)); + CHECK_RELERR(np, cdf_log_logistic(1/(2*x*x), .5, .5)); + CHECK_RELERR(np, cdf_log_logistic(2/(x*x), 2, .5)); + } + + CHECK_RELERR(p, sf_log_logistic(1/x, 1, 1)); + CHECK_RELERR(p, sf_log_logistic(1/(2*x), .5, 1)); + CHECK_RELERR(p, sf_log_logistic(2/x, 2, 1)); + CHECK_RELERR(p, sf_log_logistic(1/sqrt(x), 1, 2)); + CHECK_RELERR(p, sf_log_logistic(1/(2*sqrt(x)), .5, 2)); + CHECK_RELERR(p, sf_log_logistic(2/sqrt(x), 2, 2)); + if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) { + CHECK_RELERR(p, sf_log_logistic(1/(x*x), 1, .5)); + CHECK_RELERR(p, sf_log_logistic(1/(2*x*x), .5, .5)); + CHECK_RELERR(p, sf_log_logistic(2/(x*x), 2, .5)); + } + + CHECK_RELERR(x, icdf_log_logistic(p, 1, 1)); + CHECK_RELERR(x/2, icdf_log_logistic(p, .5, 1)); + CHECK_RELERR(x*2, icdf_log_logistic(p, 2, 1)); + CHECK_RELERR(x, icdf_log_logistic(p, 1, 1)); + CHECK_RELERR(sqrt(x)/2, icdf_log_logistic(p, .5, 2)); + CHECK_RELERR(sqrt(x)*2, icdf_log_logistic(p, 2, 2)); + CHECK_RELERR(sqrt(x), icdf_log_logistic(p, 1, 2)); + CHECK_RELERR(x*x/2, icdf_log_logistic(p, .5, .5)); + CHECK_RELERR(x*x*2, icdf_log_logistic(p, 2, .5)); + + if (np < .9) { + CHECK_RELERR(x, isf_log_logistic(np, 1, 1)); + CHECK_RELERR(x/2, isf_log_logistic(np, .5, 1)); + CHECK_RELERR(x*2, isf_log_logistic(np, 2, 1)); + CHECK_RELERR(sqrt(x), isf_log_logistic(np, 1, 2)); + CHECK_RELERR(sqrt(x)/2, isf_log_logistic(np, .5, 2)); + CHECK_RELERR(sqrt(x)*2, isf_log_logistic(np, 2, 2)); + CHECK_RELERR(x*x, isf_log_logistic(np, 1, .5)); + CHECK_RELERR(x*x/2, isf_log_logistic(np, .5, .5)); + CHECK_RELERR(x*x*2, isf_log_logistic(np, 2, .5)); + + CHECK_RELERR(1/x, icdf_log_logistic(np, 1, 1)); + CHECK_RELERR(1/(2*x), icdf_log_logistic(np, .5, 1)); + CHECK_RELERR(2/x, icdf_log_logistic(np, 2, 1)); + CHECK_RELERR(1/sqrt(x), icdf_log_logistic(np, 1, 2)); + CHECK_RELERR(1/(2*sqrt(x)), + icdf_log_logistic(np, .5, 2)); + CHECK_RELERR(2/sqrt(x), icdf_log_logistic(np, 2, 2)); + CHECK_RELERR(1/(x*x), icdf_log_logistic(np, 1, .5)); + CHECK_RELERR(1/(2*x*x), icdf_log_logistic(np, .5, .5)); + CHECK_RELERR(2/(x*x), icdf_log_logistic(np, 2, .5)); + } + + CHECK_RELERR(1/x, isf_log_logistic(p, 1, 1)); + CHECK_RELERR(1/(2*x), isf_log_logistic(p, .5, 1)); + CHECK_RELERR(2/x, isf_log_logistic(p, 2, 1)); + CHECK_RELERR(1/sqrt(x), isf_log_logistic(p, 1, 2)); + CHECK_RELERR(1/(2*sqrt(x)), isf_log_logistic(p, .5, 2)); + CHECK_RELERR(2/sqrt(x), isf_log_logistic(p, 2, 2)); + CHECK_RELERR(1/(x*x), isf_log_logistic(p, 1, .5)); + CHECK_RELERR(1/(2*x*x), isf_log_logistic(p, .5, .5)); + CHECK_RELERR(2/(x*x), isf_log_logistic(p, 2, .5)); + } + + for (i = 0; i <= 100; i++) { + double p0 = (double)i/100; + + CHECK_RELERR(0.5*p0/(1 - 0.5*p0), sample_log_logistic(0, p0)); + CHECK_RELERR((1 - 0.5*p0)/(0.5*p0), + sample_log_logistic(1, p0)); + } + + if (!ok) + printf("fail log logistic cdf/sf\n"); + + tt_assert(ok); + + done: + ; +} + +/** + * Test the cdf, sf, icdf, isf of the Weibull distribution. + */ +static void +test_weibull(void *arg) +{ + (void) arg; + + static const struct { + /* x is a point in the support of the Weibull distribution */ + double x; + /* 'p' is the probability that a random variable X for a given Weibull + * probability ditribution will take value less-or-equal to x */ + double p; + /* 'np' is the probability that a random variable X for a given Weibull + * probability distribution will take value greater-or-equal to x. */ + double np; + } cases[] = { + { 0, 0, 1 }, + { 1e-300, 1e-300, 1 }, + { 1e-17, 1e-17, 1 }, + { .1, .09516258196404043, .9048374180359595 }, + { .5, .3934693402873666, .6065306597126334 }, + { .6931471805599453, .5, .5 }, + { 1, .6321205588285577, .36787944117144233 }, + { 10, .9999546000702375, 4.5399929762484854e-5 }, + { 36, .9999999999999998, 2.319522830243569e-16 }, + { 37, .9999999999999999, 8.533047625744066e-17 }, + { 38, 1, 3.1391327920480296e-17 }, + { 100, 1, 3.720075976020836e-44 }, + { 708, 1, 3.307553003638408e-308 }, + { 710, 1, 4.47628622567513e-309 }, + { 1000, 1, 0 }, + { HUGE_VAL, 1, 0 }, + }; + double relerr_bound = 3e-15; + size_t i; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double x = cases[i].x; + double p = cases[i].p; + double np = cases[i].np; + + CHECK_RELERR(p, cdf_weibull(x, 1, 1)); + CHECK_RELERR(p, cdf_weibull(x/2, .5, 1)); + CHECK_RELERR(p, cdf_weibull(x*2, 2, 1)); + /* For 0 < x < sqrt(DBL_MIN), x^2 loses lots of bits. */ + if (x <= 0 || + sqrt(DBL_MIN) <= x) { + CHECK_RELERR(p, cdf_weibull(x*x, 1, .5)); + CHECK_RELERR(p, cdf_weibull(x*x/2, .5, .5)); + CHECK_RELERR(p, cdf_weibull(x*x*2, 2, .5)); + } + CHECK_RELERR(p, cdf_weibull(sqrt(x), 1, 2)); + CHECK_RELERR(p, cdf_weibull(sqrt(x)/2, .5, 2)); + CHECK_RELERR(p, cdf_weibull(sqrt(x)*2, 2, 2)); + CHECK_RELERR(np, sf_weibull(x, 1, 1)); + CHECK_RELERR(np, sf_weibull(x/2, .5, 1)); + CHECK_RELERR(np, sf_weibull(x*2, 2, 1)); + CHECK_RELERR(np, sf_weibull(x*x, 1, .5)); + CHECK_RELERR(np, sf_weibull(x*x/2, .5, .5)); + CHECK_RELERR(np, sf_weibull(x*x*2, 2, .5)); + if (x >= 10) { + /* + * exp amplifies the error of sqrt(x)^2 + * proportionally to exp(x); for large inputs + * this is significant. + */ + double t = -expm1(-x*(2*DBL_EPSILON + DBL_EPSILON)); + relerr_bound = t + DBL_EPSILON + t*DBL_EPSILON; + if (relerr_bound < 3e-15) + /* + * The tests are written only to 16 + * decimal places anyway even if your + * `double' is, say, i387 binary80, for + * whatever reason. + */ + relerr_bound = 3e-15; + CHECK_RELERR(np, sf_weibull(sqrt(x), 1, 2)); + CHECK_RELERR(np, sf_weibull(sqrt(x)/2, .5, 2)); + CHECK_RELERR(np, sf_weibull(sqrt(x)*2, 2, 2)); + } + + if (p <= 0.75) { + /* + * For p near 1, not enough precision near 1 to + * recover x. + */ + CHECK_RELERR(x, icdf_weibull(p, 1, 1)); + CHECK_RELERR(x/2, icdf_weibull(p, .5, 1)); + CHECK_RELERR(x*2, icdf_weibull(p, 2, 1)); + } + if (p >= 0.25 && !tor_isinf(x) && np > 0) { + /* + * For p near 0, not enough precision in np + * near 1 to recover x. For 0, isf gives inf, + * even if p is precise enough for the icdf to + * work. + */ + CHECK_RELERR(x, isf_weibull(np, 1, 1)); + CHECK_RELERR(x/2, isf_weibull(np, .5, 1)); + CHECK_RELERR(x*2, isf_weibull(np, 2, 1)); + } + } + + for (i = 0; i <= 100; i++) { + double p0 = (double)i/100; + + CHECK_RELERR(3*sqrt(-log(p0/2)), sample_weibull(0, p0, 3, 2)); + CHECK_RELERR(3*sqrt(-log1p(-p0/2)), + sample_weibull(1, p0, 3, 2)); + } + + if (!ok) + printf("fail Weibull cdf/sf\n"); + + tt_assert(ok); + + done: + ; +} + +/** + * Test the cdf, sf, icdf, and isf of the generalized Pareto + * distribution. + */ +static void +test_genpareto(void *arg) +{ + (void) arg; + + struct { + /* xi is the 'xi' parameter of the generalized Pareto distribution, and the + * rest are the same as in the above tests */ + double xi, x, p, np; + } cases[] = { + { 0, 0, 0, 1 }, + { 1e-300, .004, 3.992010656008528e-3, .9960079893439915 }, + { 1e-300, .1, .09516258196404043, .9048374180359595 }, + { 1e-300, 1, .6321205588285577, .36787944117144233 }, + { 1e-300, 10, .9999546000702375, 4.5399929762484854e-5 }, + { 1e-200, 1e-16, 9.999999999999999e-17, .9999999999999999 }, + { 1e-16, 1e-200, 9.999999999999998e-201, 1 }, + { 1e-16, 1e-16, 1e-16, 1 }, + { 1e-16, .004, 3.992010656008528e-3, .9960079893439915 }, + { 1e-16, .1, .09516258196404043, .9048374180359595 }, + { 1e-16, 1, .6321205588285577, .36787944117144233 }, + { 1e-16, 10, .9999546000702375, 4.539992976248509e-5 }, + { 1e-10, 1e-6, 9.999995000001667e-7, .9999990000005 }, + { 1e-8, 1e-8, 9.999999950000001e-9, .9999999900000001 }, + { 1, 1e-300, 1e-300, 1 }, + { 1, 1e-16, 1e-16, .9999999999999999 }, + { 1, .1, .09090909090909091, .9090909090909091 }, + { 1, 1, .5, .5 }, + { 1, 10, .9090909090909091, .0909090909090909 }, + { 1, 100, .9900990099009901, .0099009900990099 }, + { 1, 1000, .999000999000999, 9.990009990009992e-4 }, + { 10, 1e-300, 1e-300, 1 }, + { 10, 1e-16, 9.999999999999995e-17, .9999999999999999 }, + { 10, .1, .06696700846319258, .9330329915368074 }, + { 10, 1, .21320655780322778, .7867934421967723 }, + { 10, 10, .3696701667040189, .6303298332959811 }, + { 10, 100, .49886285755007337, .5011371424499267 }, + { 10, 1000, .6018968102992647, .3981031897007353 }, + }; + double xi_array[] = { -1.5, -1, -1e-30, 0, 1e-30, 1, 1.5 }; + size_t i, j; + double relerr_bound = 3e-15; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double xi = cases[i].xi; + double x = cases[i].x; + double p = cases[i].p; + double np = cases[i].np; + + CHECK_RELERR(p, cdf_genpareto(x, 0, 1, xi)); + CHECK_RELERR(p, cdf_genpareto(x*2, 0, 2, xi)); + CHECK_RELERR(p, cdf_genpareto(x/2, 0, .5, xi)); + CHECK_RELERR(np, sf_genpareto(x, 0, 1, xi)); + CHECK_RELERR(np, sf_genpareto(x*2, 0, 2, xi)); + CHECK_RELERR(np, sf_genpareto(x/2, 0, .5, xi)); + + if (p < .5) { + CHECK_RELERR(x, icdf_genpareto(p, 0, 1, xi)); + CHECK_RELERR(x*2, icdf_genpareto(p, 0, 2, xi)); + CHECK_RELERR(x/2, icdf_genpareto(p, 0, .5, xi)); + } + if (np < .5) { + CHECK_RELERR(x, isf_genpareto(np, 0, 1, xi)); + CHECK_RELERR(x*2, isf_genpareto(np, 0, 2, xi)); + CHECK_RELERR(x/2, isf_genpareto(np, 0, .5, xi)); + } + } + + for (i = 0; i < arraycount(xi_array); i++) { + for (j = 0; j <= 100; j++) { + double p0 = (j == 0 ? 2*DBL_MIN : (double)j/100); + + /* This is actually a check against 0, but we do <= so that the compiler + does not raise a -Wfloat-equal */ + if (fabs(xi_array[i]) <= 0) { + /* + * When xi == 0, the generalized Pareto + * distribution reduces to an + * exponential distribution. + */ + CHECK_RELERR(-log(p0/2), + sample_genpareto(0, p0, 0)); + CHECK_RELERR(-log1p(-p0/2), + sample_genpareto(1, p0, 0)); + } else { + CHECK_RELERR(expm1(-xi_array[i]*log(p0/2))/xi_array[i], + sample_genpareto(0, p0, xi_array[i])); + CHECK_RELERR((j == 0 ? DBL_MIN : + expm1(-xi_array[i]*log1p(-p0/2))/xi_array[i]), + sample_genpareto(1, p0, xi_array[i])); + } + + CHECK_RELERR(isf_genpareto(p0/2, 0, 1, xi_array[i]), + sample_genpareto(0, p0, xi_array[i])); + CHECK_RELERR(icdf_genpareto(p0/2, 0, 1, xi_array[i]), + sample_genpareto(1, p0, xi_array[i])); + } + } + + tt_assert(ok); + + done: + ; +} + +/** + * Test the deterministic sampler for uniform distribution on [a, b]. + * + * This currently only tests whether the outcome lies within [a, b]. + */ +static void +test_uniform_interval(void *arg) +{ + (void) arg; + struct { + /* Sample from a uniform distribution with parameters 'a' and 'b', using + * 't' as the sampling index. */ + double t, a, b; + } cases[] = { + { 0, 0, 0 }, + { 0, 0, 1 }, + { 0, 1.0000000000000007, 3.999999999999995 }, + { 0, 4000, 4000 }, + { 0.42475836677491291, 4000, 4000 }, + { 0, -DBL_MAX, DBL_MAX }, + { 0.25, -DBL_MAX, DBL_MAX }, + { 0.5, -DBL_MAX, DBL_MAX }, + }; + size_t i = 0; + bool ok = true; + + for (i = 0; i < arraycount(cases); i++) { + double t = cases[i].t; + double a = cases[i].a; + double b = cases[i].b; + + CHECK_LE(a, sample_uniform_interval(t, a, b)); + CHECK_LE(sample_uniform_interval(t, a, b), b); + + CHECK_LE(a, sample_uniform_interval(1 - t, a, b)); + CHECK_LE(sample_uniform_interval(1 - t, a, b), b); + + CHECK_LE(sample_uniform_interval(t, -b, -a), -a); + CHECK_LE(-b, sample_uniform_interval(t, -b, -a)); + + CHECK_LE(sample_uniform_interval(1 - t, -b, -a), -a); + CHECK_LE(-b, sample_uniform_interval(1 - t, -b, -a)); + } + + tt_assert(ok); + + done: + ; +} + +/********************** Stochastic tests ****************************/ + +/* + * Psi test, sometimes also called G-test. The psi test statistic, + * suitably scaled, has chi^2 distribution, but the psi test tends to + * have better statistical power in practice to detect deviations than + * the chi^2 test does. (The chi^2 test statistic is the first term of + * the Taylor expansion of the psi test statistic.) The psi test is + * generic, for any CDF; particular distributions might have higher- + * power tests to distinguish them from predictable deviations or bugs. + * + * We choose the psi critical value so that a single psi test has + * probability below alpha = 1% of spuriously failing even if all the + * code is correct. But the false positive rate for a suite of n tests + * is higher: 1 - Binom(0; n, alpha) = 1 - (1 - alpha)^n. For n = 10, + * this is about 10%, and for n = 100 it is well over 50%. + * + * Given that these tests will run with every CI job, we want to drive down the + * false positive rate. We can drive it down by running each test four times, + * and accepting it if it passes at least once; in that case, it is as if we + * used Binom(4; 2, alpha) = alpha^4 as the false positive rate for each test, + * and for n = 10 tests, it would be 9.99999959506e-08. If each CI build has 14 + * jobs, then the chance of a CI build failing is 1.39999903326e-06, which + * means that a CI build will break with probability 50% after about 495106 + * builds. + * + * The critical value for a chi^2 distribution with 100 degrees of + * freedom and false positive rate alpha = 1% was taken from: + * + * NIST/SEMATECH e-Handbook of Statistical Methods, Section + * 1.3.6.7.4 `Critical Values of the Chi-Square Distribution', + * <http://www.itl.nist.gov/div898/handbook/eda/section3/eda3674.htm>, + * retrieved 2018-10-28. + */ + +static const size_t NSAMPLES = 100000; +/* Number of chances we give to the test to succeed. */ +static const unsigned NTRIALS = 4; +/* Number of times we want the test to pass per NTRIALS. */ +static const unsigned NPASSES_MIN = 1; + +#define PSI_DF 100 /* degrees of freedom */ +static const double PSI_CRITICAL = 135.807; /* critical value, alpha = .01 */ + +/** + * Perform a psi test on an array of sample counts, C, adding up to N + * samples, and an array of log expected probabilities, logP, + * representing the null hypothesis for the distribution of samples + * counted. Return false if the psi test rejects the null hypothesis, + * true if otherwise. + */ +static bool +psi_test(const size_t C[PSI_DF], const double logP[PSI_DF], size_t N) +{ + double psi = 0; + double c = 0; /* Kahan compensation */ + double t, u; + size_t i; + + for (i = 0; i < PSI_DF; i++) { + /* + * c*log(c/(n*p)) = (1/n) * f*log(f/p) where f = c/n is + * the frequency, and f*log(f/p) ---> 0 as f ---> 0, so + * this is a reasonable choice. Further, any mass that + * _fails_ to turn up in this bin will inflate another + * bin instead, so we don't really lose anything by + * ignoring empty bins even if they have high + * probability. + */ + if (C[i] == 0) + continue; + t = C[i]*(log((double)C[i]/N) - logP[i]) - c; + u = psi + t; + c = (u - psi) - t; + psi = u; + } + psi *= 2; + + return psi <= PSI_CRITICAL; +} + +static bool +test_stochastic_geometric_impl(double p) +{ + const struct geometric geometric = { + .base = GEOMETRIC(geometric), + .p = p, + }; + double logP[PSI_DF] = {0}; + unsigned ntry = NTRIALS, npass = 0; + unsigned i; + size_t j; + + /* Compute logP[i] = Geom(i + 1; p). */ + for (i = 0; i < PSI_DF - 1; i++) + logP[i] = logpmf_geometric(i + 1, p); + + /* Compute logP[n-1] = log (1 - (P[0] + P[1] + ... + P[n-2])). */ + logP[PSI_DF - 1] = log1mexp(logsumexp(logP, PSI_DF - 1)); + + while (ntry --> 0) { + size_t C[PSI_DF] = {0}; + + for (j = 0; j < NSAMPLES; j++) { + double n_tmp = dist_sample(&geometric.base); + + /* Must be an integer. (XXX -Wfloat-equal) */ + tor_assert(ceil(n_tmp) <= n_tmp && ceil(n_tmp) >= n_tmp); + + /* Must be a positive integer. */ + tor_assert(n_tmp >= 1); + + /* Probability of getting a value in the billions is negligible. */ + tor_assert(n_tmp <= (double)UINT_MAX); + + unsigned n = (unsigned) n_tmp; + + if (n > PSI_DF) + n = PSI_DF; + C[n - 1]++; + } + + if (psi_test(C, logP, NSAMPLES)) { + if (++npass >= NPASSES_MIN) + break; + } + } + + if (npass >= NPASSES_MIN) { + /* printf("pass %s sampler\n", "geometric"); */ + return true; + } else { + printf("fail %s sampler\n", "geometric"); + return false; + } +} + +/** + * Divide the support of <b>dist</b> into histogram bins in <b>logP</b>. Start + * at the 1st percentile and ending at the 99th percentile. Pick the bin + * boundaries using linear interpolation so that they are uniformly spaced. + * + * In each bin logP[i] we insert the expected log-probability that a sampled + * value will fall into that bin. We will use this as the null hypothesis of + * the psi test. + * + * Set logP[i] = log(CDF(x_i) - CDF(x_{i-1})), where x_-1 = -inf, x_n = + * +inf, and x_i = i*(hi - lo)/(n - 2). + */ +static void +bin_cdfs(const struct dist *dist, double lo, double hi, double *logP, size_t n) +{ +#define CDF(x) dist_cdf(dist, x) +#define SF(x) dist_sf(dist, x) + const double w = (hi - lo)/(n - 2); + double halfway = dist_icdf(dist, 0.5); + double x_0, x_1; + size_t i; + size_t n2 = ceil_to_size_t((halfway - lo)/w); + + tor_assert(lo <= halfway); + tor_assert(halfway <= hi); + tor_assert(n2 <= n); + + x_1 = lo; + logP[0] = log(CDF(x_1) - 0); /* 0 = CDF(-inf) */ + for (i = 1; i < n2; i++) { + x_0 = x_1; + /* do the linear interpolation */ + x_1 = (i <= n/2 ? lo + i*w : hi - (n - 2 - i)*w); + /* set the expected log-probability */ + logP[i] = log(CDF(x_1) - CDF(x_0)); + } + x_0 = hi; + logP[n - 1] = log(SF(x_0) - 0); /* 0 = SF(+inf) = 1 - CDF(+inf) */ + + /* In this loop we are filling out the high part of the array. We are using + * SF because in these cases the CDF is near 1 where precision is lower. So + * instead we are using SF near 0 where the precision is higher. We have + * SF(t) = 1 - CDF(t). */ + for (i = 1; i < n - n2; i++) { + x_1 = x_0; + /* do the linear interpolation */ + x_0 = (i <= n/2 ? hi - i*w : lo + (n - 2 - i)*w); + /* set the expected log-probability */ + logP[n - i - 1] = log(SF(x_0) - SF(x_1)); + } +#undef SF +#undef CDF +} + +/** + * Draw NSAMPLES samples from dist, counting the number of samples x in + * the ith bin C[i] if x_{i-1} <= x < x_i, where x_-1 = -inf, x_n = + * +inf, and x_i = i*(hi - lo)/(n - 2). + */ +static void +bin_samples(const struct dist *dist, double lo, double hi, size_t *C, size_t n) +{ + const double w = (hi - lo)/(n - 2); + size_t i; + + for (i = 0; i < NSAMPLES; i++) { + double x = dist_sample(dist); + size_t bin; + + if (x < lo) + bin = 0; + else if (x < hi) + bin = 1 + floor_to_size_t((x - lo)/w); + else + bin = n - 1; + tor_assert(bin < n); + C[bin]++; + } +} + +/** + * Carry out a Psi test on <b>dist</b>. + * + * Sample NSAMPLES from dist, putting them in bins from -inf to lo to + * hi to +inf, and apply up to two psi tests. True if at least one psi + * test passes; false if not. False positive rate should be bounded by + * 0.01^2 = 0.0001. + */ +static bool +test_psi_dist_sample(const struct dist *dist) +{ + double logP[PSI_DF] = {0}; + unsigned ntry = NTRIALS, npass = 0; + double lo = dist_icdf(dist, 1/(double)(PSI_DF + 2)); + double hi = dist_isf(dist, 1/(double)(PSI_DF + 2)); + + /* Create the null hypothesis in logP */ + bin_cdfs(dist, lo, hi, logP, PSI_DF); + + /* Now run the test */ + while (ntry --> 0) { + size_t C[PSI_DF] = {0}; + bin_samples(dist, lo, hi, C, PSI_DF); + if (psi_test(C, logP, NSAMPLES)) { + if (++npass >= NPASSES_MIN) + break; + } + } + + /* Did we fail or succeed? */ + if (npass >= NPASSES_MIN) { + /* printf("pass %s sampler\n", dist_name(dist));*/ + return true; + } else { + printf("fail %s sampler\n", dist_name(dist)); + return false; + } +} + +/* This is the seed of the deterministic randomness */ +static uint8_t rng_seed[16]; +static crypto_xof_t *rng_xof = NULL; + +/** Initialize the seed of the deterministic randomness. */ +static void +init_deterministic_rand(void) +{ + crypto_rand((char*)rng_seed, sizeof(rng_seed)); + crypto_xof_free(rng_xof); + rng_xof = crypto_xof_new(); + crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed)); +} + +static void +teardown_deterministic_rand(void) +{ + crypto_xof_free(rng_xof); +} + +static void +dump_seed(void) +{ + printf("\n" + "NOTE: This is a stochastic test, and we expect it to fail from\n" + "time to time, with some low probability. If you see it fail more\n" + "than one trial in 100, though, please tell us.\n\n" + "Seed: %s\n", + hex_str((const char*)rng_seed, sizeof(rng_seed))); +} + +/** Produce deterministic randomness for the stochastic tests using the global + * deterministic_rand_counter seed + * + * This function produces deterministic data over multiple calls iff it's + * called in the same call order with the same 'n' parameter (which is the + * case for the psi test). If not, outputs will deviate. */ +static void +crypto_rand_deterministic(char *out, size_t n) +{ + /* Use a XOF to squeeze bytes out of that silly counter */ + tor_assert(rng_xof); + crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n); +} + +static void +test_stochastic_uniform(void *arg) +{ + (void) arg; + + const struct uniform uniform01 = { + .base = UNIFORM(uniform01), + .a = 0, + .b = 1, + }; + const struct uniform uniform_pos = { + .base = UNIFORM(uniform_pos), + .a = 1.23, + .b = 4.56, + }; + const struct uniform uniform_neg = { + .base = UNIFORM(uniform_neg), + .a = -10, + .b = -1, + }; + const struct uniform uniform_cross = { + .base = UNIFORM(uniform_cross), + .a = -1.23, + .b = 4.56, + }; + const struct uniform uniform_subnormal = { + .base = UNIFORM(uniform_subnormal), + .a = 4e-324, + .b = 4e-310, + }; + const struct uniform uniform_subnormal_cross = { + .base = UNIFORM(uniform_subnormal_cross), + .a = -4e-324, + .b = 4e-310, + }; + bool ok = true, tests_failed = true; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok &= test_psi_dist_sample(&uniform01.base); + ok &= test_psi_dist_sample(&uniform_pos.base); + ok &= test_psi_dist_sample(&uniform_neg.base); + ok &= test_psi_dist_sample(&uniform_cross.base); + ok &= test_psi_dist_sample(&uniform_subnormal.base); + ok &= test_psi_dist_sample(&uniform_subnormal_cross.base); + + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +static bool +test_stochastic_logistic_impl(double mu, double sigma) +{ + const struct logistic dist = { + .base = LOGISTIC(dist), + .mu = mu, + .sigma = sigma, + }; + + /* XXX Consider some fancier logistic test. */ + return test_psi_dist_sample(&dist.base); +} + +static bool +test_stochastic_log_logistic_impl(double alpha, double beta) +{ + const struct log_logistic dist = { + .base = LOG_LOGISTIC(dist), + .alpha = alpha, + .beta = beta, + }; + + /* XXX Consider some fancier log logistic test. */ + return test_psi_dist_sample(&dist.base); +} + +static bool +test_stochastic_weibull_impl(double lambda, double k) +{ + const struct weibull dist = { + .base = WEIBULL(dist), + .lambda = lambda, + .k = k, + }; + +/* + * XXX Consider applying a Tiku-Singh test: + * + * M.L. Tiku and M. Singh, `Testing the two-parameter + * Weibull distribution', Communications in Statistics -- + * Theory and Methods A10(9), 1981, 907--918. + *https://www.tandfonline.com/doi/pdf/10.1080/03610928108828082?needAccess=true + */ + return test_psi_dist_sample(&dist.base); +} + +static bool +test_stochastic_genpareto_impl(double mu, double sigma, double xi) +{ + const struct genpareto dist = { + .base = GENPARETO(dist), + .mu = mu, + .sigma = sigma, + .xi = xi, + }; + + /* XXX Consider some fancier GPD test. */ + return test_psi_dist_sample(&dist.base); +} + +static void +test_stochastic_genpareto(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_genpareto_impl(0, 1, -0.25); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, -1e-30); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, 0); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, 1e-30); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(0, 1, 0.25); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(-1, 1, -0.25); + tt_assert(ok); + ok = test_stochastic_genpareto_impl(1, 2, 0.25); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +static void +test_stochastic_geometric(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_geometric_impl(0.1); + tt_assert(ok); + ok = test_stochastic_geometric_impl(0.5); + tt_assert(ok); + ok = test_stochastic_geometric_impl(0.9); + tt_assert(ok); + ok = test_stochastic_geometric_impl(1); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +static void +test_stochastic_logistic(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_logistic_impl(0, 1); + tt_assert(ok); + ok = test_stochastic_logistic_impl(0, 1e-16); + tt_assert(ok); + ok = test_stochastic_logistic_impl(1, 10); + tt_assert(ok); + ok = test_stochastic_logistic_impl(-10, 100); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +static void +test_stochastic_log_logistic(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_log_logistic_impl(1, 1); + tt_assert(ok); + ok = test_stochastic_log_logistic_impl(1, 10); + tt_assert(ok); + ok = test_stochastic_log_logistic_impl(M_E, 1e-1); + tt_assert(ok); + ok = test_stochastic_log_logistic_impl(exp(-10), 1e-2); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +static void +test_stochastic_weibull(void *arg) +{ + bool ok = 0; + bool tests_failed = true; + (void) arg; + + init_deterministic_rand(); + MOCK(crypto_rand, crypto_rand_deterministic); + + ok = test_stochastic_weibull_impl(1, 0.5); + tt_assert(ok); + ok = test_stochastic_weibull_impl(1, 1); + tt_assert(ok); + ok = test_stochastic_weibull_impl(1, 1.5); + tt_assert(ok); + ok = test_stochastic_weibull_impl(1, 2); + tt_assert(ok); + ok = test_stochastic_weibull_impl(10, 1); + tt_assert(ok); + + tests_failed = false; + + done: + if (tests_failed) { + dump_seed(); + } + teardown_deterministic_rand(); + UNMOCK(crypto_rand); +} + +struct testcase_t prob_distr_tests[] = { + { "logit_logistics", test_logit_logistic, TT_FORK, NULL, NULL }, + { "log_logistic", test_log_logistic, TT_FORK, NULL, NULL }, + { "weibull", test_weibull, TT_FORK, NULL, NULL }, + { "genpareto", test_genpareto, TT_FORK, NULL, NULL }, + { "uniform_interval", test_uniform_interval, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + +struct testcase_t slow_stochastic_prob_distr_tests[] = { + { "stochastic_genpareto", test_stochastic_genpareto, TT_FORK, NULL, NULL }, + { "stochastic_geometric", test_stochastic_geometric, TT_FORK, NULL, NULL }, + { "stochastic_uniform", test_stochastic_uniform, TT_FORK, NULL, NULL }, + { "stochastic_logistic", test_stochastic_logistic, TT_FORK, NULL, NULL }, + { "stochastic_log_logistic", test_stochastic_log_logistic, TT_FORK, NULL, + NULL }, + { "stochastic_weibull", test_stochastic_weibull, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_process.c b/src/test/test_process.c new file mode 100644 index 0000000000..7cc01d2442 --- /dev/null +++ b/src/test/test_process.c @@ -0,0 +1,669 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process.c + * \brief Test cases for the Process API. + */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "test/test.h" +#include "lib/process/env.h" + +#define PROCESS_PRIVATE +#include "lib/process/process.h" +#define PROCESS_UNIX_PRIVATE +#include "lib/process/process_unix.h" +#define PROCESS_WIN32_PRIVATE +#include "lib/process/process_win32.h" + +static const char *stdout_read_buffer; +static const char *stderr_read_buffer; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static int +process_mocked_read_stdout(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stdout_read_buffer != NULL) { + buf_add_string(buffer, stdout_read_buffer); + stdout_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static int +process_mocked_read_stderr(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stderr_read_buffer != NULL) { + buf_add_string(buffer, stderr_read_buffer); + stderr_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static void +process_mocked_write_stdin(process_t *process, buf_t *buffer) +{ + const size_t size = buf_datalen(buffer); + + if (size == 0) + return; + + char *data = tor_malloc_zero(size + 1); + process_data_t *process_data = process_get_data(process); + + buf_get_bytes(buffer, data, size); + smartlist_add(process_data->stdin_data, data); +} + +static void +process_stdout_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static bool +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + + done: + /* Do not free up our process_t. */ + return false; +} + +static void +test_default_values(void *arg) +{ + (void)arg; + process_t *process = process_new("/path/to/nothing"); + + /* We are not running by default. */ + tt_int_op(PROCESS_STATUS_NOT_RUNNING, OP_EQ, process_get_status(process)); + + /* We use the line protocol by default. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + /* We don't set any custom data by default. */ + tt_ptr_op(NULL, OP_EQ, process_get_data(process)); + + /* Our command was given to the process_t's constructor in process_new(). */ + tt_str_op("/path/to/nothing", OP_EQ, process_get_command(process)); + + /* Make sure we are listed in the list of proccesses. */ + tt_assert(smartlist_contains(process_get_all_processes(), + process)); + + /* Default PID is 0. */ + tt_u64_op(0, OP_EQ, process_get_pid(process)); + + /* Our arguments should be empty. */ + tt_int_op(0, OP_EQ, + smartlist_len(process_get_arguments(process))); + + done: + process_free(process); +} + +static void +test_environment(void *arg) +{ + (void)arg; + + process_t *process = process_new(""); + process_environment_t *env = NULL; + + process_set_environment(process, "E", "F"); + process_set_environment(process, "C", "D"); + process_set_environment(process, "A", "B"); + + env = process_get_environment(process); + tt_mem_op(env->windows_environment_block, OP_EQ, + "A=B\0C=D\0E=F\0", 12); + tt_str_op(env->unixoid_environment_block[0], OP_EQ, + "A=B"); + tt_str_op(env->unixoid_environment_block[1], OP_EQ, + "C=D"); + tt_str_op(env->unixoid_environment_block[2], OP_EQ, + "E=F"); + tt_ptr_op(env->unixoid_environment_block[3], OP_EQ, + NULL); + process_environment_free(env); + + /* Reset our environment. */ + smartlist_t *new_env = smartlist_new(); + smartlist_add(new_env, (char *)"FOO=bar"); + smartlist_add(new_env, (char *)"HELLO=world"); + + process_reset_environment(process, new_env); + smartlist_free(new_env); + + env = process_get_environment(process); + tt_mem_op(env->windows_environment_block, OP_EQ, + "FOO=bar\0HELLO=world\0", 20); + tt_str_op(env->unixoid_environment_block[0], OP_EQ, + "FOO=bar"); + tt_str_op(env->unixoid_environment_block[1], OP_EQ, + "HELLO=world"); + tt_ptr_op(env->unixoid_environment_block[2], OP_EQ, + NULL); + + done: + process_environment_free(env); + process_free(process); +} + +static void +test_stringified_types(void *arg) +{ + (void)arg; + + /* process_protocol_t values. */ + tt_str_op("Raw", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_RAW)); + tt_str_op("Line", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_LINE)); + + /* process_status_t values. */ + tt_str_op("not running", OP_EQ, + process_status_to_string(PROCESS_STATUS_NOT_RUNNING)); + tt_str_op("running", OP_EQ, + process_status_to_string(PROCESS_STATUS_RUNNING)); + tt_str_op("error", OP_EQ, + process_status_to_string(PROCESS_STATUS_ERROR)); + + done: + return; +} + +static void +test_line_protocol_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\r\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_multi(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\r\nOnion Onion Onion\nA B C D\r\n\r\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\nFoo bar baz\nOnion Onion Onion\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(4, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Onion Onion Onion"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "A B C D"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + ""); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Foo bar baz"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Onion Onion Onion"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_partial(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout this is a partial line ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr this is a partial line ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should NOT be ready. */ + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(3, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "Foo bar baz"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Foo bar baz"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_raw_protocol_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_protocol(process, PROCESS_PROTOCOL_RAW); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the raw protocol. */ + tt_int_op(PROCESS_PROTOCOL_RAW, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello, again, stdout\nThis contains multiple lines"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello, again, stderr\nThis contains multiple lines"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout\n"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Hello, again, stdout\nThis contains multiple lines"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr\n"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Hello, again, stderr\nThis contains multiple lines"); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_write_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + MOCK(process_write_stdin, process_mocked_write_stdin); + + process_write(process, (uint8_t *)"Hello world\n", 12); + process_notify_event_stdin(process); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdin_data)); + + process_printf(process, "Hello %s !\n", "moon"); + process_notify_event_stdin(process); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdin_data)); + + done: + process_data_free(process_data); + process_free(process); + + UNMOCK(process_write_stdin); +} + +static void +test_exit_simple(void *arg) +{ + (void)arg; + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Our default is 0. */ + tt_u64_op(0, OP_EQ, process_data->exit_code); + + /* Fake that we are a running process. */ + process_set_status(process, PROCESS_STATUS_RUNNING); + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_RUNNING); + + /* Fake an exit. */ + process_notify_event_exit(process, 1337); + + /* Check if our state changed and if our callback fired. */ + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING); + tt_u64_op(1337, OP_EQ, process_data->exit_code); + + done: + process_set_data(process, process_data); + process_data_free(process_data); + process_free(process); +} + +static void +test_argv_simple(void *arg) +{ + (void)arg; + + process_t *process = process_new("/bin/cat"); + char **argv = NULL; + + /* Setup some arguments. */ + process_append_argument(process, "foo"); + process_append_argument(process, "bar"); + process_append_argument(process, "baz"); + + /* Check the number of elements. */ + tt_int_op(3, OP_EQ, + smartlist_len(process_get_arguments(process))); + + /* Let's try to convert it into a Unix style char **argv. */ + argv = process_get_argv(process); + + /* Check our values. */ + tt_str_op(argv[0], OP_EQ, "/bin/cat"); + tt_str_op(argv[1], OP_EQ, "foo"); + tt_str_op(argv[2], OP_EQ, "bar"); + tt_str_op(argv[3], OP_EQ, "baz"); + tt_ptr_op(argv[4], OP_EQ, NULL); + + done: + tor_free(argv); + process_free(process); +} + +static void +test_unix(void *arg) +{ + (void)arg; +#ifndef _WIN32 + process_t *process = process_new(""); + + /* On Unix all processes should have a Unix process handle. */ + tt_ptr_op(NULL, OP_NE, process_get_unix_process(process)); + + done: + process_free(process); +#endif +} + +static void +test_win32(void *arg) +{ + (void)arg; +#ifdef _WIN32 + process_t *process = process_new(""); + char *joined_argv = NULL; + + /* On Win32 all processes should have a Win32 process handle. */ + tt_ptr_op(NULL, OP_NE, process_get_win32_process(process)); + + /* Based on some test cases from "Parsing C++ Command-Line Arguments" in + * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline + * will try to only generate simple cases for the child process to parse; + * i.e. we never embed quoted strings in arguments. */ + + const char *argvs[][4] = { + {"a", "bb", "CCC", NULL}, // Normal + {NULL, NULL, NULL, NULL}, // Empty argument list + {"", NULL, NULL, NULL}, // Empty argument + {"\"a", "b\"b", "CCC\"", NULL}, // Quotes + {"a\tbc", "dd dd", "E", NULL}, // Whitespace + {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes + {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote + {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote + { NULL } // Terminator + }; + + const char *cmdlines[] = { + "a bb CCC", + "", + "\"\"", + "\\\"a b\\\"b CCC\\\"", + "\"a\tbc\" \"dd dd\" E", + "a\\\\\\b \"de fg\" H", + "a\\\\\\\"b \\c D\\", + "\"a\\\\b c\" d E", + NULL // Terminator + }; + + int i; + + for (i=0; cmdlines[i]!=NULL; i++) { + log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]); + joined_argv = tor_join_win_cmdline(argvs[i]); + tt_str_op(cmdlines[i],OP_EQ, joined_argv); + tor_free(joined_argv); + } + + done: + tor_free(joined_argv); + process_free(process); +#endif +} + +struct testcase_t process_tests[] = { + { "default_values", test_default_values, TT_FORK, NULL, NULL }, + { "environment", test_environment, TT_FORK, NULL, NULL }, + { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, + { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL }, + { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL }, + { "line_protocol_partial", test_line_protocol_partial, TT_FORK, NULL, NULL }, + { "raw_protocol_simple", test_raw_protocol_simple, TT_FORK, NULL, NULL }, + { "write_simple", test_write_simple, TT_FORK, NULL, NULL }, + { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL }, + { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL }, + { "unix", test_unix, TT_FORK, NULL, NULL }, + { "win32", test_win32, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c new file mode 100644 index 0000000000..d269a068ec --- /dev/null +++ b/src/test/test_process_slow.c @@ -0,0 +1,365 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process_slow.c + * \brief Slow test cases for the Process API. + */ + +#define MAINLOOP_PRIVATE +#include "orconfig.h" +#include "core/or/or.h" +#include "core/mainloop/mainloop.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/process/process.h" +#include "lib/process/waitpid.h" +#include "test/test.h" + +#ifndef BUILDDIR +#define BUILDDIR "." +#endif + +#ifdef _WIN32 +#define TEST_PROCESS "test-process.exe" +#else +#define TEST_PROCESS BUILDDIR "/src/test/test-process" +#endif /* defined(_WIN32) */ + +/** Timer that ticks once a second and stop the event loop after 5 ticks. */ +static periodic_timer_t *main_loop_timeout_timer; + +/** How many times have our timer ticked? */ +static int timer_tick_count; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; + bool did_exit; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static void +process_stdout_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, const char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static bool +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + process_status_t status; + + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + process_data->did_exit = true; + + /* Check if our process is still running? */ + status = process_get_status(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING); + + done: + /* Do not free up our process_t. */ + return false; +} + +#ifdef _WIN32 +static const char * +get_win32_test_binary_path(void) +{ + static char buffer[MAX_PATH]; + + /* Get the absolute path of our binary: \path\to\test-slow.exe. */ + GetModuleFileNameA(GetModuleHandle(0), buffer, sizeof(buffer)); + + /* Find our process name. */ + char *offset = strstr(buffer, "test-slow.exe"); + tt_ptr_op(offset, OP_NE, NULL); + + /* Change test-slow.exe to test-process.exe. */ + memcpy(offset, TEST_PROCESS, strlen(TEST_PROCESS)); + + return buffer; + done: + return NULL; +} +#endif + +static void +main_loop_timeout_cb(periodic_timer_t *timer, void *data) +{ + /* Sanity check. */ + tt_ptr_op(timer, OP_EQ, main_loop_timeout_timer); + tt_ptr_op(data, OP_NE, NULL); + + /* Our process data. */ + process_data_t *process_data = data; + + /* Our process did exit. */ + if (process_data->did_exit) + tor_shutdown_event_loop_and_exit(0); + + /* Have we been called 10 times we exit the main loop. */ + timer_tick_count++; + + tt_int_op(timer_tick_count, OP_LT, 10); + +#ifndef _WIN32 + /* Call waitpid callbacks. */ + notify_pending_waitpid_callbacks(); +#endif + + return; + done: + /* Exit with an error. */ + tor_shutdown_event_loop_and_exit(-1); +} + +static void +run_main_loop(process_data_t *process_data) +{ + int ret; + + /* Wake up after 1 seconds. */ + static const struct timeval interval = {1, 0}; + + timer_tick_count = 0; + main_loop_timeout_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + main_loop_timeout_cb, + process_data); + + /* Run our main loop. */ + ret = run_main_loop_until_done(); + + /* Clean up our main loop timeout timer. */ + tt_int_op(ret, OP_EQ, 0); + + done: + periodic_timer_free(main_loop_timeout_timer); +} + +static void +test_callbacks(void *arg) +{ + (void)arg; + const char *filename = NULL; + +#ifdef _WIN32 + filename = get_win32_test_binary_path(); +#else + filename = TEST_PROCESS; +#endif + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new(filename); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + process_set_exit_callback(process, process_exit_callback); + + /* Set environment variable. */ + process_set_environment(process, "TOR_TEST_ENV", "Hello, from Tor!"); + + /* Add some arguments. */ + process_append_argument(process, "This is the first one"); + process_append_argument(process, "Second one"); + process_append_argument(process, "Third: Foo bar baz"); + + /* Run our process. */ + process_status_t status; + + status = process_exec(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); + + /* Write some lines to stdin. */ + process_printf(process, "Hi process!\r\n"); + process_printf(process, "Can you read more than one line?\n"); + process_printf(process, "Can you read partial ..."); + process_printf(process, " lines?\r\n"); + + /* Start our main loop. */ + run_main_loop(process_data); + + /* We returned. Let's see what our event loop said. */ + tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12); + tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3); + tt_assert(process_data->did_exit); + tt_u64_op(process_data->exit_code, OP_EQ, 0); + + /* Check stdout output. */ + char argv0_expected[256]; + tor_snprintf(argv0_expected, sizeof(argv0_expected), + "argv[0] = '%s'", filename); + + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + argv0_expected); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "argv[1] = 'This is the first one'"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "argv[2] = 'Second one'"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + "argv[3] = 'Third: Foo bar baz'"); + tt_str_op(smartlist_get(process_data->stdout_data, 4), OP_EQ, + "Environment variable TOR_TEST_ENV = 'Hello, from Tor!'"); + tt_str_op(smartlist_get(process_data->stdout_data, 5), OP_EQ, + "Output on stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 6), OP_EQ, + "This is a new line"); + tt_str_op(smartlist_get(process_data->stdout_data, 7), OP_EQ, + "Partial line on stdout ...end of partial line on stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 8), OP_EQ, + "Read line from stdin: 'Hi process!'"); + tt_str_op(smartlist_get(process_data->stdout_data, 9), OP_EQ, + "Read line from stdin: 'Can you read more than one line?'"); + tt_str_op(smartlist_get(process_data->stdout_data, 10), OP_EQ, + "Read line from stdin: 'Can you read partial ... lines?'"); + tt_str_op(smartlist_get(process_data->stdout_data, 11), OP_EQ, + "We are done for here, thank you!"); + + /* Check stderr output. */ + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Output on stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "This is a new line"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Partial line on stderr ...end of partial line on stderr"); + + done: + process_data_free(process_data); + process_free(process); +} + +static void +test_callbacks_terminate(void *arg) +{ + (void)arg; + const char *filename = NULL; + +#ifdef _WIN32 + filename = get_win32_test_binary_path(); +#else + filename = TEST_PROCESS; +#endif + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new(filename); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Run our process. */ + process_status_t status; + + status = process_exec(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); + + /* Zap our process. */ + bool success; + + success = process_terminate(process); + tt_assert(success); + + /* Start our main loop. */ + run_main_loop(process_data); + + /* Check if we did exit. */ + tt_assert(process_data->did_exit); + + done: + process_data_free(process_data); + process_free(process); +} + +static void +test_nonexistent_executable(void *arg) +{ + (void)arg; + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new("binary-does-not-exist"); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Run our process. */ + process_exec(process); + + /* Start our main loop. */ + run_main_loop(process_data); + + /* Ensure that the exit callback was actually called even though the binary + * did not exist. + */ + tt_assert(process_data->did_exit); + + done: + process_data_free(process_data); + process_free(process); +} + +struct testcase_t slow_process_tests[] = { + { "callbacks", test_callbacks, 0, NULL, NULL }, + { "callbacks_terminate", test_callbacks_terminate, 0, NULL, NULL }, + { "nonexistent_executable", test_nonexistent_executable, 0, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c index 08990c0b6a..f9339e8dd3 100644 --- a/src/test/test_proto_http.c +++ b/src/test/test_proto_http.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "test/test.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/proto/proto_http.h" #include "test/log_test_helpers.h" diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c index af9cf7eee2..18669a7772 100644 --- a/src/test/test_proto_misc.c +++ b/src/test/test_proto_misc.c @@ -8,7 +8,7 @@ #include "core/or/or.h" #include "test/test.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "core/or/connection_or.h" #include "feature/relay/ext_orport.h" #include "core/proto/proto_cell.h" diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 1f9786648a..d2996f4cc3 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -8,7 +8,7 @@ #define UTIL_PRIVATE #define STATEFILE_PRIVATE #define CONTROL_PRIVATE -#define SUBPROCESS_PRIVATE +#define PROCESS_PRIVATE #include "core/or/or.h" #include "app/config/config.h" #include "app/config/confparse.h" @@ -17,12 +17,14 @@ #include "core/or/circuitbuild.h" #include "app/config/statefile.h" #include "test/test.h" -#include "lib/process/subprocess.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" +#include "lib/process/process.h" #include "app/config/or_state_st.h" +#include "test/log_test_helpers.h" + static void reset_mp(managed_proxy_t *mp) { @@ -288,41 +290,35 @@ test_pt_get_extrainfo_string(void *arg) tor_free(s); } -#ifdef _WIN32 -#define STDIN_HANDLE HANDLE* -#else -#define STDIN_HANDLE int -#endif - -static smartlist_t * -tor_get_lines_from_handle_replacement(STDIN_HANDLE handle, - enum stream_status *stream_status_out) +static int +process_read_stdout_replacement(process_t *process, buf_t *buffer) { + (void)process; static int times_called = 0; - smartlist_t *retval_sl = smartlist_new(); - - (void) handle; - (void) stream_status_out; /* Generate some dummy CMETHOD lines the first 5 times. The 6th time, send 'CMETHODS DONE' to finish configuring the proxy. */ - if (times_called++ != 5) { - smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d", + times_called++; + + if (times_called <= 5) { + buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n", times_called, times_called); - } else { - smartlist_add_strdup(retval_sl, "SMETHODS DONE"); + } else if (times_called <= 6) { + buf_add_string(buffer, "SMETHODS DONE\n"); + } else if (times_called <= 7) { + buf_add_string(buffer, "LOG SEVERITY=error MESSAGE=\"Oh noes, something " + "bad happened. What do we do!?\"\n"); + buf_add_string(buffer, "LOG SEVERITY=warning MESSAGE=\"warning msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=notice MESSAGE=\"notice msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=info MESSAGE=\"info msg\"\n"); + buf_add_string(buffer, "LOG SEVERITY=debug MESSAGE=\"debug msg\"\n"); + } else if (times_called <= 8) { + buf_add_string(buffer, "STATUS TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\n"); + buf_add_string(buffer, "STATUS TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\n"); } - return retval_sl; -} - -/* NOP mock */ -static void -tor_process_handle_destroy_replacement(process_handle_t *process_handle, - int also_terminate_process) -{ - (void) process_handle; - (void) also_terminate_process; + return (int)buf_datalen(buffer); } static or_state_t *dummy_state = NULL; @@ -357,10 +353,7 @@ test_pt_configure_proxy(void *arg) dummy_state = tor_malloc_zero(sizeof(or_state_t)); - MOCK(tor_get_lines_from_handle, - tor_get_lines_from_handle_replacement); - MOCK(tor_process_handle_destroy, - tor_process_handle_destroy_replacement); + MOCK(process_read_stdout, process_read_stdout_replacement); MOCK(get_or_state, get_or_state_replacement); MOCK(queue_control_event_string, @@ -372,24 +365,34 @@ test_pt_configure_proxy(void *arg) mp->conf_state = PT_PROTO_ACCEPTING_METHODS; mp->transports = smartlist_new(); mp->transports_to_launch = smartlist_new(); - mp->process_handle = tor_malloc_zero(sizeof(process_handle_t)); mp->argv = tor_malloc_zero(sizeof(char*)*2); mp->argv[0] = tor_strdup("<testcase>"); mp->is_server = 1; + /* Configure the process. */ + mp->process = process_new(""); + process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback); + process_set_data(mp->process, mp); + /* Test the return value of configure_proxy() by calling it some times while it is uninitialized and then finally finalizing its configuration. */ for (i = 0 ; i < 5 ; i++) { + /* force a read from our mocked stdout reader. */ + process_notify_event_stdout(mp->process); + /* try to configure our proxy. */ retval = configure_proxy(mp); /* retval should be zero because proxy hasn't finished configuring yet */ tt_int_op(retval, OP_EQ, 0); /* check the number of registered transports */ - tt_assert(smartlist_len(mp->transports) == i+1); + tt_int_op(smartlist_len(mp->transports), OP_EQ, i+1); /* check that the mp is still waiting for transports */ tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); } + /* Get the SMETHOD DONE written to the process. */ + process_notify_event_stdout(mp->process); + /* this last configure_proxy() should finalize the proxy configuration. */ retval = configure_proxy(mp); /* retval should be 1 since the proxy finished configuring */ @@ -412,6 +415,49 @@ test_pt_configure_proxy(void *arg) tt_str_op(smartlist_get(controlevent_msgs, 4), OP_EQ, "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n"); + /* Get the log message out. */ + setup_full_capture_of_logs(LOG_ERR); + process_notify_event_stdout(mp->process); + expect_single_log_msg_containing("Oh noes, something bad happened"); + teardown_capture_of_logs(); + + tt_int_op(controlevent_n, OP_EQ, 10); + tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG); + tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 10); + tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=error " + "MESSAGE=\"Oh noes, " + "something bad happened. What do we do!?\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 6), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=warning " + "MESSAGE=\"warning msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 7), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=notice " + "MESSAGE=\"notice msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 8), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=info " + "MESSAGE=\"info msg\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 9), OP_EQ, + "650 PT_LOG PT=<testcase> SEVERITY=debug " + "MESSAGE=\"debug msg\"\r\n"); + + /* Get the STATUS messages out. */ + process_notify_event_stdout(mp->process); + + tt_int_op(controlevent_n, OP_EQ, 13); + tt_int_op(controlevent_event, OP_EQ, EVENT_PT_STATUS); + tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 13); + + tt_str_op(smartlist_get(controlevent_msgs, 10), OP_EQ, + "650 PT_STATUS " + "PT=<testcase> TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 11), OP_EQ, + "650 PT_STATUS " + "PT=<testcase> TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\r\n"); + tt_str_op(smartlist_get(controlevent_msgs, 12), OP_EQ, + "650 PT_STATUS " + "PT=<testcase> TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\r\n"); + { /* check that the transport info were saved properly in the tor state */ config_line_t *transport_in_state = NULL; smartlist_t *transport_info_sl = smartlist_new(); @@ -434,9 +480,9 @@ test_pt_configure_proxy(void *arg) } done: + teardown_capture_of_logs(); or_state_free(dummy_state); - UNMOCK(tor_get_lines_from_handle); - UNMOCK(tor_process_handle_destroy); + UNMOCK(process_read_stdout); UNMOCK(get_or_state); UNMOCK(queue_control_event_string); if (controlevent_msgs) { @@ -449,7 +495,7 @@ test_pt_configure_proxy(void *arg) smartlist_free(mp->transports); } smartlist_free(mp->transports_to_launch); - tor_free(mp->process_handle); + process_free(mp->process); tor_free(mp->argv[0]); tor_free(mp->argv); tor_free(mp); diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py index 30a587858f..c9b9200b2d 100644 --- a/src/test/test_rebind.py +++ b/src/test/test_rebind.py @@ -22,9 +22,10 @@ def skip(msg): def try_connecting_to_socksport(): socks_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if socks_socket.connect_ex(('127.0.0.1', socks_port)): + e = socks_socket.connect_ex(('127.0.0.1', socks_port)) + if e: tor_process.terminate() - fail('Cannot connect to SOCKSPort') + fail('Cannot connect to SOCKSPort: error ' + os.strerror(e)) socks_socket.close() def wait_for_log(s): diff --git a/src/test/test_rng.c b/src/test/test_rng.c new file mode 100644 index 0000000000..c749de112a --- /dev/null +++ b/src/test/test_rng.c @@ -0,0 +1,59 @@ +/* Copyright (c) 2016-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* + * Example usage: + * + * ./src/test/test-rng --emit | dieharder -g 200 -a + * + * Remember, dieharder can tell you that your RNG is completely broken, but if + * your RNG is not _completely_ broken, dieharder cannot tell you whether your + * RNG is actually secure. + */ + +#include "orconfig.h" + +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#include <stdio.h> +#include <string.h> +#include <errno.h> + +#include "lib/crypt_ops/crypto_rand.h" + +int +main(int argc, char **argv) +{ + uint8_t buf[0x123]; + + if (argc != 2 || strcmp(argv[1], "--emit")) { + fprintf(stderr, "If you want me to fill stdout with a bunch of random " + "bytes, you need to say --emit.\n"); + return 1; + } + + if (crypto_seed_rng() < 0) { + fprintf(stderr, "Can't seed RNG.\n"); + return 1; + } + +#if 0 + while (1) { + crypto_rand(buf, sizeof(buf)); + if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) { + fprintf(stderr, "write() failed: %s\n", strerror(errno)); + return 1; + } + } +#endif + + crypto_fast_rng_t *rng = crypto_fast_rng_new(); + while (1) { + crypto_fast_rng_getbytes(rng, buf, sizeof(buf)); + if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) { + fprintf(stderr, "write() failed: %s\n", strerror(errno)); + return 1; + } + } +} diff --git a/src/test/test_router.c b/src/test/test_router.c index 601881a124..ea0ee3e84c 100644 --- a/src/test/test_router.c +++ b/src/test/test_router.c @@ -7,16 +7,25 @@ * \brief Unittests for code in router.c **/ +#define CONFIG_PRIVATE +#define ROUTER_PRIVATE + #include "core/or/or.h" #include "app/config/config.h" #include "core/mainloop/mainloop.h" #include "feature/hibernate/hibernate.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/networkstatus_st.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerinfo_st.h" #include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerstatus_st.h" #include "feature/relay/router.h" #include "feature/stats/rephist.h" #include "lib/crypt_ops/crypto_curve25519.h" #include "lib/crypt_ops/crypto_ed25519.h" +#include "lib/encoding/confline.h" /* Test suite stuff */ #include "test/test.h" @@ -231,11 +240,254 @@ test_router_check_descriptor_bandwidth_changed(void *arg) UNMOCK(we_are_hibernating); } +static networkstatus_t *mock_ns = NULL; +static networkstatus_t * +mock_networkstatus_get_live_consensus(time_t now) +{ + (void)now; + return mock_ns; +} + +static routerstatus_t *mock_rs = NULL; +static const routerstatus_t * +mock_networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest) +{ + (void)ns; + (void)digest; + return mock_rs; +} + +static void +test_router_mark_if_too_old(void *arg) +{ + (void)arg; + time_t now = approx_time(); + MOCK(networkstatus_get_live_consensus, + mock_networkstatus_get_live_consensus); + MOCK(networkstatus_vote_find_entry, mock_networkstatus_vote_find_entry); + + routerstatus_t rs; + networkstatus_t ns; + memset(&rs, 0, sizeof(rs)); + memset(&ns, 0, sizeof(ns)); + mock_ns = &ns; + mock_ns->valid_after = now-3600; + mock_rs = &rs; + mock_rs->published_on = now - 10; + + // no reason to mark this time. + desc_clean_since = now-10; + desc_dirty_reason = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now-10); + + // Doesn't appear in consensus? Still don't mark it. + mock_ns = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now-10); + mock_ns = &ns; + + // No new descriptor in a long time? Mark it. + desc_clean_since = now - 3600 * 96; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, "time for new descriptor"); + + // Version in consensus published a long time ago? We won't mark it + // if it's been clean for only a short time. + desc_clean_since = now - 10; + desc_dirty_reason = NULL; + mock_rs->published_on = now - 3600 * 96; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, now - 10); + + // ... but if it's been clean a while, we mark. + desc_clean_since = now - 2 * 3600; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "version listed in consensus is quite old"); + + // same deal if we're marked stale. + desc_clean_since = now - 2 * 3600; + desc_dirty_reason = NULL; + mock_rs->published_on = now - 10; + mock_rs->is_staledesc = 1; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "listed as stale in consensus"); + + // same deal if we're absent from the consensus. + desc_clean_since = now - 2 * 3600; + desc_dirty_reason = NULL; + mock_rs = NULL; + mark_my_descriptor_dirty_if_too_old(now); + tt_i64_op(desc_clean_since, OP_EQ, 0); + tt_str_op(desc_dirty_reason, OP_EQ, + "not listed in consensus"); + + done: + UNMOCK(networkstatus_get_live_consensus); + UNMOCK(networkstatus_vote_find_entry); +} + +static node_t fake_node; +static const node_t * +mock_node_get_by_nickname(const char *name, unsigned flags) +{ + (void)flags; + if (!strcasecmp(name, "crumpet")) + return &fake_node; + else + return NULL; +} + +static void +test_router_get_my_family(void *arg) +{ + (void)arg; + or_options_t *options = options_new(); + smartlist_t *sl = NULL; + char *join = NULL; + // Overwrite the result of router_get_my_identity_digest(). This + // happens to be okay, but only for testing. + set_server_identity_key_digest_testing( + (const uint8_t*)"holeinthebottomofthe"); + + setup_capture_of_logs(LOG_WARN); + + // No family listed -- so there's no list. + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_EQ, NULL); + expect_no_log_entry(); + +#define CLEAR() do { \ + if (sl) { \ + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); \ + smartlist_free(sl); \ + } \ + tor_free(join); \ + mock_clean_saved_logs(); \ + } while (0) + + // Add a single nice friendly hex member. This should be enough + // to have our own ID added. + tt_ptr_op(options->MyFamily, OP_EQ, NULL); + config_line_append(&options->MyFamily, "MyFamily", + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 2); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + expect_no_log_entry(); + CLEAR(); + + // Add a hex member with a ~. The ~ part should get removed. + config_line_append(&options->MyFamily, "MyFamily", + "$0123456789abcdef0123456789abcdef01234567~Muffin"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 3); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + expect_no_log_entry(); + CLEAR(); + + // Nickname lookup will fail, so a nickname will appear verbatim. + config_line_append(&options->MyFamily, "MyFamily", + "BAGEL"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 4); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + expect_single_log_msg_containing( + "There is a router named \"BAGEL\" in my declared family, but " + "I have no descriptor for it."); + CLEAR(); + + // A bogus digest should fail entirely. + config_line_append(&options->MyFamily, "MyFamily", + "$painauchocolat"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 4); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + // "BAGEL" is still there, but it won't make a warning, because we already + // warned about it. + expect_single_log_msg_containing( + "There is a router named \"$painauchocolat\" in my declared " + "family, but that isn't a legal digest or nickname. Skipping it."); + CLEAR(); + + // Let's introduce a node we can look up by nickname + memset(&fake_node, 0, sizeof(fake_node)); + memcpy(fake_node.identity, "whydoyouasknonononon", DIGEST_LEN); + MOCK(node_get_by_nickname, mock_node_get_by_nickname); + + config_line_append(&options->MyFamily, "MyFamily", + "CRUmpeT"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_NE, NULL); + tt_int_op(smartlist_len(sl), OP_EQ, 5); + join = smartlist_join_strings(sl, " ", 0, NULL); + tt_str_op(join, OP_EQ, + "$0123456789ABCDEF0123456789ABCDEF01234567 " + "$686F6C65696E746865626F74746F6D6F66746865 " + "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E " + "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "bagel"); + // "BAGEL" is still there, but it won't make a warning, because we already + // warned about it. Some with "$painauchocolat". + expect_single_log_msg_containing( + "There is a router named \"CRUmpeT\" in my declared " + "family, but it wasn't listed by digest. Please consider saying " + "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E instead, if that's " + "what you meant."); + CLEAR(); + UNMOCK(node_get_by_nickname); + + // Try a singleton list containing only us: It should give us NULL. + config_free_lines(options->MyFamily); + config_line_append(&options->MyFamily, "MyFamily", + "$686F6C65696E746865626F74746F6D6F66746865"); + sl = get_my_declared_family(options); + tt_ptr_op(sl, OP_EQ, NULL); + expect_no_log_entry(); + + done: + or_options_free(options); + teardown_capture_of_logs(); + CLEAR(); + UNMOCK(node_get_by_nickname); + +#undef CLEAR +} + #define ROUTER_TEST(name, flags) \ { #name, test_router_ ## name, flags, NULL, NULL } struct testcase_t router_tests[] = { ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK), ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK), + ROUTER_TEST(mark_if_too_old, TT_FORK), + ROUTER_TEST(get_my_family, TT_FORK), END_OF_TESTCASES }; diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 95c9176faa..84ec8cc462 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -46,7 +46,7 @@ #include "feature/nodelist/routerstatus_st.h" #include "lib/encoding/confline.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "test/test.h" #include "test/test_dir_common.h" @@ -265,7 +265,9 @@ test_router_pick_directory_server_impl(void *arg) /* Init SR subsystem. */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -275,7 +277,9 @@ test_router_pick_directory_server_impl(void *arg) construct_consensus(&consensus_text_md, now); tt_assert(consensus_text_md); - con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, + con_md = networkstatus_parse_vote_from_string(consensus_text_md, + strlen(consensus_text_md), + NULL, NS_TYPE_CONSENSUS); tt_assert(con_md); tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC); @@ -301,7 +305,6 @@ test_router_pick_directory_server_impl(void *arg) tt_assert(!networkstatus_consensus_is_bootstrapping(con_md->valid_until + 24*60*60)); /* These times are outside the test validity period */ - tt_assert(networkstatus_consensus_is_bootstrapping(now)); tt_assert(networkstatus_consensus_is_bootstrapping(now + 2*24*60*60)); tt_assert(networkstatus_consensus_is_bootstrapping(now - 2*24*60*60)); @@ -475,7 +478,9 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) /* Initialize the SRV subsystem */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -648,7 +653,9 @@ test_skew_common(void *arg, time_t now, unsigned long *offset) /* Initialize the SRV subsystem */ MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); sr_init(0); UNMOCK(get_my_v3_authority_cert); @@ -662,7 +669,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset) MOCK(clock_skew_warning, mock_clock_skew_warning); /* Caller will call teardown_capture_of_logs() */ setup_capture_of_logs(LOG_WARN); - retval = networkstatus_set_current_consensus(consensus, "microdesc", 0, + retval = networkstatus_set_current_consensus(consensus, strlen(consensus), + "microdesc", 0, NULL); done: diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index b4fe6eef64..5fa7e80d07 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -58,14 +58,17 @@ trusteddirserver_get_by_v3_auth_digest_m(const char *digest) } /* Setup a minimal dirauth environment by initializing the SR state and - * making sure the options are set to be an authority directory. */ + * making sure the options are set to be an authority directory. + * You must only call this function once per process. */ static void init_authority_state(void) { MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); or_options_t *options = get_options_mutable(); - mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(mock_cert); options->AuthoritativeDir = 1; tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0); @@ -310,6 +313,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 19 Apr 2015 23:00:00 UTC", &mock_consensus.valid_after); + tt_int_op(retval, OP_EQ, 0); retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:08:00 UTC", ¤t_time); @@ -424,7 +428,9 @@ test_sr_commit(void *arg) { /* Setup a minimal dirauth environment for this test */ or_options_t *options = get_options_mutable(); - auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(auth_cert); options->AuthoritativeDir = 1; @@ -835,7 +841,9 @@ test_sr_setup_commits(void) { /* Setup a minimal dirauth environment for this test */ or_options_t *options = get_options_mutable(); - auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, + strlen(AUTHORITY_CERT_1), + NULL); tt_assert(auth_cert); options->AuthoritativeDir = 1; @@ -1073,12 +1081,13 @@ test_sr_get_majority_srv_from_votes(void *arg) smartlist_free(votes); } +/* Test utils that don't depend on authority state */ static void -test_utils(void *arg) +test_utils_general(void *arg) { (void) arg; - /* Testing srv_dup(). */ + /* Testing sr_srv_dup(). */ { sr_srv_t *srv = NULL, *dup_srv = NULL; const char *srv_value = @@ -1086,7 +1095,7 @@ test_utils(void *arg) srv = tor_malloc_zero(sizeof(*srv)); srv->num_reveals = 42; memcpy(srv->value, srv_value, sizeof(srv->value)); - dup_srv = srv_dup(srv); + dup_srv = sr_srv_dup(srv); tt_assert(dup_srv); tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals); tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value)); @@ -1137,9 +1146,19 @@ test_utils(void *arg) tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit"); } + done: + return; +} + +/* Test utils that depend on authority state */ +static void +test_utils_auth(void *arg) +{ + (void)arg; + init_authority_state(); + /* Testing phase transition */ { - init_authority_state(); set_sr_phase(SR_PHASE_COMMIT); tt_int_op(is_phase_transition(SR_PHASE_REVEAL), OP_EQ, 1); tt_int_op(is_phase_transition(SR_PHASE_COMMIT), OP_EQ, 0); @@ -1150,8 +1169,193 @@ test_utils(void *arg) tt_int_op(is_phase_transition(42), OP_EQ, 1); } + /* Testing get, set, delete, clean SRVs */ + + { + /* Just set the previous SRV */ + test_sr_setup_srv(0); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + } + + { + /* Delete the SRVs one at a time */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And in the opposite order */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And both at once */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And do the gets and sets multiple times */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + state_del_previous_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + state_del_current_srv(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + } + + { + /* Now set the SRVs to NULL instead */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_current_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + sr_state_set_previous_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And in the opposite order */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_previous_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_current_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And both at once */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_clean_srvs(); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + + /* And do the gets and sets multiple times */ + test_sr_setup_srv(1); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_previous_srv(NULL); + sr_state_set_previous_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + sr_state_set_current_srv(NULL); + sr_state_set_previous_srv(NULL); + sr_state_set_current_srv(NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); + } + + { + /* Now copy the values across */ + test_sr_setup_srv(1); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Check that the content is different */ + tt_mem_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Set the current to the previous: the protocol goes the other way */ + sr_state_set_current_srv(sr_srv_dup(sr_state_get_previous_srv())); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Check that the content is the same */ + tt_mem_op(sr_state_get_previous_srv(), OP_EQ, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + } + + { + /* Now copy a value onto itself */ + test_sr_setup_srv(1); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Take a copy of the old value */ + sr_srv_t old_current_srv; + memcpy(&old_current_srv, sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Check that the content is different */ + tt_mem_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Set the current to the current: the protocol never replaces an SRV with + * the same value */ + sr_state_set_current_srv(sr_srv_dup(sr_state_get_current_srv())); + /* Check that the pointers are non-NULL, and different from each other */ + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL); + tt_ptr_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv()); + /* Check that the content is different between current and previous */ + tt_mem_op(sr_state_get_previous_srv(), OP_NE, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + /* Check that the content is the same as the old content */ + tt_mem_op(&old_current_srv, OP_EQ, + sr_state_get_current_srv(), sizeof(sr_srv_t)); + } + + /* I don't think we can say "expect a BUG()" in our tests. */ +#if 0 + { + /* Now copy a value onto itself without sr_srv_dup(). + * This should fail with a BUG() warning. */ + test_sr_setup_srv(1); + sr_state_set_current_srv(sr_state_get_current_srv()); + sr_state_set_previous_srv(sr_state_get_previous_srv()); + } +#endif + done: - return; + sr_state_free_all(); } static void @@ -1159,6 +1363,7 @@ test_state_transition(void *arg) { sr_state_t *state = NULL; time_t now = time(NULL); + sr_srv_t *cur = NULL; (void) arg; @@ -1197,44 +1402,47 @@ test_state_transition(void *arg) /* Test SRV rotation in our state. */ { - const sr_srv_t *cur, *prev; test_sr_setup_srv(1); - cur = sr_state_get_current_srv(); + tt_assert(sr_state_get_current_srv()); + /* Take a copy of the data, because the state owns the pointer */ + cur = sr_srv_dup(sr_state_get_current_srv()); tt_assert(cur); - /* After, current srv should be the previous and then set to NULL. */ + /* After, the previous SRV should be the same as the old current SRV, and + * the current SRV should be set to NULL */ state_rotate_srv(); - prev = sr_state_get_previous_srv(); - tt_assert(prev == cur); + tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t)); tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL); sr_state_clean_srvs(); + tor_free(cur); } /* New protocol run. */ { - const sr_srv_t *cur; /* Setup some new SRVs so we can confirm that a new protocol run * actually makes them rotate and compute new ones. */ test_sr_setup_srv(1); - cur = sr_state_get_current_srv(); - tt_assert(cur); + tt_assert(sr_state_get_current_srv()); + /* Take a copy of the data, because the state owns the pointer */ + cur = sr_srv_dup(sr_state_get_current_srv()); set_sr_phase(SR_PHASE_REVEAL); MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); new_protocol_run(now); UNMOCK(get_my_v3_authority_cert); /* Rotation happened. */ - tt_assert(sr_state_get_previous_srv() == cur); + tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t)); /* We are going into COMMIT phase so we had to rotate our SRVs. Usually * our current SRV would be NULL but a new protocol run should make us * compute a new SRV. */ tt_assert(sr_state_get_current_srv()); /* Also, make sure we did change the current. */ - tt_assert(sr_state_get_current_srv() != cur); + tt_mem_op(sr_state_get_current_srv(), OP_NE, cur, sizeof(sr_srv_t)); /* We should have our commitment alone. */ tt_int_op(digestmap_size(state->commits), OP_EQ, 1); tt_int_op(state->n_reveal_rounds, OP_EQ, 0); tt_int_op(state->n_commit_rounds, OP_EQ, 0); /* 46 here since we were at 45 just before. */ tt_u64_op(state->n_protocol_runs, OP_EQ, 46); + tor_free(cur); } /* Cleanup of SRVs. */ @@ -1245,6 +1453,7 @@ test_state_transition(void *arg) } done: + tor_free(cur); sr_state_free_all(); } @@ -1440,7 +1649,8 @@ struct testcase_t sr_tests[] = { { "sr_compute_srv", test_sr_compute_srv, TT_FORK, NULL, NULL }, { "sr_get_majority_srv_from_votes", test_sr_get_majority_srv_from_votes, TT_FORK, NULL, NULL }, - { "utils", test_utils, TT_FORK, NULL, NULL }, + { "utils_general", test_utils_general, TT_FORK, NULL, NULL }, + { "utils_auth", test_utils_auth, TT_FORK, NULL, NULL }, { "state_transition", test_state_transition, TT_FORK, NULL, NULL }, { "state_update", test_state_update, TT_FORK, NULL, NULL }, diff --git a/src/test/test_slow.c b/src/test/test_slow.c index bda67b2d92..c3e7edd408 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -20,7 +20,8 @@ struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, - { "slow/util/", slow_util_tests }, + { "slow/process/", slow_process_tests }, + { "slow/prob_distr/", slow_stochastic_prob_distr_tests }, END_OF_GROUPS }; diff --git a/src/test/test_socks.c b/src/test/test_socks.c index 783f4726ee..a4a768ce84 100644 --- a/src/test/test_socks.c +++ b/src/test/test_socks.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "core/mainloop/connection.h" #include "core/proto/proto_socks.h" diff --git a/src/test/test_util.c b/src/test/test_util.c index 3a0eb15157..f1ffae7af8 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -10,18 +10,20 @@ #define UTIL_PRIVATE #define UTIL_MALLOC_PRIVATE #define SOCKET_PRIVATE -#define SUBPROCESS_PRIVATE +#define PROCESS_WIN32_PRIVATE #include "lib/testsupport/testsupport.h" #include "core/or/or.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "app/config/config.h" #include "feature/control/control.h" #include "feature/client/transports.h" #include "lib/crypt_ops/crypto_format.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/defs/time.h" #include "test/test.h" #include "lib/memarea/memarea.h" #include "lib/process/waitpid.h" +#include "lib/process/process_win32.h" #include "test/log_test_helpers.h" #include "lib/compress/compress.h" #include "lib/compress/compress_zstd.h" @@ -30,7 +32,6 @@ #include "lib/fs/winlib.h" #include "lib/process/env.h" #include "lib/process/pidfile.h" -#include "lib/process/subprocess.h" #include "lib/intmath/weakrng.h" #include "lib/thread/numcpus.h" #include "lib/math/fp.h" @@ -39,6 +40,7 @@ #include "lib/time/tvdiff.h" #include "lib/encoding/confline.h" #include "lib/net/socketpair.h" +#include "lib/malloc/map_anon.h" #ifdef HAVE_PWD_H #include <pwd.h> @@ -58,6 +60,12 @@ #ifdef HAVE_UNISTD_H #include <unistd.h> #endif +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif +#ifdef HAVE_SYS_WAIT_H +#include <sys/wait.h> +#endif #ifdef _WIN32 #include <tchar.h> @@ -69,6 +77,28 @@ #define INFINITY_DBL ((double)INFINITY) #define NAN_DBL ((double)NAN) +/** Test the tor_isinf() wrapper */ +static void +test_tor_isinf(void *arg) +{ + (void) arg; + + tt_assert(tor_isinf(INFINITY_DBL)); + + tt_assert(!tor_isinf(NAN_DBL)); + tt_assert(!tor_isinf(DBL_EPSILON)); + tt_assert(!tor_isinf(DBL_MAX)); + tt_assert(!tor_isinf(DBL_MIN)); + + tt_assert(!tor_isinf(0.0)); + tt_assert(!tor_isinf(0.1)); + tt_assert(!tor_isinf(3)); + tt_assert(!tor_isinf(3.14)); + + done: + ; +} + /* XXXX this is a minimal wrapper to make the unit tests compile with the * changed tor_timegm interface. */ static time_t @@ -404,7 +434,6 @@ test_util_time(void *arg) /* Assume tv_usec is an unsigned integer until proven otherwise */ #define TV_USEC_MAX UINT_MAX -#define TOR_USEC_PER_SEC 1000000 /* Overflows in the result type */ @@ -2179,15 +2208,6 @@ test_util_strmisc(void *arg) tt_int_op(strcmp_opt(NULL, "foo"), OP_LT, 0); tt_int_op(strcmp_opt("foo", NULL), OP_GT, 0); - /* Test strcmp_len */ - tt_int_op(strcmp_len("foo", "bar", 3), OP_GT, 0); - tt_int_op(strcmp_len("foo", "bar", 2), OP_LT, 0); - tt_int_op(strcmp_len("foo2", "foo1", 4), OP_GT, 0); - tt_int_op(strcmp_len("foo2", "foo1", 3), OP_LT, 0); /* Really stop at len */ - tt_int_op(strcmp_len("foo2", "foo", 3), OP_EQ, 0); /* Really stop at len */ - tt_int_op(strcmp_len("blah", "", 4), OP_GT, 0); - tt_int_op(strcmp_len("blah", "", 0), OP_EQ, 0); - done: tor_free(cp_tmp); } @@ -4050,6 +4070,13 @@ test_util_string_is_utf8(void *ptr) tt_int_op(1, OP_EQ, string_is_utf8("ascii\x7f\n", 7)); tt_int_op(1, OP_EQ, string_is_utf8("Risqu\u00e9=1", 9)); + /* Test the utf8_no_bom function */ + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFF", 3)); + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFFFE", 3)); + tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFFlove", 7)); + tt_int_op(1, OP_EQ, string_is_utf8_no_bom("loveandrespect", + strlen("loveandrespect"))); + // Validate exactly 'len' bytes. tt_int_op(0, OP_EQ, string_is_utf8("\0\x80", 2)); tt_int_op(0, OP_EQ, string_is_utf8("Risqu\u00e9=1", 6)); @@ -4320,204 +4347,6 @@ test_util_load_win_lib(void *ptr) } #endif /* defined(_WIN32) */ -#ifndef _WIN32 -static void -clear_hex_errno(char *hex_errno) -{ - memset(hex_errno, '\0', HEX_ERRNO_SIZE + 1); -} - -static void -test_util_exit_status(void *ptr) -{ - /* Leave an extra byte for a \0 so we can do string comparison */ - char hex_errno[HEX_ERRNO_SIZE + 1]; - int n; - - (void)ptr; - - clear_hex_errno(hex_errno); - tt_str_op("",OP_EQ, hex_errno); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0, hex_errno); - tt_str_op("0/0\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - -#if SIZEOF_INT == 4 - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0x7FFFFFFF, hex_errno); - tt_str_op("0/7FFFFFFF\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0xFF, -0x80000000, hex_errno); - tt_str_op("FF/-80000000\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE); - -#elif SIZEOF_INT == 8 - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0x7FFFFFFFFFFFFFFF, hex_errno); - tt_str_op("0/7FFFFFFFFFFFFFFF\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0xFF, -0x8000000000000000, hex_errno); - tt_str_op("FF/-8000000000000000\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE); - -#endif /* SIZEOF_INT == 4 || ... */ - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0x7F, 0, hex_errno); - tt_str_op("7F/0\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0x08, -0x242, hex_errno); - tt_str_op("8/-242\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - tt_str_op("",OP_EQ, hex_errno); - - done: - ; -} -#endif /* !defined(_WIN32) */ - -#ifndef _WIN32 -static void -test_util_string_from_pipe(void *ptr) -{ - int test_pipe[2] = {-1, -1}; - int retval = 0; - enum stream_status status = IO_STREAM_TERM; - ssize_t retlen; - char buf[4] = { 0 }; - - (void)ptr; - - errno = 0; - - /* Set up a pipe to test on */ - retval = pipe(test_pipe); - tt_int_op(retval, OP_EQ, 0); - - /* Send in a string. */ - retlen = write(test_pipe[1], "ABC", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "ABC"); - errno = 0; - - /* Send in a string that contains a nul. */ - retlen = write(test_pipe[1], "AB\0", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Send in a string that contains a nul only. */ - retlen = write(test_pipe[1], "\0", 1); - tt_int_op(retlen, OP_EQ, 1); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, ""); - errno = 0; - - /* Send in a string that contains a trailing newline. */ - retlen = write(test_pipe[1], "AB\n", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Send in a string that contains a newline only. */ - retlen = write(test_pipe[1], "\n", 1); - tt_int_op(retlen, OP_EQ, 1); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, ""); - errno = 0; - - /* Send in a string and check that we nul terminate return values. */ - retlen = write(test_pipe[1], "AAA", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AAA"); - tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf)); - errno = 0; - - retlen = write(test_pipe[1], "B", 1); - tt_int_op(retlen, OP_EQ, 1); - - memset(buf, '\xff', sizeof(buf)); - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "B"); - tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf)); - errno = 0; - - /* Send in multiple lines. */ - retlen = write(test_pipe[1], "A\nB", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "A\nB"); - errno = 0; - - /* Send in a line and close */ - retlen = write(test_pipe[1], "AB", 2); - tt_int_op(retlen, OP_EQ, 2); - retval = close(test_pipe[1]); - tt_int_op(retval, OP_EQ, 0); - test_pipe[1] = -1; - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Check for EOF */ - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_CLOSED); - errno = 0; - - done: - if (test_pipe[0] != -1) - close(test_pipe[0]); - if (test_pipe[1] != -1) - close(test_pipe[1]); -} - -#endif /* !defined(_WIN32) */ - /** * Test for format_hex_number_sigsafe() */ @@ -4612,57 +4441,6 @@ test_util_format_dec_number(void *ptr) return; } -/** - * Test that we can properly format a Windows command line - */ -static void -test_util_join_win_cmdline(void *ptr) -{ - /* Based on some test cases from "Parsing C++ Command-Line Arguments" in - * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline - * will try to only generate simple cases for the child process to parse; - * i.e. we never embed quoted strings in arguments. */ - - const char *argvs[][4] = { - {"a", "bb", "CCC", NULL}, // Normal - {NULL, NULL, NULL, NULL}, // Empty argument list - {"", NULL, NULL, NULL}, // Empty argument - {"\"a", "b\"b", "CCC\"", NULL}, // Quotes - {"a\tbc", "dd dd", "E", NULL}, // Whitespace - {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes - {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote - {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote - { NULL } // Terminator - }; - - const char *cmdlines[] = { - "a bb CCC", - "", - "\"\"", - "\\\"a b\\\"b CCC\\\"", - "\"a\tbc\" \"dd dd\" E", - "a\\\\\\b \"de fg\" H", - "a\\\\\\\"b \\c D\\", - "\"a\\\\b c\" d E", - NULL // Terminator - }; - - int i; - char *joined_argv = NULL; - - (void)ptr; - - for (i=0; cmdlines[i]!=NULL; i++) { - log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]); - joined_argv = tor_join_win_cmdline(argvs[i]); - tt_str_op(cmdlines[i],OP_EQ, joined_argv); - tor_free(joined_argv); - } - - done: - tor_free(joined_argv); -} - #define MAX_SPLIT_LINE_COUNT 4 struct split_lines_test_t { const char *orig_line; // Line to be split (may contain \0's) @@ -4670,67 +4448,6 @@ struct split_lines_test_t { const char *split_line[MAX_SPLIT_LINE_COUNT]; // Split lines }; -/** - * Test that we properly split a buffer into lines - */ -static void -test_util_split_lines(void *ptr) -{ - /* Test cases. orig_line of last test case must be NULL. - * The last element of split_line[i] must be NULL. */ - struct split_lines_test_t tests[] = { - {"", 0, {NULL}}, - {"foo", 3, {"foo", NULL}}, - {"\n\rfoo\n\rbar\r\n", 12, {"foo", "bar", NULL}}, - {"fo o\r\nb\tar", 10, {"fo o", "b.ar", NULL}}, - {"\x0f""f\0o\0\n\x01""b\0r\0\r", 12, {".f.o.", ".b.r.", NULL}}, - {"line 1\r\nline 2", 14, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\nline 2", 16, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\r\r\nline 2", 18, {"line 1", "line 2", NULL}}, - {"line 1\r\n\n\n\n\rline 2", 18, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\t\r\nline 3", 18, {"line 1", ".", "line 3", NULL}}, - {"\n\t\r\t\nline 3", 11, {".", ".", "line 3", NULL}}, - {NULL, 0, { NULL }} - }; - - int i, j; - char *orig_line=NULL; - smartlist_t *sl=NULL; - - (void)ptr; - - for (i=0; tests[i].orig_line; i++) { - sl = smartlist_new(); - /* Allocate space for string and trailing NULL */ - orig_line = tor_memdup(tests[i].orig_line, tests[i].orig_length + 1); - tor_split_lines(sl, orig_line, tests[i].orig_length); - - j = 0; - log_info(LD_GENERAL, "Splitting test %d of length %d", - i, tests[i].orig_length); - SMARTLIST_FOREACH_BEGIN(sl, const char *, line) { - /* Check we have not got too many lines */ - tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j); - /* Check that there actually should be a line here */ - tt_ptr_op(tests[i].split_line[j], OP_NE, NULL); - log_info(LD_GENERAL, "Line %d of test %d, should be <%s>", - j, i, tests[i].split_line[j]); - /* Check that the line is as expected */ - tt_str_op(line,OP_EQ, tests[i].split_line[j]); - j++; - } SMARTLIST_FOREACH_END(line); - /* Check that we didn't miss some lines */ - tt_ptr_op(NULL,OP_EQ, tests[i].split_line[j]); - tor_free(orig_line); - smartlist_free(sl); - sl = NULL; - } - - done: - tor_free(orig_line); - smartlist_free(sl); -} - static void test_util_di_ops(void *arg) { @@ -5834,6 +5551,18 @@ test_util_ipv4_validation(void *arg) } static void +test_util_ipv6_validation(void *arg) +{ + (void)arg; + + tt_assert(string_is_valid_ipv6_address("2a00:1450:401b:800::200e")); + tt_assert(!string_is_valid_ipv6_address("11:22::33:44:")); + + done: + return; +} + +static void test_util_writepid(void *arg) { (void) arg; @@ -6397,6 +6126,104 @@ test_util_log_mallinfo(void *arg) tor_free(mem); } +static void +test_util_map_anon(void *arg) +{ + (void)arg; + char *ptr = NULL; + size_t sz = 16384; + + /* Basic checks. */ + ptr = tor_mmap_anonymous(sz, 0); + tt_ptr_op(ptr, OP_NE, 0); + ptr[sz-1] = 3; + tt_int_op(ptr[0], OP_EQ, 0); + tt_int_op(ptr[sz-2], OP_EQ, 0); + tt_int_op(ptr[sz-1], OP_EQ, 3); + + /* Try again, with a private (non-swappable) mapping. */ + tor_munmap_anonymous(ptr, sz); + ptr = tor_mmap_anonymous(sz, ANONMAP_PRIVATE); + tt_ptr_op(ptr, OP_NE, 0); + ptr[sz-1] = 10; + tt_int_op(ptr[0], OP_EQ, 0); + tt_int_op(ptr[sz/2], OP_EQ, 0); + tt_int_op(ptr[sz-1], OP_EQ, 10); + + /* Now let's test a drop-on-fork mapping. */ + tor_munmap_anonymous(ptr, sz); + ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT); + tt_ptr_op(ptr, OP_NE, 0); + ptr[sz-1] = 10; + tt_int_op(ptr[0], OP_EQ, 0); + tt_int_op(ptr[sz/2], OP_EQ, 0); + tt_int_op(ptr[sz-1], OP_EQ, 10); + + done: + tor_munmap_anonymous(ptr, sz); +} + +static void +test_util_map_anon_nofork(void *arg) +{ + (void)arg; +#if !defined(HAVE_MADVISE) && !defined(HAVE_MINHERIT) + /* The operating system doesn't support this. */ + tt_skip(); + done: + ; +#else + /* We have the right OS support. We're going to try marking the buffer as + * either zero-on-fork or as drop-on-fork, whichever is supported. Then we + * will fork and send a byte back to the parent process. This will either + * crash, or send zero. */ + + char *ptr = NULL; + size_t sz = 16384; + int pipefd[2] = {-1, -1}; + + tor_munmap_anonymous(ptr, sz); + ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT); + tt_ptr_op(ptr, OP_NE, 0); + memset(ptr, 0xd0, sz); + + tt_int_op(0, OP_EQ, pipe(pipefd)); + pid_t child = fork(); + if (child == 0) { + /* We're in the child. */ + close(pipefd[0]); + ssize_t r = write(pipefd[1], &ptr[sz-1], 1); /* This may crash. */ + close(pipefd[1]); + if (r < 0) + exit(1); + exit(0); + } + tt_int_op(child, OP_GT, 0); + /* In the parent. */ + close(pipefd[1]); + pipefd[1] = -1; + char buf[1]; + ssize_t r = read(pipefd[0], buf, 1); +#if defined(INHERIT_ZERO) || defined(MADV_WIPEONFORK) + tt_int_op((int)r, OP_EQ, 1); // child should send us a byte. + tt_int_op(buf[0], OP_EQ, 0); +#else + tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed. +#endif + int ws; + waitpid(child, &ws, 0); + + done: + tor_munmap_anonymous(ptr, sz); + if (pipefd[0] >= 0) { + close(pipefd[0]); + } + if (pipefd[1] >= 0) { + close(pipefd[1]); + } +#endif +} + #define UTIL_LEGACY(name) \ { #name, test_util_ ## name , 0, NULL, NULL } @@ -6490,12 +6317,8 @@ struct testcase_t util_tests[] = { UTIL_TEST(nowrap_math, 0), UTIL_TEST(num_cpus, 0), UTIL_TEST_WIN_ONLY(load_win_lib, 0), - UTIL_TEST_NO_WIN(exit_status, 0), - UTIL_TEST_NO_WIN(string_from_pipe, 0), UTIL_TEST(format_hex_number, 0), UTIL_TEST(format_dec_number, 0), - UTIL_TEST(join_win_cmdline, 0), - UTIL_TEST(split_lines, 0), UTIL_TEST(n_bits_set, 0), UTIL_TEST(eat_whitespace, 0), UTIL_TEST(sl_new_from_text_lines, 0), @@ -6512,6 +6335,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(mathlog, 0), UTIL_TEST(fraction, 0), UTIL_TEST(weak_random, 0), + { "tor_isinf", test_tor_isinf, TT_FORK, NULL, NULL }, { "socket_ipv4", test_util_socket, TT_FORK, &passthrough_setup, (void*)"4" }, { "socket_ipv6", test_util_socket, TT_FORK, @@ -6524,6 +6348,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(hostname_validation, 0), UTIL_TEST(dest_validation_edgecase, 0), UTIL_TEST(ipv4_validation, 0), + UTIL_TEST(ipv6_validation, 0), UTIL_TEST(writepid, 0), UTIL_TEST(get_avail_disk_space, 0), UTIL_TEST(touch_file, 0), @@ -6536,5 +6361,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(htonll, 0), UTIL_TEST(get_unquoted_path, 0), UTIL_TEST(log_mallinfo, 0), + UTIL_TEST(map_anon, 0), + UTIL_TEST(map_anon_nofork, TT_SKIP /* See bug #29535 */), END_OF_TESTCASES }; diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index d344d0e95c..3a0b41faa5 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -392,10 +392,13 @@ test_util_format_encoded_size(void *arg) base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0); tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0)); + tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf))); + base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, BASE64_ENCODE_MULTILINE); tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, BASE64_ENCODE_MULTILINE)); + tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf))); } done: @@ -417,4 +420,3 @@ struct testcase_t util_format_tests[] = { { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL }, END_OF_TESTCASES }; - diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c deleted file mode 100644 index 29e30eaa11..0000000000 --- a/src/test/test_util_slow.c +++ /dev/null @@ -1,396 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#include "orconfig.h" -#define UTIL_PRIVATE -#define SUBPROCESS_PRIVATE -#include "lib/crypt_ops/crypto_cipher.h" -#include "lib/log/log.h" -#include "lib/process/subprocess.h" -#include "lib/process/waitpid.h" -#include "lib/string/printf.h" -#include "lib/time/compat_time.h" -#include "test/test.h" - -#include <errno.h> -#include <string.h> - -#ifndef BUILDDIR -#define BUILDDIR "." -#endif - -#ifdef _WIN32 -#define notify_pending_waitpid_callbacks() STMT_NIL -#define TEST_CHILD "test-child.exe" -#define EOL "\r\n" -#else -#define TEST_CHILD (BUILDDIR "/src/test/test-child") -#define EOL "\n" -#endif /* defined(_WIN32) */ - -#ifdef _WIN32 -/* I've assumed Windows doesn't have the gap between fork and exec - * that causes the race condition on unix-like platforms */ -#define MATCH_PROCESS_STATUS(s1,s2) ((s1) == (s2)) - -#else /* !(defined(_WIN32)) */ -/* work around a race condition of the timing of SIGCHLD handler updates - * to the process_handle's fields, and checks of those fields - * - * TODO: Once we can signal failure to exec, change PROCESS_STATUS_RUNNING to - * PROCESS_STATUS_ERROR (and similarly with *_OR_NOTRUNNING) */ -#define PROCESS_STATUS_RUNNING_OR_NOTRUNNING (PROCESS_STATUS_RUNNING+1) -#define IS_RUNNING_OR_NOTRUNNING(s) \ - ((s) == PROCESS_STATUS_RUNNING || (s) == PROCESS_STATUS_NOTRUNNING) -/* well, this is ugly */ -#define MATCH_PROCESS_STATUS(s1,s2) \ - ( (s1) == (s2) \ - ||((s1) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING \ - && IS_RUNNING_OR_NOTRUNNING(s2)) \ - ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING \ - && IS_RUNNING_OR_NOTRUNNING(s1))) - -#endif /* defined(_WIN32) */ - -/** Helper function for testing tor_spawn_background */ -static void -run_util_spawn_background(const char *argv[], const char *expected_out, - const char *expected_err, int expected_exit, - int expected_status) -{ - int retval, exit_code; - ssize_t pos; - process_handle_t *process_handle=NULL; - char stdout_buf[100], stderr_buf[100]; - int status; - - /* Start the program */ -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - - notify_pending_waitpid_callbacks(); - - /* the race condition doesn't affect status, - * because status isn't updated by the SIGCHLD handler, - * but we still need to handle PROCESS_STATUS_RUNNING_OR_NOTRUNNING */ - tt_assert(MATCH_PROCESS_STATUS(expected_status, status)); - if (status == PROCESS_STATUS_ERROR) { - tt_ptr_op(process_handle, OP_EQ, NULL); - return; - } - - tt_ptr_op(process_handle, OP_NE, NULL); - - /* When a spawned process forks, fails, then exits very quickly, - * (this typically occurs when exec fails) - * there is a race condition between the SIGCHLD handler - * updating the process_handle's fields, and this test - * checking the process status in those fields. - * The SIGCHLD update can occur before or after the code below executes. - * This causes intermittent failures in spawn_background_fail(), - * typically when the machine is under load. - * We use PROCESS_STATUS_RUNNING_OR_NOTRUNNING to avoid this issue. */ - - /* the race condition affects the change in - * process_handle->status from RUNNING to NOTRUNNING */ - tt_assert(MATCH_PROCESS_STATUS(expected_status, process_handle->status)); - -#ifndef _WIN32 - notify_pending_waitpid_callbacks(); - /* the race condition affects the change in - * process_handle->waitpid_cb to NULL, - * so we skip the check if expected_status is ambiguous, - * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */ - tt_assert(process_handle->waitpid_cb != NULL - || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING); -#endif /* !defined(_WIN32) */ - -#ifdef _WIN32 - tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE); - tt_assert(process_handle->stderr_pipe != INVALID_HANDLE_VALUE); - tt_assert(process_handle->stdin_pipe != INVALID_HANDLE_VALUE); -#else - tt_assert(process_handle->stdout_pipe >= 0); - tt_assert(process_handle->stderr_pipe >= 0); - tt_assert(process_handle->stdin_pipe >= 0); -#endif /* defined(_WIN32) */ - - /* Check stdout */ - pos = tor_read_all_from_process_stdout(process_handle, stdout_buf, - sizeof(stdout_buf) - 1); - tt_assert(pos >= 0); - stdout_buf[pos] = '\0'; - tt_int_op(strlen(expected_out),OP_EQ, pos); - tt_str_op(expected_out,OP_EQ, stdout_buf); - - notify_pending_waitpid_callbacks(); - - /* Check it terminated correctly */ - retval = tor_get_exit_code(process_handle, 1, &exit_code); - tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval); - tt_int_op(expected_exit,OP_EQ, exit_code); - // TODO: Make test-child exit with something other than 0 - -#ifndef _WIN32 - notify_pending_waitpid_callbacks(); - tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL); -#endif - - /* Check stderr */ - pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, - sizeof(stderr_buf) - 1); - tt_assert(pos >= 0); - stderr_buf[pos] = '\0'; - tt_str_op(expected_err,OP_EQ, stderr_buf); - tt_int_op(strlen(expected_err),OP_EQ, pos); - - notify_pending_waitpid_callbacks(); - - done: - if (process_handle) - tor_process_handle_destroy(process_handle, 1); -} - -/** Check that we can launch a process and read the output */ -static void -test_util_spawn_background_ok(void *ptr) -{ - const char *argv[] = {TEST_CHILD, "--test", NULL}; - const char *expected_out = "OUT"EOL "--test"EOL "SLEEPING"EOL "DONE" EOL; - const char *expected_err = "ERR"EOL; - - (void)ptr; - - run_util_spawn_background(argv, expected_out, expected_err, 0, - PROCESS_STATUS_RUNNING); -} - -/** Check that failing to find the executable works as expected */ -static void -test_util_spawn_background_fail(void *ptr) -{ - const char *argv[] = {BUILDDIR "/src/test/no-such-file", "--test", NULL}; - const char *expected_err = ""; - char expected_out[1024]; - char code[32]; -#ifdef _WIN32 - const int expected_status = PROCESS_STATUS_ERROR; -#else - /* TODO: Once we can signal failure to exec, set this to be - * PROCESS_STATUS_RUNNING_OR_ERROR */ - const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING; -#endif /* defined(_WIN32) */ - - memset(expected_out, 0xf0, sizeof(expected_out)); - memset(code, 0xf0, sizeof(code)); - - (void)ptr; - - tor_snprintf(code, sizeof(code), "%x/%x", - 9 /* CHILD_STATE_FAILEXEC */ , ENOENT); - tor_snprintf(expected_out, sizeof(expected_out), - "ERR: Failed to spawn background process - code %s\n", code); - - run_util_spawn_background(argv, expected_out, expected_err, 255, - expected_status); -} - -/** Test that reading from a handle returns a partial read rather than - * blocking */ -static void -test_util_spawn_background_partial_read_impl(int exit_early) -{ - const int expected_exit = 0; - const int expected_status = PROCESS_STATUS_RUNNING; - - int retval, exit_code; - ssize_t pos = -1; - process_handle_t *process_handle=NULL; - int status; - char stdout_buf[100], stderr_buf[100]; - - const char *argv[] = {TEST_CHILD, "--test", NULL}; - const char *expected_out[] = { "OUT" EOL "--test" EOL "SLEEPING" EOL, - "DONE" EOL, - NULL }; - const char *expected_err = "ERR" EOL; - -#ifndef _WIN32 - int eof = 0; -#endif - int expected_out_ctr; - - if (exit_early) { - argv[1] = "--hang"; - expected_out[0] = "OUT"EOL "--hang"EOL "SLEEPING" EOL; - } - - /* Start the program */ -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - tt_int_op(expected_status,OP_EQ, status); - tt_assert(process_handle); - tt_int_op(expected_status,OP_EQ, process_handle->status); - - /* Check stdout */ - for (expected_out_ctr = 0; expected_out[expected_out_ctr] != NULL;) { -#ifdef _WIN32 - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, NULL); -#else - /* Check that we didn't read the end of file last time */ - tt_assert(!eof); - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, NULL, &eof); -#endif /* defined(_WIN32) */ - log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos); - - /* We would have blocked, keep on trying */ - if (0 == pos) - continue; - - tt_assert(pos > 0); - stdout_buf[pos] = '\0'; - tt_str_op(expected_out[expected_out_ctr],OP_EQ, stdout_buf); - tt_int_op(strlen(expected_out[expected_out_ctr]),OP_EQ, pos); - expected_out_ctr++; - } - - if (exit_early) { - tor_process_handle_destroy(process_handle, 1); - process_handle = NULL; - goto done; - } - - /* The process should have exited without writing more */ -#ifdef _WIN32 - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, - process_handle); - tt_int_op(0,OP_EQ, pos); -#else /* !(defined(_WIN32)) */ - if (!eof) { - /* We should have got all the data, but maybe not the EOF flag */ - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, - process_handle, &eof); - tt_int_op(0,OP_EQ, pos); - tt_assert(eof); - } - /* Otherwise, we got the EOF on the last read */ -#endif /* defined(_WIN32) */ - - /* Check it terminated correctly */ - retval = tor_get_exit_code(process_handle, 1, &exit_code); - tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval); - tt_int_op(expected_exit,OP_EQ, exit_code); - - // TODO: Make test-child exit with something other than 0 - - /* Check stderr */ - pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, - sizeof(stderr_buf) - 1); - tt_assert(pos >= 0); - stderr_buf[pos] = '\0'; - tt_str_op(expected_err,OP_EQ, stderr_buf); - tt_int_op(strlen(expected_err),OP_EQ, pos); - - done: - tor_process_handle_destroy(process_handle, 1); -} - -static void -test_util_spawn_background_partial_read(void *arg) -{ - (void)arg; - test_util_spawn_background_partial_read_impl(0); -} - -static void -test_util_spawn_background_exit_early(void *arg) -{ - (void)arg; - test_util_spawn_background_partial_read_impl(1); -} - -static void -test_util_spawn_background_waitpid_notify(void *arg) -{ - int retval, exit_code; - process_handle_t *process_handle=NULL; - int status; - int ms_timer; - - const char *argv[] = {TEST_CHILD, "--fast", NULL}; - - (void) arg; - -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - - tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); - tt_ptr_op(process_handle, OP_NE, NULL); - - /* We're not going to look at the stdout/stderr output this time. Instead, - * we're testing whether notify_pending_waitpid_calbacks() can report the - * process exit (on unix) and/or whether tor_get_exit_code() can notice it - * (on windows) */ - -#ifndef _WIN32 - ms_timer = 30*1000; - tt_ptr_op(process_handle->waitpid_cb, OP_NE, NULL); - while (process_handle->waitpid_cb && ms_timer > 0) { - tor_sleep_msec(100); - ms_timer -= 100; - notify_pending_waitpid_callbacks(); - } - tt_int_op(ms_timer, OP_GT, 0); - tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL); -#endif /* !defined(_WIN32) */ - - ms_timer = 30*1000; - while (((retval = tor_get_exit_code(process_handle, 0, &exit_code)) - == PROCESS_EXIT_RUNNING) && ms_timer > 0) { - tor_sleep_msec(100); - ms_timer -= 100; - } - tt_int_op(ms_timer, OP_GT, 0); - - tt_int_op(retval, OP_EQ, PROCESS_EXIT_EXITED); - - done: - tor_process_handle_destroy(process_handle, 1); -} - -#undef TEST_CHILD -#undef EOL - -#undef MATCH_PROCESS_STATUS - -#ifndef _WIN32 -#undef PROCESS_STATUS_RUNNING_OR_NOTRUNNING -#undef IS_RUNNING_OR_NOTRUNNING -#endif - -#define UTIL_TEST(name, flags) \ - { #name, test_util_ ## name, flags, NULL, NULL } - -struct testcase_t slow_util_tests[] = { - UTIL_TEST(spawn_background_ok, 0), - UTIL_TEST(spawn_background_fail, 0), - UTIL_TEST(spawn_background_partial_read, 0), - UTIL_TEST(spawn_background_exit_early, 0), - UTIL_TEST(spawn_background_waitpid_notify, 0), - END_OF_TESTCASES -}; diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c new file mode 100644 index 0000000000..5c9eebd00e --- /dev/null +++ b/src/test/test_voting_flags.c @@ -0,0 +1,191 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#define VOTEFLAGS_PRIVATE + +#include "core/or/or.h" + +#include "feature/dirauth/voteflags.h" +#include "feature/nodelist/node_st.h" +#include "feature/nodelist/routerstatus_st.h" +#include "feature/nodelist/routerinfo_st.h" + +#include "app/config/config.h" + +#include "test/test.h" + +typedef struct { + time_t now; + routerinfo_t ri; + node_t node; + + routerstatus_t expected; +} flag_vote_test_cfg_t; + +static void +setup_cfg(flag_vote_test_cfg_t *c) +{ + memset(c, 0, sizeof(*c)); + + c->now = approx_time(); + + c->ri.nickname = (char *) "testing100"; + strlcpy(c->expected.nickname, "testing100", sizeof(c->expected.nickname)); + + memset(c->ri.cache_info.identity_digest, 0xff, DIGEST_LEN); + memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST_LEN); + + c->ri.cache_info.published_on = c->now - 100; + c->expected.published_on = c->now - 100; + + c->ri.addr = 0x7f010105; + c->expected.addr = 0x7f010105; + c->ri.or_port = 9090; + c->expected.or_port = 9090; + + tor_addr_make_null(&c->ri.ipv6_addr, AF_INET6); + tor_addr_make_null(&c->expected.ipv6_addr, AF_INET6); + + // By default we have no loaded information about stability or speed, + // so we'll default to voting "yeah sure." on these two. + c->expected.is_fast = 1; + c->expected.is_stable = 1; +} + +static bool +check_result(flag_vote_test_cfg_t *c) +{ + bool result = false; + routerstatus_t rs; + memset(&rs, 0, sizeof(rs)); + set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0); + + tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on); + tt_str_op(rs.nickname, OP_EQ, c->expected.nickname); + + // identity_digest and descriptor_digest are not set here. + + tt_uint_op(rs.addr, OP_EQ, c->expected.addr); + tt_uint_op(rs.or_port, OP_EQ, c->expected.or_port); + tt_uint_op(rs.dir_port, OP_EQ, c->expected.dir_port); + + tt_assert(tor_addr_eq(&rs.ipv6_addr, &c->expected.ipv6_addr)); + tt_uint_op(rs.ipv6_orport, OP_EQ, c->expected.ipv6_orport); + +#define FLAG(flagname) \ + tt_uint_op(rs.flagname, OP_EQ, c->expected.flagname) + + FLAG(is_authority); + FLAG(is_exit); + FLAG(is_stable); + FLAG(is_fast); + FLAG(is_flagged_running); + FLAG(is_named); + FLAG(is_unnamed); + FLAG(is_valid); + FLAG(is_possible_guard); + FLAG(is_bad_exit); + FLAG(is_hs_dir); + FLAG(is_v2_dir); + FLAG(is_staledesc); + FLAG(has_bandwidth); + FLAG(has_exitsummary); + FLAG(bw_is_unmeasured); + + result = true; + + done: + return result; +} + +static void +test_voting_flags_minimal(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + (void) check_result(cfg); +} + +static void +test_voting_flags_ipv6(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + + tt_assert(tor_addr_parse(&cfg->ri.ipv6_addr, "f00::b42") == AF_INET6); + cfg->ri.ipv6_orport = 9091; + // no change in expected results, since we aren't set up with ipv6 + // connectivity. + if (!check_result(cfg)) + goto done; + + get_options_mutable()->AuthDirHasIPv6Connectivity = 1; + // no change in expected results, since last_reachable6 won't be set. + if (!check_result(cfg)) + goto done; + + cfg->node.last_reachable6 = cfg->now - 10; + // now that lastreachable6 is set, we expect to see the result. + tt_assert(tor_addr_parse(&cfg->expected.ipv6_addr, "f00::b42") == AF_INET6); + cfg->expected.ipv6_orport = 9091; + if (!check_result(cfg)) + goto done; + done: + ; +} + +static void +test_voting_flags_staledesc(void *arg) +{ + flag_vote_test_cfg_t *cfg = arg; + time_t now = cfg->now; + + cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL + 10; + cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL + 10; + // no change in expectations for is_staledesc + if (!check_result(cfg)) + goto done; + + cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL - 10; + cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL - 10; + cfg->expected.is_staledesc = 1; + if (!check_result(cfg)) + goto done; + + done: + ; +} + +static void * +setup_voting_flags_test(const struct testcase_t *testcase) +{ + (void)testcase; + flag_vote_test_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + setup_cfg(cfg); + return cfg; +} + +static int +teardown_voting_flags_test(const struct testcase_t *testcase, void *arg) +{ + (void)testcase; + flag_vote_test_cfg_t *cfg = arg; + tor_free(cfg); + return 1; +} + +static const struct testcase_setup_t voting_flags_setup = { + .setup_fn = setup_voting_flags_test, + .cleanup_fn = teardown_voting_flags_test, +}; + +#define T(name,flags) \ + { #name, test_voting_flags_##name, (flags), &voting_flags_setup, NULL } + +struct testcase_t voting_flags_tests[] = { + T(minimal, 0), + T(ipv6, TT_FORK), + // TODO: Add more of these tests. + T(staledesc, TT_FORK), + END_OF_TESTCASES +}; diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 62d40a42fa..8fc8ef7830 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -25,6 +25,8 @@ #include "lib/compress/compress.h" #include "lib/evloop/compat_libevent.h" #include "lib/crypt_ops/crypto_init.h" +#include "lib/version/torversion.h" +#include "app/main/subsysmgr.h" #include <stdio.h> #ifdef HAVE_FCNTL_H @@ -230,12 +232,12 @@ void tinytest_prefork(void) { free_pregenerated_keys(); - crypto_prefork(); + subsystems_prefork(); } void tinytest_postfork(void) { - crypto_postfork(); + subsystems_postfork(); init_pregenerated_keys(); } @@ -259,24 +261,15 @@ main(int c, const char **v) int loglevel = LOG_ERR; int accel_crypto = 0; - /* We must initialise logs before we call tor_assert() */ - init_logging(1); + subsystems_init_upto(SUBSYS_LEVEL_LIBS); - update_approx_time(time(NULL)); options = options_new(); - tor_threads_init(); - tor_compress_init(); - - network_init(); - - monotime_init(); struct tor_libevent_cfg cfg; memset(&cfg, 0, sizeof(cfg)); tor_libevent_initialize(&cfg); control_initialize_event_queue(); - configure_backtrace_handler(get_version()); for (i_out = i = 1; i < c; ++i) { if (!strcmp(v[i], "--warn")) { @@ -312,6 +305,7 @@ main(int c, const char **v) s.masks[LOG_WARN-LOG_ERR] |= LD_BUG; add_callback_log(&s, log_callback_failure); } + flush_log_messages_from_startup(); init_protocol_warning_severity_level(); options->command = CMD_RUN_UNITTESTS; @@ -352,8 +346,6 @@ main(int c, const char **v) free_pregenerated_keys(); - crypto_global_cleanup(); - if (have_failed) return 1; else diff --git a/src/tools/include.am b/src/tools/include.am index f7aa7e0d1e..72dfe6017c 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -5,9 +5,11 @@ noinst_PROGRAMS+= src/tools/tor-cov-resolve endif src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c -src_tools_tor_resolve_LDFLAGS = +src_tools_tor_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@ src_tools_tor_resolve_LDADD = \ + src/trunnel/libor-trunnel.a \ $(TOR_UTIL_LIBS) \ + $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\ $(rust_ldadd) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@ @@ -15,8 +17,11 @@ if COVERAGE_ENABLED src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +src_tools_tor_cov_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@ src_tools_tor_cov_resolve_LDADD = \ + src/trunnel/libor-trunnel.a \ $(TOR_UTIL_TESTING_LIBS) \ + $(TOR_CRYPTO_TESTING_LIBS) $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_MATH@ @TOR_LIB_WS32@ endif diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c index 6a84abe557..98b3a4a74c 100644 --- a/src/tools/tor-resolve.c +++ b/src/tools/tor-resolve.c @@ -15,6 +15,7 @@ #include "lib/string/util_string.h" #include "lib/net/socks5_status.h" +#include "trunnel/socks5.h" #include <stdio.h> #include <stdlib.h> @@ -49,68 +50,204 @@ static void usage(void) ATTR_NORETURN; +/** + * Set <b>out</b> to a pointer to newly allocated buffer containing + * SOCKS4a RESOLVE request with <b>username</b> and <b>hostname</b>. + * Return number of bytes in the buffer if succeeded or -1 if failed. + */ +static ssize_t +build_socks4a_resolve_request(uint8_t **out, + const char *username, + const char *hostname) +{ + tor_assert(out); + tor_assert(username); + tor_assert(hostname); + + const char *errmsg = NULL; + uint8_t *output = NULL; + socks4_client_request_t *rq = socks4_client_request_new(); + + socks4_client_request_set_version(rq, 4); + socks4_client_request_set_command(rq, CMD_RESOLVE); + socks4_client_request_set_port(rq, 0); + socks4_client_request_set_addr(rq, 0x00000001u); + socks4_client_request_set_username(rq, username); + socks4_client_request_set_socks4a_addr_hostname(rq, hostname); + + errmsg = socks4_client_request_check(rq); + if (errmsg) { + goto cleanup; + } + + ssize_t encoded_len = socks4_client_request_encoded_len(rq); + if (encoded_len <= 0) { + errmsg = "socks4_client_request_encoded_len failed"; + goto cleanup; + } + + output = tor_malloc(encoded_len); + memset(output, 0, encoded_len); + + encoded_len = socks4_client_request_encode(output, encoded_len, rq); + if (encoded_len <= 0) { + errmsg = "socks4_client_request_encode failed"; + goto cleanup; + } + + *out = output; + + cleanup: + socks4_client_request_free(rq); + if (errmsg) { + log_err(LD_NET, "build_socks4a_resolve_request failed: %s", errmsg); + *out = NULL; + tor_free(output); + } + return errmsg ? -1 : encoded_len; +} + +#define SOCKS5_ATYPE_HOSTNAME 0x03 +#define SOCKS5_ATYPE_IPV4 0x01 +#define SOCKS5_ATYPE_IPV6 0x04 + +/** + * Set <b>out</b> to pointer to newly allocated buffer containing + * SOCKS5 RESOLVE/RESOLVE_PTR request with given <b>hostname<b>. + * Generate a reverse request if <b>reverse</b> is true. + * Return the number of bytes in the buffer if succeeded or -1 if failed. + */ +static ssize_t +build_socks5_resolve_request(uint8_t **out, + const char *hostname, + int reverse) +{ + const char *errmsg = NULL; + uint8_t *outbuf = NULL; + int is_ip_address; + tor_addr_t addr; + size_t addrlen; + int ipv6; + is_ip_address = tor_addr_parse(&addr, hostname) != -1; + if (!is_ip_address && reverse) { + log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!"); + return -1; + } + ipv6 = reverse && tor_addr_family(&addr) == AF_INET6; + addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname); + if (addrlen > UINT8_MAX) { + log_err(LD_GENERAL, "Hostname is too long!"); + return -1; + } + + socks5_client_request_t *rq = socks5_client_request_new(); + + socks5_client_request_set_version(rq, 5); + /* RESOLVE_PTR or RESOLVE */ + socks5_client_request_set_command(rq, reverse ? CMD_RESOLVE_PTR : + CMD_RESOLVE); + socks5_client_request_set_reserved(rq, 0); + + uint8_t atype = SOCKS5_ATYPE_HOSTNAME; + if (reverse) + atype = ipv6 ? SOCKS5_ATYPE_IPV6 : SOCKS5_ATYPE_IPV4; + + socks5_client_request_set_atype(rq, atype); + + switch (atype) { + case SOCKS5_ATYPE_IPV4: { + socks5_client_request_set_dest_addr_ipv4(rq, + tor_addr_to_ipv4h(&addr)); + } break; + case SOCKS5_ATYPE_IPV6: { + uint8_t *ipv6_array = + socks5_client_request_getarray_dest_addr_ipv6(rq); + + tor_assert(ipv6_array); + + memcpy(ipv6_array, tor_addr_to_in6_addr8(&addr), 16); + } break; + + case SOCKS5_ATYPE_HOSTNAME: { + domainname_t *dn = domainname_new(); + domainname_set_len(dn, addrlen - 1); + domainname_setlen_name(dn, addrlen - 1); + char *dn_buf = domainname_getarray_name(dn); + memcpy(dn_buf, hostname, addrlen - 1); + + errmsg = domainname_check(dn); + + if (errmsg) { + domainname_free(dn); + goto cleanup; + } else { + socks5_client_request_set_dest_addr_domainname(rq, dn); + } + } break; + default: + tor_assert_unreached(); + break; + } + + socks5_client_request_set_dest_port(rq, 0); + + errmsg = socks5_client_request_check(rq); + if (errmsg) { + goto cleanup; + } + + ssize_t encoded_len = socks5_client_request_encoded_len(rq); + if (encoded_len < 0) { + errmsg = "Cannot predict encoded length"; + goto cleanup; + } + + outbuf = tor_malloc(encoded_len); + memset(outbuf, 0, encoded_len); + + encoded_len = socks5_client_request_encode(outbuf, encoded_len, rq); + if (encoded_len < 0) { + errmsg = "encoding failed"; + goto cleanup; + } + + *out = outbuf; + + cleanup: + socks5_client_request_free(rq); + if (errmsg) { + tor_free(outbuf); + log_err(LD_NET, "build_socks5_resolve_request failed with error: %s", + errmsg); + } + + return errmsg ? -1 : encoded_len; +} + /** Set *<b>out</b> to a newly allocated SOCKS4a resolve request with * <b>username</b> and <b>hostname</b> as provided. Return the number * of bytes in the request. */ static ssize_t -build_socks_resolve_request(char **out, +build_socks_resolve_request(uint8_t **out, const char *username, const char *hostname, int reverse, int version) { - size_t len = 0; tor_assert(out); tor_assert(username); tor_assert(hostname); + tor_assert(version == 4 || version == 5); + if (version == 4) { - len = 8 + strlen(username) + 1 + strlen(hostname) + 1; - *out = tor_malloc(len); - (*out)[0] = 4; /* SOCKS version 4 */ - (*out)[1] = '\xF0'; /* Command: resolve. */ - set_uint16((*out)+2, htons(0)); /* port: 0. */ - set_uint32((*out)+4, htonl(0x00000001u)); /* addr: 0.0.0.1 */ - memcpy((*out)+8, username, strlen(username)+1); - memcpy((*out)+8+strlen(username)+1, hostname, strlen(hostname)+1); + return build_socks4a_resolve_request(out, username, hostname); } else if (version == 5) { - int is_ip_address; - tor_addr_t addr; - size_t addrlen; - int ipv6; - is_ip_address = tor_addr_parse(&addr, hostname) != -1; - if (!is_ip_address && reverse) { - log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!"); - return -1; - } - ipv6 = reverse && tor_addr_family(&addr) == AF_INET6; - addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname); - if (addrlen > UINT8_MAX) { - log_err(LD_GENERAL, "Hostname is too long!"); - return -1; - } - len = 6 + addrlen; - *out = tor_malloc(len); - (*out)[0] = 5; /* SOCKS version 5 */ - (*out)[1] = reverse ? '\xF1' : '\xF0'; /* RESOLVE_PTR or RESOLVE */ - (*out)[2] = 0; /* reserved. */ - if (reverse) { - (*out)[3] = ipv6 ? 4 : 1; - if (ipv6) - memcpy((*out)+4, tor_addr_to_in6_addr8(&addr), 16); - else - set_uint32((*out)+4, tor_addr_to_ipv4n(&addr)); - } else { - (*out)[3] = 3; - (*out)[4] = (char)(uint8_t)(addrlen - 1); - memcpy((*out)+5, hostname, addrlen - 1); - } - set_uint16((*out)+4+addrlen, 0); /* port */ - } else { - tor_assert(0); + return build_socks5_resolve_request(out, hostname, reverse); } - return len; + tor_assert_unreached(); + return -1; } static void @@ -134,34 +271,50 @@ parse_socks4a_resolve_response(const char *hostname, const char *response, size_t len, tor_addr_t *addr_out) { + int result = 0; uint8_t status; tor_assert(response); tor_assert(addr_out); - if (len < RESPONSE_LEN_4) { + socks4_server_reply_t *reply; + + ssize_t parsed = socks4_server_reply_parse(&reply, + (const uint8_t *)response, + len); + + if (parsed == -1) { + log_warn(LD_PROTOCOL, "Failed parsing SOCKS4a response"); + result = -1; goto cleanup; + } + + if (parsed == -2) { log_warn(LD_PROTOCOL,"Truncated socks response."); - return -1; + result = -1; goto cleanup; } - if (((uint8_t)response[0])!=0) { /* version: 0 */ + + if (socks4_server_reply_get_version(reply) != 0) { /* version: 0 */ log_warn(LD_PROTOCOL,"Nonzero version in socks response: bad format."); - return -1; + result = -1; goto cleanup; } - status = (uint8_t)response[1]; - if (get_uint16(response+2)!=0) { /* port: 0 */ + if (socks4_server_reply_get_port(reply) != 0) { /* port: 0 */ log_warn(LD_PROTOCOL,"Nonzero port in socks response: bad format."); - return -1; + result = -1; goto cleanup; } + status = socks4_server_reply_get_status(reply); if (status != 90) { log_warn(LD_NET,"Got status response '%d': socks request failed.", status); if (!strcasecmpend(hostname, ".onion")) { onion_warning(hostname); - return -1; + result = -1; goto cleanup; } - return -1; + result = -1; goto cleanup; } - tor_addr_from_ipv4n(addr_out, get_uint32(response+4)); - return 0; + tor_addr_from_ipv4h(addr_out, socks4_server_reply_get_addr(reply)); + + cleanup: + socks4_server_reply_free(reply); + return result; } /* It would be nice to let someone know what SOCKS5 issue a user may have */ @@ -205,7 +358,7 @@ do_resolve(const char *hostname, int s = -1; struct sockaddr_storage ss; socklen_t socklen; - char *req = NULL; + uint8_t *req = NULL; ssize_t len = 0; tor_assert(hostname); @@ -230,23 +383,57 @@ do_resolve(const char *hostname, } if (version == 5) { - char method_buf[2]; - if (write_all_to_socket(s, "\x05\x01\x00", 3) != 3) { + socks5_client_version_t *v = socks5_client_version_new(); + + socks5_client_version_set_version(v, 5); + socks5_client_version_set_n_methods(v, 1); + socks5_client_version_setlen_methods(v, 1); + socks5_client_version_set_methods(v, 0, 0x00); + + tor_assert(!socks5_client_version_check(v)); + ssize_t encoded_len = socks5_client_version_encoded_len(v); + tor_assert(encoded_len > 0); + + uint8_t *buf = tor_malloc(encoded_len); + encoded_len = socks5_client_version_encode(buf, encoded_len, v); + tor_assert(encoded_len > 0); + + socks5_client_version_free(v); + + if (write_all_to_socket(s, (const char *)buf, + encoded_len) != encoded_len) { log_err(LD_NET, "Error sending SOCKS5 method list."); + tor_free(buf); + goto err; } - if (read_all_from_socket(s, method_buf, 2) != 2) { + + tor_free(buf); + + uint8_t method_buf[2]; + + if (read_all_from_socket(s, (char *)method_buf, 2) != 2) { log_err(LD_NET, "Error reading SOCKS5 methods."); goto err; } - if (method_buf[0] != '\x05') { - log_err(LD_NET, "Unrecognized socks version: %u", - (unsigned)method_buf[0]); + + socks5_server_method_t *m; + ssize_t parsed = socks5_server_method_parse(&m, method_buf, + sizeof(method_buf)); + + if (parsed < 2) { + log_err(LD_NET, "Failed to parse SOCKS5 method selection " + "message"); goto err; } - if (method_buf[1] != '\x00') { + + uint8_t method = socks5_server_method_get_method(m); + + socks5_server_method_free(m); + + if (method != 0x00) { log_err(LD_NET, "Unrecognized socks authentication method: %u", - (unsigned)method_buf[1]); + (unsigned)method); goto err; } } @@ -257,7 +444,7 @@ do_resolve(const char *hostname, tor_assert(!req); goto err; } - if (write_all_to_socket(s, req, len) != len) { + if (write_all_to_socket(s, (const char *)req, len) != len) { log_sock_error("sending SOCKS request", s); tor_free(req); goto err; @@ -276,55 +463,59 @@ do_resolve(const char *hostname, goto err; } } else { - char reply_buf[16]; - if (read_all_from_socket(s, reply_buf, 4) != 4) { - log_err(LD_NET, "Error reading SOCKS5 response."); + uint8_t reply_buf[512]; + + len = read_all_from_socket(s, (char *)reply_buf, + sizeof(reply_buf)); + + socks5_server_reply_t *reply; + + ssize_t parsed = socks5_server_reply_parse(&reply, + reply_buf, + len); + if (parsed == -1) { + log_err(LD_NET, "Failed parsing SOCKS5 response"); goto err; } - if (reply_buf[0] != 5) { - log_err(LD_NET, "Bad SOCKS5 reply version."); + + if (parsed == -2) { + log_err(LD_NET, "Truncated SOCKS5 response"); goto err; } + /* Give a user some useful feedback about SOCKS5 errors */ - if (reply_buf[1] != 0) { + uint8_t reply_field = socks5_server_reply_get_reply(reply); + if (reply_field != 0) { log_warn(LD_NET,"Got SOCKS5 status response '%u': %s", - (unsigned)reply_buf[1], - socks5_reason_to_string(reply_buf[1])); - if (reply_buf[1] == 4 && !strcasecmpend(hostname, ".onion")) { + (unsigned)reply_field, + socks5_reason_to_string(reply_field)); + if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) { onion_warning(hostname); } + + socks5_server_reply_free(reply); goto err; } - if (reply_buf[3] == 1) { + + uint8_t atype = socks5_server_reply_get_atype(reply); + + if (atype == SOCKS5_ATYPE_IPV4) { /* IPv4 address */ - if (read_all_from_socket(s, reply_buf, 4) != 4) { - log_err(LD_NET, "Error reading address in socks5 response."); - goto err; - } - tor_addr_from_ipv4n(result_addr, get_uint32(reply_buf)); - } else if (reply_buf[3] == 4) { + tor_addr_from_ipv4h(result_addr, + socks5_server_reply_get_bind_addr_ipv4(reply)); + } else if (atype == SOCKS5_ATYPE_IPV6) { /* IPv6 address */ - if (read_all_from_socket(s, reply_buf, 16) != 16) { - log_err(LD_NET, "Error reading address in socks5 response."); - goto err; - } - tor_addr_from_ipv6_bytes(result_addr, reply_buf); - } else if (reply_buf[3] == 3) { + tor_addr_from_ipv6_bytes(result_addr, + (const char *)socks5_server_reply_getarray_bind_addr_ipv6(reply)); + } else if (atype == SOCKS5_ATYPE_HOSTNAME) { /* Domain name */ - size_t result_len; - if (read_all_from_socket(s, reply_buf, 1) != 1) { - log_err(LD_NET, "Error reading address_length in socks5 response."); - goto err; - } - result_len = *(uint8_t*)(reply_buf); - *result_hostname = tor_malloc(result_len+1); - if (read_all_from_socket(s, *result_hostname, result_len) - != (int) result_len) { - log_err(LD_NET, "Error reading hostname in socks5 response."); - goto err; - } - (*result_hostname)[result_len] = '\0'; + domainname_t *dn = + socks5_server_reply_get_bind_addr_domainname(reply); + + *result_hostname = tor_strdup(domainname_getstr_name(dn)); } + + socks5_server_reply_free(reply); } tor_close_socket(s); diff --git a/src/trunnel/circpad_negotiation.c b/src/trunnel/circpad_negotiation.c new file mode 100644 index 0000000000..236be06ada --- /dev/null +++ b/src/trunnel/circpad_negotiation.c @@ -0,0 +1,549 @@ +/* circpad_negotiation.c -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include <stdlib.h> +#include "trunnel-impl.h" + +#include "circpad_negotiation.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int circpadnegotiation_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || circpadnegotiation_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +circpad_negotiate_t * +circpad_negotiate_new(void) +{ + circpad_negotiate_t *val = trunnel_calloc(1, sizeof(circpad_negotiate_t)); + if (NULL == val) + return NULL; + val->command = CIRCPAD_COMMAND_START; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +circpad_negotiate_clear(circpad_negotiate_t *obj) +{ + (void) obj; +} + +void +circpad_negotiate_free(circpad_negotiate_t *obj) +{ + if (obj == NULL) + return; + circpad_negotiate_clear(obj); + trunnel_memwipe(obj, sizeof(circpad_negotiate_t)); + trunnel_free_(obj); +} + +uint8_t +circpad_negotiate_get_version(const circpad_negotiate_t *inp) +{ + return inp->version; +} +int +circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val) +{ + if (! ((val == 0))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +uint8_t +circpad_negotiate_get_command(const circpad_negotiate_t *inp) +{ + return inp->command; +} +int +circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val) +{ + if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->command = val; + return 0; +} +uint8_t +circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp) +{ + return inp->machine_type; +} +int +circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val) +{ + inp->machine_type = val; + return 0; +} +uint8_t +circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp) +{ + return inp->echo_request; +} +int +circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val) +{ + if (! ((val == 0 || val == 1))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->echo_request = val; + return 0; +} +const char * +circpad_negotiate_check(const circpad_negotiate_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0)) + return "Integer out of bounds"; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + return "Integer out of bounds"; + if (! (obj->echo_request == 0 || obj->echo_request == 1)) + return "Integer out of bounds"; + return NULL; +} + +ssize_t +circpad_negotiate_encoded_len(const circpad_negotiate_t *obj) +{ + ssize_t result = 0; + + if (NULL != circpad_negotiate_check(obj)) + return -1; + + + /* Length of u8 version IN [0] */ + result += 1; + + /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + result += 1; + + /* Length of u8 machine_type */ + result += 1; + + /* Length of u8 echo_request IN [0, 1] */ + result += 1; + return result; +} +int +circpad_negotiate_clear_errors(circpad_negotiate_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +circpad_negotiate_encode(uint8_t *output, const size_t avail, const circpad_negotiate_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = circpad_negotiate_encoded_len(obj); +#endif + + if (NULL != (msg = circpad_negotiate_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->command)); + written += 1; ptr += 1; + + /* Encode u8 machine_type */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->machine_type)); + written += 1; ptr += 1; + + /* Encode u8 echo_request IN [0, 1] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->echo_request)); + written += 1; ptr += 1; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As circpad_negotiate_parse(), but do not allocate the output + * object. + */ +static ssize_t +circpad_negotiate_parse_into(circpad_negotiate_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0)) + goto fail; + + /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + CHECK_REMAINING(1, truncated); + obj->command = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + goto fail; + + /* Parse u8 machine_type */ + CHECK_REMAINING(1, truncated); + obj->machine_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse u8 echo_request IN [0, 1] */ + CHECK_REMAINING(1, truncated); + obj->echo_request = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->echo_request == 0 || obj->echo_request == 1)) + goto fail; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = circpad_negotiate_new(); + if (NULL == *output) + return -1; + result = circpad_negotiate_parse_into(*output, input, len_in); + if (result < 0) { + circpad_negotiate_free(*output); + *output = NULL; + } + return result; +} +circpad_negotiated_t * +circpad_negotiated_new(void) +{ + circpad_negotiated_t *val = trunnel_calloc(1, sizeof(circpad_negotiated_t)); + if (NULL == val) + return NULL; + val->command = CIRCPAD_COMMAND_START; + val->response = CIRCPAD_RESPONSE_ERR; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +circpad_negotiated_clear(circpad_negotiated_t *obj) +{ + (void) obj; +} + +void +circpad_negotiated_free(circpad_negotiated_t *obj) +{ + if (obj == NULL) + return; + circpad_negotiated_clear(obj); + trunnel_memwipe(obj, sizeof(circpad_negotiated_t)); + trunnel_free_(obj); +} + +uint8_t +circpad_negotiated_get_version(const circpad_negotiated_t *inp) +{ + return inp->version; +} +int +circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val) +{ + if (! ((val == 0))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->version = val; + return 0; +} +uint8_t +circpad_negotiated_get_command(const circpad_negotiated_t *inp) +{ + return inp->command; +} +int +circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val) +{ + if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->command = val; + return 0; +} +uint8_t +circpad_negotiated_get_response(const circpad_negotiated_t *inp) +{ + return inp->response; +} +int +circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val) +{ + if (! ((val == CIRCPAD_RESPONSE_ERR || val == CIRCPAD_RESPONSE_OK))) { + TRUNNEL_SET_ERROR_CODE(inp); + return -1; + } + inp->response = val; + return 0; +} +uint8_t +circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp) +{ + return inp->machine_type; +} +int +circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val) +{ + inp->machine_type = val; + return 0; +} +const char * +circpad_negotiated_check(const circpad_negotiated_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + if (! (obj->version == 0)) + return "Integer out of bounds"; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + return "Integer out of bounds"; + if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK)) + return "Integer out of bounds"; + return NULL; +} + +ssize_t +circpad_negotiated_encoded_len(const circpad_negotiated_t *obj) +{ + ssize_t result = 0; + + if (NULL != circpad_negotiated_check(obj)) + return -1; + + + /* Length of u8 version IN [0] */ + result += 1; + + /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + result += 1; + + /* Length of u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */ + result += 1; + + /* Length of u8 machine_type */ + result += 1; + return result; +} +int +circpad_negotiated_clear_errors(circpad_negotiated_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +circpad_negotiated_encode(uint8_t *output, const size_t avail, const circpad_negotiated_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = circpad_negotiated_encoded_len(obj); +#endif + + if (NULL != (msg = circpad_negotiated_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 version IN [0] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->version)); + written += 1; ptr += 1; + + /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->command)); + written += 1; ptr += 1; + + /* Encode u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->response)); + written += 1; ptr += 1; + + /* Encode u8 machine_type */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->machine_type)); + written += 1; ptr += 1; + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As circpad_negotiated_parse(), but do not allocate the output + * object. + */ +static ssize_t +circpad_negotiated_parse_into(circpad_negotiated_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 version IN [0] */ + CHECK_REMAINING(1, truncated); + obj->version = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->version == 0)) + goto fail; + + /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */ + CHECK_REMAINING(1, truncated); + obj->command = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP)) + goto fail; + + /* Parse u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */ + CHECK_REMAINING(1, truncated); + obj->response = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK)) + goto fail; + + /* Parse u8 machine_type */ + CHECK_REMAINING(1, truncated); + obj->machine_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = circpad_negotiated_new(); + if (NULL == *output) + return -1; + result = circpad_negotiated_parse_into(*output, input, len_in); + if (result < 0) { + circpad_negotiated_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/circpad_negotiation.h b/src/trunnel/circpad_negotiation.h new file mode 100644 index 0000000000..d09080dc16 --- /dev/null +++ b/src/trunnel/circpad_negotiation.h @@ -0,0 +1,195 @@ +/* circpad_negotiation.h -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_CIRCPAD_NEGOTIATION_H +#define TRUNNEL_CIRCPAD_NEGOTIATION_H + +#include <stdint.h> +#include "trunnel.h" + +#define CIRCPAD_COMMAND_STOP 1 +#define CIRCPAD_COMMAND_START 2 +#define CIRCPAD_RESPONSE_OK 1 +#define CIRCPAD_RESPONSE_ERR 2 +#define CIRCPAD_MACHINE_CIRC_SETUP 1 +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATE) +struct circpad_negotiate_st { + uint8_t version; + uint8_t command; + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + uint8_t machine_type; + /** If true, send a relay_drop reply.. */ + uint8_t echo_request; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct circpad_negotiate_st circpad_negotiate_t; +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATED) +struct circpad_negotiated_st { + uint8_t version; + uint8_t command; + uint8_t response; + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + uint8_t machine_type; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct circpad_negotiated_st circpad_negotiated_t; +/** Return a newly allocated circpad_negotiate with all elements set + * to zero. + */ +circpad_negotiate_t *circpad_negotiate_new(void); +/** Release all storage held by the circpad_negotiate in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void circpad_negotiate_free(circpad_negotiate_t *victim); +/** Try to parse a circpad_negotiate from the buffer in 'input', using + * up to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * circpad_negotiate_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * circpad_negotiate in 'obj'. On failure, return a negative value. + * Note that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t circpad_negotiate_encoded_len(const circpad_negotiate_t *obj); +/** Try to encode the circpad_negotiate from 'input' into the buffer + * at 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t circpad_negotiate_encode(uint8_t *output, size_t avail, const circpad_negotiate_t *input); +/** Check whether the internal state of the circpad_negotiate in 'obj' + * is consistent. Return NULL if it is, and a short message if it is + * not. + */ +const char *circpad_negotiate_check(const circpad_negotiate_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int circpad_negotiate_clear_errors(circpad_negotiate_t *obj); +/** Return the value of the version field of the circpad_negotiate_t + * in 'inp' + */ +uint8_t circpad_negotiate_get_version(const circpad_negotiate_t *inp); +/** Set the value of the version field of the circpad_negotiate_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val); +/** Return the value of the command field of the circpad_negotiate_t + * in 'inp' + */ +uint8_t circpad_negotiate_get_command(const circpad_negotiate_t *inp); +/** Set the value of the command field of the circpad_negotiate_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val); +/** Return the value of the machine_type field of the + * circpad_negotiate_t in 'inp' + */ +uint8_t circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp); +/** Set the value of the machine_type field of the circpad_negotiate_t + * in 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val); +/** Return the value of the echo_request field of the + * circpad_negotiate_t in 'inp' + */ +uint8_t circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp); +/** Set the value of the echo_request field of the circpad_negotiate_t + * in 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val); +/** Return a newly allocated circpad_negotiated with all elements set + * to zero. + */ +circpad_negotiated_t *circpad_negotiated_new(void); +/** Release all storage held by the circpad_negotiated in 'victim'. + * (Do nothing if 'victim' is NULL.) + */ +void circpad_negotiated_free(circpad_negotiated_t *victim); +/** Try to parse a circpad_negotiated from the buffer in 'input', + * using up to 'len_in' bytes from the input buffer. On success, + * return the number of bytes consumed and set *output to the newly + * allocated circpad_negotiated_t. On failure, return -2 if the input + * appears truncated, and -1 if the input is otherwise invalid. + */ +ssize_t circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * circpad_negotiated in 'obj'. On failure, return a negative value. + * Note that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t circpad_negotiated_encoded_len(const circpad_negotiated_t *obj); +/** Try to encode the circpad_negotiated from 'input' into the buffer + * at 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t circpad_negotiated_encode(uint8_t *output, size_t avail, const circpad_negotiated_t *input); +/** Check whether the internal state of the circpad_negotiated in + * 'obj' is consistent. Return NULL if it is, and a short message if + * it is not. + */ +const char *circpad_negotiated_check(const circpad_negotiated_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int circpad_negotiated_clear_errors(circpad_negotiated_t *obj); +/** Return the value of the version field of the circpad_negotiated_t + * in 'inp' + */ +uint8_t circpad_negotiated_get_version(const circpad_negotiated_t *inp); +/** Set the value of the version field of the circpad_negotiated_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val); +/** Return the value of the command field of the circpad_negotiated_t + * in 'inp' + */ +uint8_t circpad_negotiated_get_command(const circpad_negotiated_t *inp); +/** Set the value of the command field of the circpad_negotiated_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val); +/** Return the value of the response field of the circpad_negotiated_t + * in 'inp' + */ +uint8_t circpad_negotiated_get_response(const circpad_negotiated_t *inp); +/** Set the value of the response field of the circpad_negotiated_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val); +/** Return the value of the machine_type field of the + * circpad_negotiated_t in 'inp' + */ +uint8_t circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp); +/** Set the value of the machine_type field of the + * circpad_negotiated_t in 'inp' to 'val'. Return 0 on success; return + * -1 and set the error code on 'inp' on failure. + */ +int circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val); + + +#endif diff --git a/src/trunnel/circpad_negotiation.trunnel b/src/trunnel/circpad_negotiation.trunnel new file mode 100644 index 0000000000..abbc929cc5 --- /dev/null +++ b/src/trunnel/circpad_negotiation.trunnel @@ -0,0 +1,44 @@ +/* These are the padding negotiation commands */ +const CIRCPAD_COMMAND_STOP = 1; +const CIRCPAD_COMMAND_START = 2; + +/* Responses to commands */ +const CIRCPAD_RESPONSE_OK = 1; +const CIRCPAD_RESPONSE_ERR = 2; + +/* Built-in machine types */ + +/* 1) Machine that obscures circuit setup */ +const CIRCPAD_MACHINE_CIRC_SETUP = 1; + +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +struct circpad_negotiate { + u8 version IN [0]; + u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP]; + + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + u8 machine_type; + + /** If true, send a relay_drop reply.. */ + // FIXME-MP-AP: Maybe we just say to transition to the first state + // here instead.. Also what about delay before responding? + u8 echo_request IN [0,1]; +}; + +/** + * This command tells the relay to alter its min and max netflow + * timeout range values, and send padding at that rate (resuming + * if stopped). */ +struct circpad_negotiated { + u8 version IN [0]; + u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP]; + u8 response IN [CIRCPAD_RESPONSE_OK, CIRCPAD_RESPONSE_ERR]; + + /** Machine type is left unbounded because we can specify + * new machines in the consensus */ + u8 machine_type; +}; diff --git a/src/trunnel/include.am b/src/trunnel/include.am index 03c1753e96..4f4f1d3624 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -11,7 +11,8 @@ TRUNNELINPUTS = \ src/trunnel/link_handshake.trunnel \ src/trunnel/pwbox.trunnel \ src/trunnel/channelpadding_negotiation.trunnel \ - src/trunner/socks5.trunnel + src/trunnel/socks5.trunnel \ + src/trunnel/circpad_negotiation.trunnel TRUNNELSOURCES = \ src/ext/trunnel/trunnel.c \ @@ -23,7 +24,9 @@ TRUNNELSOURCES = \ src/trunnel/hs/cell_introduce1.c \ src/trunnel/hs/cell_rendezvous.c \ src/trunnel/channelpadding_negotiation.c \ - src/trunnel/socks5.c + src/trunnel/socks5.c \ + src/trunnel/netinfo.c \ + src/trunnel/circpad_negotiation.c TRUNNELHEADERS = \ src/ext/trunnel/trunnel.h \ @@ -37,7 +40,9 @@ TRUNNELHEADERS = \ src/trunnel/hs/cell_introduce1.h \ src/trunnel/hs/cell_rendezvous.h \ src/trunnel/channelpadding_negotiation.h \ - src/trunnel/socks5.h + src/trunnel/socks5.h \ + src/trunnel/netinfo.h \ + src/trunnel/circpad_negotiation.h src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES) src_trunnel_libor_trunnel_a_CPPFLAGS = \ diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c new file mode 100644 index 0000000000..5d815b9b12 --- /dev/null +++ b/src/trunnel/netinfo.c @@ -0,0 +1,723 @@ +/* netinfo.c -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#include <stdlib.h> +#include "trunnel-impl.h" + +#include "netinfo.h" + +#define TRUNNEL_SET_ERROR_CODE(obj) \ + do { \ + (obj)->trunnel_error_code_ = 1; \ + } while (0) + +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* If we're running a static analysis tool, we don't want it to complain + * that some of our remaining-bytes checks are dead-code. */ +int netinfo_deadcode_dummy__ = 0; +#define OR_DEADCODE_DUMMY || netinfo_deadcode_dummy__ +#else +#define OR_DEADCODE_DUMMY +#endif + +#define CHECK_REMAINING(nbytes, label) \ + do { \ + if (remaining < (nbytes) OR_DEADCODE_DUMMY) { \ + goto label; \ + } \ + } while (0) + +netinfo_addr_t * +netinfo_addr_new(void) +{ + netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +netinfo_addr_clear(netinfo_addr_t *obj) +{ + (void) obj; +} + +void +netinfo_addr_free(netinfo_addr_t *obj) +{ + if (obj == NULL) + return; + netinfo_addr_clear(obj); + trunnel_memwipe(obj, sizeof(netinfo_addr_t)); + trunnel_free_(obj); +} + +uint8_t +netinfo_addr_get_addr_type(const netinfo_addr_t *inp) +{ + return inp->addr_type; +} +int +netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val) +{ + inp->addr_type = val; + return 0; +} +uint8_t +netinfo_addr_get_len(const netinfo_addr_t *inp) +{ + return inp->len; +} +int +netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val) +{ + inp->len = val; + return 0; +} +uint32_t +netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp) +{ + return inp->addr_ipv4; +} +int +netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val) +{ + inp->addr_ipv4 = val; + return 0; +} +size_t +netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp) +{ + (void)inp; return 16; +} + +uint8_t +netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx) +{ + trunnel_assert(idx < 16); + return inp->addr_ipv6[idx]; +} + +uint8_t +netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx) +{ + return netinfo_addr_get_addr_ipv6((netinfo_addr_t*)inp, idx); +} +int +netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt) +{ + trunnel_assert(idx < 16); + inp->addr_ipv6[idx] = elt; + return 0; +} + +uint8_t * +netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp) +{ + return inp->addr_ipv6; +} +const uint8_t * +netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp) +{ + return (const uint8_t *)netinfo_addr_getarray_addr_ipv6((netinfo_addr_t*)inp); +} +const char * +netinfo_addr_check(const netinfo_addr_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + break; + + case NETINFO_ADDR_TYPE_IPV6: + break; + + default: + break; + } + return NULL; +} + +ssize_t +netinfo_addr_encoded_len(const netinfo_addr_t *obj) +{ + ssize_t result = 0; + + if (NULL != netinfo_addr_check(obj)) + return -1; + + + /* Length of u8 addr_type */ + result += 1; + + /* Length of u8 len */ + result += 1; + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Length of u32 addr_ipv4 */ + result += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Length of u8 addr_ipv6[16] */ + result += 16; + break; + + default: + break; + } + return result; +} +int +netinfo_addr_clear_errors(netinfo_addr_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = netinfo_addr_encoded_len(obj); +#endif + + uint8_t *backptr_len = NULL; + + if (NULL != (msg = netinfo_addr_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u8 addr_type */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->addr_type)); + written += 1; ptr += 1; + + /* Encode u8 len */ + backptr_len = ptr; + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->len)); + written += 1; ptr += 1; + { + size_t written_before_union = written; + + /* Encode union addr[addr_type] */ + trunnel_assert(written <= avail); + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Encode u32 addr_ipv4 */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4)); + written += 4; ptr += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Encode u8 addr_ipv6[16] */ + trunnel_assert(written <= avail); + if (avail - written < 16) + goto truncated; + memcpy(ptr, obj->addr_ipv6, 16); + written += 16; ptr += 16; + break; + + default: + break; + } + /* Write the length field back to len */ + trunnel_assert(written >= written_before_union); +#if UINT8_MAX < SIZE_MAX + if (written - written_before_union > UINT8_MAX) + goto check_failed; +#endif + trunnel_set_uint8(backptr_len, (written - written_before_union)); + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As netinfo_addr_parse(), but do not allocate the output object. + */ +static ssize_t +netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u8 addr_type */ + CHECK_REMAINING(1, truncated); + obj->addr_type = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse u8 len */ + CHECK_REMAINING(1, truncated); + obj->len = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + { + size_t remaining_after; + CHECK_REMAINING(obj->len, truncated); + remaining_after = remaining - obj->len; + remaining = obj->len; + + /* Parse union addr[addr_type] */ + switch (obj->addr_type) { + + case NETINFO_ADDR_TYPE_IPV4: + + /* Parse u32 addr_ipv4 */ + CHECK_REMAINING(4, fail); + obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + break; + + case NETINFO_ADDR_TYPE_IPV6: + + /* Parse u8 addr_ipv6[16] */ + CHECK_REMAINING(16, fail); + memcpy(obj->addr_ipv6, ptr, 16); + remaining -= 16; ptr += 16; + break; + + default: + /* Skip to end of union */ + ptr += remaining; remaining = 0; + break; + } + if (remaining != 0) + goto fail; + remaining = remaining_after; + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + fail: + result = -1; + return result; +} + +ssize_t +netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = netinfo_addr_new(); + if (NULL == *output) + return -1; + result = netinfo_addr_parse_into(*output, input, len_in); + if (result < 0) { + netinfo_addr_free(*output); + *output = NULL; + } + return result; +} +netinfo_cell_t * +netinfo_cell_new(void) +{ + netinfo_cell_t *val = trunnel_calloc(1, sizeof(netinfo_cell_t)); + if (NULL == val) + return NULL; + return val; +} + +/** Release all storage held inside 'obj', but do not free 'obj'. + */ +static void +netinfo_cell_clear(netinfo_cell_t *obj) +{ + (void) obj; + netinfo_addr_free(obj->other_addr); + obj->other_addr = NULL; + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + netinfo_addr_free(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + } + } + TRUNNEL_DYNARRAY_WIPE(&obj->my_addrs); + TRUNNEL_DYNARRAY_CLEAR(&obj->my_addrs); +} + +void +netinfo_cell_free(netinfo_cell_t *obj) +{ + if (obj == NULL) + return; + netinfo_cell_clear(obj); + trunnel_memwipe(obj, sizeof(netinfo_cell_t)); + trunnel_free_(obj); +} + +uint32_t +netinfo_cell_get_timestamp(const netinfo_cell_t *inp) +{ + return inp->timestamp; +} +int +netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val) +{ + inp->timestamp = val; + return 0; +} +struct netinfo_addr_st * +netinfo_cell_get_other_addr(netinfo_cell_t *inp) +{ + return inp->other_addr; +} +const struct netinfo_addr_st * +netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp) +{ + return netinfo_cell_get_other_addr((netinfo_cell_t*) inp); +} +int +netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val) +{ + if (inp->other_addr && inp->other_addr != val) + netinfo_addr_free(inp->other_addr); + return netinfo_cell_set0_other_addr(inp, val); +} +int +netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val) +{ + inp->other_addr = val; + return 0; +} +uint8_t +netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp) +{ + return inp->n_my_addrs; +} +int +netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val) +{ + inp->n_my_addrs = val; + return 0; +} +size_t +netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp) +{ + return TRUNNEL_DYNARRAY_LEN(&inp->my_addrs); +} + +struct netinfo_addr_st * +netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx) +{ + return TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx); +} + + const struct netinfo_addr_st * +netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx) +{ + return netinfo_cell_get_my_addrs((netinfo_cell_t*)inp, idx); +} +int +netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt) +{ + netinfo_addr_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx); + if (oldval && oldval != elt) + netinfo_addr_free(oldval); + return netinfo_cell_set0_my_addrs(inp, idx, elt); +} +int +netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt) +{ + TRUNNEL_DYNARRAY_SET(&inp->my_addrs, idx, elt); + return 0; +} +int +netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt) +{ +#if SIZE_MAX >= UINT8_MAX + if (inp->my_addrs.n_ == UINT8_MAX) + goto trunnel_alloc_failed; +#endif + TRUNNEL_DYNARRAY_ADD(struct netinfo_addr_st *, &inp->my_addrs, elt, {}); + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} + +struct netinfo_addr_st * * +netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp) +{ + return inp->my_addrs.elts_; +} +const struct netinfo_addr_st * const * +netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp) +{ + return (const struct netinfo_addr_st * const *)netinfo_cell_getarray_my_addrs((netinfo_cell_t*)inp); +} +int +netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen) +{ + struct netinfo_addr_st * *newptr; +#if UINT8_MAX < SIZE_MAX + if (newlen > UINT8_MAX) + goto trunnel_alloc_failed; +#endif + newptr = trunnel_dynarray_setlen(&inp->my_addrs.allocated_, + &inp->my_addrs.n_, inp->my_addrs.elts_, newlen, + sizeof(inp->my_addrs.elts_[0]), (trunnel_free_fn_t) netinfo_addr_free, + &inp->trunnel_error_code_); + if (newlen != 0 && newptr == NULL) + goto trunnel_alloc_failed; + inp->my_addrs.elts_ = newptr; + return 0; + trunnel_alloc_failed: + TRUNNEL_SET_ERROR_CODE(inp); + return -1; +} +const char * +netinfo_cell_check(const netinfo_cell_t *obj) +{ + if (obj == NULL) + return "Object was NULL"; + if (obj->trunnel_error_code_) + return "A set function failed on this object"; + { + const char *msg; + if (NULL != (msg = netinfo_addr_check(obj->other_addr))) + return msg; + } + { + const char *msg; + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + if (NULL != (msg = netinfo_addr_check(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)))) + return msg; + } + } + if (TRUNNEL_DYNARRAY_LEN(&obj->my_addrs) != obj->n_my_addrs) + return "Length mismatch for my_addrs"; + return NULL; +} + +ssize_t +netinfo_cell_encoded_len(const netinfo_cell_t *obj) +{ + ssize_t result = 0; + + if (NULL != netinfo_cell_check(obj)) + return -1; + + + /* Length of u32 timestamp */ + result += 4; + + /* Length of struct netinfo_addr other_addr */ + result += netinfo_addr_encoded_len(obj->other_addr); + + /* Length of u8 n_my_addrs */ + result += 1; + + /* Length of struct netinfo_addr my_addrs[n_my_addrs] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + result += netinfo_addr_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + } + } + return result; +} +int +netinfo_cell_clear_errors(netinfo_cell_t *obj) +{ + int r = obj->trunnel_error_code_; + obj->trunnel_error_code_ = 0; + return r; +} +ssize_t +netinfo_cell_encode(uint8_t *output, const size_t avail, const netinfo_cell_t *obj) +{ + ssize_t result = 0; + size_t written = 0; + uint8_t *ptr = output; + const char *msg; +#ifdef TRUNNEL_CHECK_ENCODED_LEN + const ssize_t encoded_len = netinfo_cell_encoded_len(obj); +#endif + + if (NULL != (msg = netinfo_cell_check(obj))) + goto check_failed; + +#ifdef TRUNNEL_CHECK_ENCODED_LEN + trunnel_assert(encoded_len >= 0); +#endif + + /* Encode u32 timestamp */ + trunnel_assert(written <= avail); + if (avail - written < 4) + goto truncated; + trunnel_set_uint32(ptr, trunnel_htonl(obj->timestamp)); + written += 4; ptr += 4; + + /* Encode struct netinfo_addr other_addr */ + trunnel_assert(written <= avail); + result = netinfo_addr_encode(ptr, avail - written, obj->other_addr); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + + /* Encode u8 n_my_addrs */ + trunnel_assert(written <= avail); + if (avail - written < 1) + goto truncated; + trunnel_set_uint8(ptr, (obj->n_my_addrs)); + written += 1; ptr += 1; + + /* Encode struct netinfo_addr my_addrs[n_my_addrs] */ + { + + unsigned idx; + for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) { + trunnel_assert(written <= avail); + result = netinfo_addr_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx)); + if (result < 0) + goto fail; /* XXXXXXX !*/ + written += result; ptr += result; + } + } + + + trunnel_assert(ptr == output + written); +#ifdef TRUNNEL_CHECK_ENCODED_LEN + { + trunnel_assert(encoded_len >= 0); + trunnel_assert((size_t)encoded_len == written); + } + +#endif + + return written; + + truncated: + result = -2; + goto fail; + check_failed: + (void)msg; + result = -1; + goto fail; + fail: + trunnel_assert(result < 0); + return result; +} + +/** As netinfo_cell_parse(), but do not allocate the output object. + */ +static ssize_t +netinfo_cell_parse_into(netinfo_cell_t *obj, const uint8_t *input, const size_t len_in) +{ + const uint8_t *ptr = input; + size_t remaining = len_in; + ssize_t result = 0; + (void)result; + + /* Parse u32 timestamp */ + CHECK_REMAINING(4, truncated); + obj->timestamp = trunnel_ntohl(trunnel_get_uint32(ptr)); + remaining -= 4; ptr += 4; + + /* Parse struct netinfo_addr other_addr */ + result = netinfo_addr_parse(&obj->other_addr, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + + /* Parse u8 n_my_addrs */ + CHECK_REMAINING(1, truncated); + obj->n_my_addrs = (trunnel_get_uint8(ptr)); + remaining -= 1; ptr += 1; + + /* Parse struct netinfo_addr my_addrs[n_my_addrs] */ + TRUNNEL_DYNARRAY_EXPAND(netinfo_addr_t *, &obj->my_addrs, obj->n_my_addrs, {}); + { + netinfo_addr_t * elt; + unsigned idx; + for (idx = 0; idx < obj->n_my_addrs; ++idx) { + result = netinfo_addr_parse(&elt, ptr, remaining); + if (result < 0) + goto relay_fail; + trunnel_assert((size_t)result <= remaining); + remaining -= result; ptr += result; + TRUNNEL_DYNARRAY_ADD(netinfo_addr_t *, &obj->my_addrs, elt, {netinfo_addr_free(elt);}); + } + } + trunnel_assert(ptr + remaining == input + len_in); + return len_in - remaining; + + truncated: + return -2; + relay_fail: + trunnel_assert(result < 0); + return result; + trunnel_alloc_failed: + return -1; +} + +ssize_t +netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in) +{ + ssize_t result; + *output = netinfo_cell_new(); + if (NULL == *output) + return -1; + result = netinfo_cell_parse_into(*output, input, len_in); + if (result < 0) { + netinfo_cell_free(*output); + *output = NULL; + } + return result; +} diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h new file mode 100644 index 0000000000..ac46e603ba --- /dev/null +++ b/src/trunnel/netinfo.h @@ -0,0 +1,226 @@ +/* netinfo.h -- generated by Trunnel v1.5.2. + * https://gitweb.torproject.org/trunnel.git + * You probably shouldn't edit this file. + */ +#ifndef TRUNNEL_NETINFO_H +#define TRUNNEL_NETINFO_H + +#include <stdint.h> +#include "trunnel.h" + +#define NETINFO_ADDR_TYPE_IPV4 4 +#define NETINFO_ADDR_TYPE_IPV6 6 +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_ADDR) +struct netinfo_addr_st { + uint8_t addr_type; + uint8_t len; + uint32_t addr_ipv4; + uint8_t addr_ipv6[16]; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct netinfo_addr_st netinfo_addr_t; +#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_CELL) +struct netinfo_cell_st { + uint32_t timestamp; + struct netinfo_addr_st *other_addr; + uint8_t n_my_addrs; + TRUNNEL_DYNARRAY_HEAD(, struct netinfo_addr_st *) my_addrs; + uint8_t trunnel_error_code_; +}; +#endif +typedef struct netinfo_cell_st netinfo_cell_t; +/** Return a newly allocated netinfo_addr with all elements set to + * zero. + */ +netinfo_addr_t *netinfo_addr_new(void); +/** Release all storage held by the netinfo_addr in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void netinfo_addr_free(netinfo_addr_t *victim); +/** Try to parse a netinfo_addr from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * netinfo_addr_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * netinfo_addr in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t netinfo_addr_encoded_len(const netinfo_addr_t *obj); +/** Try to encode the netinfo_addr from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t netinfo_addr_encode(uint8_t *output, size_t avail, const netinfo_addr_t *input); +/** Check whether the internal state of the netinfo_addr in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *netinfo_addr_check(const netinfo_addr_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int netinfo_addr_clear_errors(netinfo_addr_t *obj); +/** Return the value of the addr_type field of the netinfo_addr_t in + * 'inp' + */ +uint8_t netinfo_addr_get_addr_type(const netinfo_addr_t *inp); +/** Set the value of the addr_type field of the netinfo_addr_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val); +/** Return the value of the len field of the netinfo_addr_t in 'inp' + */ +uint8_t netinfo_addr_get_len(const netinfo_addr_t *inp); +/** Set the value of the len field of the netinfo_addr_t in 'inp' to + * 'val'. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val); +/** Return the value of the addr_ipv4 field of the netinfo_addr_t in + * 'inp' + */ +uint32_t netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp); +/** Set the value of the addr_ipv4 field of the netinfo_addr_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val); +/** Return the (constant) length of the array holding the addr_ipv6 + * field of the netinfo_addr_t in 'inp'. + */ +size_t netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp); +/** Return the element at position 'idx' of the fixed array field + * addr_ipv6 of the netinfo_addr_t in 'inp'. + */ +uint8_t netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx); +/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer + */ +uint8_t netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx); +/** Change the element at position 'idx' of the fixed array field + * addr_ipv6 of the netinfo_addr_t in 'inp', so that it will hold the + * value 'elt'. + */ +int netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt); +/** Return a pointer to the 16-element array field addr_ipv6 of 'inp'. + */ +uint8_t * netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp); +/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer + */ +const uint8_t * netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp); +/** Return a newly allocated netinfo_cell with all elements set to + * zero. + */ +netinfo_cell_t *netinfo_cell_new(void); +/** Release all storage held by the netinfo_cell in 'victim'. (Do + * nothing if 'victim' is NULL.) + */ +void netinfo_cell_free(netinfo_cell_t *victim); +/** Try to parse a netinfo_cell from the buffer in 'input', using up + * to 'len_in' bytes from the input buffer. On success, return the + * number of bytes consumed and set *output to the newly allocated + * netinfo_cell_t. On failure, return -2 if the input appears + * truncated, and -1 if the input is otherwise invalid. + */ +ssize_t netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in); +/** Return the number of bytes we expect to need to encode the + * netinfo_cell in 'obj'. On failure, return a negative value. Note + * that this value may be an overestimate, and can even be an + * underestimate for certain unencodeable objects. + */ +ssize_t netinfo_cell_encoded_len(const netinfo_cell_t *obj); +/** Try to encode the netinfo_cell from 'input' into the buffer at + * 'output', using up to 'avail' bytes of the output buffer. On + * success, return the number of bytes used. On failure, return -2 if + * the buffer was not long enough, and -1 if the input was invalid. + */ +ssize_t netinfo_cell_encode(uint8_t *output, size_t avail, const netinfo_cell_t *input); +/** Check whether the internal state of the netinfo_cell in 'obj' is + * consistent. Return NULL if it is, and a short message if it is not. + */ +const char *netinfo_cell_check(const netinfo_cell_t *obj); +/** Clear any errors that were set on the object 'obj' by its setter + * functions. Return true iff errors were cleared. + */ +int netinfo_cell_clear_errors(netinfo_cell_t *obj); +/** Return the value of the timestamp field of the netinfo_cell_t in + * 'inp' + */ +uint32_t netinfo_cell_get_timestamp(const netinfo_cell_t *inp); +/** Set the value of the timestamp field of the netinfo_cell_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val); +/** Return the value of the other_addr field of the netinfo_cell_t in + * 'inp' + */ +struct netinfo_addr_st * netinfo_cell_get_other_addr(netinfo_cell_t *inp); +/** As netinfo_cell_get_other_addr, but take and return a const + * pointer + */ +const struct netinfo_addr_st * netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp); +/** Set the value of the other_addr field of the netinfo_cell_t in + * 'inp' to 'val'. Free the old value if any. Steals the referenceto + * 'val'.Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val); +/** As netinfo_cell_set_other_addr, but does not free the previous + * value. + */ +int netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val); +/** Return the value of the n_my_addrs field of the netinfo_cell_t in + * 'inp' + */ +uint8_t netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp); +/** Set the value of the n_my_addrs field of the netinfo_cell_t in + * 'inp' to 'val'. Return 0 on success; return -1 and set the error + * code on 'inp' on failure. + */ +int netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val); +/** Return the length of the dynamic array holding the my_addrs field + * of the netinfo_cell_t in 'inp'. + */ +size_t netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp); +/** Return the element at position 'idx' of the dynamic array field + * my_addrs of the netinfo_cell_t in 'inp'. + */ +struct netinfo_addr_st * netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx); +/** As netinfo_cell_get_my_addrs, but take and return a const pointer + */ + const struct netinfo_addr_st * netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx); +/** Change the element at position 'idx' of the dynamic array field + * my_addrs of the netinfo_cell_t in 'inp', so that it will hold the + * value 'elt'. Free the previous value, if any. + */ +int netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt); +/** As netinfo_cell_set_my_addrs, but does not free the previous + * value. + */ +int netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt); +/** Append a new element 'elt' to the dynamic array field my_addrs of + * the netinfo_cell_t in 'inp'. + */ +int netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt); +/** Return a pointer to the variable-length array field my_addrs of + * 'inp'. + */ +struct netinfo_addr_st * * netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp); +/** As netinfo_cell_get_my_addrs, but take and return a const pointer + */ +const struct netinfo_addr_st * const * netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp); +/** Change the length of the variable-length array field my_addrs of + * 'inp' to 'newlen'.Fill extra elements with NULL; free removed + * elements. Return 0 on success; return -1 and set the error code on + * 'inp' on failure. + */ +int netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen); + + +#endif diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel new file mode 100644 index 0000000000..2c4b7a7591 --- /dev/null +++ b/src/trunnel/netinfo.trunnel @@ -0,0 +1,24 @@ +// Warning: make sure these values are consistent with RESOLVED_TYPE_* +// constants in Tor code and numbers in Section 6.4 of tor-spec.txt. + +const NETINFO_ADDR_TYPE_IPV4 = 4; +const NETINFO_ADDR_TYPE_IPV6 = 6; + +struct netinfo_addr { + u8 addr_type; + u8 len; + union addr[addr_type] with length len { + NETINFO_ADDR_TYPE_IPV4: u32 ipv4; + NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16]; + default: ignore; + }; + +} + +struct netinfo_cell { + u32 timestamp; + struct netinfo_addr other_addr; + u8 n_my_addrs; + struct netinfo_addr my_addrs[n_my_addrs]; +} + diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c index 9e5f6fcfed..057a52b042 100644 --- a/src/trunnel/socks5.c +++ b/src/trunnel/socks5.c @@ -1694,7 +1694,6 @@ socks4_server_reply_new(void) socks4_server_reply_t *val = trunnel_calloc(1, sizeof(socks4_server_reply_t)); if (NULL == val) return NULL; - val->version = 4; return val; } @@ -1724,7 +1723,7 @@ socks4_server_reply_get_version(const socks4_server_reply_t *inp) int socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val) { - if (! ((val == 4))) { + if (! ((val == 0 || val == 4))) { TRUNNEL_SET_ERROR_CODE(inp); return -1; } @@ -1771,7 +1770,7 @@ socks4_server_reply_check(const socks4_server_reply_t *obj) return "Object was NULL"; if (obj->trunnel_error_code_) return "A set function failed on this object"; - if (! (obj->version == 4)) + if (! (obj->version == 0 || obj->version == 4)) return "Integer out of bounds"; return NULL; } @@ -1785,7 +1784,7 @@ socks4_server_reply_encoded_len(const socks4_server_reply_t *obj) return -1; - /* Length of u8 version IN [4] */ + /* Length of u8 version IN [0, 4] */ result += 1; /* Length of u8 status */ @@ -1823,7 +1822,7 @@ socks4_server_reply_encode(uint8_t *output, const size_t avail, const socks4_ser trunnel_assert(encoded_len >= 0); #endif - /* Encode u8 version IN [4] */ + /* Encode u8 version IN [0, 4] */ trunnel_assert(written <= avail); if (avail - written < 1) goto truncated; @@ -1886,11 +1885,11 @@ socks4_server_reply_parse_into(socks4_server_reply_t *obj, const uint8_t *input, ssize_t result = 0; (void)result; - /* Parse u8 version IN [4] */ + /* Parse u8 version IN [0, 4] */ CHECK_REMAINING(1, truncated); obj->version = (trunnel_get_uint8(ptr)); remaining -= 1; ptr += 1; - if (! (obj->version == 4)) + if (! (obj->version == 0 || obj->version == 4)) goto fail; /* Parse u8 status */ diff --git a/src/trunnel/socks5.trunnel b/src/trunnel/socks5.trunnel index b86ec03b9d..17fa2ed996 100644 --- a/src/trunnel/socks5.trunnel +++ b/src/trunnel/socks5.trunnel @@ -86,7 +86,7 @@ struct socks4_client_request { } struct socks4_server_reply { - u8 version IN [4]; + u8 version IN [0,4]; u8 status; u16 port; u32 addr; diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index baad733d0f..bf746a19ac 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.3.5.9-dev" +#define VERSION "0.4.0.6-dev" |