diff options
58 files changed, 5033 insertions, 161 deletions
diff --git a/.gitignore b/.gitignore index a40cda02dd..0865f981b1 100644 --- a/.gitignore +++ b/.gitignore @@ -168,6 +168,8 @@ uptime-*.json /src/lib/libtor-crypt-ops-testing.a /src/lib/libtor-ctime.a /src/lib/libtor-ctime-testing.a +/src/lib/libtor-dispatch.a +/src/lib/libtor-dispatch-testing.a /src/lib/libtor-encoding.a /src/lib/libtor-encoding-testing.a /src/lib/libtor-evloop.a @@ -200,6 +202,8 @@ uptime-*.json /src/lib/libtor-osinfo-testing.a /src/lib/libtor-process.a /src/lib/libtor-process-testing.a +/src/lib/libtor-pubsub.a +/src/lib/libtor-pubsub-testing.a /src/lib/libtor-sandbox.a /src/lib/libtor-sandbox-testing.a /src/lib/libtor-string.a diff --git a/Makefile.am b/Makefile.am index eea7c3a12f..673643cd31 100644 --- a/Makefile.am +++ b/Makefile.am @@ -41,6 +41,8 @@ TOR_UTIL_LIBS = \ src/lib/libtor-geoip.a \ src/lib/libtor-process.a \ src/lib/libtor-buf.a \ + src/lib/libtor-pubsub.a \ + src/lib/libtor-dispatch.a \ src/lib/libtor-time.a \ src/lib/libtor-fs.a \ src/lib/libtor-encoding.a \ @@ -72,6 +74,8 @@ TOR_UTIL_TESTING_LIBS = \ src/lib/libtor-geoip-testing.a \ src/lib/libtor-process-testing.a \ src/lib/libtor-buf-testing.a \ + src/lib/libtor-pubsub-testing.a \ + src/lib/libtor-dispatch-testing.a \ src/lib/libtor-time-testing.a \ src/lib/libtor-fs-testing.a \ src/lib/libtor-encoding-testing.a \ diff --git a/changes/pubsub b/changes/pubsub new file mode 100644 index 0000000000..f67b36b988 --- /dev/null +++ b/changes/pubsub @@ -0,0 +1,5 @@ + o Major features (code organization): + - Tor now includes a generic publish-subscribe message-passing subsystem + that we can use to organize intermodule dependencies. We hope to use + this to reduce dependencies between modules that don't need to be + related, and to generally simplify our codebase. Closes ticket 28226. diff --git a/doc/tor.1.txt b/doc/tor.1.txt index ac79ed231b..889b397e51 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -679,7 +679,7 @@ GENERAL OPTIONS The currently recognized domains are: general, crypto, net, config, fs, protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge, acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, - process, pt, and btrack. + process, pt, btrack, and mesg. Domain names are case-insensitive. + + For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends diff --git a/src/app/main/main.c b/src/app/main/main.c index 83eef0dc70..184b9e91da 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -15,68 +15,51 @@ #include "app/config/statefile.h" #include "app/main/main.h" #include "app/main/ntmain.h" +#include "app/main/shutdown.h" #include "app/main/subsysmgr.h" #include "core/mainloop/connection.h" #include "core/mainloop/cpuworker.h" #include "core/mainloop/mainloop.h" +#include "core/mainloop/mainloop_pubsub.h" #include "core/mainloop/netstatus.h" #include "core/or/channel.h" #include "core/or/channelpadding.h" #include "core/or/circuitpadding.h" -#include "core/or/channeltls.h" #include "core/or/circuitlist.h" -#include "core/or/circuitmux_ewma.h" #include "core/or/command.h" -#include "core/or/connection_edge.h" #include "core/or/connection_or.h" -#include "core/or/dos.h" -#include "core/or/policies.h" -#include "core/or/protover.h" #include "core/or/relay.h" -#include "core/or/scheduler.h" #include "core/or/status.h" -#include "core/or/versions.h" #include "feature/api/tor_api.h" #include "feature/api/tor_api_internal.h" #include "feature/client/addressmap.h" -#include "feature/client/bridges.h" -#include "feature/client/entrynodes.h" -#include "feature/client/transports.h" #include "feature/control/control.h" #include "feature/control/control_auth.h" #include "feature/control/control_events.h" -#include "feature/dirauth/bwauth.h" #include "feature/dirauth/keypin.h" #include "feature/dirauth/process_descs.h" #include "feature/dircache/consdiffmgr.h" -#include "feature/dircache/dirserv.h" #include "feature/dirparse/routerparse.h" #include "feature/hibernate/hibernate.h" -#include "feature/hs/hs_cache.h" #include "feature/nodelist/authcert.h" -#include "feature/nodelist/microdesc.h" #include "feature/nodelist/networkstatus.h" -#include "feature/nodelist/nodelist.h" #include "feature/nodelist/routerlist.h" #include "feature/relay/dns.h" #include "feature/relay/ext_orport.h" -#include "feature/relay/onion_queue.h" #include "feature/relay/routerkeys.h" #include "feature/relay/routermode.h" #include "feature/rend/rendcache.h" -#include "feature/rend/rendclient.h" #include "feature/rend/rendservice.h" -#include "feature/stats/geoip_stats.h" #include "feature/stats/predict_ports.h" #include "feature/stats/rephist.h" #include "lib/compress/compress.h" #include "lib/buf/buffers.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/crypt_ops/crypto_s2k.h" -#include "lib/geoip/geoip.h" #include "lib/net/resolve.h" #include "lib/process/waitpid.h" +#include "lib/pubsub/pubsub_build.h" #include "lib/meminfo/meminfo.h" #include "lib/osinfo/uname.h" @@ -92,7 +75,6 @@ #include <event2/event.h> -#include "feature/dirauth/dirvote.h" #include "feature/dirauth/authmode.h" #include "feature/dirauth/shared_random.h" @@ -113,8 +95,6 @@ #include <systemd/sd-daemon.h> #endif /* defined(HAVE_SYSTEMD) */ -void evdns_shutdown(int); - #ifdef HAVE_RUST // helper function defined in Rust to output a log message indicating if tor is // running with Rust enabled. See src/rust/tor_util @@ -744,86 +724,6 @@ release_lockfile(void) } } -/** Free all memory that we might have allocated somewhere. - * If <b>postfork</b>, we are a worker process and we want to free - * only the parts of memory that we won't touch. If !<b>postfork</b>, - * Tor is shutting down and we should free everything. - * - * Helps us find the real leaks with sanitizers and the like. Also valgrind - * should then report 0 reachable in its leak report (in an ideal world -- - * in practice libevent, SSL, libc etc never quite free everything). */ -void -tor_free_all(int postfork) -{ - if (!postfork) { - evdns_shutdown(1); - } - geoip_free_all(); - geoip_stats_free_all(); - dirvote_free_all(); - routerlist_free_all(); - networkstatus_free_all(); - addressmap_free_all(); - dirserv_free_fingerprint_list(); - dirserv_free_all(); - dirserv_clear_measured_bw_cache(); - rend_cache_free_all(); - rend_service_authorization_free_all(); - rep_hist_free_all(); - dns_free_all(); - clear_pending_onions(); - circuit_free_all(); - circpad_machines_free(); - entry_guards_free_all(); - pt_free_all(); - channel_tls_free_all(); - channel_free_all(); - connection_free_all(); - connection_edge_free_all(); - scheduler_free_all(); - nodelist_free_all(); - microdesc_free_all(); - routerparse_free_all(); - ext_orport_free_all(); - control_free_all(); - protover_free_all(); - bridges_free_all(); - consdiffmgr_free_all(); - hs_free_all(); - dos_free_all(); - circuitmux_ewma_free_all(); - accounting_free_all(); - protover_summary_cache_free_all(); - - if (!postfork) { - config_free_all(); - or_state_free_all(); - router_free_all(); - routerkeys_free_all(); - policies_free_all(); - } - if (!postfork) { -#ifndef _WIN32 - tor_getpwnam(NULL); -#endif - } - /* stuff in main.c */ - - tor_mainloop_free_all(); - - if (!postfork) { - release_lockfile(); - } - tor_libevent_free_all(); - - subsystems_shutdown(); - - /* Stuff in util.c and address.c*/ - if (!postfork) { - esc_router_info(NULL); - } -} - /** * Remove the specified file, and log a warning if the operation fails for * any reason other than the file not existing. Ignores NULL filenames. @@ -837,50 +737,6 @@ tor_remove_file(const char *filename) } } -/** Do whatever cleanup is necessary before shutting Tor down. */ -void -tor_cleanup(void) -{ - const or_options_t *options = get_options(); - if (options->command == CMD_RUN_TOR) { - time_t now = time(NULL); - /* Remove our pid file. We don't care if there was an error when we - * unlink, nothing we could do about it anyways. */ - tor_remove_file(options->PidFile); - /* Remove control port file */ - tor_remove_file(options->ControlPortWriteToFile); - /* Remove cookie authentication file */ - { - char *cookie_fname = get_controller_cookie_file_name(); - tor_remove_file(cookie_fname); - tor_free(cookie_fname); - } - /* Remove Extended ORPort cookie authentication file */ - { - char *cookie_fname = get_ext_or_auth_cookie_file_name(); - tor_remove_file(cookie_fname); - tor_free(cookie_fname); - } - if (accounting_is_enabled(options)) - accounting_record_bandwidth_usage(now, get_or_state()); - or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */ - or_state_save(now); - if (authdir_mode(options)) { - sr_save_and_cleanup(); - } - if (authdir_mode_tests_reachability(options)) - rep_hist_record_mtbf_data(now, 0); - keypin_close_journal(); - } - - timers_shutdown(); - - tor_free_all(0); /* We could move tor_free_all back into the ifdef below - later, if it makes shutdown unacceptably slow. But for - now, leave it here: it's helped us catch bugs in the - past. */ -} - /** Read/create keys as needed, and echo our fingerprint to stdout. */ static int do_list_fingerprint(void) @@ -1378,6 +1234,30 @@ run_tor_main_loop(void) return do_main_loop(); } +/** Install the publish/subscribe relationships for all the subsystems. */ +static void +pubsub_install(void) +{ + pubsub_builder_t *builder = pubsub_builder_new(); + int r = subsystems_add_pubsub(builder); + tor_assert(r == 0); + r = tor_mainloop_connect_pubsub(builder); // consumes builder + tor_assert(r == 0); +} + +/** Connect the mainloop to its publish/subscribe message delivery events if + * appropriate, and configure the global channels appropriately. */ +static void +pubsub_connect(void) +{ + if (get_options()->command == CMD_RUN_TOR) { + tor_mainloop_connect_pubsub_events(); + /* XXXX For each pubsub channel, its delivery strategy should be set at + * this XXXX point, using tor_mainloop_set_delivery_strategy(). + */ + } +} + /* Main entry point for the Tor process. Called from tor_main(), and by * anybody embedding Tor. */ int @@ -1409,6 +1289,9 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) } } #endif /* defined(NT_SERVICE) */ + + pubsub_install(); + { int init_rv = tor_init(argc, argv); if (init_rv) { @@ -1418,6 +1301,8 @@ tor_run_main(const tor_main_configuration_t *tor_cfg) } } + pubsub_connect(); + if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) { sandbox_cfg_t* cfg = sandbox_init_filter(); diff --git a/src/app/main/main.h b/src/app/main/main.h index bbbbf984fb..9dfaf4b8ef 100644 --- a/src/app/main/main.h +++ b/src/app/main/main.h @@ -21,9 +21,6 @@ void release_lockfile(void); void tor_remove_file(const char *filename); -void tor_cleanup(void); -void tor_free_all(int postfork); - int tor_init(int argc, char **argv); int run_tor_main_loop(void); diff --git a/src/app/main/ntmain.c b/src/app/main/ntmain.c index 05d203b0be..f00b712702 100644 --- a/src/app/main/ntmain.c +++ b/src/app/main/ntmain.c @@ -24,6 +24,7 @@ #include "app/config/config.h" #include "app/main/main.h" #include "app/main/ntmain.h" +#include "app/main/shutdown.h" #include "core/mainloop/mainloop.h" #include "lib/evloop/compat_libevent.h" #include "lib/fs/winlib.h" diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c new file mode 100644 index 0000000000..314e33f228 --- /dev/null +++ b/src/app/main/shutdown.c @@ -0,0 +1,192 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file shutdown.c + * @brief Code to free global resources used by Tor. + * + * In the future, this should all be handled by the subsystem manager. */ + +#include "core/or/or.h" + +#include "app/config/config.h" +#include "app/config/statefile.h" +#include "app/main/main.h" +#include "app/main/shutdown.h" +#include "app/main/subsysmgr.h" +#include "core/mainloop/connection.h" +#include "core/mainloop/mainloop.h" +#include "core/mainloop/mainloop_pubsub.h" +#include "core/or/channeltls.h" +#include "core/or/circuitlist.h" +#include "core/or/circuitmux_ewma.h" +#include "core/or/circuitpadding.h" +#include "core/or/connection_edge.h" +#include "core/or/dos.h" +#include "core/or/policies.h" +#include "core/or/protover.h" +#include "core/or/scheduler.h" +#include "core/or/versions.h" +#include "feature/client/addressmap.h" +#include "feature/client/bridges.h" +#include "feature/client/entrynodes.h" +#include "feature/client/transports.h" +#include "feature/control/control.h" +#include "feature/control/control_auth.h" +#include "feature/dirauth/authmode.h" +#include "feature/dirauth/bwauth.h" +#include "feature/dirauth/dirvote.h" +#include "feature/dirauth/keypin.h" +#include "feature/dirauth/process_descs.h" +#include "feature/dirauth/shared_random.h" +#include "feature/dircache/consdiffmgr.h" +#include "feature/dircache/dirserv.h" +#include "feature/dirparse/routerparse.h" +#include "feature/hibernate/hibernate.h" +#include "feature/hs/hs_common.h" +#include "feature/nodelist/microdesc.h" +#include "feature/nodelist/networkstatus.h" +#include "feature/nodelist/nodelist.h" +#include "feature/nodelist/routerlist.h" +#include "feature/nodelist/routerlist.h" +#include "feature/relay/dns.h" +#include "feature/relay/ext_orport.h" +#include "feature/relay/onion_queue.h" +#include "feature/relay/routerkeys.h" +#include "feature/rend/rendcache.h" +#include "feature/rend/rendclient.h" +#include "feature/stats/geoip_stats.h" +#include "feature/stats/rephist.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/geoip/geoip.h" +#include "src/feature/relay/router.h" + +void evdns_shutdown(int); + +/** Do whatever cleanup is necessary before shutting Tor down. */ +void +tor_cleanup(void) +{ + const or_options_t *options = get_options(); + if (options->command == CMD_RUN_TOR) { + time_t now = time(NULL); + /* Remove our pid file. We don't care if there was an error when we + * unlink, nothing we could do about it anyways. */ + tor_remove_file(options->PidFile); + /* Remove control port file */ + tor_remove_file(options->ControlPortWriteToFile); + /* Remove cookie authentication file */ + { + char *cookie_fname = get_controller_cookie_file_name(); + tor_remove_file(cookie_fname); + tor_free(cookie_fname); + } + /* Remove Extended ORPort cookie authentication file */ + { + char *cookie_fname = get_ext_or_auth_cookie_file_name(); + tor_remove_file(cookie_fname); + tor_free(cookie_fname); + } + if (accounting_is_enabled(options)) + accounting_record_bandwidth_usage(now, get_or_state()); + or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */ + or_state_save(now); + if (authdir_mode(options)) { + sr_save_and_cleanup(); + } + if (authdir_mode_tests_reachability(options)) + rep_hist_record_mtbf_data(now, 0); + keypin_close_journal(); + } + + timers_shutdown(); + + tor_free_all(0); /* We could move tor_free_all back into the ifdef below + later, if it makes shutdown unacceptably slow. But for + now, leave it here: it's helped us catch bugs in the + past. */ +} + +/** Free all memory that we might have allocated somewhere. + * If <b>postfork</b>, we are a worker process and we want to free + * only the parts of memory that we won't touch. If !<b>postfork</b>, + * Tor is shutting down and we should free everything. + * + * Helps us find the real leaks with sanitizers and the like. Also valgrind + * should then report 0 reachable in its leak report (in an ideal world -- + * in practice libevent, SSL, libc etc never quite free everything). */ +void +tor_free_all(int postfork) +{ + if (!postfork) { + evdns_shutdown(1); + } + geoip_free_all(); + geoip_stats_free_all(); + dirvote_free_all(); + routerlist_free_all(); + networkstatus_free_all(); + addressmap_free_all(); + dirserv_free_fingerprint_list(); + dirserv_free_all(); + dirserv_clear_measured_bw_cache(); + rend_cache_free_all(); + rend_service_authorization_free_all(); + rep_hist_free_all(); + dns_free_all(); + clear_pending_onions(); + circuit_free_all(); + circpad_machines_free(); + entry_guards_free_all(); + pt_free_all(); + channel_tls_free_all(); + channel_free_all(); + connection_free_all(); + connection_edge_free_all(); + scheduler_free_all(); + nodelist_free_all(); + microdesc_free_all(); + routerparse_free_all(); + ext_orport_free_all(); + control_free_all(); + protover_free_all(); + bridges_free_all(); + consdiffmgr_free_all(); + hs_free_all(); + dos_free_all(); + circuitmux_ewma_free_all(); + accounting_free_all(); + protover_summary_cache_free_all(); + + if (!postfork) { + config_free_all(); + or_state_free_all(); + router_free_all(); + routerkeys_free_all(); + policies_free_all(); + } + if (!postfork) { +#ifndef _WIN32 + tor_getpwnam(NULL); +#endif + } + /* stuff in main.c */ + + tor_mainloop_disconnect_pubsub(); + tor_mainloop_free_all(); + + if (!postfork) { + release_lockfile(); + } + tor_libevent_free_all(); + + subsystems_shutdown(); + + /* Stuff in util.c and address.c*/ + if (!postfork) { + esc_router_info(NULL); + } +} diff --git a/src/app/main/shutdown.h b/src/app/main/shutdown.h new file mode 100644 index 0000000000..1bca96a0aa --- /dev/null +++ b/src/app/main/shutdown.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file shutdown.h + * \brief Header file for shutdown.c. + **/ + +#ifndef TOR_SHUTDOWN_H +#define TOR_SHUTDOWN_H + +void tor_cleanup(void); +void tor_free_all(int postfork); + +#endif /* !defined(TOR_SHUTDOWN_H) */ diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c index e0ca3ce4df..5aa4fd76c9 100644 --- a/src/app/main/subsysmgr.c +++ b/src/app/main/subsysmgr.c @@ -5,9 +5,14 @@ #include "orconfig.h" #include "app/main/subsysmgr.h" -#include "lib/err/torerr.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib |