diff options
Diffstat (limited to 'src/lib')
324 files changed, 16878 insertions, 2894 deletions
diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h index fa82241b28..4756ca2beb 100644 --- a/src/lib/arch/bytes.h +++ b/src/lib/arch/bytes.h @@ -129,7 +129,7 @@ tor_ntohll(uint64_t a) { return a; } -#else +#else /* !defined(WORDS_BIGENDIAN) */ static inline uint16_t tor_htons(uint16_t a) { @@ -177,6 +177,6 @@ tor_ntohll(uint64_t a) { return tor_htonll(a); } -#endif +#endif /* defined(WORDS_BIGENDIAN) */ -#endif +#endif /* !defined(TOR_BYTES_H) */ diff --git a/src/lib/arch/include.am b/src/lib/arch/include.am index f92ee9222f..c5926c6330 100644 --- a/src/lib/arch/include.am +++ b/src/lib/arch/include.am @@ -1,3 +1,4 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/arch/bytes.h diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include new file mode 100644 index 0000000000..c4be73bce2 --- /dev/null +++ b/src/lib/buf/.may_include @@ -0,0 +1,10 @@ +orconfig.h + +lib/buf/*.h +lib/cc/*.h +lib/ctime/*.h +lib/malloc/*.h +lib/testsupport/*.h +lib/log/*.h +lib/string/*.h +lib/time/*.h diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c index fe4cf7c385..ace5bdc4a4 100644 --- a/src/lib/container/buffers.c +++ b/src/lib/buf/buffers.c @@ -25,7 +25,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include <stddef.h> -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/cc/torint.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" @@ -158,7 +158,7 @@ chunk_new_with_alloc_size(size_t alloc) static inline chunk_t * chunk_grow(chunk_t *chunk, size_t sz) { - off_t offset; + ptrdiff_t offset; const size_t memlen_orig = chunk->memlen; const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig); const size_t new_alloc = CHUNK_ALLOC_SIZE(sz); @@ -440,7 +440,7 @@ chunk_copy(const chunk_t *in_chunk) #endif newch->next = NULL; if (in_chunk->data) { - off_t offset = in_chunk->data - in_chunk->mem; + ptrdiff_t offset = in_chunk->data - in_chunk->mem; newch->data = newch->mem + offset; } return newch; @@ -712,7 +712,8 @@ buf_move_all(buf_t *buf_out, buf_t *buf_in) /** Internal structure: represents a position in a buffer. */ typedef struct buf_pos_t { const chunk_t *chunk; /**< Which chunk are we pointing to? */ - int pos;/**< Which character inside the chunk's data are we pointing to? */ + ptrdiff_t pos;/**< Which character inside the chunk's data are we pointing + * to? */ size_t chunk_pos; /**< Total length of all previous chunks. */ } buf_pos_t; @@ -728,15 +729,15 @@ buf_pos_init(const buf_t *buf, buf_pos_t *out) /** Advance <b>out</b> to the first appearance of <b>ch</b> at the current * position of <b>out</b>, or later. Return -1 if no instances are found; * otherwise returns the absolute position of the character. */ -static off_t +static ptrdiff_t buf_find_pos_of_char(char ch, buf_pos_t *out) { const chunk_t *chunk; - int pos; + ptrdiff_t pos; tor_assert(out); if (out->chunk) { if (out->chunk->datalen) { - tor_assert(out->pos < (off_t)out->chunk->datalen); + tor_assert(out->pos < (ptrdiff_t)out->chunk->datalen); } else { tor_assert(out->pos == 0); } @@ -764,7 +765,7 @@ buf_pos_inc(buf_pos_t *pos) { tor_assert(pos->pos < INT_MAX - 1); ++pos->pos; - if (pos->pos == (off_t)pos->chunk->datalen) { + if (pos->pos == (ptrdiff_t)pos->chunk->datalen) { if (!pos->chunk->next) return -1; pos->chunk_pos += pos->chunk->datalen; @@ -838,11 +839,11 @@ buf_peek_startswith(const buf_t *buf, const char *cmd) /** Return the index within <b>buf</b> at which <b>ch</b> first appears, * or -1 if <b>ch</b> does not appear on buf. */ -static off_t +static ptrdiff_t buf_find_offset_of_char(buf_t *buf, char ch) { chunk_t *chunk; - off_t offset = 0; + ptrdiff_t offset = 0; tor_assert(buf->datalen < INT_MAX); for (chunk = buf->head; chunk; chunk = chunk->next) { char *cp = memchr(chunk->data, ch, chunk->datalen); @@ -865,7 +866,7 @@ int buf_get_line(buf_t *buf, char *data_out, size_t *data_len) { size_t sz; - off_t offset; + ptrdiff_t offset; if (!buf->head) return 0; diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h index c103b93a82..c103b93a82 100644 --- a/src/lib/container/buffers.h +++ b/src/lib/buf/buffers.h diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am new file mode 100644 index 0000000000..27430d1d38 --- /dev/null +++ b/src/lib/buf/include.am @@ -0,0 +1,19 @@ + +noinst_LIBRARIES += src/lib/libtor-buf.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-buf-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_buf_a_SOURCES = \ + src/lib/buf/buffers.c + +src_lib_libtor_buf_testing_a_SOURCES = \ + $(src_lib_libtor_buf_a_SOURCES) +src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/buf/buffers.h diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include index 2b06e8519c..fa1478ce46 100644 --- a/src/lib/cc/.may_include +++ b/src/lib/cc/.may_include @@ -1 +1,2 @@ orconfig.h +lib/cc/*.h
\ No newline at end of file diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h index fbe6a38f1f..d9ccabb2b3 100644 --- a/src/lib/cc/compat_compiler.h +++ b/src/lib/cc/compat_compiler.h @@ -88,7 +88,7 @@ # define ENABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) #endif /* defined(__clang__) || GCC_VERSION >= 406 */ -#else /* !(defined(__GNUC__)) */ +#else /* !defined(__GNUC__) */ /* not gcc at all */ # define DISABLE_GCC_WARNING(warning) # define ENABLE_GCC_WARNING(warning) @@ -201,7 +201,7 @@ * structure <b>st</b>. Example: * <pre> * struct a { int foo; int bar; } x; - * off_t bar_offset = offsetof(struct a, bar); + * ptrdiff_t bar_offset = offsetof(struct a, bar); * int *bar_p = STRUCT_VAR_P(&x, bar_offset); * *bar_p = 3; * </pre> @@ -223,4 +223,16 @@ /** Macro: Yields the number of elements in array x. */ #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0])) -#endif /* !defined(TOR_COMPAT_H) */ +/** + * "Eat" a semicolon that somebody puts at the end of a top-level macro. + * + * Frequently, we want to declare a macro that people will use at file scope, + * and we want to allow people to put a semicolon after the macro. + * + * This declaration of a struct can be repeated any number of times, and takes + * a trailing semicolon afterwards. + **/ +#define EAT_SEMICOLON \ + struct dummy_semicolon_eater__ + +#endif /* !defined(TOR_COMPAT_COMPILER_H) */ diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h new file mode 100644 index 0000000000..bedf0b83a6 --- /dev/null +++ b/src/lib/cc/ctassert.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2018 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ctassert.h + * + * \brief Compile-time assertions: CTASSERT(expression). + */ + +#ifndef TOR_CTASSERT_H +#define TOR_CTASSERT_H + +#include "lib/cc/compat_compiler.h" + +/** + * CTASSERT(expression) + * + * Trigger a compiler error if expression is false. + */ +#if __STDC_VERSION__ >= 201112L + +/* If C11 is available, just use _Static_assert. */ +#define CTASSERT(x) _Static_assert((x), #x) + +#else /* !(__STDC_VERSION__ >= 201112L) */ + +/* + * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__ + * and __LINE__, or just __LINE__, with an intermediate preprocessor + * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the + * expansions together into a unique name. + * + * We use this name as a typedef of an array type with a positive + * length if the assertion is true, and a negative length of the + * assertion is false, which is invalid and hence triggers a compiler + * error. + */ +#if defined(__COUNTER__) +#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__) +#elif defined(__INCLUDE_LEVEL__) +#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__) +#else +/* hope it's unique enough */ +#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__) +#endif /* defined(__COUNTER__) || ... */ + +#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b) +#define CTASSERT_DECL(x, a, b) \ + typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED + +#endif /* __STDC_VERSION__ >= 201112L */ + +#endif /* !defined(TOR_CTASSERT_H) */ diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am index 2ae90f97dd..1aa722dd82 100644 --- a/src/lib/cc/include.am +++ b/src/lib/cc/include.am @@ -1,4 +1,6 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/cc/compat_compiler.h \ + src/lib/cc/ctassert.h \ src/lib/cc/torint.h diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h index c9b2d329f2..94b79d30a1 100644 --- a/src/lib/cc/torint.h +++ b/src/lib/cc/torint.h @@ -96,9 +96,9 @@ typedef int32_t ssize_t; # else # define TOR_PRIuSZ PRIu32 # endif -#else +#else /* !defined(_WIN32) */ # define TOR_PRIuSZ "zu" -#endif +#endif /* defined(_WIN32) */ #ifdef _WIN32 # ifdef _WIN64 @@ -106,9 +106,9 @@ typedef int32_t ssize_t; # else # define TOR_PRIdSZ PRId32 # endif -#else +#else /* !defined(_WIN32) */ # define TOR_PRIdSZ "zd" -#endif +#endif /* defined(_WIN32) */ #ifndef SSIZE_MAX #if (SIZEOF_SIZE_T == 4) @@ -125,4 +125,13 @@ typedef int32_t ssize_t; /** Any size_t larger than this amount is likely to be an underflow. */ #define SIZE_T_CEILING ((size_t)(SSIZE_MAX-16)) +#if SIZEOF_INT > SIZEOF_VOID_P +#error "sizeof(int) > sizeof(void *) - Tor cannot be built on this platform!" +#endif + +#if SIZEOF_UNSIGNED_INT > SIZEOF_VOID_P +#error "sizeof(unsigned int) > sizeof(void *) - Tor cannot be built on this \ +platform!" +#endif + #endif /* !defined(TOR_TORINT_H) */ diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include index 68fe9f1c54..6cd80086e6 100644 --- a/src/lib/compress/.may_include +++ b/src/lib/compress/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/compress/*.h lib/container/*.h @@ -8,5 +9,6 @@ lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c index 95fd73bb32..51591410a2 100644 --- a/src/lib/compress/compress.c +++ b/src/lib/compress/compress.c @@ -29,10 +29,12 @@ #include "lib/compress/compress.h" #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_none.h" +#include "lib/compress/compress_sys.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/intmath/cmp.h" #include "lib/malloc/malloc.h" +#include "lib/subsys/subsys.h" #include "lib/thread/threads.h" /** Total number of bytes allocated for compression state overhead. */ @@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state) } /** Initialize all compression modules. */ -void +int tor_compress_init(void) { atomic_counter_init(&total_compress_allocation); @@ -668,6 +670,8 @@ tor_compress_init(void) tor_zlib_init(); tor_lzma_init(); tor_zstd_init(); + + return 0; } /** Warn if we had any problems while setting up our compression libraries. @@ -677,5 +681,20 @@ tor_compress_init(void) void tor_compress_log_init_warnings(void) { + // XXXX can we move this into tor_compress_init() after all? log.c queues + // XXXX log messages at startup. tor_zstd_warn_if_version_mismatched(); } + +static int +subsys_compress_initialize(void) +{ + return tor_compress_init(); +} + +const subsys_fns_t sys_compress = { + .name = "compress", + .supported = true, + .level = -70, + .initialize = subsys_compress_initialize, +}; diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h index 5f16a2ab27..8cea4ead60 100644 --- a/src/lib/compress/compress.h +++ b/src/lib/compress/compress.h @@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); -void tor_compress_init(void); +int tor_compress_init(void); void tor_compress_log_init_warnings(void); struct buf_t; diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c index 198128b261..2e704466f2 100644 --- a/src/lib/compress/compress_buf.c +++ b/src/lib/compress/compress_buf.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/cc/compat_compiler.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/compress/compress.h" #include "lib/log/util_bug.h" diff --git a/src/lib/compress/compress_lzma.c b/src/lib/compress/compress_lzma.c index 2dab37e433..915f4949ae 100644 --- a/src/lib/compress/compress_lzma.c +++ b/src/lib/compress/compress_lzma.c @@ -221,7 +221,7 @@ tor_lzma_compress_new(int compress, tor_free(result); return NULL; /* LCOV_EXCL_STOP */ -#else /* !(defined(HAVE_LZMA)) */ +#else /* !defined(HAVE_LZMA) */ (void)compress; (void)method; (void)level; @@ -312,7 +312,7 @@ tor_lzma_compress_process(tor_lzma_compress_state_t *state, lzma_error_str(retval)); return TOR_COMPRESS_ERROR; } -#else /* !(defined(HAVE_LZMA)) */ +#else /* !defined(HAVE_LZMA) */ (void)state; (void)out; (void)out_len; diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h new file mode 100644 index 0000000000..6181072315 --- /dev/null +++ b/src/lib/compress/compress_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_sys.h + * \brief Declare subsystem object for the compress module + **/ + +#ifndef TOR_COMPRESS_SYS_H +#define TOR_COMPRESS_SYS_H + +extern const struct subsys_fns_t sys_compress; + +#endif /* !defined(TOR_COMPRESS_SYS_H) */ diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index 45d0d4d602..9076665295 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -25,7 +25,7 @@ * all invocations of zstd's static-only functions in a check to make sure * that the compile-time version matches the run-time version. */ #define ZSTD_STATIC_LINKING_ONLY -#endif +#endif /* defined(ENABLE_ZSTD_ADVANCED_APIS) */ #ifdef HAVE_ZSTD #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE @@ -35,7 +35,7 @@ DISABLE_GCC_WARNING(unused-const-variable) #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE ENABLE_GCC_WARNING(unused-const-variable) #endif -#endif +#endif /* defined(HAVE_ZSTD) */ /** Total number of bytes allocated for Zstandard state. */ static atomic_counter_t total_zstd_allocation; @@ -77,7 +77,7 @@ tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number) version_number / 100 % 100, version_number % 100); } -#endif +#endif /* defined(HAVE_ZSTD) */ #define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */ @@ -93,7 +93,7 @@ tor_zstd_get_version_str(void) ZSTD_versionNumber()); return version_str; -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ return NULL; #endif /* defined(HAVE_ZSTD) */ } @@ -125,9 +125,9 @@ tor_zstd_can_use_static_apis(void) } #endif return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber()); -#else +#else /* !(defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)) */ return 0; -#endif +#endif /* defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD) */ } /** Internal Zstandard state for incremental compression/decompression. @@ -237,7 +237,7 @@ tor_zstd_state_size_precalc(int compress, int preset) #endif } } -#endif +#endif /* defined(ZSTD_STATIC_LINKING_ONLY) */ return tor_zstd_state_size_precalc_fake(compress, preset); } #endif /* defined(HAVE_ZSTD) */ @@ -317,7 +317,7 @@ tor_zstd_compress_new(int compress, tor_free(result); return NULL; // LCOV_EXCL_STOP -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ (void)compress; (void)method; (void)level; @@ -454,7 +454,7 @@ tor_zstd_compress_process(tor_zstd_compress_state_t *state, return TOR_COMPRESS_OK; } -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ (void)state; (void)out; (void)out_len; @@ -527,7 +527,7 @@ tor_zstd_warn_if_version_mismatched(void) "For safety, we'll avoid using advanced zstd functionality.", header_version, runtime_version); } -#endif +#endif /* defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS) */ } #ifdef TOR_UNIT_TESTS @@ -538,4 +538,4 @@ tor_zstd_set_static_apis_disabled_for_testing(int disabled) { static_apis_disable_for_testing = disabled; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am index 75c9032bd2..60dd447d4e 100644 --- a/src/lib/compress/include.am +++ b/src/lib/compress/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-compress-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_compress_a_SOURCES = \ src/lib/compress/compress.c \ src/lib/compress/compress_buf.c \ @@ -18,9 +19,11 @@ src_lib_libtor_compress_testing_a_SOURCES = \ src_lib_libtor_compress_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_compress_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/compress/compress.h \ src/lib/compress/compress_lzma.h \ src/lib/compress/compress_none.h \ + src/lib/compress/compress_sys.h \ src/lib/compress/compress_zlib.h \ src/lib/compress/compress_zstd.h diff --git a/src/lib/conf/.may_include b/src/lib/conf/.may_include new file mode 100644 index 0000000000..629e2f897d --- /dev/null +++ b/src/lib/conf/.may_include @@ -0,0 +1,3 @@ +orconfig.h +lib/cc/*.h +lib/conf/*.h diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h new file mode 100644 index 0000000000..68121891f1 --- /dev/null +++ b/src/lib/conf/confmacros.h @@ -0,0 +1,67 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file confmacros.h + * @brief Macro definitions for declaring configuration variables + **/ + +#ifndef TOR_LIB_CONF_CONFMACROS_H +#define TOR_LIB_CONF_CONFMACROS_H + +#include "orconfig.h" +#include "lib/conf/conftesting.h" + +/** + * Used to indicate the end of an array of configuration variables. + **/ +#define END_OF_CONFIG_VARS \ + { .member = { .name = NULL } DUMMY_CONF_TEST_MEMBERS } + +/** + * Declare a config_var_t as a member named <b>membername</b> of the structure + * <b>structtype</b>, whose user-visible name is <b>varname</b>, whose + * type corresponds to the config_type_t member CONFIG_TYPE_<b>vartype</b>, + * and whose initial value is <b>intval</b>. + * + * Most modules that use this macro should wrap it in a local macro that + * sets structtype to the local configuration type. + **/ +#define CONFIG_VAR_ETYPE(structtype, varname, vartype, membername, \ + varflags, initval) \ + { .member = \ + { .name = varname, \ + .type = CONFIG_TYPE_ ## vartype, \ + .offset = offsetof(structtype, membername), \ + }, \ + .flags = varflags, \ + .initvalue = initval \ + CONF_TEST_MEMBERS(structtype, vartype, membername) \ + } + +/** + * As CONFIG_VAR_XTYPE, but declares a value using an extension type whose + * type definition is <b>vartype</b>_type_defn. + **/ +#define CONFIG_VAR_DEFN(structtype, varname, vartype, membername, \ + varflags, initval) \ + { .member = \ + { .name = varname, \ + .type = CONFIG_TYPE_EXTENDED, \ + .type_def = &vartype ## _type_defn, \ + .offset = offsetof(structtype, membername), \ + }, \ + .flags = varflags, \ + .initvalue = initval \ + CONF_TEST_MEMBERS(structtype, vartype, membername) \ + } + +#define CONFIG_VAR_OBSOLETE(varname) \ + { .member = { .name = varname, .type = CONFIG_TYPE_OBSOLETE }, \ + .flags = CFLG_GROUP_OBSOLETE \ + } + +#endif /* !defined(TOR_LIB_CONF_CONFMACROS_H) */ diff --git a/src/lib/conf/conftesting.h b/src/lib/conf/conftesting.h new file mode 100644 index 0000000000..f01f52d59e --- /dev/null +++ b/src/lib/conf/conftesting.h @@ -0,0 +1,86 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file conftesting.h + * @brief Macro and type declarations for testing + **/ + +#ifndef TOR_LIB_CONF_CONFTESTING_H +#define TOR_LIB_CONF_CONFTESTING_H + +#ifdef TOR_UNIT_TESTS +/** + * Union used when building in test mode typechecking the members of a type + * used with confparse.c. See CONF_CHECK_VAR_TYPE for a description of how + * it is used. */ +typedef union { + char **STRING; + char **FILENAME; + int *POSINT; /* yes, this is really an int, and not an unsigned int. For + * historical reasons, many configuration values are restricted + * to the range [0,INT_MAX], and stored in signed ints. + */ + uint64_t *UINT64; + int *INT; + int *INTERVAL; + int *MSEC_INTERVAL; + uint64_t *MEMUNIT; + double *DOUBLE; + int *BOOL; + int *AUTOBOOL; + time_t *ISOTIME; + struct smartlist_t **CSV; + int *CSV_INTERVAL; + struct config_line_t **LINELIST; + struct config_line_t **LINELIST_S; + struct config_line_t **LINELIST_V; + // XXXX this doesn't belong at this level of abstraction. + struct routerset_t **ROUTERSET; +} confparse_dummy_values_t; +#endif /* defined(TOR_UNIT_TESTS) */ + +/* Macros to define extra members inside config_var_t fields, and at the + * end of a list of them. + */ +#ifdef TOR_UNIT_TESTS +/* This is a somewhat magic type-checking macro for users of confparse.c. + * It initializes a union member "confparse_dummy_values_t.conftype" with + * the address of a static member "tp_dummy.member". This + * will give a compiler warning unless the member field is of the correct + * type. + * + * (This warning is mandatory, because a type mismatch here violates the type + * compatibility constraint for simple assignment, and requires a diagnostic, + * according to the C spec.) + * + * For example, suppose you say: + * "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)". + * Then this macro will evaluate to: + * { .STRING = &or_options_t_dummy.Address } + * And since confparse_dummy_values_t.STRING has type "char **", that + * expression will create a warning unless or_options_t.Address also + * has type "char *". + */ +#define CONF_CHECK_VAR_TYPE(tp, conftype, member) \ + { . conftype = &tp ## _dummy . member } +#define CONF_TEST_MEMBERS(tp, conftype, member) \ + , .var_ptr_dummy=CONF_CHECK_VAR_TYPE(tp, conftype, member) +#define DUMMY_CONF_TEST_MEMBERS , .var_ptr_dummy={ .INT=NULL } +#define DUMMY_TYPECHECK_INSTANCE(tp) \ + static tp tp ## _dummy + +#else /* !defined(TOR_UNIT_TESTS) */ + +#define CONF_TEST_MEMBERS(tp, conftype, member) +/* Repeatedly declarable incomplete struct to absorb redundant semicolons */ +#define DUMMY_TYPECHECK_INSTANCE(tp) \ + struct tor_semicolon_eater +#define DUMMY_CONF_TEST_MEMBERS + +#endif /* defined(TOR_UNIT_TESTS) */ + +#endif /* !defined(TOR_LIB_CONF_CONFTESTING_H) */ diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h new file mode 100644 index 0000000000..274065cff2 --- /dev/null +++ b/src/lib/conf/conftypes.h @@ -0,0 +1,202 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file conftypes.h + * @brief Types used to specify configurable options. + * + * This header defines the types that different modules will use in order to + * declare their configuration and state variables, and tell the configuration + * management code about those variables. From the individual module's point + * of view, its configuration and state are simply data structures. + * + * For defining new variable types, see var_type_def_st.h. + * + * For the code that manipulates variables defined via this module, see + * lib/confmgt/, especially typedvar.h and (later) structvar.h. The + * configuration manager is responsible for encoding, decoding, and + * maintaining the configuration structures used by the various modules. + * + * STATUS NOTE: This is a work in process refactoring. It is not yet possible + * for modules to define their own variables, and much of the configuration + * management code is still in src/app/config/. + **/ + +#ifndef TOR_SRC_LIB_CONF_CONFTYPES_H +#define TOR_SRC_LIB_CONF_CONFTYPES_H + +#include "lib/cc/torint.h" +#ifdef TOR_UNIT_TESTS +#include "lib/conf/conftesting.h" +#endif + +#include <stddef.h> + +/** Enumeration of types which option values can take */ +typedef enum config_type_t { + CONFIG_TYPE_STRING = 0, /**< An arbitrary string. */ + CONFIG_TYPE_FILENAME, /**< A filename: some prefixes get expanded. */ + CONFIG_TYPE_POSINT, /**< A non-negative integer less than MAX_INT */ + CONFIG_TYPE_INT, /**< Any integer. */ + CONFIG_TYPE_UINT64, /**< A value in range 0..UINT64_MAX */ + CONFIG_TYPE_INTERVAL, /**< A number of seconds, with optional units*/ + CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional + * units */ + CONFIG_TYPE_MEMUNIT, /**< A number of bytes, with optional units*/ + CONFIG_TYPE_DOUBLE, /**< A floating-point value */ + CONFIG_TYPE_BOOL, /**< A boolean value, expressed as 0 or 1. */ + CONFIG_TYPE_AUTOBOOL, /**< A boolean+auto value, expressed 0 for false, + * 1 for true, and -1 for auto */ + CONFIG_TYPE_ISOTIME, /**< An ISO-formatted time relative to UTC. */ + CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and + * optional whitespace. */ + CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and + * optional whitespace, representing intervals in + * seconds, with optional units. We allow + * multiple values here for legacy reasons, but + * ignore every value after the first. */ + CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ + CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, + * mixed with other keywords. */ + CONFIG_TYPE_LINELIST_V, /**< Catch-all "virtual" option to summarize + * context-sensitive config lines when fetching. + */ + CONFIG_TYPE_OBSOLETE, /**< Obsolete (ignored) option. */ + /** + * Extended type: definition appears in the <b>type_def</b> pointer + * of the corresponding struct_member_t. + * + * For some types, we cannot define them as particular values of this + * enumeration, since those types are abstractions defined at a higher level + * than this module. (For example, parsing a routerset_t is higher-level + * than this module.) To handle this, we use CONFIG_TYPE_EXTENDED for those + * types, and give a definition for them in the struct_member_t.type_def. + **/ + CONFIG_TYPE_EXTENDED, +} config_type_t; + +/* Forward delcaration for var_type_def_t, for extended types. */ +struct var_type_def_t; + +/** Structure to specify a named, typed member within a structure. */ +typedef struct struct_member_t { + /** Name of the field. */ + const char *name; + /** + * Type of the field, according to the config_type_t enumeration. + * + * For any type not otherwise listed in config_type_t, this field's value + * should be CONFIG_TYPE_EXTENDED. When it is, the <b>type_def</b> pointer + * must be set. + **/ + /* + * NOTE: In future refactoring, we might remove this field entirely, along + * with its corresponding enumeration. In that case, we will require that + * type_def be set in all cases. If we do, we will also need a new mechanism + * to enforce consistency between configuration variable types and their + * corresponding structures, since our current design in + * lib/conf/conftesting.h won't work any more. + */ + config_type_t type; + /** + * Pointer to a type definition for the type of this field. Overrides + * <b>type</b> if it is not NULL. Must be set when <b>type</b> is + * CONFIG_TYPE_EXTENDED. + **/ + const struct var_type_def_t *type_def; + /** + * Offset of this field within the structure. Compute this with + * offsetof(structure, fieldname). + **/ + ptrdiff_t offset; +} struct_member_t; + +/** + * Structure to describe the location and preferred value of a "magic number" + * field within a structure. + * + * These 'magic numbers' are 32-bit values used to tag objects to make sure + * that they have the correct type. + */ +typedef struct struct_magic_decl_t { + /** The name of the structure */ + const char *typename; + /** A value used to recognize instances of this structure. */ + uint32_t magic_val; + /** The location within the structure at which we expect to find + * <b>magic_val</b>. */ + ptrdiff_t magic_offset; +} struct_magic_decl_t; + +/** + * Flag to indicate that an option or type is "undumpable". An + * undumpable option is never saved to disk. + * + * For historical reasons its name is usually is prefixed with __. + **/ +#define CFLG_NODUMP (1u<<0) +/** + * Flag to indicate that an option or type is "unlisted". + * + * We don't tell the controller about unlisted options when it asks for a + * list of them. + **/ +#define CFLG_NOLIST (1u<<1) +/** + * Flag to indicate that an option or type is "unsettable". + * + * An unsettable option can never be set directly by name. + **/ +#define CFLG_NOSET (1u<<2) +/** + * Flag to indicate that an option or type does not need to be copied when + * copying the structure that contains it. + * + * (Usually, if an option does not need to be copied, then either it contains + * no data, or the data that it does contain is completely contained within + * another option.) + **/ +#define CFLG_NOCOPY (1u<<3) +/** + * Flag to indicate that an option or type does not need to be compared + * when telling the controller about the differences between two + * configurations. + * + * (Usually, if an option does not need to be compared, then either it + * contains no data, or the data that it does contain is completely contained + * within another option.) + **/ +#define CFLG_NOCMP (1u<<4) +/** + * Flag to indicate that an option or type should not be replaced when setting + * it. + * + * For most options, setting them replaces their old value. For some options, + * however, setting them appends to their old value. + */ +#define CFLG_NOREPLACE (1u<<5) + +/** + * A group of flags that should be set on all obsolete options and types. + **/ +#define CFLG_GROUP_OBSOLETE \ + (CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET|CFLG_NOLIST) + +/** A variable allowed in the configuration file or on the command line. */ +typedef struct config_var_t { + struct_member_t member; /** A struct member corresponding to this + * variable. */ + const char *initvalue; /**< String (or null) describing initial value. */ + uint32_t flags; /**< One or more flags describing special handling for this + * variable */ +#ifdef TOR_UNIT_TESTS + /** Used for compiler-magic to typecheck the corresponding field in the + * corresponding struct. Only used in unit test mode, at compile-time. */ + confparse_dummy_values_t var_ptr_dummy; +#endif +} config_var_t; + +#endif /* !defined(TOR_SRC_LIB_CONF_CONFTYPES_H) */ diff --git a/src/lib/conf/include.am b/src/lib/conf/include.am new file mode 100644 index 0000000000..cb7126184d --- /dev/null +++ b/src/lib/conf/include.am @@ -0,0 +1,6 @@ + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/conf/conftesting.h \ + src/lib/conf/conftypes.h \ + src/lib/conf/confmacros.h diff --git a/src/lib/confmgt/.may_include b/src/lib/confmgt/.may_include new file mode 100644 index 0000000000..2564133917 --- /dev/null +++ b/src/lib/confmgt/.may_include @@ -0,0 +1,11 @@ +orconfig.h +lib/cc/*.h +lib/conf/*.h +lib/confmgt/*.h +lib/container/*.h +lib/encoding/*.h +lib/log/*.h +lib/malloc/*.h +lib/string/*.h +lib/testsupport/*.h +ext/*.h diff --git a/src/lib/confmgt/confparse.c b/src/lib/confmgt/confparse.c new file mode 100644 index 0000000000..08e562f654 --- /dev/null +++ b/src/lib/confmgt/confparse.c @@ -0,0 +1,1239 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file confparse.c + * + * \brief Back-end for parsing and generating key-value files, used to + * implement the torrc file format and the state file. + * + * This module is used by config.c to parse and encode torrc + * configuration files, and by statefile.c to parse and encode the + * $DATADIR/state file. + * + * To use this module, its callers provide an instance of + * config_format_t to describe the mappings from a set of configuration + * options to a number of fields in a C structure. With this mapping, + * the functions here can convert back and forth between the C structure + * specified, and a linked list of key-value pairs. + */ + +#define CONFPARSE_PRIVATE +#include "orconfig.h" +#include "lib/confmgt/confparse.h" + +#include "lib/confmgt/structvar.h" +#include "lib/confmgt/unitparse.h" +#include "lib/container/bitarray.h" +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" + +#include "ext/siphash.h" + +/** + * A managed_var_t is an internal wrapper around a config_var_t in + * a config_format_t structure. It is used by config_mgr_t to + * keep track of which option goes with which structure. */ +typedef struct managed_var_t { + /** + * A pointer to the config_var_t for this option. + */ + const config_var_t *cvar; + /** + * The index of the object in which this option is stored. It is + * IDX_TOPLEVEL to indicate that the object is the top-level object. + **/ + int object_idx; +} managed_var_t; + +static void config_reset(const config_mgr_t *fmt, void *options, + const managed_var_t *var, int use_defaults); +static void config_mgr_register_fmt(config_mgr_t *mgr, + const config_format_t *fmt, + int object_idx); + +/** Release all storage held in a managed_var_t. */ +static void +managed_var_free_(managed_var_t *mv) +{ + if (!mv) + return; + tor_free(mv); +} +#define managed_var_free(mv) \ + FREE_AND_NULL(managed_var_t, managed_var_free_, (mv)) + +struct config_suite_t { + /** A list of configuration objects managed by a given configuration + * manager. They are stored in the same order as the config_format_t + * objects in the manager's list of subformats. */ + smartlist_t *configs; +}; + +/** + * Allocate a new empty config_suite_t. + **/ +static config_suite_t * +config_suite_new(void) +{ + config_suite_t *suite = tor_malloc_zero(sizeof(config_suite_t)); + suite->configs = smartlist_new(); + return suite; +} + +/** Release all storage held by a config_suite_t. (Does not free + * any configuration objects it holds; the caller must do that first.) */ +static void +config_suite_free_(config_suite_t *suite) +{ + if (!suite) + return; + smartlist_free(suite->configs); + tor_free(suite); +} + +#define config_suite_free(suite) \ + FREE_AND_NULL(config_suite_t, config_suite_free_, (suite)) + +struct config_mgr_t { + /** The 'top-level' configuration format. This one is used for legacy + * options that have not yet been assigned to different sub-modules. + * + * (NOTE: for now, this is the only config_format_t that a config_mgr_t + * contains. A subsequent commit will add more. XXXX) + */ + const config_format_t *toplevel; + /** + * List of second-level configuration format objects that this manager + * also knows about. + */ + smartlist_t *subconfigs; + /** A smartlist of managed_var_t objects for all configuration formats. */ + smartlist_t *all_vars; + /** A smartlist of config_abbrev_t objects for all configuration + * formats. These objects are used to track synonyms and abbreviations for + * different configuration options. */ + smartlist_t *all_abbrevs; + /** A smartlist of config_deprecation_t for all configuration formats. */ + smartlist_t *all_deprecations; + /** True if this manager has been frozen and cannot have any more formats + * added to it. A manager must be frozen before it can be used to construct + * or manipulate objects. */ + bool frozen; + /** A replacement for the magic number of the toplevel object. We override + * that number to make it unique for this particular config_mgr_t, so that + * an object constructed with one mgr can't be used with another, even if + * those managers' contents are equal. + */ + struct_magic_decl_t toplevel_magic; +}; + +#define IDX_TOPLEVEL (-1) + +/** Create a new config_mgr_t to manage a set of configuration objects to be + * wrapped under <b>toplevel_fmt</b>. */ +config_mgr_t * +config_mgr_new(const config_format_t *toplevel_fmt) +{ + config_mgr_t *mgr = tor_malloc_zero(sizeof(config_mgr_t)); + mgr->subconfigs = smartlist_new(); + mgr->all_vars = smartlist_new(); + mgr->all_abbrevs = smartlist_new(); + mgr->all_deprecations = smartlist_new(); + + config_mgr_register_fmt(mgr, toplevel_fmt, IDX_TOPLEVEL); + mgr->toplevel = toplevel_fmt; + + return mgr; +} + +/** Add a config_format_t to a manager, with a specified (unique) index. */ +static void +config_mgr_register_fmt(config_mgr_t *mgr, + const config_format_t *fmt, + int object_idx) +{ + int i; + + tor_assertf(!mgr->frozen, + "Tried to add a format to a configuration manager after " + "it had been frozen."); + + if (object_idx != IDX_TOPLEVEL) { + tor_assertf(fmt->config_suite_offset < 0, + "Tried to register a toplevel format in a non-toplevel position"); + } + tor_assertf(fmt != mgr->toplevel && + ! smartlist_contains(mgr->subconfigs, fmt), + "Tried to register an already-registered format."); + + /* register variables */ + for (i = 0; fmt->vars[i].member.name; ++i) { + managed_var_t *mv = tor_malloc_zero(sizeof(managed_var_t)); + mv->cvar = &fmt->vars[i]; + mv->object_idx = object_idx; + smartlist_add(mgr->all_vars, mv); + } + + /* register abbrevs */ + if (fmt->abbrevs) { + for (i = 0; fmt->abbrevs[i].abbreviated; ++i) { + smartlist_add(mgr->all_abbrevs, (void*)&fmt->abbrevs[i]); + } + } + + /* register deprecations. */ + if (fmt->deprecations) { + const config_deprecation_t *d; + for (d = fmt->deprecations; d->name; ++d) { + smartlist_add(mgr->all_deprecations, (void*)d); + } + } +} + +/** + * Add a new format to this configuration object. Asserts on failure. + * + * Returns an internal "index" value used to identify this format within + * all of those formats contained in <b>mgr</b>. This index value + * should not generally be used outside of this module. + **/ +int +config_mgr_add_format(config_mgr_t *mgr, + const config_format_t *fmt) +{ + tor_assert(mgr); + int idx = smartlist_len(mgr->subconfigs); + config_mgr_register_fmt(mgr, fmt, idx); + smartlist_add(mgr->subconfigs, (void *)fmt); + return idx; +} + +/** Return a pointer to the config_suite_t * pointer inside a + * configuration object; returns NULL if there is no such member. */ +static inline config_suite_t ** +config_mgr_get_suite_ptr(const config_mgr_t *mgr, void *toplevel) +{ + if (mgr->toplevel->config_suite_offset < 0) + return NULL; + return STRUCT_VAR_P(toplevel, mgr->toplevel->config_suite_offset); +} + +/** + * Return a pointer to the configuration object within <b>toplevel</b> whose + * index is <b>idx</b>. + * + * NOTE: XXXX Eventually, there will be multiple objects supported within the + * toplevel object. For example, the or_options_t will contain pointers + * to configuration objects for other modules. This function gets + * the sub-object for a particular module. + */ +STATIC void * +config_mgr_get_obj_mutable(const config_mgr_t *mgr, void *toplevel, int idx) +{ + tor_assert(mgr); + tor_assert(toplevel); + if (idx == IDX_TOPLEVEL) + return toplevel; + + tor_assertf(idx >= 0 && idx < smartlist_len(mgr->subconfigs), + "Index %d is out of range.", idx); + config_suite_t **suite = config_mgr_get_suite_ptr(mgr, toplevel); + tor_assert(suite); + tor_assert(smartlist_len(mgr->subconfigs) == + smartlist_len((*suite)->configs)); + + return smartlist_get((*suite)->configs, idx); +} + +/** As config_mgr_get_obj_mutable(), but return a const pointer. */ +STATIC const void * +config_mgr_get_obj(const config_mgr_t *mgr, const void *toplevel, int idx) +{ + return config_mgr_get_obj_mutable(mgr, (void*)toplevel, idx); +} + +/** Sorting helper for smartlist of managed_var_t */ +static int +managed_var_cmp(const void **a, const void **b) +{ + const managed_var_t *mv1 = *(const managed_var_t**)a; + const managed_var_t *mv2 = *(const managed_var_t**)b; + + return strcasecmp(mv1->cvar->member.name, mv2->cvar->member.name); +} + +/** + * Mark a configuration manager as "frozen", so that no more formats can be + * added, and so that it can be used for manipulating configuration objects. + **/ +void +config_mgr_freeze(config_mgr_t *mgr) +{ + static uint64_t mgr_count = 0; + + smartlist_sort(mgr->all_vars, managed_var_cmp); + memcpy(&mgr->toplevel_magic, &mgr->toplevel->magic, + sizeof(struct_magic_decl_t)); + uint64_t magic_input[3] = { mgr->toplevel_magic.magic_val, + (uint64_t) (uintptr_t) mgr, + ++mgr_count }; + mgr->toplevel_magic.magic_val = + (uint32_t)siphash24g(magic_input, sizeof(magic_input)); + mgr->frozen = true; +} + +/** Release all storage held in <b>mgr</b> */ +void +config_mgr_free_(config_mgr_t *mgr) +{ + if (!mgr) + return; + SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv, managed_var_free(mv)); + smartlist_free(mgr->all_vars); + smartlist_free(mgr->all_abbrevs); + smartlist_free(mgr->all_deprecations); + smartlist_free(mgr->subconfigs); + memset(mgr, 0, sizeof(*mgr)); + tor_free(mgr); +} + +/** Return a new smartlist_t containing a config_var_t for every variable that + * <b>mgr</b> knows about. The elements of this smartlist do not need + * to be freed; they have the same lifespan as <b>mgr</b>. */ +smartlist_t * +config_mgr_list_vars(const config_mgr_t *mgr) +{ + smartlist_t *result = smartlist_new(); + tor_assert(mgr); + SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv, + smartlist_add(result, (void*) mv->cvar)); + return result; +} + +/** Return a new smartlist_t containing the names of all deprecated variables. + * The elements of this smartlist do not need to be freed; they have the same + * lifespan as <b>mgr</b>. + */ +smartlist_t * +config_mgr_list_deprecated_vars(const config_mgr_t *mgr) +{ + smartlist_t *result = smartlist_new(); + tor_assert(mgr); + SMARTLIST_FOREACH(mgr->all_deprecations, config_deprecation_t *, d, + smartlist_add(result, (char*)d->name)); + return result; +} + +/** Assert that the magic fields in <b>options</b> and its subsidiary + * objects are all okay. */ +static void +config_mgr_assert_magic_ok(const config_mgr_t *mgr, + const void *options) +{ + tor_assert(mgr); + tor_assert(options); + tor_assert(mgr->frozen); + struct_check_magic(options, &mgr->toplevel_magic); + + config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, (void*)options); + if (suitep == NULL) { + tor_assert(smartlist_len(mgr->subconfigs) == 0); + return; + } + + tor_assert(smartlist_len((*suitep)->configs) == + smartlist_len(mgr->subconfigs)); + SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) { + void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx); + tor_assert(obj); + struct_check_magic(obj, &fmt->magic); + } SMARTLIST_FOREACH_END(fmt); +} + +/** Macro: assert that <b>cfg</b> has the right magic field for + * <b>mgr</b>. */ +#define CONFIG_CHECK(mgr, cfg) STMT_BEGIN \ + config_mgr_assert_magic_ok((mgr), (cfg)); \ + STMT_END + +/** Allocate an empty configuration object of a given format type. */ +void * +config_new(const config_mgr_t *mgr) +{ + tor_assert(mgr->frozen); + void *opts = tor_malloc_zero(mgr->toplevel->size); + struct_set_magic(opts, &mgr->toplevel_magic); + config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, opts); + if (suitep) { + *suitep = config_suite_new(); + SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) { + void *obj = tor_malloc_zero(fmt->size); + struct_set_magic(obj, &fmt->magic); + smartlist_add((*suitep)->configs, obj); + } SMARTLIST_FOREACH_END(fmt); + } + CONFIG_CHECK(mgr, opts); + return opts; +} + +/* + * Functions to parse config options + */ + +/** If <b>option</b> is an official abbreviation for a longer option, + * return the longer option. Otherwise return <b>option</b>. + * If <b>command_line</b> is set, apply all abbreviations. Otherwise, only + * apply abbreviations that work for the config file and the command line. + * If <b>warn_obsolete</b> is set, warn about deprecated names. */ +const char * +config_expand_abbrev(const config_mgr_t *mgr, const char *option, + int command_line, int warn_obsolete) +{ + SMARTLIST_FOREACH_BEGIN(mgr->all_abbrevs, const config_abbrev_t *, abbrev) { + /* Abbreviations are case insensitive. */ + if (!strcasecmp(option, abbrev->abbreviated) && + (command_line || !abbrev->commandline_only)) { + if (warn_obsolete && abbrev->warn) { + log_warn(LD_CONFIG, + "The configuration option '%s' is deprecated; " + "use '%s' instead.", + abbrev->abbreviated, + abbrev->full); + } + /* Keep going through the list in case we want to rewrite it more. + * (We could imagine recursing here, but I don't want to get the + * user into an infinite loop if we craft our list wrong.) */ + option = abbrev->full; + } + } SMARTLIST_FOREACH_END(abbrev); + return option; +} + +/** If <b>key</b> is a deprecated configuration option, return the message + * explaining why it is deprecated (which may be an empty string). Return NULL + * if it is not deprecated. The <b>key</b> field must be fully expanded. */ +const char * +config_find_deprecation(const config_mgr_t *mgr, const char *key) +{ + if (BUG(mgr == NULL) || BUG(key == NULL)) + return NULL; // LCOV_EXCL_LINE + + SMARTLIST_FOREACH_BEGIN(mgr->all_deprecations, const config_deprecation_t *, + d) { + if (!strcasecmp(d->name, key)) { + return d->why_deprecated ? d->why_deprecated : ""; + } + } SMARTLIST_FOREACH_END(d); + return NULL; +} + +/** + * Find the managed_var_t object for a variable whose name is <b>name</b> + * according to <b>mgr</b>. Return that object, or NULL if none exists. + * + * If <b>allow_truncated</b> is true, then accept any variable whose + * name begins with <b>name</b>. + * + * If <b>idx_out</b> is not NULL, set *<b>idx_out</b> to the position of + * that variable within mgr->all_vars, or to -1 if the variable is + * not found. + */ +static const managed_var_t * +config_mgr_find_var(const config_mgr_t *mgr, + const char *key, + bool allow_truncated, int *idx_out) +{ + const size_t keylen = strlen(key); + if (idx_out) + *idx_out = -1; + + if (!keylen) + return NULL; /* if they say "--" on the command line, it's not an option */ + + /* First, check for an exact (case-insensitive) match */ + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!strcasecmp(mv->cvar->member.name, key)) { + if (idx_out) + *idx_out = mv_sl_idx; + return mv; + } + } SMARTLIST_FOREACH_END(mv); + + if (!allow_truncated) + return NULL; + + /* If none, check for an abbreviated match */ + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!strncasecmp(key, mv->cvar->member.name, keylen)) { + log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. " + "Please use '%s' instead", + key, mv->cvar->member.name); + if (idx_out) + *idx_out = mv_sl_idx; + return mv; + } + } SMARTLIST_FOREACH_END(mv); + + /* Okay, unrecognized option */ + return NULL; +} + +/** + * If <b>key</b> is a name or an abbreviation configuration option, return + * the corresponding canonical name for it. Warn if the abbreviation is + * non-standard. Return NULL if the option does not exist. + */ +const char * +config_find_option_name(const config_mgr_t *mgr, const char *key) +{ + key = config_expand_abbrev(mgr, key, 0, 0); + const managed_var_t *mv = config_mgr_find_var(mgr, key, true, NULL); + if (mv) + return mv->cvar->member.name; + else + return NULL; +} + +/** Return the number of option entries in <b>fmt</b>. */ +static int +config_count_options(const config_mgr_t *mgr) +{ + return smartlist_len(mgr->all_vars); +} + +/** + * Return true iff at least one bit from <b>flag</b> is set on <b>var</b>, + * either in <b>var</b>'s flags, or on the flags of its type. + **/ +static bool +config_var_has_flag(const config_var_t *var, uint32_t flag) +{ + uint32_t have_flags = var->flags | struct_var_get_flags(&var->member); + + return (have_flags & flag) != 0; +} + +/** + * Return true if assigning a value to <b>var</b> replaces the previous + * value. Return false if assigning a value to <b>var</b> appends + * to the previous value. + **/ +static bool +config_var_is_replaced_on_set(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOREPLACE); +} + +/** + * Return true iff <b>var</b> may be assigned by name (e.g., via the + * CLI, the configuration files, or the controller API). + **/ +bool +config_var_is_settable(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOSET); +} + +/** + * Return true iff the controller is allowed to fetch the value of + * <b>var</b>. + **/ +static bool +config_var_is_gettable(const config_var_t *var) +{ + /* Arguably, invisible or obsolete options should not be gettable. However, + * they have been gettable for a long time, and making them ungettable could + * have compatibility effects. For now, let's leave them alone. + */ + + // return ! config_var_has_flag(var, CVFLAG_OBSOLETE|CFGLAGS_INVISIBLE); + (void)var; + return true; +} + +/** + * Return true iff we need to check <b>var</b> for changes when we are + * comparing config options for changes. + * + * A false result might mean that the variable is a derived variable, and that + * comparing the variable it derives from compares this one too-- or it might + * mean that there is no data to compare. + **/ +static bool +config_var_should_list_changes(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOCMP); +} + +/** + * Return true iff we need to copy the data for <b>var</b> when we are + * copying a config option. + * + * A false option might mean that the variable is a derived variable, and that + * copying the variable it derives from copies it-- or it might mean that + * there is no data to copy. + **/ +static bool +config_var_needs_copy(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOCOPY); +} + +/** + * Return true iff variable <b>var</b> should appear on list of variable + * names given to the controller or the CLI. + * + * (Note that this option is imperfectly obeyed. The + * --list-torrc-options command looks at the "settable" flag, whereas + * "GETINFO config/defaults" and "list_deprecated_*()" do not filter + * their results. It would be good for consistency to try to converge + * these behaviors in the future.) + **/ +bool +config_var_is_listable(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOLIST); +} + +/** + * Return true iff variable <b>var</b> should be written out when we + * are writing our configuration to disk, to a controller, or via the + * --dump-config command. + * + * This option may be set because a variable is hidden, or because it is + * derived from another variable which will already be written out. + **/ +static bool +config_var_is_dumpable(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NODUMP); +} + +/* + * Functions to assign config options. + */ + +/** <b>c</b>-\>key is known to be a real key. Update <b>options</b> + * with <b>c</b>-\>value and return 0, or return -1 if bad value. + * + * Called from config_assign_line() and option_reset(). + */ +static int +config_assign_value(const config_mgr_t *mgr, void *options, + config_line_t *c, char **msg) +{ + const managed_var_t *var; + + CONFIG_CHECK(mgr, options); + + var = config_mgr_find_var(mgr, c->key, true, NULL); + tor_assert(var); + tor_assert(!strcmp(c->key, var->cvar->member.name)); + void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx); + + return struct_var_kvassign(object, c, msg, &var->cvar->member); +} + +/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments + * to it will replace old ones. */ +static void +config_mark_lists_fragile(const config_mgr_t *mgr, void *options) +{ + tor_assert(mgr); + tor_assert(options); + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + void *object = config_mgr_get_obj_mutable(mgr, options, mv->object_idx); + struct_var_mark_fragile(object, &mv->cvar->member); + } SMARTLIST_FOREACH_END(mv); +} + +/** + * Log a warning that declaring that the option called <b>what</b> + * is deprecated because of the reason in <b>why</b>. + * + * (Both arguments must be non-NULL.) + **/ +void +warn_deprecated_option(const char *what, const char *why) +{ + const char *space = (why && strlen(why)) ? " " : ""; + log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely " + "be removed in a future version of Tor.%s%s (If you think this is " + "a mistake, please let us know!)", + what, space, why); +} + +/** If <b>c</b> is a syntactically valid configuration line, update + * <b>options</b> with its value and return 0. Otherwise return -1 for bad + * key, -2 for bad value. + * + * If <b>clear_first</b> is set, clear the value first. Then if + * <b>use_defaults</b> is set, set the value to the default. + * + * Called from config_assign(). + */ +static int +config_assign_line(const config_mgr_t *mgr, void *options, + config_line_t *c, unsigned flags, + bitarray_t *options_seen, char **msg) +{ + const unsigned use_defaults = flags & CAL_USE_DEFAULTS; + const unsigned clear_first = flags & CAL_CLEAR_FIRST; + const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS; + const managed_var_t *mvar; + + CONFIG_CHECK(mgr, options); + + int var_index = -1; + mvar = config_mgr_find_var(mgr, c->key, true, &var_index); + if (!mvar) { + const config_format_t *fmt = mgr->toplevel; + if (fmt->extra) { + void *lvalue = STRUCT_VAR_P(options, fmt->extra->offset); + log_info(LD_CONFIG, + "Found unrecognized option '%s'; saving it.", c->key); + config_line_append((config_line_t**)lvalue, c->key, c->value); + return 0; + } else { + tor_asprintf(msg, + "Unknown option '%s'. Failing.", c->key); + return -1; + } + } + + const config_var_t *cvar = mvar->cvar; + tor_assert(cvar); + + /* Put keyword into canonical case. */ + if (strcmp(cvar->member.name, c->key)) { + tor_free(c->key); + c->key = tor_strdup(cvar->member.name); + } + + const char *deprecation_msg; + if (warn_deprecations && + (deprecation_msg = config_find_deprecation(mgr, cvar->member.name))) { + warn_deprecated_option(cvar->member.name, deprecation_msg); + } + + if (!strlen(c->value)) { + /* reset or clear it, then return */ + if (!clear_first) { + if (! config_var_is_replaced_on_set(cvar) && + c->command != CONFIG_LINE_CLEAR) { + /* We got an empty linelist from the torrc or command line. + As a special case, call this an error. Warn and ignore. */ + log_warn(LD_CONFIG, + "Linelist option '%s' has no value. Skipping.", c->key); + } else { /* not already cleared */ + config_reset(mgr, options, mvar, use_defaults); + } + } + return 0; + } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) { + // This block is unreachable, since a CLEAR line always has an + // empty value, and so will trigger be handled by the previous + // "if (!strlen(c->value))" block. + + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + config_reset(mgr, options, mvar, use_defaults); + // LCOV_EXCL_STOP + } + + if (options_seen && config_var_is_replaced_on_set(cvar)) { + /* We're tracking which options we've seen, and this option is not + * supposed to occur more than once. */ + tor_assert(var_index >= 0); + if (bitarray_is_set(options_seen, var_index)) { + log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last " + "value will be ignored.", cvar->member.name); + } + bitarray_set(options_seen, var_index); + } + + if (config_assign_value(mgr, options, c, msg) < 0) + return -2; + return 0; +} + +/** Restore the option named <b>key</b> in options to its default value. + * Called from config_assign(). */ +STATIC void +config_reset_line(const config_mgr_t *mgr, void *options, + const char *key, int use_defaults) +{ + const managed_var_t *var; + + CONFIG_CHECK(mgr, options); + + var = config_mgr_find_var(mgr, key, true, NULL); + if (!var) + return; /* give error on next pass. */ + + config_reset(mgr, options, var, use_defaults); +} + +/** Return true iff value needs to be quoted and escaped to be used in + * a configuration file. */ +static int +config_value_needs_escape(const char *value) +{ + if (*value == '\"') + return 1; + while (*value) { + switch (*value) + { + case '\r': + case '\n': + case '#': + /* Note: quotes and backspaces need special handling when we are using + * quotes, not otherwise, so they don't trigger escaping on their + * own. */ + return 1; + default: + if (!TOR_ISPRINT(*value)) + return 1; + } + ++value; + } + return 0; +} + +/** Return newly allocated line or lines corresponding to <b>key</b> in the + * configuration <b>options</b>. If <b>escape_val</b> is true and a + * value needs to be quoted before it's put in a config file, quote and + * escape that value. Return NULL if no such key exists. */ +config_line_t * +config_get_assigned_option(const config_mgr_t *mgr, const void *options, + const char *key, int escape_val) +{ + const managed_var_t *var; + config_line_t *result; + + tor_assert(options && key); + + CONFIG_CHECK(mgr, options); + + var = config_mgr_find_var(mgr, key, true, NULL); + if (!var) { + log_warn(LD_CONFIG, "Unknown option '%s'. Failing.", key); + return NULL; + } + if (! config_var_is_gettable(var->cvar)) { + log_warn(LD_CONFIG, "Option '%s' is obsolete or unfetchable. Failing.", + key); + return NULL; + } + const void *object = config_mgr_get_obj(mgr, options, var->object_idx); + + result = struct_var_kvencode(object, &var->cvar->member); + + if (escape_val) { + config_line_t *line; + for (line = result; line; line = line->next) { + if (line->value && config_value_needs_escape(line->value)) { + char *newval = esc_for_log(line->value); + tor_free(line->value); + line->value = newval; + } + } + } + + return result; +} +/** Iterate through the linked list of requested options <b>list</b>. + * For each item, convert as appropriate and assign to <b>options</b>. + * If an item is unrecognized, set *msg and return -1 immediately, + * else return 0 for success. + * + * If <b>clear_first</b>, interpret config options as replacing (not + * extending) their previous values. If <b>clear_first</b> is set, + * then <b>use_defaults</b> to decide if you set to defaults after + * clearing, or make the value 0 or NULL. + * + * Here are the use cases: + * 1. A non-empty AllowInvalid line in your torrc. Appends to current + * if linelist, replaces current if csv. + * 2. An empty AllowInvalid line in your torrc. Should clear it. + * 3. "RESETCONF AllowInvalid" sets it to default. + * 4. "SETCONF AllowInvalid" makes it NULL. + * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo". + * + * Use_defaults Clear_first + * 0 0 "append" + * 1 0 undefined, don't use + * 0 1 "set to null first" + * 1 1 "set to defaults first" + * Return 0 on success, -1 on bad key, -2 on bad value. + * + * As an additional special case, if a LINELIST config option has + * no value and clear_first is 0, then warn and ignore it. + */ + +/* +There are three call cases for config_assign() currently. + +Case one: Torrc entry +options_init_from_torrc() calls config_assign(0, 0) + calls config_assign_line(0, 0). + if value is empty, calls config_reset(0) and returns. + calls config_assign_value(), appends. + +Case two: setconf +options_trial_assign() calls config_assign(0, 1) + calls config_reset_line(0) + calls config_reset(0) + calls option_clear(). + calls config_assign_line(0, 1). + if value is empty, returns. + calls config_assign_value(), appends. + +Case three: resetconf +options_trial_assign() calls config_assign(1, 1) + calls config_reset_line(1) + calls config_reset(1) + calls option_clear(). + calls config_assign_value(default) + calls config_assign_line(1, 1). + returns. +*/ +int +config_assign(const config_mgr_t *mgr, void *options, config_line_t *list, + unsigned config_assign_flags, char **msg) +{ + config_line_t *p; + bitarray_t *options_seen; + const int n_options = config_count_options(mgr); + const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST; + const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS; + + CONFIG_CHECK(mgr, options); + + /* pass 1: normalize keys */ + for (p = list; p; p = p->next) { + const char *full = config_expand_abbrev(mgr, p->key, 0, 1); + if (strcmp(full,p->key)) { + tor_free(p->key); + p->key = tor_strdup(full); + } + } + + /* pass 2: if we're reading from a resetting source, clear all + * mentioned config options, and maybe set to their defaults. */ + if (clear_first) { + for (p = list; p; p = p->next) + config_reset_line(mgr, options, p->key, use_defaults); + } + + options_seen = bitarray_init_zero(n_options); + /* pass 3: assign. */ + while (list) { + int r; + if ((r=config_assign_line(mgr, options, list, config_assign_flags, + options_seen, msg))) { + bitarray_free(options_seen); + return r; + } + list = list->next; + } + bitarray_free(options_seen); + + /** Now we're done assigning a group of options to the configuration. + * Subsequent group assignments should _replace_ linelists, not extend + * them. */ + config_mark_lists_fragile(mgr, options); + + return 0; +} + +/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent. + * Called from config_reset() and config_free(). */ +static void +config_clear(const config_mgr_t *mgr, void *options, const managed_var_t *var) +{ + void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx); + struct_var_free(object, &var->cvar->member); +} + +/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if + * <b>use_defaults</b>, set it to its default value. + * Called by config_init() and option_reset_line() and option_assign_line(). */ +static void +config_reset(const config_mgr_t *mgr, void *options, + const managed_var_t *var, int use_defaults) +{ + config_line_t *c; + char *msg = NULL; + CONFIG_CHECK(mgr, options); + config_clear(mgr, options, var); /* clear it first */ + + if (!use_defaults) + return; /* all done */ + + if (var->cvar->initvalue) { + c = tor_malloc_zero(sizeof(config_line_t)); + c->key = tor_strdup(var->cvar->member.name); + c->value = tor_strdup(var->cvar->initvalue); + if (config_assign_value(mgr, options, c, &msg) < 0) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Failed to assign default: %s", msg); + tor_free(msg); /* if this happens it's a bug */ + // LCOV_EXCL_STOP + } + config_free_lines(c); + } +} + +/** Release storage held by <b>options</b>. */ +void +config_free_(const config_mgr_t *mgr, void *options) +{ + if (!options) + return; + + tor_assert(mgr); + + if (mgr->toplevel->clear_fn) { + mgr->toplevel->clear_fn(mgr, options); + } + config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, options); + if (suitep) { + tor_assert(smartlist_len((*suitep)->configs) == + smartlist_len(mgr->subconfigs)); + SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) { + void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx); + if (fmt->clear_fn) { + fmt->clear_fn(mgr, obj); + } + } SMARTLIST_FOREACH_END(fmt); + } + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + config_clear(mgr, options, mv); + } SMARTLIST_FOREACH_END(mv); + + if (mgr->toplevel->extra) { + config_line_t **linep = STRUCT_VAR_P(options, + mgr->toplevel->extra->offset); + config_free_lines(*linep); + *linep = NULL; + } + + if (suitep) { + SMARTLIST_FOREACH((*suitep)->configs, void *, obj, tor_free(obj)); + config_suite_free(*suitep); + } + + tor_free(options); +} + +/** Return true iff the option <b>name</b> has the same value in <b>o1</b> + * and <b>o2</b>. Must not be called for LINELIST_S or OBSOLETE options. + */ +int +config_is_same(const config_mgr_t *mgr, + const void *o1, const void *o2, + const char *name) +{ + CONFIG_CHECK(mgr, o1); + CONFIG_CHECK(mgr, o2); + + const managed_var_t *var = config_mgr_find_var(mgr, name, true, NULL); + if (!var) { + return true; + } + const void *obj1 = config_mgr_get_obj(mgr, o1, var->object_idx); + const void *obj2 = config_mgr_get_obj(mgr, o2, var->object_idx); + + return struct_var_eq(obj1, obj2, &var->cvar->member); +} + +/** + * Return a list of the options which have changed between <b>options1</b> and + * <b>options2</b>. If an option has reverted to its default value, it has a + * value entry of NULL. + * + * <b>options1</b> and <b>options2</b> must be top-level configuration objects + * of the type managed by <b>mgr</b>. + **/ +config_line_t * +config_get_changes(const config_mgr_t *mgr, + const void *options1, const void *options2) +{ + config_line_t *result = NULL; + config_line_t **next = &result; + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) { + if (! config_var_should_list_changes(mv->cvar)) { + /* something else will check this var, or it doesn't need checking */ + continue; + } + const void *obj1 = config_mgr_get_obj(mgr, options1, mv->object_idx); + const void *obj2 = config_mgr_get_obj(mgr, options2, mv->object_idx); + + if (struct_var_eq(obj1, obj2, &mv->cvar->member)) { + continue; + } + + const char *varname = mv->cvar->member.name; + config_line_t *line = + config_get_assigned_option(mgr, options2, varname, 1); + + if (line) { + *next = line; + } else { + *next = tor_malloc_zero(sizeof(config_line_t)); + (*next)->key = tor_strdup(varname); + } + while (*next) + next = &(*next)->next; + } SMARTLIST_FOREACH_END(mv); + + return result; +} + +/** Copy storage held by <b>old</b> into a new or_options_t and return it. */ +void * +config_dup(const config_mgr_t *mgr, const void *old) +{ + void *newopts; + + newopts = config_new(mgr); + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) { + if (! config_var_needs_copy(mv->cvar)) { + // Something else will copy this option, or it doesn't need copying. + continue; + } + const void *oldobj = config_mgr_get_obj(mgr, old, mv->object_idx); + void *newobj = config_mgr_get_obj_mutable(mgr, newopts, mv->object_idx); + if (struct_var_copy(newobj, oldobj, &mv->cvar->member) < 0) { + // LCOV_EXCL_START + log_err(LD_BUG, "Unable to copy value for %s.", + mv->cvar->member.name); + tor_assert_unreached(); + // LCOV_EXCL_STOP + } + } SMARTLIST_FOREACH_END(mv); + + return newopts; +} +/** Set all vars in the configuration object <b>options</b> to their default + * values. */ +void +config_init(const config_mgr_t *mgr, void *options) +{ + CONFIG_CHECK(mgr, options); + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!mv->cvar->initvalue) + continue; /* defaults to NULL or 0 */ + config_reset(mgr, options, mv, 1); + } SMARTLIST_FOREACH_END(mv); +} + +/** Allocate and return a new string holding the written-out values of the vars + * in 'options'. If 'minimal', do not write out any default-valued vars. + * Else, if comment_defaults, write default values as comments. + */ +char * +config_dump(const config_mgr_t *mgr, const void *default_options, + const void *options, int minimal, + int comment_defaults) +{ + const config_format_t *fmt = mgr->toplevel; + smartlist_t *elements; + const void *defaults = default_options; + void *defaults_tmp = NULL; + config_line_t *line, *assigned; + char *result; + char *msg = NULL; + + if (defaults == NULL) { + defaults = defaults_tmp = config_new(mgr); + config_init(mgr, defaults_tmp); + } + + /* XXX use a 1 here so we don't add a new log line while dumping */ + if (default_options == NULL) { + if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) { + // LCOV_EXCL_START + log_err(LD_BUG, "Failed to validate default config: %s", msg); + tor_free(msg); + tor_assert(0); + // LCOV_EXCL_STOP + } + } + + elements = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) { + int comment_option = 0; + /* Don't save 'hidden' control variables. */ + if (! config_var_is_dumpable(mv->cvar)) + continue; + const char *name = mv->cvar->member.name; + if (minimal && config_is_same(mgr, options, defaults, name)) + continue; + else if (comment_defaults && + config_is_same(mgr, options, defaults, name)) + comment_option = 1; + + line = assigned = + config_get_assigned_option(mgr, options, name, 1); + + for (; line; line = line->next) { + if (!strcmpstart(line->key, "__")) { + /* This check detects "hidden" variables inside LINELIST_V structures. + */ + continue; + } + smartlist_add_asprintf(elements, "%s%s %s\n", + comment_option ? "# " : "", + line->key, line->value); + } + config_free_lines(assigned); + } SMARTLIST_FOREACH_END(mv); + + if (fmt->extra) { + line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->offset); + for (; line; line = line->next) { + smartlist_add_asprintf(elements, "%s %s\n", line->key, line->value); + } + } + + result = smartlist_join_strings(elements, "", 0, NULL); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + config_free(mgr, defaults_tmp); + return result; +} + +/** + * Return true if every member of <b>options</b> is in-range and well-formed. + * Return false otherwise. Log errors at level <b>severity</b>. + */ +bool +config_check_ok(const config_mgr_t *mgr, const void *options, int severity) +{ + bool all_ok = true; + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!struct_var_ok(options, &mv->cvar->member)) { + log_fn(severity, LD_BUG, "Invalid value for %s", + mv->cvar->member.name); + all_ok = false; + } + } SMARTLIST_FOREACH_END(mv); + + return all_ok; +} diff --git a/src/lib/confmgt/confparse.h b/src/lib/confmgt/confparse.h new file mode 100644 index 0000000000..2332f69790 --- /dev/null +++ b/src/lib/confmgt/confparse.h @@ -0,0 +1,212 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file confparse.h + * + * \brief Header for confparse.c. + */ + +#ifndef TOR_CONFPARSE_H +#define TOR_CONFPARSE_H + +#include "lib/conf/conftypes.h" +#include "lib/conf/confmacros.h" +#include "lib/testsupport/testsupport.h" + +/** + * An abbreviation or alias for a configuration option. + **/ +typedef struct config_abbrev_t { + /** The option name as abbreviated. Not case-sensitive. */ + const char *abbreviated; + /** The full name of the option. Not case-sensitive. */ + const char *full; + /** True if this abbreviation should only be allowed on the command line. */ + int commandline_only; + /** True if we should warn whenever this abbreviation is used. */ + int warn; +} config_abbrev_t; + +/** + * A note that a configuration option is deprecated, with an explanation why. + */ +typedef struct config_deprecation_t { + /** The option that is deprecated. */ + const char *name; + /** A user-facing string explaining why the option is deprecated. */ + const char *why_deprecated; +} config_deprecation_t; + +/** + * Handy macro for declaring "In the config file or on the command line, you + * can abbreviate <b>tok</b>s as <b>tok</b>". Used inside an array of + * config_abbrev_t. + * + * For example, to declare "NumCpu" as an abbreviation for "NumCPUs", + * you can say PLURAL(NumCpu). + **/ +#define PLURAL(tok) { #tok, #tok "s", 0, 0 } + +/** + * Type of a callback to validate whether a given configuration is + * well-formed and consistent. + * + * The configuration to validate is passed as <b>newval</b>. The previous + * configuration, if any, is provided in <b>oldval</b>. The + * <b>default_val</b> argument receives a configuration object initialized + * with default values for all its fields. The <b>from_setconf</b> argument + * is true iff the input comes from a SETCONF controller command. + * + * On success, return 0. On failure, set *<b>msg_out</b> to a newly allocated + * error message, and return -1. + * + * REFACTORING NOTE: Currently, this callback type is only used from inside + * config_dump(); later in our refactoring, it will be cleaned up and used + * more generally. + */ +typedef int (*validate_fn_t)(void *oldval, + void *newval, + void *default_val, + int from_setconf, + char **msg_out); + +struct config_mgr_t; + +/** + * Callback to clear all non-managed fields of a configuration object. + * + * <b>obj</b> is the configuration object whose non-managed fields should be + * cleared. + * + * (Regular fields get cleared by config_reset(), but you might have fields + * in the object that do not correspond to configuration variables. If those + * fields need to be cleared or freed, this is where to do it.) + */ +typedef void (*clear_cfg_fn_t)(const struct config_mgr_t *mgr, void *obj); + +/** Information on the keys, value types, key-to-struct-member mappings, + * variable descriptions, validation functions, and abbreviations for a + * configuration or storage format. */ +typedef struct config_format_t { + size_t size; /**< Size of the struct that everything gets parsed into. */ + struct_magic_decl_t magic; /**< Magic number info for this struct. */ + const config_abbrev_t *abbrevs; /**< List of abbreviations that we expand + * when parsing this format. */ + const config_deprecation_t *deprecations; /** List of deprecated options */ + const config_var_t *vars; /**< List of variables we recognize, their default + * values, and where we stick them in the + * structure. */ + validate_fn_t validate_fn; /**< Function to validate config. */ + clear_cfg_fn_t clear_fn; /**< Function to clear the configuration. */ + /** If present, extra denotes a LINELIST variable for unrecognized + * lines. Otherwise, unrecognized lines are an error. */ + const struct_member_t *extra; + /** The position of a config_suite_t pointer within the toplevel object, + * or -1 if there is no such pointer. */ + ptrdiff_t config_suite_offset; +} config_format_t; + +/** + * A collection of config_format_t objects to describe several objects + * that are all configured with the same configuration file. + * + * (NOTE: for now, this only handles a single config_format_t.) + **/ +typedef struct config_mgr_t config_mgr_t; + +config_mgr_t *config_mgr_new(const config_format_t *toplevel_fmt); +void config_mgr_free_(config_mgr_t *mgr); +int config_mgr_add_format(config_mgr_t *mgr, + const config_format_t *fmt); +void config_mgr_freeze(config_mgr_t *mgr); +#define config_mgr_free(mgr) \ + FREE_AND_NULL(config_mgr_t, config_mgr_free_, (mgr)) +struct smartlist_t *config_mgr_list_vars(const config_mgr_t *mgr); +struct smartlist_t *config_mgr_list_deprecated_vars(const config_mgr_t *mgr); + +/** A collection of managed configuration objects. */ +typedef struct config_suite_t config_suite_t; + +/** + * Flag for config_assign: if set, then "resetting" an option changes it to + * its default value, as specified in the config_var_t. Otherwise, + * "resetting" an option changes it to a type-dependent null value -- + * typically 0 or NULL. + * + * (An option is "reset" when it is set to an empty value, or as described in + * CAL_CLEAR_FIRST). + **/ +#define CAL_USE_DEFAULTS (1u<<0) +/** + * Flag for config_assign: if set, then we reset every provided config + * option before we set it. + * + * For example, if this flag is not set, then passing a multi-line option to + * config_assign will cause any previous value to be extended. But if this + * flag is set, then a multi-line option will replace any previous value. + **/ +#define CAL_CLEAR_FIRST (1u<<1) +/** + * Flag for config_assign: if set, we warn about deprecated options. + **/ +#define CAL_WARN_DEPRECATIONS (1u<<2) + +void *config_new(const config_mgr_t *fmt); +void config_free_(const config_mgr_t *fmt, void *options); +#define config_free(mgr, options) do { \ + config_free_((mgr), (options)); \ + (options) = NULL; \ + } while (0) + +struct config_line_t *config_get_assigned_option(const config_mgr_t *mgr, + const void *options, const char *key, + int escape_val); +int config_is_same(const config_mgr_t *fmt, + const void *o1, const void *o2, + const char *name); +struct config_line_t *config_get_changes(const config_mgr_t *mgr, + const void *options1, const void *options2); +void config_init(const config_mgr_t *mgr, void *options); +void *config_dup(const config_mgr_t *mgr, const void *old); +char *config_dump(const config_mgr_t *mgr, const void *default_options, + const void *options, int minimal, + int comment_defaults); +bool config_check_ok(const config_mgr_t *mgr, const void *options, + int severity); +int config_assign(const config_mgr_t *mgr, void *options, + struct config_line_t *list, + unsigned flags, char **msg); +const char *config_find_deprecation(const config_mgr_t *mgr, + const char *key); +const char *config_find_option_name(const config_mgr_t *mgr, + const char *key); +const char *config_expand_abbrev(const config_mgr_t *mgr, + const char *option, + int command_line, int warn_obsolete); +void warn_deprecated_option(const char *what, const char *why); + +bool config_var_is_settable(const config_var_t *var); +bool config_var_is_listable(const config_var_t *var); + +/* Helper macros to compare an option across two configuration objects */ +#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt) +#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt) +#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt)) +#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt) +#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt) +#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt) + +#ifdef CONFPARSE_PRIVATE +STATIC void config_reset_line(const config_mgr_t *mgr, void *options, + const char *key, int use_defaults); +STATIC void *config_mgr_get_obj_mutable(const config_mgr_t *mgr, + void *toplevel, int idx); +STATIC const void *config_mgr_get_obj(const config_mgr_t *mgr, + const void *toplevel, int idx); +#endif /* defined(CONFPARSE_PRIVATE) */ + +#endif /* !defined(TOR_CONFPARSE_H) */ diff --git a/src/lib/confmgt/include.am b/src/lib/confmgt/include.am new file mode 100644 index 0000000000..81cd868e5e --- /dev/null +++ b/src/lib/confmgt/include.am @@ -0,0 +1,27 @@ +noinst_LIBRARIES += src/lib/libtor-confmgt.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-confmgt-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_confmgt_a_SOURCES = \ + src/lib/confmgt/confparse.c \ + src/lib/confmgt/structvar.c \ + src/lib/confmgt/type_defs.c \ + src/lib/confmgt/typedvar.c \ + src/lib/confmgt/unitparse.c + +src_lib_libtor_confmgt_testing_a_SOURCES = \ + $(src_lib_libtor_confmgt_a_SOURCES) +src_lib_libtor_confmgt_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_confmgt_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/confmgt/confparse.h \ + src/lib/confmgt/structvar.h \ + src/lib/confmgt/type_defs.h \ + src/lib/confmgt/typedvar.h \ + src/lib/confmgt/unitparse.h \ + src/lib/confmgt/var_type_def_st.h diff --git a/src/lib/confmgt/structvar.c b/src/lib/confmgt/structvar.c new file mode 100644 index 0000000000..7a3b8c7df2 --- /dev/null +++ b/src/lib/confmgt/structvar.c @@ -0,0 +1,221 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file structvar.c + * @brief Functions to manipulate named and typed elements of + * a structure. + * + * These functions represent a low-level API for accessing a member of a + * structure. They use typedvar.c to work, and they are used in turn by the + * configuration system to examine and set fields in configuration objects + * used by individual modules. + * + * Almost no code should call these directly. + **/ + +#include "orconfig.h" +#include "lib/confmgt/structvar.h" +#include "lib/cc/compat_compiler.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/typedvar.h" +#include "lib/log/util_bug.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include <stddef.h> + +/** + * Set the 'magic number' on <b>object</b> to correspond to decl. + **/ +void +struct_set_magic(void *object, const struct_magic_decl_t *decl) +{ + tor_assert(object); + tor_assert(decl); + uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset); + *ptr = decl->magic_val; +} + +/** + * Assert that the 'magic number' on <b>object</b> to corresponds to decl. + **/ +void +struct_check_magic(const void *object, const struct_magic_decl_t *decl) +{ + tor_assert(object); + tor_assert(decl); + + const uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset); + tor_assertf(*ptr == decl->magic_val, + "Bad magic number on purported %s object. " + "Expected %"PRIu32"x but got %"PRIu32"x.", + decl->typename, decl->magic_val, *ptr); +} + +/** + * Return a mutable pointer to the member of <b>object</b> described + * by <b>member</b>. + **/ +void * +struct_get_mptr(void *object, const struct_member_t *member) +{ + tor_assert(object); + return STRUCT_VAR_P(object, member->offset); +} + +/** + * Return a const pointer to the member of <b>object</b> described + * by <b>member</b>. + **/ +const void * +struct_get_ptr(const void *object, const struct_member_t *member) +{ + tor_assert(object); + return STRUCT_VAR_P(object, member->offset); +} + +/** + * Helper: given a struct_member_t, look up the type definition for its + * variable. + */ +static const var_type_def_t * +get_type_def(const struct_member_t *member) +{ + if (member->type_def) + return member->type_def; + + return lookup_type_def(member->type); +} + +/** + * (As typed_var_free, but free and clear the member of <b>object</b> defined + * by <b>member</b>.) + **/ +void +struct_var_free(void *object, const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + + typed_var_free(p, def); +} + +/** + * (As typed_var_copy, but copy from <b>src</b> to <b>dest</b> the member + * defined by <b>member</b>.) + **/ +int +struct_var_copy(void *dest, const void *src, const struct_member_t *member) +{ + void *p_dest = struct_get_mptr(dest, member); + const void *p_src = struct_get_ptr(src, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_copy(p_dest, p_src, def); +} + +/** + * (As typed_var_eq, but compare the members of <b>a</b> and <b>b</b> + * defined by <b>member</b>.) + **/ +bool +struct_var_eq(const void *a, const void *b, const struct_member_t *member) +{ + const void *p_a = struct_get_ptr(a, member); + const void *p_b = struct_get_ptr(b, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_eq(p_a, p_b, def); +} + +/** + * (As typed_var_ok, but validate the member of <b>object</b> defined by + * <b>member</b>.) + **/ +bool +struct_var_ok(const void *object, const struct_member_t *member) +{ + const void *p = struct_get_ptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_ok(p, def); +} + +/** + * (As typed_var_kvassign, but assign a value to the member of <b>object</b> + * defined by <b>member</b>.) + **/ +int +struct_var_kvassign(void *object, const struct config_line_t *line, + char **errmsg, + const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_kvassign(p, line, errmsg, def); +} + +/** + * (As typed_var_kvencode, but encode the value of the member of <b>object</b> + * defined by <b>member</b>.) + **/ +struct config_line_t * +struct_var_kvencode(const void *object, const struct_member_t *member) +{ + const void *p = struct_get_ptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_kvencode(member->name, p, def); +} + +/** + * Mark the field in <b>object</b> determined by <b>member</b> -- a variable + * that ordinarily would be extended by assignment -- as "fragile", so that it + * will get replaced by the next assignment instead. + */ +void +struct_var_mark_fragile(void *object, const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + return typed_var_mark_fragile(p, def); +} + +/** + * Return the official name of this struct member. + **/ +const char * +struct_var_get_name(const struct_member_t *member) +{ + return member->name; +} + +/** + * Return the type name for this struct member. + * + * Do not use the output of this function to inspect a type within Tor. It is + * suitable for debugging, informing the controller or user of a variable's + * type, etc. + **/ +const char * +struct_var_get_typename(const struct_member_t *member) +{ + const var_type_def_t *def = get_type_def(member); + + return def ? def->name : NULL; +} + +/** Return all of the flags set for this struct member. */ +uint32_t +struct_var_get_flags(const struct_member_t *member) +{ + const var_type_def_t *def = get_type_def(member); + + return def ? def->flags : 0; +} diff --git a/src/lib/confmgt/structvar.h b/src/lib/confmgt/structvar.h new file mode 100644 index 0000000000..bcb4b58c3f --- /dev/null +++ b/src/lib/confmgt/structvar.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file structvar.h + * @brief Header for lib/confmgt/structvar.c + **/ + +#ifndef TOR_LIB_CONFMGT_STRUCTVAR_H +#define TOR_LIB_CONFMGT_STRUCTVAR_H + +struct struct_magic_decl_t; +struct struct_member_t; +struct config_line_t; + +#include <stdbool.h> +#include "lib/cc/torint.h" + +void struct_set_magic(void *object, + const struct struct_magic_decl_t *decl); +void struct_check_magic(const void *object, + const struct struct_magic_decl_t *decl); + +void *struct_get_mptr(void *object, + const struct struct_member_t *member); +const void *struct_get_ptr(const void *object, + const struct struct_member_t *member); + +void struct_var_free(void *object, + const struct struct_member_t *member); +int struct_var_copy(void *dest, const void *src, + const struct struct_member_t *member); +bool struct_var_eq(const void *a, const void *b, + const struct struct_member_t *member); +bool struct_var_ok(const void *object, + const struct struct_member_t *member); +void struct_var_mark_fragile(void *object, + const struct struct_member_t *member); + +const char *struct_var_get_name(const struct struct_member_t *member); +const char *struct_var_get_typename(const struct struct_member_t *member); +uint32_t struct_var_get_flags(const struct struct_member_t *member); + +int struct_var_kvassign(void *object, const struct config_line_t *line, + char **errmsg, + const struct struct_member_t *member); +struct config_line_t *struct_var_kvencode( + const void *object, + const struct struct_member_t *member); + +#endif /* !defined(TOR_LIB_CONFMGT_STRUCTVAR_H) */ diff --git a/src/lib/confmgt/type_defs.c b/src/lib/confmgt/type_defs.c new file mode 100644 index 0000000000..5066e12265 --- /dev/null +++ b/src/lib/confmgt/type_defs.c @@ -0,0 +1,791 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file type_defs.c + * @brief Definitions for various low-level configuration types. + * + * This module creates a number of var_type_def_t objects, to be used by + * typedvar.c in manipulating variables. + * + * The types here are common types that can be implemented with Tor's + * low-level functionality. To define new types, see var_type_def_st.h. + **/ + +#include "orconfig.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/typedvar.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/unitparse.h" + +#include "lib/cc/compat_compiler.h" +#include "lib/conf/conftypes.h" +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/time_fmt.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/parse_int.h" +#include "lib/string/printf.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include <stddef.h> +#include <string.h> +#include <errno.h> + +////// +// CONFIG_TYPE_STRING +// CONFIG_TYPE_FILENAME +// +// These two types are the same for now, but they have different names. +// +// Warning: For this type, the default value (NULL) and "" are considered +// different values. That is generally risky, and best avoided for other +// types in the future. +////// + +static int +string_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)params; + (void)errmsg; + (void)key; + char **p = (char**)target; + *p = tor_strdup(value); + return 0; +} + +static char * +string_encode(const void *value, const void *params) +{ + (void)params; + const char **p = (const char**)value; + return *p ? tor_strdup(*p) : NULL; +} + +static void +string_clear(void *value, const void *params) +{ + (void)params; + char **p = (char**)value; + tor_free(*p); // sets *p to NULL. +} + +static const var_type_fns_t string_fns = { + .parse = string_parse, + .encode = string_encode, + .clear = string_clear, +}; + +///// +// CONFIG_TYPE_INT +// CONFIG_TYPE_POSINT +// +// These types are implemented as int, possibly with a restricted range. +///// + +typedef struct int_type_params_t { + int minval; + int maxval; +} int_parse_params_t; + +static const int_parse_params_t INT_PARSE_UNRESTRICTED = { + .minval = INT_MIN, + .maxval = INT_MAX, +}; + +static const int_parse_params_t INT_PARSE_POSINT = { + .minval = 0, + .maxval = INT_MAX, +}; + +static int +int_parse(void *target, const char *value, char **errmsg, const void *params, + const char *key) +{ + (void)key; + const int_parse_params_t *pp; + if (params) { + pp = params; + } else { + pp = &INT_PARSE_UNRESTRICTED; + } + int *p = target; + int ok=0; + *p = (int)tor_parse_long(value, 10, pp->minval, pp->maxval, &ok, NULL); + if (!ok) { + tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.", + value); + return -1; + } + return 0; +} + +static char * +int_encode(const void *value, const void *params) +{ + (void)params; + int v = *(int*)value; + char *result; + tor_asprintf(&result, "%d", v); + return result; +} + +static void +int_clear(void *value, const void *params) +{ + (void)params; + *(int*)value = 0; +} + +static bool +int_ok(const void *value, const void *params) +{ + const int_parse_params_t *pp = params; + if (pp) { + int v = *(int*)value; + return pp->minval <= v && v <= pp->maxval; + } else { + return true; + } +} + +static const var_type_fns_t int_fns = { + .parse = int_parse, + .encode = int_encode, + .clear = int_clear, + .ok = int_ok, +}; + +///// +// CONFIG_TYPE_UINT64 +// +// This type is an unrestricted u64. +///// + +static int +uint64_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)params; + (void)errmsg; + (void)key; + uint64_t *p = target; + int ok=0; + *p = tor_parse_uint64(value, 10, 0, UINT64_MAX, &ok, NULL); + if (!ok) { + tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.", + value); + return -1; + } + return 0; +} + +static char * +uint64_encode(const void *value, const void *params) +{ + (void)params; + uint64_t v = *(uint64_t*)value; + char *result; + tor_asprintf(&result, "%"PRIu64, v); + return result; +} + +static void +uint64_clear(void *value, const void *params) +{ + (void)params; + *(uint64_t*)value = 0; +} + +static const var_type_fns_t uint64_fns = { + .parse = uint64_parse, + .encode = uint64_encode, + .clear = uint64_clear, +}; + +///// +// CONFIG_TYPE_INTERVAL +// CONFIG_TYPE_MSEC_INTERVAL +// CONFIG_TYPE_MEMUNIT +// +// These types are implemented using the config_parse_units() function. +// The intervals are stored as ints, whereas memory units are stored as +// uint64_ts. +///// + +static int +units_parse_u64(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)key; + const unit_table_t *table = params; + tor_assert(table); + uint64_t *v = (uint64_t*)target; + int ok=1; + *v = config_parse_units(value, table, &ok); + if (!ok) { + *errmsg = tor_strdup("Provided value is malformed or out of bounds."); + return -1; + } + return 0; +} + +static int +units_parse_int(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)key; + const unit_table_t *table = params; + tor_assert(table); + int *v = (int*)target; + int ok=1; + uint64_t u64 = config_parse_units(value, table, &ok); + if (!ok) { + *errmsg = tor_strdup("Provided value is malformed or out of bounds."); + return -1; + } + if (u64 > INT_MAX) { + tor_asprintf(errmsg, "Provided value %s is too large", value); + return -1; + } + *v = (int) u64; + return 0; +} + +static bool +units_ok_int(const void *value, const void *params) +{ + (void)params; + int v = *(int*)value; + return v >= 0; +} + +static const var_type_fns_t memunit_fns = { + .parse = units_parse_u64, + .encode = uint64_encode, // doesn't use params + .clear = uint64_clear, // doesn't use params +}; + +static const var_type_fns_t interval_fns = { + .parse = units_parse_int, + .encode = int_encode, // doesn't use params + .clear = int_clear, // doesn't use params, + .ok = units_ok_int // can't use int_ok, since that expects int params. +}; + +///// +// CONFIG_TYPE_DOUBLE +// +// This is a nice simple double. +///// + +static int +double_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)params; + (void)errmsg; + (void)key; + double *v = (double*)target; + char *endptr=NULL; + errno = 0; + *v = strtod(value, &endptr); + if (endptr == value || *endptr != '\0') { + // Either there are no converted characters, or there were some characters + // that didn't get converted. + tor_asprintf(errmsg, "Could not convert %s to a number.", escaped(value)); + return -1; + } + if (errno == ERANGE) { + // strtod will set errno to ERANGE on underflow or overflow. + bool underflow = -.00001 < *v && *v < .00001; + tor_asprintf(errmsg, + "%s is too %s to express as a floating-point number.", + escaped(value), underflow ? "small" : "large"); + return -1; + } + return 0; +} + +static char * +double_encode(const void *value, const void *params) +{ + (void)params; + double v = *(double*)value; + char *result; + tor_asprintf(&result, "%f", v); + return result; +} + +static void +double_clear(void *value, const void *params) +{ + (void)params; + double *v = (double *)value; + *v = 0.0; +} + +static const var_type_fns_t double_fns = { + .parse = double_parse, + .encode = double_encode, + .clear = double_clear, +}; + +///// +// CONFIG_TYPE_BOOL +// CONFIG_TYPE_AUTOBOOL +// +// These types are implemented as a case-insensitive string-to-integer +// mapping. +///// + +typedef struct enumeration_table_t { + const char *name; + int value; +} enumeration_table_t; + +static int +enum_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)key; + const enumeration_table_t *table = params; + int *p = (int *)target; + for (; table->name; ++table) { + if (!strcasecmp(value, table->name)) { + *p = table->value; + return 0; + } + } + tor_asprintf(errmsg, "Unrecognized value %s.", value); + return -1; +} + +static char * +enum_encode(const void *value, const void *params) +{ + int v = *(const int*)value; + const enumeration_table_t *table = params; + for (; table->name; ++table) { + if (v == table->value) + return tor_strdup(table->name); + } + return NULL; // error. +} + +static void +enum_clear(void *value, const void *params) +{ + int *p = (int*)value; + const enumeration_table_t *table = params; + tor_assert(table->name); + *p = table->value; +} + +static bool +enum_ok(const void *value, const void *params) +{ + int v = *(const int*)value; + const enumeration_table_t *table = params; + for (; table->name; ++table) { + if (v == table->value) + return true; + } + return false; +} + +static const enumeration_table_t enum_table_bool[] = { + { "0", 0 }, + { "1", 1 }, + { NULL, 0 }, +}; + +static const enumeration_table_t enum_table_autobool[] = { + { "0", 0 }, + { "1", 1 }, + { "auto", -1 }, + { NULL, 0 }, +}; + +static const var_type_fns_t enum_fns = { + .parse = enum_parse, + .encode = enum_encode, + .clear = enum_clear, + .ok = enum_ok, +}; + +///// +// CONFIG_TYPE_ISOTIME +// +// This is a time_t, encoded in ISO8601 format. +///// + +static int +time_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void) params; + (void) key; + time_t *p = target; + if (parse_iso_time(value, p) < 0) { + tor_asprintf(errmsg, "Invalid time %s", escaped(value)); + return -1; + } + return 0; +} + +static char * +time_encode(const void *value, const void *params) +{ + (void)params; + time_t v = *(const time_t *)value; + char *result = tor_malloc(ISO_TIME_LEN+1); + format_iso_time(result, v); + return result; +} + +static void +time_clear(void *value, const void *params) +{ + (void)params; + time_t *t = value; + *t = 0; +} + +static const var_type_fns_t time_fns = { + .parse = time_parse, + .encode = time_encode, + .clear = time_clear, +}; + +///// +// CONFIG_TYPE_CSV +// +// This type is a comma-separated list of strings, stored in a smartlist_t. +// An empty list may be encoded either as an empty smartlist, or as NULL. +///// + +static int +csv_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)params; + (void)errmsg; + (void)key; + smartlist_t **sl = (smartlist_t**)target; + *sl = smartlist_new(); + smartlist_split_string(*sl, value, ",", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + return 0; +} + +static char * +csv_encode(const void *value, const void *params) +{ + (void)params; + const smartlist_t *sl = *(const smartlist_t **)value; + if (! sl) + return tor_strdup(""); + + return smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL); +} + +static void +csv_clear(void *value, const void *params) +{ + (void)params; + smartlist_t **sl = (smartlist_t**)value; + if (!*sl) + return; + SMARTLIST_FOREACH(*sl, char *, cp, tor_free(cp)); + smartlist_free(*sl); // clears pointer. +} + +static const var_type_fns_t csv_fns = { + .parse = csv_parse, + .encode = csv_encode, + .clear = csv_clear, +}; + +///// +// CONFIG_TYPE_CSV_INTERVAL +// +// This type used to be a list of time intervals, used to determine a download +// schedule. Now, only the first interval counts: everything after the first +// comma is discarded. +///// + +static int +legacy_csv_interval_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)params; + /* We used to have entire smartlists here. But now that all of our + * download schedules use exponential backoff, only the first part + * matters. */ + const char *comma = strchr(value, ','); + const char *val = value; + char *tmp = NULL; + if (comma) { + tmp = tor_strndup(val, comma - val); + val = tmp; + } + + int rv = units_parse_int(target, val, errmsg, &time_units, key); + tor_free(tmp); + return rv; +} + +static const var_type_fns_t legacy_csv_interval_fns = { + .parse = legacy_csv_interval_parse, + .encode = int_encode, + .clear = int_clear, +}; + +///// +// CONFIG_TYPE_LINELIST +// CONFIG_TYPE_LINELIST_S +// CONFIG_TYPE_LINELIST_V +// +// A linelist is a raw config_line_t list. Order is preserved. +// +// The LINELIST type is used for homogeneous lists, where all the lines +// have the same key. +// +// The LINELIST_S and LINELIST_V types are used for the case where multiple +// lines of different keys are kept in a single list, to preserve their +// relative order. The unified list is stored as a "virtual" variable whose +// type is LINELIST_V; the individual sublists are treated as variables of +// type LINELIST_S. +// +// A linelist may be fragile or non-fragile. Assigning a line to a fragile +// linelist replaces the list with the line. If the line has the "APPEND" +// command set on it, or if the list is non-fragile, the line is appended. +// Either way, the new list is non-fragile. +///// + +static int +linelist_kv_parse(void *target, const struct config_line_t *line, + char **errmsg, const void *params) +{ + (void)params; + (void)errmsg; + config_line_t **lines = target; + + if (*lines && (*lines)->fragile) { + if (line->command == CONFIG_LINE_APPEND) { + (*lines)->fragile = 0; + } else { + config_free_lines(*lines); // sets it to NULL + } + } + + config_line_append(lines, line->key, line->value); + return 0; +} + +static int +linelist_kv_virt_noparse(void *target, const struct config_line_t *line, + char **errmsg, const void *params) +{ + (void)target; + (void)line; + (void)params; + *errmsg = tor_strdup("Cannot assign directly to virtual option."); + return -1; +} + +static struct config_line_t * +linelist_kv_encode(const char *key, const void *value, + const void *params) +{ + (void)key; + (void)params; + config_line_t *lines = *(config_line_t **)value; + return config_lines_dup(lines); +} + +static struct config_line_t * +linelist_s_kv_encode(const char *key, const void *value, + const void *params) +{ + (void)params; + config_line_t *lines = *(config_line_t **)value; + return config_lines_dup_and_filter(lines, key); +} + +static void +linelist_clear(void *target, const void *params) +{ + (void)params; + config_line_t **lines = target; + config_free_lines(*lines); // sets it to NULL +} + +static bool +linelist_eq(const void *a, const void *b, const void *params) +{ + (void)params; + const config_line_t *lines_a = *(const config_line_t **)a; + const config_line_t *lines_b = *(const config_line_t **)b; + return config_lines_eq(lines_a, lines_b); +} + +static int +linelist_copy(void *target, const void *value, const void *params) +{ + (void)params; + config_line_t **ptr = (config_line_t **)target; + const config_line_t *val = *(const config_line_t **)value; + config_free_lines(*ptr); + *ptr = config_lines_dup(val); + return 0; +} + +static void +linelist_mark_fragile(void *target, const void *params) +{ + (void)params; + config_line_t **ptr = (config_line_t **)target; + if (*ptr) + (*ptr)->fragile = 1; +} + +static const var_type_fns_t linelist_fns = { + .kv_parse = linelist_kv_parse, + .kv_encode = linelist_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, + .mark_fragile = linelist_mark_fragile, +}; + +static const var_type_fns_t linelist_v_fns = { + .kv_parse = linelist_kv_virt_noparse, + .kv_encode = linelist_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, + .mark_fragile = linelist_mark_fragile, +}; + +static const var_type_fns_t linelist_s_fns = { + .kv_parse = linelist_kv_parse, + .kv_encode = linelist_s_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, +}; + +///// +// CONFIG_TYPE_ROUTERSET +// +// XXXX This type is not implemented here, since routerset_t is not available +// XXXX to this module. +///// + +///// +// CONFIG_TYPE_OBSOLETE +// +// Used to indicate an obsolete option. +// +// XXXX This is not a type, and should be handled at a higher level of +// XXXX abstraction. +///// + +static int +ignore_parse(void *target, const char *value, char **errmsg, + const void *params, const char *key) +{ + (void)target; + (void)value; + (void)errmsg; + (void)params; + // XXXX move this to a higher level, once such a level exists. + log_warn(LD_GENERAL, "Skipping obsolete configuration option%s%s%s", + key && *key ? " \"" : "", + key && *key ? key : "", + key && *key ? "\"." : "."); + return 0; +} + +static char * +ignore_encode(const void *value, const void *params) +{ + (void)value; + (void)params; + return NULL; +} + +static const var_type_fns_t ignore_fns = { + .parse = ignore_parse, + .encode = ignore_encode, +}; + +/** + * Table mapping conf_type_t values to var_type_def_t objects. + **/ +static const var_type_def_t type_definitions_table[] = { + [CONFIG_TYPE_STRING] = { .name="String", .fns=&string_fns }, + [CONFIG_TYPE_FILENAME] = { .name="Filename", .fns=&string_fns }, + [CONFIG_TYPE_INT] = { .name="SignedInteger", .fns=&int_fns, + .params=&INT_PARSE_UNRESTRICTED }, + [CONFIG_TYPE_POSINT] = { .name="Integer", .fns=&int_fns, + .params=&INT_PARSE_POSINT }, + [CONFIG_TYPE_UINT64] = { .name="Integer", .fns=&uint64_fns, }, + [CONFIG_TYPE_MEMUNIT] = { .name="DataSize", .fns=&memunit_fns, + .params=&memory_units }, + [CONFIG_TYPE_INTERVAL] = { .name="TimeInterval", .fns=&interval_fns, + .params=&time_units }, + [CONFIG_TYPE_MSEC_INTERVAL] = { .name="TimeMsecInterval", + .fns=&interval_fns, + .params=&time_msec_units }, + [CONFIG_TYPE_DOUBLE] = { .name="Float", .fns=&double_fns, }, + [CONFIG_TYPE_BOOL] = { .name="Boolean", .fns=&enum_fns, + .params=&enum_table_bool }, + [CONFIG_TYPE_AUTOBOOL] = { .name="Boolean+Auto", .fns=&enum_fns, + .params=&enum_table_autobool }, + [CONFIG_TYPE_ISOTIME] = { .name="Time", .fns=&time_fns, }, + [CONFIG_TYPE_CSV] = { .name="CommaList", .fns=&csv_fns, }, + [CONFIG_TYPE_CSV_INTERVAL] = { .name="TimeInterval", + .fns=&legacy_csv_interval_fns, }, + [CONFIG_TYPE_LINELIST] = { .name="LineList", .fns=&linelist_fns, + .flags=CFLG_NOREPLACE }, + /* + * A "linelist_s" is a derived view of a linelist_v: inspecting + * it gets part of a linelist_v, and setting it adds to the linelist_v. + */ + [CONFIG_TYPE_LINELIST_S] = { .name="Dependent", .fns=&linelist_s_fns, + .flags=CFLG_NOREPLACE| + /* The operations we disable here are + * handled by the linelist_v. */ + CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP }, + [CONFIG_TYPE_LINELIST_V] = { .name="Virtual", .fns=&linelist_v_fns, + .flags=CFLG_NOREPLACE|CFLG_NOSET }, + [CONFIG_TYPE_OBSOLETE] = { + .name="Obsolete", .fns=&ignore_fns, + .flags=CFLG_GROUP_OBSOLETE, + } +}; + +/** + * Return a pointer to the var_type_def_t object for the given + * config_type_t value, or NULL if no such type definition exists. + **/ +const var_type_def_t * +lookup_type_def(config_type_t type) +{ + int t = type; + tor_assert(t >= 0); + if (t >= (int)ARRAY_LENGTH(type_definitions_table)) + return NULL; + return &type_definitions_table[t]; +} diff --git a/src/lib/confmgt/type_defs.h b/src/lib/confmgt/type_defs.h new file mode 100644 index 0000000000..ecf040529e --- /dev/null +++ b/src/lib/confmgt/type_defs.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file type_defs.h + * @brief Header for lib/confmgt/type_defs.c + **/ + +#ifndef TOR_LIB_CONFMGT_TYPE_DEFS_H +#define TOR_LIB_CONFMGT_TYPE_DEFS_H + +const struct var_type_def_t *lookup_type_def(config_type_t type); + +#endif /* !defined(TOR_LIB_CONFMGT_TYPE_DEFS_H) */ diff --git a/src/lib/confmgt/typedvar.c b/src/lib/confmgt/typedvar.c new file mode 100644 index 0000000000..ce11a69379 --- /dev/null +++ b/src/lib/confmgt/typedvar.c @@ -0,0 +1,228 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file typedvar.c + * @brief Functions for accessing a pointer as an object of a given type. + * + * These functions represent a low-level API for accessing a typed variable. + * They are used in the configuration system to examine and set fields in + * configuration objects used by individual modules. + * + * Almost no code should call these directly. + **/ + +#include "orconfig.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/typedvar.h" +#include "lib/encoding/confline.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/util_string.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include <stddef.h> +#include <string.h> + +/** + * Try to parse a string in <b>value</b> that encodes an object of the type + * defined by <b>def</b>. If not NULL, <b>key</b> is the name of the option, + * which may be used for logging. + * + * On success, adjust the lvalue pointed to by <b>target</b> to hold that + * value, and return 0. On failure, set *<b>errmsg</b> to a newly allocated + * string holding an error message, and return -1. + **/ +int +typed_var_assign(void *target, const char *value, char **errmsg, + const var_type_def_t *def, const char *key) +{ + if (BUG(!def)) + return -1; // LCOV_EXCL_LINE + // clear old value if needed. + typed_var_free(target, def); + + tor_assert(def->fns->parse); + return def->fns->parse(target, value, errmsg, def->params, key); +} + +/** + * Try to parse a single line from the head of<b>line</b> that encodes an + * object of the type defined in <b>def</b>. On success and failure, behave as + * typed_var_assign(). + * + * All types for which keys are significant should use this function. + * + * Note that although multiple lines may be provided in <b>line</b>, + * only the first one is handled by this function. + **/ +int +typed_var_kvassign(void *target, const config_line_t *line, + char **errmsg, const var_type_def_t *def) +{ + if (BUG(!def)) + return -1; // LCOV_EXCL_LINE + + if (def->fns->kv_parse) { + // We do _not_ free the old value here, since linelist options + // sometimes have append semantics. + return def->fns->kv_parse(target, line, errmsg, def->params); + } + + return typed_var_assign(target, line->value, errmsg, def, line->key); +} + +/** + * Release storage held by a variable in <b>target</b> of type defined by + * <b>def</b>, and set <b>target</b> to a reasonable default. + **/ +void +typed_var_free(void *target, const var_type_def_t *def) +{ + if (BUG(!def)) + return; // LCOV_EXCL_LINE + if (def->fns->clear) { + def->fns->clear(target, def->params); + } +} + +/** + * Encode a value of type <b>def</b> pointed to by <b>value</b>, and return + * its result in a newly allocated string. The string may need to be escaped. + * + * Returns NULL if this option has a NULL value, or on internal error. + **/ +char * +typed_var_encode(const void *value, const var_type_def_t *def) +{ + if (BUG(!def)) + return NULL; // LCOV_EXCL_LINE + tor_assert(def->fns->encode); + return def->fns->encode(value, def->params); +} + +/** + * As typed_var_encode(), but returns a newly allocated config_line_t + * object. The provided <b>key</b> is used as the key of the lines, unless + * the type is one (line a linelist) that encodes its own keys. + * + * This function may return a list of multiple lines. + * + * Returns NULL if there are no lines to encode, or on internal error. + */ +config_line_t * +typed_var_kvencode(const char *key, const void *value, + const var_type_def_t *def) +{ + if (BUG(!def)) + return NULL; // LCOV_EXCL_LINE + if (def->fns->kv_encode) { + return def->fns->kv_encode(key, value, def->params); + } + char *encoded_value = typed_var_encode(value, def); + if (!encoded_value) + return NULL; + + config_line_t *result = tor_malloc_zero(sizeof(config_line_t)); + result->key = tor_strdup(key); + result->value = encoded_value; + return result; +} + +/** + * Set <b>dest</b> to contain the same value as <b>src</b>. Both types + * must be as defined by <b>def</b>. + * + * Return 0 on success, and -1 on failure. + **/ +int +typed_var_copy(void *dest, const void *src, const var_type_def_t *def) +{ + if (BUG(!def)) + return -1; // LCOV_EXCL_LINE + if (def->fns->copy) { + // If we have been provided a copy fuction, use it. + return def->fns->copy(dest, src, def); + } + + // Otherwise, encode 'src' and parse the result into 'def'. + char *enc = typed_var_encode(src, def); + if (!enc) { + typed_var_free(dest, def); + return 0; + } + char *err = NULL; + int rv = typed_var_assign(dest, enc, &err, def, NULL); + if (BUG(rv < 0)) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Encoded value %s was not parseable as a %s: %s", + escaped(enc), def->name, err?err:""); + // LCOV_EXCL_STOP + } + tor_free(err); + tor_free(enc); + return rv; +} + +/** + * Return true if <b>a</b> and <b>b</b> are semantically equivalent. + * Both types must be as defined by <b>def</b>. + **/ +bool +typed_var_eq(const void *a, const void *b, const var_type_def_t *def) +{ + if (BUG(!def)) + return false; // LCOV_EXCL_LINE + + if (def->fns->eq) { + // Use a provided eq function if we got one. + return def->fns->eq(a, b, def->params); + } + + // Otherwise, encode the values and compare them. + char *enc_a = typed_var_encode(a, def); + char *enc_b = typed_var_encode(b, def); + bool eq = !strcmp_opt(enc_a,enc_b); + tor_free(enc_a); + tor_free(enc_b); + return eq; +} + +/** + * Check whether <b>value</b> encodes a valid value according to the + * type definition in <b>def</b>. + */ +bool +typed_var_ok(const void *value, const var_type_def_t *def) +{ + if (BUG(!def)) + return false; // LCOV_EXCL_LINE + + if (def->fns->ok) + return def->fns->ok(value, def->params); + + return true; +} + +/** + * Mark <b>value</b> -- a variable that ordinarily would be extended by + * assignment -- as "fragile", so that it will get replaced by the next + * assignment instead. + **/ +void +typed_var_mark_fragile(void *value, const var_type_def_t *def) +{ + if (BUG(!def)) { + return; // LCOV_EXCL_LINE + } + + if (def->fns->mark_fragile) + def->fns->mark_fragile(value, def->params); +} diff --git a/src/lib/confmgt/typedvar.h b/src/lib/confmgt/typedvar.h new file mode 100644 index 0000000000..4382613833 --- /dev/null +++ b/src/lib/confmgt/typedvar.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file typedvar.h + * @brief Header for lib/confmgt/typedvar.c + **/ + +#ifndef TOR_LIB_CONFMGT_TYPEDVAR_H +#define TOR_LIB_CONFMGT_TYPEDVAR_H + +#include <stdbool.h> + +enum config_type_t; +struct config_line_t; + +typedef struct var_type_fns_t var_type_fns_t; +typedef struct var_type_def_t var_type_def_t; + +int typed_var_assign(void *target, const char *value, char **errmsg, + const var_type_def_t *def, const char *key); +void typed_var_free(void *target, const var_type_def_t *def); +char *typed_var_encode(const void *value, const var_type_def_t *def); +int typed_var_copy(void *dest, const void *src, const var_type_def_t *def); +bool typed_var_eq(const void *a, const void *b, const var_type_def_t *def); +bool typed_var_ok(const void *value, const var_type_def_t *def); + +int typed_var_kvassign(void *target, const struct config_line_t *line, + char **errmsg, const var_type_def_t *def); +struct config_line_t *typed_var_kvencode(const char *key, const void *value, + const var_type_def_t *def); + +void typed_var_mark_fragile(void *value, const var_type_def_t *def); + +#endif /* !defined(TOR_LIB_CONFMGT_TYPEDVAR_H) */ diff --git a/src/lib/confmgt/unitparse.c b/src/lib/confmgt/unitparse.c new file mode 100644 index 0000000000..c3ed8285a4 --- /dev/null +++ b/src/lib/confmgt/unitparse.c @@ -0,0 +1,206 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file unitparse.c + * @brief Functions for parsing values with units from a configuration file. + **/ + +#include "orconfig.h" +#include "lib/confmgt/unitparse.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/string/parse_int.h" +#include "lib/string/util_string.h" + +#include <string.h> + +/** Table to map the names of memory units to the number of bytes they + * contain. */ +const struct unit_table_t memory_units[] = { + { "", 1 }, + { "b", 1<< 0 }, + { "byte", 1<< 0 }, + { "bytes", 1<< 0 }, + { "kb", 1<<10 }, + { "kbyte", 1<<10 }, + { "kbytes", 1<<10 }, + { "kilobyte", 1<<10 }, + { "kilobytes", 1<<10 }, + { "kilobits", 1<<7 }, + { "kilobit", 1<<7 }, + { "kbits", 1<<7 }, + { "kbit", 1<<7 }, + { "m", 1<<20 }, + { "mb", 1<<20 }, + { "mbyte", 1<<20 }, + { "mbytes", 1<<20 }, + { "megabyte", 1<<20 }, + { "megabytes", 1<<20 }, + { "megabits", 1<<17 }, + { "megabit", 1<<17 }, + { "mbits", 1<<17 }, + { "mbit", 1<<17 }, + { "gb", 1<<30 }, + { "gbyte", 1<<30 }, + { "gbytes", 1<<30 }, + { "gigabyte", 1<<30 }, + { "gigabytes", 1<<30 }, + { "gigabits", 1<<27 }, + { "gigabit", 1<<27 }, + { "gbits", 1<<27 }, + { "gbit", 1<<27 }, + { "tb", UINT64_C(1)<<40 }, + { "tbyte", UINT64_C(1)<<40 }, + { "tbytes", UINT64_C(1)<<40 }, + { "terabyte", UINT64_C(1)<<40 }, + { "terabytes", UINT64_C(1)<<40 }, + { "terabits", UINT64_C(1)<<37 }, + { "terabit", UINT64_C(1)<<37 }, + { "tbits", UINT64_C(1)<<37 }, + { "tbit", UINT64_C(1)<<37 }, + { NULL, 0 }, +}; + +/** Table to map the names of time units to the number of seconds they + * contain. */ +const struct unit_table_t time_units[] = { + { "", 1 }, + { "second", 1 }, + { "seconds", 1 }, + { "minute", 60 }, + { "minutes", 60 }, + { "hour", 60*60 }, + { "hours", 60*60 }, + { "day", 24*60*60 }, + { "days", 24*60*60 }, + { "week", 7*24*60*60 }, + { "weeks", 7*24*60*60 }, + { "month", 2629728, }, /* about 30.437 days */ + { "months", 2629728, }, + { NULL, 0 }, +}; + +/** Table to map the names of time units to the number of milliseconds + * they contain. */ +const struct unit_table_t time_msec_units[] = { + { "", 1 }, + { "msec", 1 }, + { "millisecond", 1 }, + { "milliseconds", 1 }, + { "second", 1000 }, + { "seconds", 1000 }, + { "minute", 60*1000 }, + { "minutes", 60*1000 }, + { "hour", 60*60*1000 }, + { "hours", 60*60*1000 }, + { "day", 24*60*60*1000 }, + { "days", 24*60*60*1000 }, + { "week", 7*24*60*60*1000 }, + { "weeks", 7*24*60*60*1000 }, + { NULL, 0 }, +}; + +/** Parse a string <b>val</b> containing a number, zero or more + * spaces, and an optional unit string. If the unit appears in the + * table <b>u</b>, then multiply the number by the unit multiplier. + * On success, set *<b>ok</b> to 1 and return this product. + * Otherwise, set *<b>ok</b> to 0. + */ +uint64_t +config_parse_units(const char *val, const unit_table_t *u, int *ok) +{ + uint64_t v = 0; + double d = 0; + int use_float = 0; + char *cp; + + tor_assert(ok); + + v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); + if (!*ok || (cp && *cp == '.')) { + d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp); + if (!*ok) + goto done; + use_float = 1; + } + + if (BUG(!cp)) { + // cp should always be non-NULL if the parse operation succeeds. + + // LCOV_EXCL_START + *ok = 1; + v = use_float ? ((uint64_t)d) : v; + goto done; + // LCOV_EXCL_STOP + } + + cp = (char*) eat_whitespace(cp); + + for ( ;u->unit;++u) { + if (!strcasecmp(u->unit, cp)) { + if (use_float) + v = (uint64_t)(u->multiplier * d); + else + v *= u->multiplier; + *ok = 1; + goto done; + } + } + log_warn(LD_CONFIG, "Unknown unit '%s'.", cp); + *ok = 0; + done: + + if (*ok) + return v; + else + return 0; +} + +/** Parse a string in the format "number unit", where unit is a unit of + * information (byte, KB, M, etc). On success, set *<b>ok</b> to true + * and return the number of bytes specified. Otherwise, set + * *<b>ok</b> to false and return 0. */ +uint64_t +config_parse_memunit(const char *s, int *ok) +{ + uint64_t u = config_parse_units(s, memory_units, ok); + return u; +} + +/** Parse a string in the format "number unit", where unit is a unit of + * time in milliseconds. On success, set *<b>ok</b> to true and return + * the number of milliseconds in the provided interval. Otherwise, set + * *<b>ok</b> to 0 and return -1. */ +int +config_parse_msec_interval(const char *s, int *ok) +{ + uint64_t r; + r = config_parse_units(s, time_msec_units, ok); + if (r > INT_MAX) { + log_warn(LD_CONFIG, "Msec interval '%s' is too long", s); + *ok = 0; + return -1; + } + return (int)r; +} + +/** Parse a string in the format "number unit", where unit is a unit of time. + * On success, set *<b>ok</b> to true and return the number of seconds in + * the provided interval. Otherwise, set *<b>ok</b> to 0 and return -1. + */ +int +config_parse_interval(const char *s, int *ok) +{ + uint64_t r; + r = config_parse_units(s, time_units, ok); + if (r > INT_MAX) { + log_warn(LD_CONFIG, "Interval '%s' is too long", s); + *ok = 0; + return -1; + } + return (int)r; +} diff --git a/src/lib/confmgt/unitparse.h b/src/lib/confmgt/unitparse.h new file mode 100644 index 0000000000..216361a7d4 --- /dev/null +++ b/src/lib/confmgt/unitparse.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file unitparse.h + * @brief Header for lib/confmgt/unitparse.c + **/ + +#ifndef TOR_LIB_CONFMGT_UNITPARSE_H +#define TOR_LIB_CONFMGT_UNITPARSE_H + +#include <lib/cc/torint.h> + +/** Mapping from a unit name to a multiplier for converting that unit into a + * base unit. Used by config_parse_unit. */ +typedef struct unit_table_t { + const char *unit; /**< The name of the unit */ + uint64_t multiplier; /**< How many of the base unit appear in this unit */ +} unit_table_t; + +extern const unit_table_t memory_units[]; +extern const unit_table_t time_units[]; +extern const struct unit_table_t time_msec_units[]; + +uint64_t config_parse_units(const char *val, const unit_table_t *u, int *ok); + +uint64_t config_parse_memunit(const char *s, int *ok); +int config_parse_msec_interval(const char *s, int *ok); +int config_parse_interval(const char *s, int *ok); + +#endif /* !defined(TOR_LIB_CONFMGT_UNITPARSE_H) */ diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h new file mode 100644 index 0000000000..aa9ded39e9 --- /dev/null +++ b/src/lib/confmgt/var_type_def_st.h @@ -0,0 +1,170 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file var_type_def_st.h + * @brief Structure declarations for typedvar type definitions. + * + * This structure is used for defining new variable types. If you are not + * defining a new variable type for use by the configuration management + * system, you don't need this structure. + * + * For defining new variables, see the types in conftypes.h. + * + * For data-driven access to configuration variables, see the other members of + * lib/confmgt/. + * + * STATUS NOTE: It is not yet possible to actually define new variables + * outside of config.c, and many of the types that will eventually be used + * to do so are not yet moved. This will change as more of #29211 is + * completed. + **/ + +#ifndef TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H +#define TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H + +#include <stdbool.h> + +struct config_line_t; + +/** + * A structure full of functions pointers to implement a variable type. + * + * Every type MUST implement parse or kv_parse and encode or kv_encode; + * the other functions pointers MAY be NULL. + * + * All functions here take a <b>params</b> argument, whose value + * is determined by the type definition. Two types may have the + * same functions, but differ only in parameters. + * + * Implementation considerations: If "" encodes a valid value for a type, try + * to make sure that it encodes the same thing as the default value for the + * type (that is, the value that is set by config_clear() or memset(0)). If + * this is not the case, you need to make extra certain that your parse/encode + * implementations preserve the NULL/"" distinction. + **/ +struct var_type_fns_t { + /** + * Try to parse a string in <b>value</b> that encodes an object of this + * type. On success, adjust the lvalue pointed to by <b>target</b> to hold + * that value, and return 0. On failure, set *<b>errmsg</b> to a newly + * allocated string holding an error message, and return -1. + * + * If not NULL, <b>key</b> is the name of the option, which may be used for + * logging. + **/ + int (*parse)(void *target, const char *value, char **errmsg, + const void *params, const char *key); + /** + * Try to parse a single line from the head of<b>line</b> that encodes + * an object of this type. On success and failure, behave as in the parse() + * function. + * + * If this function is absent, it is implemented in terms of parse(). + * + * All types for which keys are significant should use this method. For + * example, a "linelist" type records the actual keys that are given + * for each line, and so should use this method. + * + * Note that although multiple lines may be provided in <b>line</b>, + * only the first one should be handled by this function. + **/ + int (*kv_parse)(void *target, const struct config_line_t *line, + char **errmsg, const void *params); + /** + * Encode a value pointed to by <b>value</b> and return its result + * in a newly allocated string. The string may need to be escaped. + * + * If this function is absent, it is implemented in terms of kv_encode(). + * + * Returns NULL if this option has a NULL value, or on internal error. + * + * Requirement: all strings generated by encode() should produce a + * semantically equivalent value when given to parse(). + **/ + char *(*encode)(const void *value, const void *params); + /** + * As encode(), but returns a newly allocated config_line_t object. The + * provided <b>key</b> is used as the key of the lines, unless the type is + * one that encodes its own keys. + * + * Unlike kv_parse(), this function will return a list of multiple lines, + * if <b>value</b> is such that it must be encoded by multiple lines. + * + * Returns NULL if there are no lines to encode, or on internal error. + * + * If this function is absent, it is implemented in terms of encode(). + **/ + struct config_line_t *(*kv_encode)(const char *key, const void *value, + const void *params); + /** + * Free all storage held in <b>arg</b>, and set <b>arg</b> to a default + * value -- usually zero or NULL. + * + * If this function is absent, the default implementation does nothing. + **/ + void (*clear)(void *arg, const void *params); + /** + * Return true if <b>a</b> and <b>b</b> hold the same value, and false + * otherwise. + * + * If this function is absent, it is implemented by encoding both a and + * b and comparing their encoded strings for equality. + **/ + bool (*eq)(const void *a, const void *b, const void *params); + /** + * Try to copy the value from <b>value</b> into <b>target</b>. + * On success return 0; on failure return -1. + * + * If this function is absent, it is implemented by encoding the value + * into a string, and then parsing it into the target. + **/ + int (*copy)(void *target, const void *value, const void *params); + /** + * Check whether <b>value</b> holds a valid value according to the + * rules of this type; return true if it does and false if it doesn't. + * + * The default implementation for this function assumes that all + * values are valid. + **/ + bool (*ok)(const void *value, const void *params); + /** + * Mark a value of this variable as "fragile", so that future attempts to + * assign to this variable will replace rather than extending it. + * + * The default implementation for this function does nothing. + * + * Only meaningful for types with is_cumulative set. + **/ + void (*mark_fragile)(void *value, const void *params); +}; + +/** + * A structure describing a type that can be manipulated with the typedvar_* + * functions. + **/ +struct var_type_def_t { + /** + * The name of this type. Should not include spaces. Used for + * debugging, log messages, and the controller API. */ + const char *name; + /** + * A function table for this type. + */ + const struct var_type_fns_t *fns; + /** + * A pointer to a value that should be passed as the 'params' argument when + * calling the functions in this type's function table. + */ + const void *params; + /** + * A bitwise OR of one or more VTFLAG_* values, describing properties + * for all values of this type. + **/ + uint32_t flags; +}; + +#endif /* !defined(TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H) */ diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include index 90de5eda40..81507527d3 100644 --- a/src/lib/container/.may_include +++ b/src/lib/container/.may_include @@ -7,12 +7,9 @@ lib/malloc/*.h lib/err/*.h lib/smartlist_core/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/intmath/*.h lib/log/*.h -# XXXX I am unsure about this one. It's only here for buffers.c -lib/time/*.h - -ht.h -siphash.h +ext/ht.h +ext/siphash.h diff --git a/src/lib/container/bitarray.h b/src/lib/container/bitarray.h index 910d5fea65..45992796af 100644 --- a/src/lib/container/bitarray.h +++ b/src/lib/container/bitarray.h @@ -83,4 +83,4 @@ bitarray_is_set(bitarray_t *b, int bit) return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK)); } -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_BITARRAY_H) */ diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c index 9aa9b1ee56..8c61db81d6 100644 --- a/src/lib/container/bloomfilt.c +++ b/src/lib/container/bloomfilt.c @@ -14,7 +14,7 @@ #include "lib/container/bloomfilt.h" #include "lib/intmath/bits.h" #include "lib/log/util_bug.h" -#include "siphash.h" +#include "ext/siphash.h" /** How many bloom-filter bits we set per address. This is twice the * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit diff --git a/src/lib/container/include.am b/src/lib/container/include.am index e6492098b5..00d7b8e587 100644 --- a/src/lib/container/include.am +++ b/src/lib/container/include.am @@ -5,10 +5,11 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-container-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_container_a_SOURCES = \ src/lib/container/bloomfilt.c \ - src/lib/container/buffers.c \ src/lib/container/map.c \ + src/lib/container/namemap.c \ src/lib/container/order.c \ src/lib/container/smartlist.c @@ -17,11 +18,13 @@ src_lib_libtor_container_testing_a_SOURCES = \ src_lib_libtor_container_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/container/bitarray.h \ src/lib/container/bloomfilt.h \ - src/lib/container/buffers.h \ src/lib/container/handles.h \ src/lib/container/map.h \ + src/lib/container/namemap.h \ + src/lib/container/namemap_st.h \ src/lib/container/order.h \ src/lib/container/smartlist.h diff --git a/src/lib/container/map.c b/src/lib/container/map.c index d213ad50bf..fde33d6ace 100644 --- a/src/lib/container/map.c +++ b/src/lib/container/map.c @@ -21,7 +21,7 @@ #include <stdlib.h> #include <string.h> -#include "ht.h" +#include "ext/ht.h" /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called <b>maptype</b>. The key part of each diff --git a/src/lib/container/map.h b/src/lib/container/map.h index a2d1b01d12..9da1d3072c 100644 --- a/src/lib/container/map.h +++ b/src/lib/container/map.h @@ -15,7 +15,7 @@ #include "lib/testsupport/testsupport.h" #include "lib/cc/torint.h" -#include "siphash.h" +#include "ext/siphash.h" #define DECLARE_MAP_FNS(maptype, keytype, prefix) \ typedef struct maptype maptype; \ @@ -258,4 +258,4 @@ void* strmap_remove_lc(strmap_t *map, const char *key); return digestmap_iter_done((digestmap_iter_t*)iter); \ } -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_MAP_H) */ diff --git a/src/lib/container/namemap.c b/src/lib/container/namemap.c new file mode 100644 index 0000000000..a90057b32c --- /dev/null +++ b/src/lib/container/namemap.c @@ -0,0 +1,184 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "lib/container/smartlist.h" +#include "lib/container/namemap.h" +#include "lib/container/namemap_st.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +#include "ext/siphash.h" + +#include <string.h> + +/** Helper for namemap hashtable implementation: compare two entries. */ +static inline int +mapped_name_eq(const mapped_name_t *a, const mapped_name_t *b) +{ + return !strcmp(a->name, b->name); +} + +/** Helper for namemap hashtable implementation: hash an entry. */ +static inline unsigned +mapped_name_hash(const mapped_name_t *a) +{ + return (unsigned) siphash24g(a->name, strlen(a->name)); +} + +HT_PROTOTYPE(namemap_ht, mapped_name_t, node, mapped_name_hash, + mapped_name_eq) +HT_GENERATE2(namemap_ht, mapped_name_t, node, mapped_name_hash, + mapped_name_eq, 0.6, tor_reallocarray_, tor_free_) + +/** Set up an uninitialized <b>map</b>. */ +void +namemap_init(namemap_t *map) +{ + memset(map, 0, sizeof(*map)); + HT_INIT(namemap_ht, &map->ht); + map->names = smartlist_new(); +} + +/** Return the name that <b>map</b> associates with a given <b>id</b>, or + * NULL if there is no such name. */ +const char * +namemap_get_name(const namemap_t *map, unsigned id) +{ + if (map->names && id < (unsigned)smartlist_len(map->names)) { + mapped_name_t *name = smartlist_get(map->names, (int)id); + return name->name; + } else { + return NULL; + } +} + +/** + * Return the name that <b>map</b> associates with a given <b>id</b>, or a + * pointer to a statically allocated string describing the value of <b>id</b> + * if no such name exists. + **/ +const char * +namemap_fmt_name(const namemap_t *map, unsigned id) +{ + static char buf[32]; + + const char *name = namemap_get_name(map, id); + if (name) + return name; + + tor_snprintf(buf, sizeof(buf), "{%u}", id); + + return buf; +} + +/** + * Helper: As namemap_get_id(), but requires that <b>name</b> is + * <b>namelen</b> charaters long, and that <b>namelen</b> is no more than + * MAX_NAMEMAP_NAME_LEN. + */ +static unsigned +namemap_get_id_unchecked(const namemap_t *map, + const char *name, + size_t namelen) +{ + union { + mapped_name_t n; + char storage[MAX_NAMEMAP_NAME_LEN + sizeof(mapped_name_t) + 1]; + } u; + memcpy(u.n.name, name, namelen); + u.n.name[namelen] = 0; + const mapped_name_t *found = HT_FIND(namemap_ht, &map->ht, &u.n); + if (found) { + tor_assert(map->names); + tor_assert(smartlist_get(map->names, found->intval) == found); + return found->intval; + } + + return NAMEMAP_ERR; +} + +/** + * Return the identifier currently associated by <b>map</b> with the name + * <b>name</b>, or NAMEMAP_ERR if no such identifier exists. + **/ +unsigned +namemap_get_id(const namemap_t *map, + const char *name) +{ + size_t namelen = strlen(name); + if (namelen > MAX_NAMEMAP_NAME_LEN) { + return NAMEMAP_ERR; + } + + return namemap_get_id_unchecked(map, name, namelen); +} + +/** + * Return the identifier associated by <b>map</b> with the name + * <b>name</b>, allocating a new identifier in <b>map</b> if none exists. + * + * Return NAMEMAP_ERR if <b>name</b> is too long, or if there are no more + * identifiers we can allocate. + **/ +unsigned +namemap_get_or_create_id(namemap_t *map, + const char *name) +{ + size_t namelen = strlen(name); + if (namelen > MAX_NAMEMAP_NAME_LEN) { + return NAMEMAP_ERR; + } + + if (PREDICT_UNLIKELY(map->names == NULL)) + map->names = smartlist_new(); + + unsigned found = namemap_get_id_unchecked(map, name, namelen); + if (found != NAMEMAP_ERR) + return found; + + unsigned new_id = (unsigned)smartlist_len(map->names); + if (new_id == NAMEMAP_ERR) + return NAMEMAP_ERR; /* Can't allocate any more. */ + + mapped_name_t *insert = tor_malloc_zero( + offsetof(mapped_name_t, name) + namelen + 1); + memcpy(insert->name, name, namelen+1); + insert->intval = new_id; + + HT_INSERT(namemap_ht, &map->ht, insert); + smartlist_add(map->names, insert); + + return new_id; +} + +/** Return the number of entries in 'names' */ +size_t +namemap_get_size(const namemap_t *map) +{ + if (PREDICT_UNLIKELY(map->names == NULL)) + return 0; + + return smartlist_len(map->names); +} + +/** + * Release all storage held in <b>map</b>. + */ +void +namemap_clear(namemap_t *map) +{ + if (!map) + return; + + HT_CLEAR(namemap_ht, &map->ht); + if (map->names) { + SMARTLIST_FOREACH(map->names, mapped_name_t *, n, + tor_free(n)); + smartlist_free(map->names); + } + memset(map, 0, sizeof(*map)); +} diff --git a/src/lib/container/namemap.h b/src/lib/container/namemap.h new file mode 100644 index 0000000000..b96bc13f3a --- /dev/null +++ b/src/lib/container/namemap.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_NAMEMAP_H +#define TOR_NAMEMAP_H + +/** + * \file namemap.h + * + * \brief Header for namemap.c + **/ + +#include "lib/cc/compat_compiler.h" +#include "ext/ht.h" + +#include <stddef.h> + +typedef struct namemap_t namemap_t; + +/** Returned in place of an identifier when an error occurs. */ +#define NAMEMAP_ERR UINT_MAX + +void namemap_init(namemap_t *map); +const char *namemap_get_name(const namemap_t *map, unsigned id); +const char *namemap_fmt_name(const namemap_t *map, unsigned id); +unsigned namemap_get_id(const namemap_t *map, + const char *name); +unsigned namemap_get_or_create_id(namemap_t *map, + const char *name); +size_t namemap_get_size(const namemap_t *map); +void namemap_clear(namemap_t *map); + +#endif /* !defined(TOR_NAMEMAP_H) */ diff --git a/src/lib/container/namemap_st.h b/src/lib/container/namemap_st.h new file mode 100644 index 0000000000..5008fd5855 --- /dev/null +++ b/src/lib/container/namemap_st.h @@ -0,0 +1,34 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef NAMEMAP_ST_H +#define NAMEMAP_ST_H + +#include "lib/cc/compat_compiler.h" +#include "ext/ht.h" + +struct smartlist_t; + +/** Longest allowed name that's allowed in a namemap_t. */ +#define MAX_NAMEMAP_NAME_LEN 128 + +/** An entry inside a namemap_t. Maps a string to a numeric identifier. */ +typedef struct mapped_name_t { + HT_ENTRY(mapped_name_t) node; + unsigned intval; + char name[FLEXIBLE_ARRAY_MEMBER]; +} mapped_name_t; + +/** A structure that allocates small numeric identifiers for names and maps + * back and forth between them. */ +struct namemap_t { + HT_HEAD(namemap_ht, mapped_name_t) ht; + struct smartlist_t *names; +}; + +/** Macro to initialize a namemap. */ +#define NAMEMAP_INIT() { HT_INITIALIZER(), NULL } + +#endif /* !defined(NAMEMAP_ST_H) */ diff --git a/src/lib/container/order.h b/src/lib/container/order.h index a176d6d8a6..3f2fd054a0 100644 --- a/src/lib/container/order.h +++ b/src/lib/container/order.h @@ -57,4 +57,4 @@ third_quartile_uint32(uint32_t *array, int n_elements) return find_nth_uint32(array, n_elements, (n_elements*3)/4); } -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_ORDER_H) */ diff --git a/src/lib/container/smartlist.c b/src/lib/container/smartlist.c index 3ab2797d68..2b71c11287 100644 --- a/src/lib/container/smartlist.c +++ b/src/lib/container/smartlist.c @@ -678,7 +678,7 @@ smartlist_sort_pointers(smartlist_t *sl) static inline void smartlist_heapify(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, int idx) { while (1) { @@ -725,7 +725,7 @@ smartlist_heapify(smartlist_t *sl, void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item) { int idx; @@ -754,7 +754,7 @@ smartlist_pqueue_add(smartlist_t *sl, void * smartlist_pqueue_pop(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset) + ptrdiff_t idx_field_offset) { void *top; tor_assert(sl->num_used); @@ -778,7 +778,7 @@ smartlist_pqueue_pop(smartlist_t *sl, void smartlist_pqueue_remove(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item) { int idx = IDX_OF_ITEM(item); @@ -802,7 +802,7 @@ smartlist_pqueue_remove(smartlist_t *sl, void smartlist_pqueue_assert_ok(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset) + ptrdiff_t idx_field_offset) { int i; for (i = sl->num_used - 1; i >= 0; --i) { diff --git a/src/lib/container/smartlist.h b/src/lib/container/smartlist.h index 77682db03e..25638e4b22 100644 --- a/src/lib/container/smartlist.h +++ b/src/lib/container/smartlist.h @@ -13,6 +13,7 @@ **/ #include <stdarg.h> +#include <stddef.h> #include "lib/smartlist_core/smartlist_core.h" #include "lib/smartlist_core/smartlist_foreach.h" @@ -72,18 +73,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key, void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item); void *smartlist_pqueue_pop(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset); + ptrdiff_t idx_field_offset); void smartlist_pqueue_remove(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item); void smartlist_pqueue_assert_ok(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset); + ptrdiff_t idx_field_offset); char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out) ATTR_MALLOC; @@ -165,4 +166,4 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, } \ STMT_END -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_SMARTLIST_H) */ diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include index a0fa4ec05c..0739699686 100644 --- a/src/lib/crypt_ops/.may_include +++ b/src/lib/crypt_ops/.may_include @@ -12,7 +12,8 @@ lib/malloc/*.h lib/intmath/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/subsys/*.h +lib/testsupport/*.h lib/thread/*.h lib/log/*.h @@ -21,4 +22,4 @@ trunnel/pwbox.h keccak-tiny/*.h ed25519/*.h -siphash.h +ext/siphash.h diff --git a/src/lib/crypt_ops/aes_openssl.c b/src/lib/crypt_ops/aes_openssl.c index 2f985d4512..64564892ad 100644 --- a/src/lib/crypt_ops/aes_openssl.c +++ b/src/lib/crypt_ops/aes_openssl.c @@ -148,7 +148,7 @@ evaluate_ctr_for_aes(void) { return 0; } -#else /* !(defined(USE_EVP_AES_CTR)) */ +#else /* !defined(USE_EVP_AES_CTR) */ /*======================================================================*/ /* Interface to AES code, and counter implementation */ diff --git a/src/lib/crypt_ops/compat_openssl.h b/src/lib/crypt_ops/compat_openssl.h index 9c10386c34..61ca51315f 100644 --- a/src/lib/crypt_ops/compat_openssl.h +++ b/src/lib/crypt_ops/compat_openssl.h @@ -45,7 +45,7 @@ ((st) == SSL3_ST_SW_SRVR_HELLO_B)) #define OSSL_HANDSHAKE_STATE int #define CONST_IF_OPENSSL_1_1_API -#else /* !(!defined(OPENSSL_1_1_API)) */ +#else /* defined(OPENSSL_1_1_API) */ #define STATE_IS_SW_SERVER_HELLO(st) \ ((st) == TLS_ST_SW_SRVR_HELLO) #define CONST_IF_OPENSSL_1_1_API const diff --git a/src/lib/crypt_ops/crypto_cipher.h b/src/lib/crypt_ops/crypto_cipher.h index cc4fbf7a41..88d63c1df2 100644 --- a/src/lib/crypt_ops/crypto_cipher.h +++ b/src/lib/crypt_ops/crypto_cipher.h @@ -54,4 +54,4 @@ int crypto_cipher_decrypt_with_iv(const char *key, char *to, size_t tolen, const char *from, size_t fromlen); -#endif /* !defined(TOR_CRYPTO_H) */ +#endif /* !defined(TOR_CRYPTO_CIPHER_H) */ diff --git a/src/lib/crypt_ops/crypto_curve25519.h b/src/lib/crypt_ops/crypto_curve25519.h index 061a7a3505..cd23169cd5 100644 --- a/src/lib/crypt_ops/crypto_curve25519.h +++ b/src/lib/crypt_ops/crypto_curve25519.h @@ -76,8 +76,8 @@ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret); int curve25519_public_from_base64(curve25519_public_key_t *pkey, const char *input); -int curve25519_public_to_base64(char *output, - const curve25519_public_key_t *pkey); +void curve25519_public_to_base64(char *output, + const curve25519_public_key_t *pkey); void curve25519_set_impl_params(int use_ed); void curve25519_init(void); diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c index 8c6388fd5d..8ae97373e8 100644 --- a/src/lib/crypt_ops/crypto_dh_openssl.c +++ b/src/lib/crypt_ops/crypto_dh_openssl.c @@ -34,7 +34,7 @@ static int tor_check_dh_key(int severity, const BIGNUM *bn); struct crypto_dh_t { DH *dh; /**< The openssl DH object */ }; -#endif +#endif /* !defined(ENABLE_NSS) */ static DH *new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g); @@ -68,7 +68,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g) goto out; if (!DH_set0_pqg(dh, dh_p, NULL, dh_g)) goto out; -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ if (!(dh->p = BN_dup(p))) goto out; if (!(dh->g = BN_dup(g))) @@ -100,7 +100,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g) DH_free(dh); return ret; } -#endif +#endif /* 0 */ /** * Helper: convert <b>hex<b> to a bignum, and return it. Assert that the @@ -202,7 +202,7 @@ crypto_dh_new(int dh_type) tor_free(res); // sets res to NULL. return res; } -#endif +#endif /* !defined(ENABLE_NSS) */ /** Create and return a new openssl DH from a given prime and generator. */ static DH * @@ -231,7 +231,7 @@ new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g) if (!DH_set_length(res_dh, DH_PRIVATE_KEY_BITS)) goto err; -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ res_dh->p = dh_p; res_dh->g = dh_g; res_dh->length = DH_PRIVATE_KEY_BITS; @@ -298,7 +298,7 @@ crypto_dh_generate_public(crypto_dh_t *dh) "the-universe chances really do happen. Treating as a failure."); return -1; } -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { /* LCOV_EXCL_START * If this happens, then openssl's DH implementation is busted. */ @@ -461,7 +461,7 @@ crypto_dh_free_(crypto_dh_t *dh) DH_free(dh->dh); tor_free(dh); } -#endif +#endif /* !defined(ENABLE_NSS) */ void crypto_dh_free_all_openssl(void) diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c index de81b87b7e..ba226f8756 100644 --- a/src/lib/crypt_ops/crypto_digest.c +++ b/src/lib/crypt_ops/crypto_digest.c @@ -23,171 +23,6 @@ #include "lib/arch/bytes.h" -#ifdef ENABLE_NSS -DISABLE_GCC_WARNING(strict-prototypes) -#include <pk11pub.h> -ENABLE_GCC_WARNING(strict-prototypes) -#else - -#include "lib/crypt_ops/crypto_openssl_mgt.h" - -DISABLE_GCC_WARNING(redundant-decls) - -#include <openssl/hmac.h> -#include <openssl/sha.h> - -ENABLE_GCC_WARNING(redundant-decls) -#endif - -#ifdef ENABLE_NSS -/** - * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS). - * On failure, return SEC_OID_UNKNOWN. */ -static SECOidTag -digest_alg_to_nss_oid(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: return SEC_OID_SHA1; - case DIGEST_SHA256: return SEC_OID_SHA256; - case DIGEST_SHA512: return SEC_OID_SHA512; - case DIGEST_SHA3_256: FALLTHROUGH; - case DIGEST_SHA3_512: FALLTHROUGH; - default: - return SEC_OID_UNKNOWN; - } -} - -/* Helper: get an unkeyed digest via pk11wrap */ -static int -digest_nss_internal(SECOidTag alg, - char *digest, unsigned len_out, - const char *msg, size_t msg_len) -{ - if (alg == SEC_OID_UNKNOWN) - return -1; - tor_assert(msg_len <= UINT_MAX); - - int rv = -1; - SECStatus s; - PK11Context *ctx = PK11_CreateDigestContext(alg); - if (!ctx) - return -1; - - s = PK11_DigestBegin(ctx); - if (s != SECSuccess) - goto done; - - s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len); - if (s != SECSuccess) - goto done; - - unsigned int len = 0; - s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out); - if (s != SECSuccess) - goto done; - - rv = 0; - done: - PK11_DestroyContext(ctx, PR_TRUE); - return rv; -} - -/** True iff alg is implemented in our crypto library, and we want to use that - * implementation */ -static bool -library_supports_digest(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: FALLTHROUGH; - case DIGEST_SHA256: FALLTHROUGH; - case DIGEST_SHA512: - return true; - case DIGEST_SHA3_256: FALLTHROUGH; - case DIGEST_SHA3_512: FALLTHROUGH; - default: - return false; - } -} -#endif - -/* Crypto digest functions */ - -/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in - * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. - * Return 0 on success, -1 on failure. - */ -MOCK_IMPL(int, -crypto_digest,(char *digest, const char *m, size_t len)) -{ - tor_assert(m); - tor_assert(digest); -#ifdef ENABLE_NSS - return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len); -#else - if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) { - return -1; - } -#endif - return 0; -} - -/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, - * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result - * into <b>digest</b>. Return 0 on success, -1 on failure. */ -int -crypto_digest256(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - - int ret = 0; - if (algorithm == DIGEST_SHA256) { -#ifdef ENABLE_NSS - return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len); -#else - ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); -#endif - } else { - ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) - > -1); - } - - if (!ret) - return -1; - return 0; -} - -/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, - * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result - * into <b>digest</b>. Return 0 on success, -1 on failure. */ -int -crypto_digest512(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - - int ret = 0; - if (algorithm == DIGEST_SHA512) { -#ifdef ENABLE_NSS - return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len); -#else - ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) - != NULL); -#endif - } else { - ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) - > -1); - } - - if (!ret) - return -1; - return 0; -} - /** Set the common_digests_t in <b>ds_out</b> to contain every digest on the * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on * success, -1 on failure. */ @@ -267,485 +102,6 @@ crypto_digest_algorithm_get_length(digest_algorithm_t alg) } } -/** Intermediate information about the digest of a stream of data. */ -struct crypto_digest_t { - digest_algorithm_t algorithm; /**< Which algorithm is in use? */ - /** State for the digest we're using. Only one member of the - * union is usable, depending on the value of <b>algorithm</b>. Note also - * that space for other members might not even be allocated! - */ - union { -#ifdef ENABLE_NSS - PK11Context *ctx; -#else - SHA_CTX sha1; /**< state for SHA1 */ - SHA256_CTX sha2; /**< state for SHA256 */ - SHA512_CTX sha512; /**< state for SHA512 */ -#endif - keccak_state sha3; /**< state for SHA3-[256,512] */ - } d; -}; - -#ifdef TOR_UNIT_TESTS - -digest_algorithm_t -crypto_digest_get_algorithm(crypto_digest_t *digest) -{ - tor_assert(digest); - - return digest->algorithm; -} - -#endif /* defined(TOR_UNIT_TESTS) */ - -/** - * Return the number of bytes we need to malloc in order to get a - * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe - * when we free one. - */ -static size_t -crypto_digest_alloc_bytes(digest_algorithm_t alg) -{ - /* Helper: returns the number of bytes in the 'f' field of 'st' */ -#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) - /* Gives the length of crypto_digest_t through the end of the field 'd' */ -#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ - STRUCT_FIELD_SIZE(crypto_digest_t, f)) - switch (alg) { -#ifdef ENABLE_NSS - case DIGEST_SHA1: FALLTHROUGH; - case DIGEST_SHA256: FALLTHROUGH; - case DIGEST_SHA512: - return END_OF_FIELD(d.ctx); -#else - case DIGEST_SHA1: - return END_OF_FIELD(d.sha1); - case DIGEST_SHA256: - return END_OF_FIELD(d.sha2); - case DIGEST_SHA512: - return END_OF_FIELD(d.sha512); -#endif - case DIGEST_SHA3_256: - case DIGEST_SHA3_512: - return END_OF_FIELD(d.sha3); - default: - tor_assert(0); // LCOV_EXCL_LINE - return 0; // LCOV_EXCL_LINE - } -#undef END_OF_FIELD -#undef STRUCT_FIELD_SIZE -} - -/** - * Internal function: create and return a new digest object for 'algorithm'. - * Does not typecheck the algorithm. - */ -static crypto_digest_t * -crypto_digest_new_internal(digest_algorithm_t algorithm) -{ - crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); - r->algorithm = algorithm; - - switch (algorithm) - { -#ifdef ENABLE_NSS - case DIGEST_SHA1: FALLTHROUGH; - case DIGEST_SHA256: FALLTHROUGH; - case DIGEST_SHA512: - r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm)); - if (BUG(!r->d.ctx)) { - tor_free(r); - return NULL; - } - if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) { - crypto_digest_free(r); - return NULL; - } - break; -#else - case DIGEST_SHA1: - SHA1_Init(&r->d.sha1); - break; - case DIGEST_SHA256: - SHA256_Init(&r->d.sha2); - break; - case DIGEST_SHA512: - SHA512_Init(&r->d.sha512); - break; -#endif - case DIGEST_SHA3_256: - keccak_digest_init(&r->d.sha3, 256); - break; - case DIGEST_SHA3_512: - keccak_digest_init(&r->d.sha3, 512); - break; - default: - tor_assert_unreached(); - } - - return r; -} - -/** Allocate and return a new digest object to compute SHA1 digests. - */ -crypto_digest_t * -crypto_digest_new(void) -{ - return crypto_digest_new_internal(DIGEST_SHA1); -} - -/** Allocate and return a new digest object to compute 256-bit digests - * using <b>algorithm</b>. - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` - * C_RUST_COUPLED: `crypto::digest::Sha256::default` - */ -crypto_digest_t * -crypto_digest256_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - return crypto_digest_new_internal(algorithm); -} - -/** Allocate and return a new digest object to compute 512-bit digests - * using <b>algorithm</b>. */ -crypto_digest_t * -crypto_digest512_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - return crypto_digest_new_internal(algorithm); -} - -/** Deallocate a digest object. - */ -void -crypto_digest_free_(crypto_digest_t *digest) -{ - if (!digest) - return; -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - PK11_DestroyContext(digest->d.ctx, PR_TRUE); - } -#endif - size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); - memwipe(digest, 0, bytes); - tor_free(digest); -} - -/** Add <b>len</b> bytes from <b>data</b> to the digest object. - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` - * C_RUST_COUPLED: `crypto::digest::Sha256::process` - */ -void -crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, - size_t len) -{ - tor_assert(digest); - tor_assert(data); - /* Using the SHA*_*() calls directly means we don't support doing - * SHA in hardware. But so far the delay of getting the question - * to the hardware, and hearing the answer, is likely higher than - * just doing it ourselves. Hashes are fast. - */ - switch (digest->algorithm) { -#ifdef ENABLE_NSS - case DIGEST_SHA1: FALLTHROUGH; - case DIGEST_SHA256: FALLTHROUGH; - case DIGEST_SHA512: - tor_assert(len <= UINT_MAX); - SECStatus s = PK11_DigestOp(digest->d.ctx, - (const unsigned char *)data, - (unsigned int)len); - tor_assert(s == SECSuccess); - break; -#else - case DIGEST_SHA1: - SHA1_Update(&digest->d.sha1, (void*)data, len); - break; - case DIGEST_SHA256: - SHA256_Update(&digest->d.sha2, (void*)data, len); - break; - case DIGEST_SHA512: - SHA512_Update(&digest->d.sha512, (void*)data, len); - break; -#endif - case DIGEST_SHA3_256: FALLTHROUGH; - case DIGEST_SHA3_512: - keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); - break; - default: - /* LCOV_EXCL_START */ - tor_fragile_assert(); - break; - /* LCOV_EXCL_STOP */ - } -} - -/** Compute the hash of the data that has been passed to the digest - * object; write the first out_len bytes of the result to <b>out</b>. - * <b>out_len</b> must be \<= DIGEST512_LEN. - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` - * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` - */ -void -crypto_digest_get_digest(crypto_digest_t *digest, - char *out, size_t out_len) -{ - unsigned char r[DIGEST512_LEN]; - tor_assert(digest); - tor_assert(out); - tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); - - /* The SHA-3 code handles copying into a temporary ctx, and also can handle - * short output buffers by truncating appropriately. */ - if (digest->algorithm == DIGEST_SHA3_256 || - digest->algorithm == DIGEST_SHA3_512) { - keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); - return; - } - -#ifdef ENABLE_NSS - /* Copy into a temporary buffer since DigestFinal (alters) the context */ - unsigned char buf[1024]; - unsigned int saved_len = 0; - unsigned rlen; - unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx, - buf, sizeof(buf), - &saved_len); - tor_assert(saved); - SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r)); - tor_assert(s == SECSuccess); - tor_assert(rlen >= out_len); - s = PK11_RestoreContext(digest->d.ctx, saved, saved_len); - tor_assert(s == SECSuccess); - if (saved != buf) { - PORT_ZFree(saved, saved_len); - } -#else - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - crypto_digest_t tmpenv; - /* memcpy into a temporary ctx, since SHA*_Final clears the context */ - memcpy(&tmpenv, digest, alloc_bytes); - switch (digest->algorithm) { - case DIGEST_SHA1: - SHA1_Final(r, &tmpenv.d.sha1); - break; - case DIGEST_SHA256: - SHA256_Final(r, &tmpenv.d.sha2); - break; - case DIGEST_SHA512: - SHA512_Final(r, &tmpenv.d.sha512); - break; -//LCOV_EXCL_START - case DIGEST_SHA3_256: FALLTHROUGH; - case DIGEST_SHA3_512: - default: - log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); - /* This is fatal, because it should never happen. */ - tor_assert_unreached(); - break; -//LCOV_EXCL_STOP - } -#endif - memcpy(out, r, out_len); - memwipe(r, 0, sizeof(r)); -} - -/** Allocate and return a new digest object with the same state as - * <b>digest</b> - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` - * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` - */ -crypto_digest_t * -crypto_digest_dup(const crypto_digest_t *digest) -{ - tor_assert(digest); - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - crypto_digest_t *result = tor_memdup(digest, alloc_bytes); -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - result->d.ctx = PK11_CloneContext(digest->d.ctx); - } -#endif - return result; -} - -/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>. - * Asserts that <b>digest</b> is a SHA1 digest object. - */ -void -crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, - const crypto_digest_t *digest) -{ - const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); - tor_assert(bytes <= sizeof(checkpoint->mem)); -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - unsigned char *allocated; - allocated = PK11_SaveContextAlloc(digest->d.ctx, - (unsigned char *)checkpoint->mem, - sizeof(checkpoint->mem), - &checkpoint->bytes_used); - /* No allocation is allowed here. */ - tor_assert(allocated == checkpoint->mem); - return; - } -#endif - memcpy(checkpoint->mem, digest, bytes); -} - -/** Restore the state of <b>digest</b> from <b>checkpoint</b>. - * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the - * state was previously stored with crypto_digest_checkpoint() */ -void -crypto_digest_restore(crypto_digest_t *digest, - const crypto_digest_checkpoint_t *checkpoint) -{ - const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - SECStatus s = PK11_RestoreContext(digest->d.ctx, - (unsigned char *)checkpoint->mem, - checkpoint->bytes_used); - tor_assert(s == SECSuccess); - return; - } -#endif - memcpy(digest, checkpoint->mem, bytes); -} - -/** Replace the state of the digest object <b>into</b> with the state - * of the digest object <b>from</b>. Requires that 'into' and 'from' - * have the same digest type. - */ -void -crypto_digest_assign(crypto_digest_t *into, - const crypto_digest_t *from) -{ - tor_assert(into); - tor_assert(from); - tor_assert(into->algorithm == from->algorithm); - const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); -#ifdef ENABLE_NSS - if (library_supports_digest(from->algorithm)) { - PK11_DestroyContext(into->d.ctx, PR_TRUE); - into->d.ctx = PK11_CloneContext(from->d.ctx); - return; - } -#endif - memcpy(into,from,alloc_bytes); -} - -/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest - * at <b>digest_out</b> to the hash of the concatenation of those strings, - * plus the optional string <b>append</b>, computed with the algorithm - * <b>alg</b>. - * <b>out_len</b> must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist(char *digest_out, size_t len_out, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); -} - -/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest - * at <b>digest_out</b> to the hash of the concatenation of: the - * optional string <b>prepend</b>, those strings, - * and the optional string <b>append</b>, computed with the algorithm - * <b>alg</b>. - * <b>len_out</b> must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, - const char *prepend, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_t *d = crypto_digest_new_internal(alg); - if (prepend) - crypto_digest_add_bytes(d, prepend, strlen(prepend)); - SMARTLIST_FOREACH(lst, const char *, cp, - crypto_digest_add_bytes(d, cp, strlen(cp))); - if (append) - crypto_digest_add_bytes(d, append, strlen(append)); - crypto_digest_get_digest(d, digest_out, len_out); - crypto_digest_free(d); -} - -/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using - * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte - * result in <b>hmac_out</b>. Asserts on failure. - */ -void -crypto_hmac_sha256(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len) -{ - /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ - tor_assert(key_len < INT_MAX); - tor_assert(msg_len < INT_MAX); - tor_assert(hmac_out); -#ifdef ENABLE_NSS - PK11SlotInfo *slot = NULL; - PK11SymKey *symKey = NULL; - PK11Context *hmac = NULL; - - int ok = 0; - SECStatus s; - SECItem keyItem, paramItem; - keyItem.data = (unsigned char *)key; - keyItem.len = (unsigned)key_len; - paramItem.type = siBuffer; - paramItem.data = NULL; - paramItem.len = 0; - - slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL); - if (!slot) - goto done; - symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC, - PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL); - if (!symKey) - goto done; - - hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey, - ¶mItem); - if (!hmac) - goto done; - s = PK11_DigestBegin(hmac); - if (s != SECSuccess) - goto done; - s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len); - if (s != SECSuccess) - goto done; - unsigned int len=0; - s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN); - if (s != SECSuccess || len != DIGEST256_LEN) - goto done; - ok = 1; - - done: - if (hmac) - PK11_DestroyContext(hmac, PR_TRUE); - if (symKey) - PK11_FreeSymKey(symKey); - if (slot) - PK11_FreeSlot(slot); - - tor_assert(ok); -#else - unsigned char *rv = NULL; - rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, - (unsigned char*)hmac_out, NULL); - tor_assert(rv); -#endif -} - /** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in @@ -779,7 +135,23 @@ crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, /** Internal state for a eXtendable-Output Function (XOF). */ struct crypto_xof_t { +#ifdef OPENSSL_HAS_SHAKE3_EVP + /* XXXX We can't enable this yet, because OpenSSL's + * DigestFinalXOF function can't be called repeatedly on the same + * XOF. + * + * We could in theory use the undocumented SHA3_absorb and SHA3_squeeze + * functions, but let's not mess with undocumented OpenSSL internals any + * more than we have to. + * + * We could also revise our XOF code so that it only allows a single + * squeeze operation; we don't require streaming squeeze operations + * outside the tests yet. + */ + EVP_MD_CTX *ctx; +#else /* !defined(OPENSSL_HAS_SHAKE3_EVP) */ keccak_state s; +#endif /* defined(OPENSSL_HAS_SHAKE3_EVP) */ }; /** Allocate a new XOF object backed by SHAKE-256. The security level @@ -792,7 +164,14 @@ crypto_xof_new(void) { crypto_xof_t *xof; xof = tor_malloc(sizeof(crypto_xof_t)); +#ifdef OPENSSL_HAS_SHAKE256 + xof->ctx = EVP_MD_CTX_new(); + tor_assert(xof->ctx); + int r = EVP_DigestInit(xof->ctx, EVP_shake256()); + tor_assert(r == 1); +#else /* !defined(OPENSSL_HAS_SHAKE256) */ keccak_xof_init(&xof->s, 256); +#endif /* defined(OPENSSL_HAS_SHAKE256) */ return xof; } @@ -803,8 +182,13 @@ crypto_xof_new(void) void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) { +#ifdef OPENSSL_HAS_SHAKE256 + int r = EVP_DigestUpdate(xof->ctx, data, len); + tor_assert(r == 1); +#else int i = keccak_xof_absorb(&xof->s, data, len); tor_assert(i == 0); +#endif /* defined(OPENSSL_HAS_SHAKE256) */ } /** Squeeze bytes out of a XOF object. Calling this routine will render @@ -813,8 +197,13 @@ crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len) { +#ifdef OPENSSL_HAS_SHAKE256 + int r = EVP_DigestFinalXOF(xof->ctx, out, len); + tor_assert(r == 1); +#else int i = keccak_xof_squeeze(&xof->s, out, len); tor_assert(i == 0); +#endif /* defined(OPENSSL_HAS_SHAKE256) */ } /** Cleanse and deallocate a XOF object. */ @@ -823,6 +212,34 @@ crypto_xof_free_(crypto_xof_t *xof) { if (!xof) return; +#ifdef OPENSSL_HAS_SHAKE256 + if (xof->ctx) + EVP_MD_CTX_free(xof->ctx); +#endif memwipe(xof, 0, sizeof(crypto_xof_t)); tor_free(xof); } + +/** Compute the XOF (SHAKE256) of a <b>input_len</b> bytes at <b>input</b>, + * putting <b>output_len</b> bytes at <b>output</b>. */ +void +crypto_xof(uint8_t *output, size_t output_len, + const uint8_t *input, size_t input_len) +{ +#ifdef OPENSSL_HAS_SHA3 + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + tor_assert(ctx); + int r = EVP_DigestInit(ctx, EVP_shake256()); + tor_assert(r == 1); + r = EVP_DigestUpdate(ctx, input, input_len); + tor_assert(r == 1); + r = EVP_DigestFinalXOF(ctx, output, output_len); + tor_assert(r == 1); + EVP_MD_CTX_free(ctx); +#else /* !defined(OPENSSL_HAS_SHA3) */ + crypto_xof_t *xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, input, input_len); + crypto_xof_squeeze_bytes(xof, output, output_len); + crypto_xof_free(xof); +#endif /* defined(OPENSSL_HAS_SHA3) */ +} diff --git a/src/lib/crypt_ops/crypto_digest.h b/src/lib/crypt_ops/crypto_digest.h index 47e60ce617..5869db7800 100644 --- a/src/lib/crypt_ops/crypto_digest.h +++ b/src/lib/crypt_ops/crypto_digest.h @@ -124,6 +124,8 @@ void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); void crypto_xof_free_(crypto_xof_t *xof); #define crypto_xof_free(xof) \ FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof)) +void crypto_xof(uint8_t *output, size_t output_len, + const uint8_t *input, size_t input_len); #ifdef TOR_UNIT_TESTS digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest); diff --git a/src/lib/crypt_ops/crypto_digest_nss.c b/src/lib/crypt_ops/crypto_digest_nss.c new file mode 100644 index 0000000000..afa8f6d5ef --- /dev/null +++ b/src/lib/crypt_ops/crypto_digest_nss.c @@ -0,0 +1,559 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest_nss.c + * \brief Block of functions related with digest and xof utilities and + * operations (NSS specific implementations). + **/ + +#include "lib/container/smartlist.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" + +#include "keccak-tiny/keccak-tiny.h" + +#include <stdlib.h> +#include <string.h> + +#include "lib/arch/bytes.h" + +DISABLE_GCC_WARNING(strict-prototypes) +#include <pk11pub.h> +ENABLE_GCC_WARNING(strict-prototypes) + +/** + * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS). + * On failure, return SEC_OID_UNKNOWN. */ +static SECOidTag +digest_alg_to_nss_oid(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: return SEC_OID_SHA1; + case DIGEST_SHA256: return SEC_OID_SHA256; + case DIGEST_SHA512: return SEC_OID_SHA512; + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: FALLTHROUGH; + default: + return SEC_OID_UNKNOWN; + } +} + +/* Helper: get an unkeyed digest via pk11wrap */ +static int +digest_nss_internal(SECOidTag alg, + char *digest, unsigned len_out, + const char *msg, size_t msg_len) +{ + if (alg == SEC_OID_UNKNOWN) + return -1; + tor_assert(msg_len <= UINT_MAX); + + int rv = -1; + SECStatus s; + PK11Context *ctx = PK11_CreateDigestContext(alg); + if (!ctx) + return -1; + + s = PK11_DigestBegin(ctx); + if (s != SECSuccess) + goto done; + + s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len); + if (s != SECSuccess) + goto done; + + unsigned int len = 0; + s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out); + if (s != SECSuccess) + goto done; + + rv = 0; + done: + PK11_DestroyContext(ctx, PR_TRUE); + return rv; +} + +/** True iff alg is implemented in our crypto library, and we want to use that + * implementation */ +static bool +library_supports_digest(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: FALLTHROUGH; + case DIGEST_SHA256: FALLTHROUGH; + case DIGEST_SHA512: + return true; + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: FALLTHROUGH; + default: + return false; + } +} + +/* Crypto digest functions */ + +/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in + * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. + * Return 0 on success, -1 on failure. + */ +MOCK_IMPL(int, +crypto_digest,(char *digest, const char *m, size_t len)) +{ + tor_assert(m); + tor_assert(digest); + return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len); +} + +/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; + if (algorithm == DIGEST_SHA256) { + return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len); + } else { + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); + } + + if (!ret) + return -1; + return 0; +} + +/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; + if (algorithm == DIGEST_SHA512) { + return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len); + } else { + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); + } + + if (!ret) + return -1; + return 0; +} + +/** Intermediate information about the digest of a stream of data. */ +struct crypto_digest_t { + digest_algorithm_t algorithm; /**< Which algorithm is in use? */ + /** State for the digest we're using. Only one member of the + * union is usable, depending on the value of <b>algorithm</b>. Note also + * that space for other members might not even be allocated! + */ + union { + PK11Context *ctx; + keccak_state sha3; /**< state for SHA3-[256,512] */ + } d; +}; + +#ifdef TOR_UNIT_TESTS + +digest_algorithm_t +crypto_digest_get_algorithm(crypto_digest_t *digest) +{ + tor_assert(digest); + + return digest->algorithm; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Return the number of bytes we need to malloc in order to get a + * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe + * when we free one. + */ +static size_t +crypto_digest_alloc_bytes(digest_algorithm_t alg) +{ + /* Helper: returns the number of bytes in the 'f' field of 'st' */ +#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) + /* Gives the length of crypto_digest_t through the end of the field 'd' */ +#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ + STRUCT_FIELD_SIZE(crypto_digest_t, f)) + switch (alg) { + case DIGEST_SHA1: FALLTHROUGH; + case DIGEST_SHA256: FALLTHROUGH; + case DIGEST_SHA512: + return END_OF_FIELD(d.ctx); + case DIGEST_SHA3_256: + case DIGEST_SHA3_512: + return END_OF_FIELD(d.sha3); + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +#undef END_OF_FIELD +#undef STRUCT_FIELD_SIZE +} + +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: FALLTHROUGH; + case DIGEST_SHA256: FALLTHROUGH; + case DIGEST_SHA512: + r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm)); + if (BUG(!r->d.ctx)) { + tor_free(r); + return NULL; + } + if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) { + crypto_digest_free(r); + return NULL; + } + break; + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; + default: + tor_assert_unreached(); + } + + return r; +} + +/** Allocate and return a new digest object to compute SHA1 digests. + */ +crypto_digest_t * +crypto_digest_new(void) +{ + return crypto_digest_new_internal(DIGEST_SHA1); +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using <b>algorithm</b>. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` + * C_RUST_COUPLED: `crypto::digest::Sha256::default` + */ +crypto_digest_t * +crypto_digest256_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + return crypto_digest_new_internal(algorithm); +} + +/** Allocate and return a new digest object to compute 512-bit digests + * using <b>algorithm</b>. */ +crypto_digest_t * +crypto_digest512_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + return crypto_digest_new_internal(algorithm); +} + +/** Deallocate a digest object. + */ +void +crypto_digest_free_(crypto_digest_t *digest) +{ + if (!digest) + return; + if (library_supports_digest(digest->algorithm)) { + PK11_DestroyContext(digest->d.ctx, PR_TRUE); + } + size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memwipe(digest, 0, bytes); + tor_free(digest); +} + +/** Add <b>len</b> bytes from <b>data</b> to the digest object. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` + * C_RUST_COUPLED: `crypto::digest::Sha256::process` + */ +void +crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len) +{ + tor_assert(digest); + tor_assert(data); + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question + * to the hardware, and hearing the answer, is likely higher than + * just doing it ourselves. Hashes are fast. + */ + switch (digest->algorithm) { + case DIGEST_SHA1: FALLTHROUGH; + case DIGEST_SHA256: FALLTHROUGH; + case DIGEST_SHA512: + tor_assert(len <= UINT_MAX); + SECStatus s = PK11_DigestOp(digest->d.ctx, + (const unsigned char *)data, + (unsigned int)len); + tor_assert(s == SECSuccess); + break; + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: + keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); + break; + default: + /* LCOV_EXCL_START */ + tor_fragile_assert(); + break; + /* LCOV_EXCL_STOP */ + } +} + +/** Compute the hash of the data that has been passed to the digest + * object; write the first out_len bytes of the result to <b>out</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` + * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` + */ +void +crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len) +{ + unsigned char r[DIGEST512_LEN]; + tor_assert(digest); + tor_assert(out); + tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); + + /* The SHA-3 code handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); + return; + } + + /* Copy into a temporary buffer since DigestFinal (alters) the context */ + unsigned char buf[1024]; + unsigned int saved_len = 0; + unsigned rlen; + unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx, + buf, sizeof(buf), + &saved_len); + tor_assert(saved); + SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r)); + tor_assert(s == SECSuccess); + tor_assert(rlen >= out_len); + s = PK11_RestoreContext(digest->d.ctx, saved, saved_len); + tor_assert(s == SECSuccess); + + if (saved != buf) { + PORT_ZFree(saved, saved_len); + } + memcpy(out, r, out_len); + memwipe(r, 0, sizeof(r)); +} + +/** Allocate and return a new digest object with the same state as + * <b>digest</b> + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` + * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` + */ +crypto_digest_t * +crypto_digest_dup(const crypto_digest_t *digest) +{ + tor_assert(digest); + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + crypto_digest_t *result = tor_memdup(digest, alloc_bytes); + + if (library_supports_digest(digest->algorithm)) { + result->d.ctx = PK11_CloneContext(digest->d.ctx); + } + + return result; +} + +/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. + */ +void +crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + tor_assert(bytes <= sizeof(checkpoint->mem)); + if (library_supports_digest(digest->algorithm)) { + unsigned char *allocated; + allocated = PK11_SaveContextAlloc(digest->d.ctx, + (unsigned char *)checkpoint->mem, + sizeof(checkpoint->mem), + &checkpoint->bytes_used); + /* No allocation is allowed here. */ + tor_assert(allocated == checkpoint->mem); + return; + } + memcpy(checkpoint->mem, digest, bytes); +} + +/** Restore the state of <b>digest</b> from <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the + * state was previously stored with crypto_digest_checkpoint() */ +void +crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + if (library_supports_digest(digest->algorithm)) { + SECStatus s = PK11_RestoreContext(digest->d.ctx, + (unsigned char *)checkpoint->mem, + checkpoint->bytes_used); + tor_assert(s == SECSuccess); + return; + } + memcpy(digest, checkpoint->mem, bytes); +} + +/** Replace the state of the digest object <b>into</b> with the state + * of the digest object <b>from</b>. Requires that 'into' and 'from' + * have the same digest type. + */ +void +crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from) +{ + tor_assert(into); + tor_assert(from); + tor_assert(into->algorithm == from->algorithm); + const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); + if (library_supports_digest(from->algorithm)) { + PK11_DestroyContext(into->d.ctx, PR_TRUE); + into->d.ctx = PK11_CloneContext(from->d.ctx); + return; + } + memcpy(into,from,alloc_bytes); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * plus the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist(char *digest_out, size_t len_out, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of: the + * optional string <b>prepend</b>, those strings, + * and the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>len_out</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_t *d = crypto_digest_new_internal(alg); + if (prepend) + crypto_digest_add_bytes(d, prepend, strlen(prepend)); + SMARTLIST_FOREACH(lst, const char *, cp, + crypto_digest_add_bytes(d, cp, strlen(cp))); + if (append) + crypto_digest_add_bytes(d, append, strlen(append)); + crypto_digest_get_digest(d, digest_out, len_out); + crypto_digest_free(d); +} + +/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using + * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte + * result in <b>hmac_out</b>. Asserts on failure. + */ +void +crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len) +{ + /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ + tor_assert(key_len < INT_MAX); + tor_assert(msg_len < INT_MAX); + tor_assert(hmac_out); + + PK11SlotInfo *slot = NULL; + PK11SymKey *symKey = NULL; + PK11Context *hmac = NULL; + + int ok = 0; + SECStatus s; + SECItem keyItem, paramItem; + keyItem.data = (unsigned char *)key; + keyItem.len = (unsigned)key_len; + paramItem.type = siBuffer; + paramItem.data = NULL; + paramItem.len = 0; + + slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL); + if (!slot) + goto done; + symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC, + PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL); + if (!symKey) + goto done; + + hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey, + ¶mItem); + if (!hmac) + goto done; + s = PK11_DigestBegin(hmac); + if (s != SECSuccess) + goto done; + s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len); + if (s != SECSuccess) + goto done; + unsigned int len=0; + s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN); + if (s != SECSuccess || len != DIGEST256_LEN) + goto done; + ok = 1; + + done: + if (hmac) + PK11_DestroyContext(hmac, PR_TRUE); + if (symKey) + PK11_FreeSymKey(symKey); + if (slot) + PK11_FreeSlot(slot); + + tor_assert(ok); +} diff --git a/src/lib/crypt_ops/crypto_digest_openssl.c b/src/lib/crypt_ops/crypto_digest_openssl.c new file mode 100644 index 0000000000..50a8ff4d27 --- /dev/null +++ b/src/lib/crypt_ops/crypto_digest_openssl.c @@ -0,0 +1,522 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest_openssl.c + * \brief Block of functions related with digest and xof utilities and + * operations (OpenSSL specific implementations). + **/ + +#include "lib/container/smartlist.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" + +#include "keccak-tiny/keccak-tiny.h" + +#include <stdlib.h> +#include <string.h> + +#include "lib/arch/bytes.h" + +#include "lib/crypt_ops/crypto_openssl_mgt.h" + +DISABLE_GCC_WARNING(redundant-decls) + +#include <openssl/hmac.h> +#include <openssl/sha.h> + +ENABLE_GCC_WARNING(redundant-decls) + +/* Crypto digest functions */ + +/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in + * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. + * Return 0 on success, -1 on failure. + */ +MOCK_IMPL(int, +crypto_digest,(char *digest, const char *m, size_t len)) +{ + tor_assert(m); + tor_assert(digest); + if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) { + return -1; + } + return 0; +} + +/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; + if (algorithm == DIGEST_SHA256) { + ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); + } else { +#ifdef OPENSSL_HAS_SHA3 + unsigned int dlen = DIGEST256_LEN; + ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_256(), NULL); +#else + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); +#endif /* defined(OPENSSL_HAS_SHA3) */ + } + + if (!ret) + return -1; + return 0; +} + +/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; + if (algorithm == DIGEST_SHA512) { + ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) + != NULL); + } else { +#ifdef OPENSSL_HAS_SHA3 + unsigned int dlen = DIGEST512_LEN; + ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_512(), NULL); +#else + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); +#endif /* defined(OPENSSL_HAS_SHA3) */ + } + + if (!ret) + return -1; + return 0; +} + +/** Intermediate information about the digest of a stream of data. */ +struct crypto_digest_t { + digest_algorithm_t algorithm; /**< Which algorithm is in use? */ + /** State for the digest we're using. Only one member of the + * union is usable, depending on the value of <b>algorithm</b>. Note also + * that space for other members might not even be allocated! + */ + union { + SHA_CTX sha1; /**< state for SHA1 */ + SHA256_CTX sha2; /**< state for SHA256 */ + SHA512_CTX sha512; /**< state for SHA512 */ +#ifdef OPENSSL_HAS_SHA3 + EVP_MD_CTX *md; +#else + keccak_state sha3; /**< state for SHA3-[256,512] */ +#endif + } d; +}; + +#ifdef TOR_UNIT_TESTS + +digest_algorithm_t +crypto_digest_get_algorithm(crypto_digest_t *digest) +{ + tor_assert(digest); + + return digest->algorithm; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Return the number of bytes we need to malloc in order to get a + * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe + * when we free one. + */ +static size_t +crypto_digest_alloc_bytes(digest_algorithm_t alg) +{ + /* Helper: returns the number of bytes in the 'f' field of 'st' */ +#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) + /* Gives the length of crypto_digest_t through the end of the field 'd' */ +#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ + STRUCT_FIELD_SIZE(crypto_digest_t, f)) + switch (alg) { + case DIGEST_SHA1: + return END_OF_FIELD(d.sha1); + case DIGEST_SHA256: + return END_OF_FIELD(d.sha2); + case DIGEST_SHA512: + return END_OF_FIELD(d.sha512); +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: + return END_OF_FIELD(d.md); +#else + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: + return END_OF_FIELD(d.sha3); +#endif /* defined(OPENSSL_HAS_SHA3) */ + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +#undef END_OF_FIELD +#undef STRUCT_FIELD_SIZE +} + +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: + SHA1_Init(&r->d.sha1); + break; + case DIGEST_SHA256: + SHA256_Init(&r->d.sha2); + break; + case DIGEST_SHA512: + SHA512_Init(&r->d.sha512); + break; +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: + r->d.md = EVP_MD_CTX_new(); + if (!EVP_DigestInit(r->d.md, EVP_sha3_256())) { + crypto_digest_free(r); + return NULL; + } + break; + case DIGEST_SHA3_512: + r->d.md = EVP_MD_CTX_new(); + if (!EVP_DigestInit(r->d.md, EVP_sha3_512())) { + crypto_digest_free(r); + return NULL; + } + break; +#else /* !defined(OPENSSL_HAS_SHA3) */ + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; +#endif /* defined(OPENSSL_HAS_SHA3) */ + default: + tor_assert_unreached(); + } + + return r; +} + +/** Allocate and return a new digest object to compute SHA1 digests. + */ +crypto_digest_t * +crypto_digest_new(void) +{ + return crypto_digest_new_internal(DIGEST_SHA1); +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using <b>algorithm</b>. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` + * C_RUST_COUPLED: `crypto::digest::Sha256::default` + */ +crypto_digest_t * +crypto_digest256_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + return crypto_digest_new_internal(algorithm); +} + +/** Allocate and return a new digest object to compute 512-bit digests + * using <b>algorithm</b>. */ +crypto_digest_t * +crypto_digest512_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + return crypto_digest_new_internal(algorithm); +} + +/** Deallocate a digest object. + */ +void +crypto_digest_free_(crypto_digest_t *digest) +{ + if (!digest) + return; +#ifdef OPENSSL_HAS_SHA3 + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + if (digest->d.md) { + EVP_MD_CTX_free(digest->d.md); + } + } +#endif /* defined(OPENSSL_HAS_SHA3) */ + size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memwipe(digest, 0, bytes); + tor_free(digest); +} + +/** Add <b>len</b> bytes from <b>data</b> to the digest object. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` + * C_RUST_COUPLED: `crypto::digest::Sha256::process` + */ +void +crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len) +{ + tor_assert(digest); + tor_assert(data); + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question + * to the hardware, and hearing the answer, is likely higher than + * just doing it ourselves. Hashes are fast. + */ + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + case DIGEST_SHA512: + SHA512_Update(&digest->d.sha512, (void*)data, len); + break; +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: { + int r = EVP_DigestUpdate(digest->d.md, data, len); + tor_assert(r); + } + break; +#else /* !defined(OPENSSL_HAS_SHA3) */ + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: + keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); + break; +#endif /* defined(OPENSSL_HAS_SHA3) */ + default: + /* LCOV_EXCL_START */ + tor_fragile_assert(); + break; + /* LCOV_EXCL_STOP */ + } +} + +/** Compute the hash of the data that has been passed to the digest + * object; write the first out_len bytes of the result to <b>out</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` + * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` + */ +void +crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len) +{ + unsigned char r[DIGEST512_LEN]; + tor_assert(digest); + tor_assert(out); + tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); + + /* The SHA-3 code handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { +#ifdef OPENSSL_HAS_SHA3 + unsigned dlen = (unsigned) + crypto_digest_algorithm_get_length(digest->algorithm); + EVP_MD_CTX *tmp = EVP_MD_CTX_new(); + EVP_MD_CTX_copy(tmp, digest->d.md); + memset(r, 0xff, sizeof(r)); + int res = EVP_DigestFinal(tmp, r, &dlen); + EVP_MD_CTX_free(tmp); + tor_assert(res == 1); + goto done; +#else /* !defined(OPENSSL_HAS_SHA3) */ + /* Tiny-Keccak handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); + return; +#endif /* defined(OPENSSL_HAS_SHA3) */ + } + + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + crypto_digest_t tmpenv; + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, alloc_bytes); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Final(r, &tmpenv.d.sha1); + break; + case DIGEST_SHA256: + SHA256_Final(r, &tmpenv.d.sha2); + break; + case DIGEST_SHA512: + SHA512_Final(r, &tmpenv.d.sha512); + break; +//LCOV_EXCL_START + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: + default: + log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); + /* This is fatal, because it should never happen. */ + tor_assert_unreached(); + break; +//LCOV_EXCL_STOP + } +#ifdef OPENSSL_HAS_SHA3 + done: +#endif + memcpy(out, r, out_len); + memwipe(r, 0, sizeof(r)); +} + +/** Allocate and return a new digest object with the same state as + * <b>digest</b> + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` + * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` + */ +crypto_digest_t * +crypto_digest_dup(const crypto_digest_t *digest) +{ + tor_assert(digest); + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + crypto_digest_t *result = tor_memdup(digest, alloc_bytes); + +#ifdef OPENSSL_HAS_SHA3 + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + result->d.md = EVP_MD_CTX_new(); + EVP_MD_CTX_copy(result->d.md, digest->d.md); + } +#endif /* defined(OPENSSL_HAS_SHA3) */ + return result; +} + +/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. + */ +void +crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + tor_assert(bytes <= sizeof(checkpoint->mem)); + memcpy(checkpoint->mem, digest, bytes); +} + +/** Restore the state of <b>digest</b> from <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the + * state was previously stored with crypto_digest_checkpoint() */ +void +crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memcpy(digest, checkpoint->mem, bytes); +} + +/** Replace the state of the digest object <b>into</b> with the state + * of the digest object <b>from</b>. Requires that 'into' and 'from' + * have the same digest type. + */ +void +crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from) +{ + tor_assert(into); + tor_assert(from); + tor_assert(into->algorithm == from->algorithm); + const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); + +#ifdef OPENSSL_HAS_SHA3 + if (from->algorithm == DIGEST_SHA3_256 || + from->algorithm == DIGEST_SHA3_512) { + EVP_MD_CTX_copy(into->d.md, from->d.md); + return; + } +#endif /* defined(OPENSSL_HAS_SHA3) */ + + memcpy(into,from,alloc_bytes); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * plus the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist(char *digest_out, size_t len_out, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of: the + * optional string <b>prepend</b>, those strings, + * and the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>len_out</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_t *d = crypto_digest_new_internal(alg); + if (prepend) + crypto_digest_add_bytes(d, prepend, strlen(prepend)); + SMARTLIST_FOREACH(lst, const char *, cp, + crypto_digest_add_bytes(d, cp, strlen(cp))); + if (append) + crypto_digest_add_bytes(d, append, strlen(append)); + crypto_digest_get_digest(d, digest_out, len_out); + crypto_digest_free(d); +} + +/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using + * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte + * result in <b>hmac_out</b>. Asserts on failure. + */ +void +crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len) +{ + /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ + tor_assert(key_len < INT_MAX); + tor_assert(msg_len < INT_MAX); + tor_assert(hmac_out); + unsigned char *rv = NULL; + rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, + (unsigned char*)hmac_out, NULL); + tor_assert(rv); +} + diff --git a/src/lib/crypt_ops/crypto_ed25519.c b/src/lib/crypt_ops/crypto_ed25519.c index 0a442bb739..c28111a5a5 100644 --- a/src/lib/crypt_ops/crypto_ed25519.c +++ b/src/lib/crypt_ops/crypto_ed25519.c @@ -226,7 +226,7 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong) int ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey) { - return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN); + return safe_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN); } /* Return a heap-allocated array that contains <b>msg</b> prefixed by the diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index 84f73e5272..118cd79045 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -104,7 +104,7 @@ crypto_read_tagged_contents_from_file(const char *fname, prefix[32] = 0; /* Check type, extract tag. */ if (strcmpstart(prefix, "== ") || strcmpend(prefix, " ==") || - ! tor_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) { + ! fast_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) { saved_errno = EINVAL; goto end; } @@ -131,20 +131,27 @@ crypto_read_tagged_contents_from_file(const char *fname, return r; } -/** Encode <b>pkey</b> as a base64-encoded string, without trailing "=" +/** Encode <b>pkey</b> as a base64-encoded string, including trailing "=" * characters, in the buffer <b>output</b>, which must have at least - * CURVE25519_BASE64_PADDED_LEN+1 bytes available. Return 0 on success, -1 on - * failure. */ -int + * CURVE25519_BASE64_PADDED_LEN+1 bytes available. + * Can not fail. + * + * Careful! CURVE25519_BASE64_PADDED_LEN is one byte longer than + * ED25519_BASE64_LEN. + */ +void curve25519_public_to_base64(char *output, const curve25519_public_key_t *pkey) { char buf[128]; - base64_encode(buf, sizeof(buf), - (const char*)pkey->public_key, CURVE25519_PUBKEY_LEN, 0); - buf[CURVE25519_BASE64_PADDED_LEN] = '\0'; + int n = base64_encode(buf, sizeof(buf), + (const char*)pkey->public_key, + CURVE25519_PUBKEY_LEN, 0); + /* These asserts should always succeed, unless there is a bug in + * base64_encode(). */ + tor_assert(n == CURVE25519_BASE64_PADDED_LEN); + tor_assert(buf[CURVE25519_BASE64_PADDED_LEN] == '\0'); memcpy(output, buf, CURVE25519_BASE64_PADDED_LEN+1); - return 0; } /** Try to decode a base64-encoded curve25519 public key from <b>input</b> @@ -181,8 +188,7 @@ ed25519_fmt(const ed25519_public_key_t *pkey) if (ed25519_public_key_is_zero(pkey)) { strlcpy(formatted, "<unset>", sizeof(formatted)); } else { - int r = ed25519_public_to_base64(formatted, pkey); - tor_assert(!r); + ed25519_public_to_base64(formatted, pkey); } } else { strlcpy(formatted, "<null>", sizeof(formatted)); @@ -202,28 +208,35 @@ ed25519_public_from_base64(ed25519_public_key_t *pkey, /** Encode the public key <b>pkey</b> into the buffer at <b>output</b>, * which must have space for ED25519_BASE64_LEN bytes of encoded key, - * plus one byte for a terminating NUL. Return 0 on success, -1 on failure. + * plus one byte for a terminating NUL. + * Can not fail. + * + * Careful! ED25519_BASE64_LEN is one byte shorter than + * CURVE25519_BASE64_PADDED_LEN. */ -int +void ed25519_public_to_base64(char *output, const ed25519_public_key_t *pkey) { - return digest256_to_base64(output, (const char *)pkey->pubkey); + digest256_to_base64(output, (const char *)pkey->pubkey); } /** Encode the signature <b>sig</b> into the buffer at <b>output</b>, * which must have space for ED25519_SIG_BASE64_LEN bytes of encoded signature, - * plus one byte for a terminating NUL. Return 0 on success, -1 on failure. + * plus one byte for a terminating NUL. + * Can not fail. */ -int +void ed25519_signature_to_base64(char *output, const ed25519_signature_t *sig) { char buf[256]; int n = base64_encode_nopad(buf, sizeof(buf), sig->sig, ED25519_SIG_LEN); + /* These asserts should always succeed, unless there is a bug in + * base64_encode_nopad(). */ tor_assert(n == ED25519_SIG_BASE64_LEN); + tor_assert(buf[ED25519_SIG_BASE64_LEN] == '\0'); memcpy(output, buf, ED25519_SIG_BASE64_LEN+1); - return 0; } /** Try to decode the string <b>input</b> into an ed25519 signature. On @@ -233,16 +246,11 @@ int ed25519_signature_from_base64(ed25519_signature_t *sig, const char *input) { - if (strlen(input) != ED25519_SIG_BASE64_LEN) return -1; - char buf[ED25519_SIG_BASE64_LEN+3]; - memcpy(buf, input, ED25519_SIG_BASE64_LEN); - buf[ED25519_SIG_BASE64_LEN+0] = '='; - buf[ED25519_SIG_BASE64_LEN+1] = '='; - buf[ED25519_SIG_BASE64_LEN+2] = 0; char decoded[128]; - int n = base64_decode(decoded, sizeof(decoded), buf, strlen(buf)); + int n = base64_decode(decoded, sizeof(decoded), input, + ED25519_SIG_BASE64_LEN); if (n < 0 || n != ED25519_SIG_LEN) return -1; memcpy(sig->sig, decoded, ED25519_SIG_LEN); @@ -250,24 +258,26 @@ ed25519_signature_from_base64(ed25519_signature_t *sig, return 0; } -/** Base64 encode DIGEST_LINE bytes from <b>digest</b>, remove the trailing = +/** Base64 encode DIGEST_LEN bytes from <b>digest</b>, remove the trailing = * characters, and store the nul-terminated result in the first - * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>. */ -/* XXXX unify with crypto_format.c code */ -int + * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>. + * Can not fail. */ +void digest_to_base64(char *d64, const char *digest) { char buf[256]; - base64_encode(buf, sizeof(buf), digest, DIGEST_LEN, 0); - buf[BASE64_DIGEST_LEN] = '\0'; + int n = base64_encode_nopad(buf, sizeof(buf), + (const uint8_t *)digest, DIGEST_LEN); + /* These asserts should always succeed, unless there is a bug in + * base64_encode_nopad(). */ + tor_assert(n == BASE64_DIGEST_LEN); + tor_assert(buf[BASE64_DIGEST_LEN] == '\0'); memcpy(d64, buf, BASE64_DIGEST_LEN+1); - return 0; } /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without * trailing newline or = characters), decode it and store the result in the * first DIGEST_LEN bytes at <b>digest</b>. */ -/* XXXX unify with crypto_format.c code */ int digest_from_base64(char *digest, const char *d64) { @@ -279,22 +289,24 @@ digest_from_base64(char *digest, const char *d64) /** Base64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the * trailing = characters, and store the nul-terminated result in the first - * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. */ - /* XXXX unify with crypto_format.c code */ -int + * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. + * Can not fail. */ +void digest256_to_base64(char *d64, const char *digest) { char buf[256]; - base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN, 0); - buf[BASE64_DIGEST256_LEN] = '\0'; + int n = base64_encode_nopad(buf, sizeof(buf), + (const uint8_t *)digest, DIGEST256_LEN); + /* These asserts should always succeed, unless there is a bug in + * base64_encode_nopad(). */ + tor_assert(n == BASE64_DIGEST256_LEN); + tor_assert(buf[BASE64_DIGEST256_LEN] == '\0'); memcpy(d64, buf, BASE64_DIGEST256_LEN+1); - return 0; } /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without * trailing newline or = characters), decode it and store the result in the * first DIGEST256_LEN bytes at <b>digest</b>. */ -/* XXXX unify with crypto_format.c code */ int digest256_from_base64(char *digest, const char *d64) { diff --git a/src/lib/crypt_ops/crypto_format.h b/src/lib/crypt_ops/crypto_format.h index fe852e6a61..b4b3aa189c 100644 --- a/src/lib/crypt_ops/crypto_format.h +++ b/src/lib/crypt_ops/crypto_format.h @@ -33,18 +33,18 @@ ssize_t crypto_read_tagged_contents_from_file(const char *fname, int ed25519_public_from_base64(struct ed25519_public_key_t *pkey, const char *input); -int ed25519_public_to_base64(char *output, - const struct ed25519_public_key_t *pkey); +void ed25519_public_to_base64(char *output, + const struct ed25519_public_key_t *pkey); const char *ed25519_fmt(const struct ed25519_public_key_t *pkey); int ed25519_signature_from_base64(struct ed25519_signature_t *sig, const char *input); -int ed25519_signature_to_base64(char *output, - const struct ed25519_signature_t *sig); +void ed25519_signature_to_base64(char *output, + const struct ed25519_signature_t *sig); -int digest_to_base64(char *d64, const char *digest); +void digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); -int digest256_to_base64(char *d64, const char *digest); +void digest256_to_base64(char *d64, const char *digest); int digest256_from_base64(char *digest, const char *d64); #endif /* !defined(TOR_CRYPTO_FORMAT_H) */ diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c index fd2e701651..e0d241d4ea 100644 --- a/src/lib/crypt_ops/crypto_hkdf.c +++ b/src/lib/crypt_ops/crypto_hkdf.c @@ -25,7 +25,7 @@ #include <openssl/kdf.h> #define HAVE_OPENSSL_HKDF 1 #endif -#endif +#endif /* defined(ENABLE_OPENSSL) */ #include <string.h> @@ -109,7 +109,7 @@ crypto_expand_key_material_rfc5869_sha256_openssl( return 0; } -#else +#else /* !defined(HAVE_OPENSSL_HKDF) */ /** * Perform RFC5869 HKDF computation using our own legacy implementation. @@ -166,7 +166,7 @@ crypto_expand_key_material_rfc5869_sha256_legacy( memwipe(mac, 0, sizeof(mac)); return 0; } -#endif +#endif /* defined(HAVE_OPENSSL_HKDF) */ /** Expand some secret key material according to RFC5869, using SHA256 as the * underlying hash. The <b>key_in_len</b> bytes at <b>key_in</b> are the @@ -191,11 +191,11 @@ crypto_expand_key_material_rfc5869_sha256( salt_in_len, info_in, info_in_len, key_out, key_out_len); -#else +#else /* !defined(HAVE_OPENSSL_HKDF) */ return crypto_expand_key_material_rfc5869_sha256_legacy(key_in, key_in_len, salt_in, salt_in_len, info_in, info_in_len, key_out, key_out_len); -#endif +#endif /* defined(HAVE_OPENSSL_HKDF) */ } diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 329c264af6..a16bf4e11a 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -12,6 +12,8 @@ #include "orconfig.h" +#define CRYPTO_PRIVATE + #include "lib/crypt_ops/crypto_init.h" #include "lib/crypt_ops/crypto_curve25519.h" @@ -20,8 +22,11 @@ #include "lib/crypt_ops/crypto_openssl_mgt.h" #include "lib/crypt_ops/crypto_nss_mgt.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_sys.h" + +#include "lib/subsys/subsys.h" -#include "siphash.h" +#include "ext/siphash.h" /** Boolean: has our crypto library been initialized? (early phase) */ static int crypto_early_initialized_ = 0; @@ -66,6 +71,8 @@ crypto_early_init(void) if (crypto_init_siphash_key() < 0) return -1; + crypto_rand_fast_init(); + curve25519_init(); ed25519_init(); } @@ -92,7 +99,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) (void)useAccel; (void)accelName; (void)accelDir; -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS if (crypto_nss_late_init() < 0) return -1; @@ -108,6 +115,7 @@ crypto_thread_cleanup(void) #ifdef ENABLE_OPENSSL crypto_openssl_thread_cleanup(); #endif + destroy_thread_fast_rng(); } /** @@ -126,6 +134,8 @@ crypto_global_cleanup(void) crypto_nss_global_cleanup(); #endif + crypto_rand_fast_shutdown(); + crypto_early_initialized_ = 0; crypto_global_initialized_ = 0; have_seeded_siphash = 0; @@ -142,6 +152,12 @@ crypto_prefork(void) #ifdef ENABLE_NSS crypto_nss_prefork(); #endif + /* It is not safe to share a fast_rng object across a fork boundary unless + * we actually have zero-on-fork support in map_anon.c. If we have + * drop-on-fork support, we will crash; if we have neither, we will yield + * a copy of the parent process's rng, which is scary and insecure. + */ + destroy_thread_fast_rng(); } /** Run operations that the crypto library requires to be happy again @@ -202,3 +218,47 @@ tor_is_using_nss(void) return 0; #endif } + +static int +subsys_crypto_initialize(void) +{ + if (crypto_early_init() < 0) + return -1; + crypto_dh_init(); + return 0; +} + +static void +subsys_crypto_shutdown(void) +{ + crypto_global_cleanup(); +} + +static void +subsys_crypto_prefork(void) +{ + crypto_prefork(); +} + +static void +subsys_crypto_postfork(void) +{ + crypto_postfork(); +} + +static void +subsys_crypto_thread_cleanup(void) +{ + crypto_thread_cleanup(); +} + +const struct subsys_fns_t sys_crypto = { + .name = "crypto", + .supported = true, + .level = -60, + .initialize = subsys_crypto_initialize, + .shutdown = subsys_crypto_shutdown, + .prefork = subsys_crypto_prefork, + .postfork = subsys_crypto_postfork, + .thread_cleanup = subsys_crypto_thread_cleanup, +}; diff --git a/src/lib/crypt_ops/crypto_init.h b/src/lib/crypt_ops/crypto_init.h index 540d08eb56..8de3eb03ed 100644 --- a/src/lib/crypt_ops/crypto_init.h +++ b/src/lib/crypt_ops/crypto_init.h @@ -33,4 +33,4 @@ const char *crypto_get_header_version_string(void); int tor_is_using_nss(void); -#endif /* !defined(TOR_CRYPTO_H) */ +#endif /* !defined(TOR_CRYPTO_INIT_H) */ diff --git a/src/lib/crypt_ops/crypto_nss_mgt.h b/src/lib/crypt_ops/crypto_nss_mgt.h index 72fd2a1229..4cfa9b42a4 100644 --- a/src/lib/crypt_ops/crypto_nss_mgt.h +++ b/src/lib/crypt_ops/crypto_nss_mgt.h @@ -29,6 +29,6 @@ void crypto_nss_global_cleanup(void); void crypto_nss_prefork(void); void crypto_nss_postfork(void); -#endif +#endif /* defined(ENABLE_NSS) */ -#endif /* !defined(TOR_CRYPTO_NSS_H) */ +#endif /* !defined(TOR_CRYPTO_NSS_MGT_H) */ diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c index 2186d2a939..ed832d852e 100644 --- a/src/lib/crypt_ops/crypto_ope.c +++ b/src/lib/crypt_ops/crypto_ope.c @@ -57,9 +57,9 @@ ope_val_from_le(ope_val_t x) ((x) >> 8) | (((x)&0xff) << 8); } -#else +#else /* !defined(WORDS_BIGENDIAN) */ #define ope_val_from_le(x) (x) -#endif +#endif /* defined(WORDS_BIGENDIAN) */ /** * Return a new AES256-CTR stream cipher object for <b>ope</b>, ready to yield diff --git a/src/lib/crypt_ops/crypto_ope.h b/src/lib/crypt_ops/crypto_ope.h index 610d956335..9778dfe0f0 100644 --- a/src/lib/crypt_ops/crypto_ope.h +++ b/src/lib/crypt_ops/crypto_ope.h @@ -41,6 +41,6 @@ struct aes_cnt_cipher; STATIC struct aes_cnt_cipher *ope_get_cipher(const crypto_ope_t *ope, uint32_t initial_idx); STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher *c, size_t n); -#endif +#endif /* defined(CRYPTO_OPE_PRIVATE) */ -#endif +#endif /* !defined(CRYPTO_OPE_H) */ diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c index c97815f9a4..f51309219a 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.c +++ b/src/lib/crypt_ops/crypto_openssl_mgt.c @@ -176,6 +176,10 @@ crypto_openssl_free_all(void) tor_free(crypto_openssl_version_str); tor_free(crypto_openssl_header_version_str); + /* Destroying a locked mutex is undefined behaviour. This mutex may be + * locked, because multiple threads can access it. But we need to destroy + * it, otherwise re-initialisation will trigger undefined behaviour. + * See #31735 for details. */ #ifndef NEW_THREAD_API if (n_openssl_mutexes_) { int n = n_openssl_mutexes_; @@ -200,10 +204,10 @@ crypto_openssl_early_init(void) OPENSSL_INIT_LOAD_CRYPTO_STRINGS | OPENSSL_INIT_ADD_ALL_CIPHERS | OPENSSL_INIT_ADD_ALL_DIGESTS, NULL); -#else +#else /* !defined(OPENSSL_1_1_API) */ ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); -#endif +#endif /* defined(OPENSSL_1_1_API) */ setup_openssl_threading(); diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.h b/src/lib/crypt_ops/crypto_openssl_mgt.h index a3dd03aa04..111a2d12ed 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.h +++ b/src/lib/crypt_ops/crypto_openssl_mgt.h @@ -84,6 +84,6 @@ int crypto_openssl_late_init(int useAccel, const char *accelName, void crypto_openssl_thread_cleanup(void); void crypto_openssl_global_cleanup(void); -#endif /* ENABLE_OPENSSL */ +#endif /* defined(ENABLE_OPENSSL) */ #endif /* !defined(TOR_CRYPTO_OPENSSL_H) */ diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index 915fe0870d..afbafbfa35 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -11,7 +11,6 @@ * number generators, and working with randomness. **/ -#ifndef CRYPTO_RAND_PRIVATE #define CRYPTO_RAND_PRIVATE #include "lib/crypt_ops/crypto_rand.h" @@ -37,6 +36,7 @@ #include "lib/defs/digest_sizes.h" #include "lib/crypt_ops/crypto_digest.h" +#include "lib/ctime/di_ops.h" #ifdef ENABLE_NSS #include "lib/crypt_ops/crypto_nss_mgt.h" @@ -47,7 +47,7 @@ DISABLE_GCC_WARNING(redundant-decls) #include <openssl/rand.h> #include <openssl/sha.h> ENABLE_GCC_WARNING(redundant-decls) -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS #include <pk11pub.h> @@ -248,7 +248,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) (void)out; (void)out_len; return -1; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ static const char *filenames[] = { "/dev/srandom", "/dev/urandom", "/dev/random", NULL }; @@ -315,7 +315,7 @@ crypto_strongest_rand_raw(uint8_t *out, size_t out_len) } } - if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len)) + if ((out_len < sanity_min_size) || !safe_mem_is_zero((char*)out, out_len)) return 0; } @@ -419,7 +419,7 @@ crypto_seed_openssl_rng(void) else return -1; } -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS /** @@ -442,7 +442,7 @@ crypto_seed_nss_rng(void) return load_entropy_ok ? 0 : -1; } -#endif +#endif /* defined(ENABLE_NSS) */ /** * Seed the RNG for any and all crypto libraries that we're using with bytes @@ -520,121 +520,24 @@ crypto_rand_unmocked(char *to, size_t n) #undef BUFLEN } -#else +#else /* !defined(ENABLE_NSS) */ int r = RAND_bytes((unsigned char*)to, (int)n); /* We consider a PRNG failure non-survivable. Let's assert so that we get a * stack trace about where it happened. */ tor_assert(r >= 0); -#endif +#endif /* defined(ENABLE_NSS) */ } /** - * Return a pseudorandom integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and - * INT_MAX+1, inclusive. + * Draw an unsigned 32-bit integer uniformly at random. */ -int -crypto_rand_int(unsigned int max) -{ - unsigned int val; - unsigned int cutoff; - tor_assert(max <= ((unsigned int)INT_MAX)+1); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT_MAX - (UINT_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** - * Return a pseudorandom integer, chosen uniformly from the values i such - * that min <= i < max. - * - * <b>min</b> MUST be in range [0, <b>max</b>). - * <b>max</b> MUST be in range (min, INT_MAX]. - **/ -int -crypto_rand_int_range(unsigned int min, unsigned int max) -{ - tor_assert(min < max); - tor_assert(max <= INT_MAX); - - /* The overflow is avoided here because crypto_rand_int() returns a value - * between 0 and (max - min) inclusive. */ - return min + crypto_rand_int(max - min); -} - -/** - * As crypto_rand_int_range, but supports uint64_t. - **/ -uint64_t -crypto_rand_uint64_range(uint64_t min, uint64_t max) +uint32_t +crypto_rand_u32(void) { - tor_assert(min < max); - return min + crypto_rand_uint64(max - min); -} - -/** - * As crypto_rand_int_range, but supports time_t. - **/ -time_t -crypto_rand_time_range(time_t min, time_t max) -{ - tor_assert(min < max); - return min + (time_t)crypto_rand_uint64(max - min); -} - -/** - * Return a pseudorandom 64-bit integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. - **/ -uint64_t -crypto_rand_uint64(uint64_t max) -{ - uint64_t val; - uint64_t cutoff; - tor_assert(max < UINT64_MAX); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT64_MAX - (UINT64_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** - * Return a pseudorandom double d, chosen uniformly from the range - * 0.0 <= d < 1.0. - **/ -double -crypto_rand_double(void) -{ - /* We just use an unsigned int here; we don't really care about getting - * more than 32 bits of resolution */ - unsigned int u; - crypto_rand((char*)&u, sizeof(u)); -#if SIZEOF_INT == 4 -#define UINT_MAX_AS_DOUBLE 4294967296.0 -#elif SIZEOF_INT == 8 -#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 -#else -#error SIZEOF_INT is neither 4 nor 8 -#endif /* SIZEOF_INT == 4 || ... */ - return ((double)u) / UINT_MAX_AS_DOUBLE; + uint32_t rand; + crypto_rand((void*)&rand, sizeof(rand)); + return rand; } /** @@ -724,8 +627,6 @@ crypto_force_rand_ssleay(void) RAND_set_rand_method(default_method); return 1; } -#endif +#endif /* defined(ENABLE_OPENSSL) */ return 0; } - -#endif /* !defined(CRYPTO_RAND_PRIVATE) */ diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 86fa20faa3..a019287aa9 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -16,6 +16,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "lib/testsupport/testsupport.h" +#include "lib/malloc/malloc.h" /* random numbers */ int crypto_seed_rng(void) ATTR_WUR; @@ -24,9 +25,11 @@ void crypto_rand_unmocked(char *to, size_t n); void crypto_strongest_rand(uint8_t *out, size_t out_len); MOCK_DECL(void,crypto_strongest_rand_,(uint8_t *out, size_t out_len)); int crypto_rand_int(unsigned int max); +unsigned crypto_rand_uint(unsigned limit); int crypto_rand_int_range(unsigned int min, unsigned int max); uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); time_t crypto_rand_time_range(time_t min, time_t max); +uint32_t crypto_rand_u32(void); uint64_t crypto_rand_uint64(uint64_t max); double crypto_rand_double(void); struct tor_weak_rng_t; @@ -40,6 +43,61 @@ void *smartlist_choose(const struct smartlist_t *sl); void smartlist_shuffle(struct smartlist_t *sl); int crypto_force_rand_ssleay(void); +/** + * A fast PRNG, for use when the PRNG provided by our crypto library isn't + * fast enough. This one _should_ be cryptographically strong, but + * has seen less auditing than the PRNGs in OpenSSL and NSS. Use with + * caution. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +typedef struct crypto_fast_rng_t crypto_fast_rng_t; +/** + * Number of bytes used to seed a crypto_rand_fast_t. + **/ +crypto_fast_rng_t *crypto_fast_rng_new(void); +#define CRYPTO_FAST_RNG_SEED_LEN 48 +crypto_fast_rng_t *crypto_fast_rng_new_from_seed(const uint8_t *seed); +void crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n); +void crypto_fast_rng_free_(crypto_fast_rng_t *); +#define crypto_fast_rng_free(c) \ + FREE_AND_NULL(crypto_fast_rng_t, crypto_fast_rng_free_, (c)) + +unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit); +uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit); +uint32_t crypto_fast_rng_get_u32(crypto_fast_rng_t *rng); +uint64_t crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng, + uint64_t min, uint64_t max); +double crypto_fast_rng_get_double(crypto_fast_rng_t *rng); + +/** + * Using the fast_rng <b>rng</b>, yield true with probability + * 1/<b>n</b>. Otherwise yield false. + * + * <b>n</b> must not be zero. + **/ +#define crypto_fast_rng_one_in_n(rng, n) \ + (0 == (crypto_fast_rng_get_uint((rng), (n)))) + +crypto_fast_rng_t *get_thread_fast_rng(void); + +#ifdef CRYPTO_PRIVATE +/* These are only used from crypto_init.c */ +void destroy_thread_fast_rng(void); +void crypto_rand_fast_init(void); +void crypto_rand_fast_shutdown(void); +#endif /* defined(CRYPTO_PRIVATE) */ + +#if defined(TOR_UNIT_TESTS) +/* Used for white-box testing */ +size_t crypto_fast_rng_get_bytes_used_per_stream(void); +/* For deterministic prng implementations */ +void crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng); +/* To override the prng for testing. */ +crypto_fast_rng_t *crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng); +#endif /* defined(TOR_UNIT_TESTS) */ + #ifdef CRYPTO_RAND_PRIVATE STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c new file mode 100644 index 0000000000..e6ceb42ccb --- /dev/null +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -0,0 +1,439 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand_fast.c + * + * \brief A fast strong PRNG for use when our underlying cryptographic + * library's PRNG isn't fast enough. + **/ + +/* This library is currently implemented to use the same implementation + * technique as libottery, using AES-CTR-256 as our underlying stream cipher. + * It's backtracking-resistant immediately, and prediction-resistant after + * a while. + * + * Here's how it works: + * + * We generate pseudorandom bytes using AES-CTR-256. We generate BUFLEN bytes + * at a time. When we do this, we keep the first SEED_LEN bytes as the key + * and the IV for our next invocation of AES_CTR, and yield the remaining + * BUFLEN - SEED_LEN bytes to the user as they invoke the PRNG. As we yield + * bytes to the user, we clear them from the buffer. + * + * After we have refilled the buffer RESEED_AFTER times, we mix in an + * additional SEED_LEN bytes from our strong PRNG into the seed. + * + * If the user ever asks for a huge number of bytes at once, we pull SEED_LEN + * bytes from the PRNG and use them with our stream cipher to fill the user's + * request. + */ + +#define CRYPTO_RAND_FAST_PRIVATE +#define CRYPTO_PRIVATE + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_cipher.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/intmath/cmp.h" +#include "lib/cc/ctassert.h" +#include "lib/malloc/map_anon.h" +#include "lib/thread/threads.h" + +#include "lib/log/util_bug.h" + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#include <string.h> + +#ifdef NOINHERIT_CAN_FAIL +#define CHECK_PID +#endif + +#ifdef CHECK_PID +#define PID_FIELD_LEN sizeof(pid_t) +#else +#define PID_FIELD_LEN 0 +#endif + +/* Alias for CRYPTO_FAST_RNG_SEED_LEN to make our code shorter. + */ +#define SEED_LEN (CRYPTO_FAST_RNG_SEED_LEN) + +/* The amount of space that we mmap for a crypto_fast_rng_t. + */ +#define MAPLEN 4096 + +/* The number of random bytes that we can yield to the user after each + * time we fill a crypto_fast_rng_t's buffer. + */ +#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN - PID_FIELD_LEN) + +/* The number of buffer refills after which we should fetch more + * entropy from crypto_strongest_rand(). + */ +#define RESEED_AFTER 16 + +/* The length of the stream cipher key we will use for the PRNG, in bytes. + */ +#define KEY_LEN (CRYPTO_FAST_RNG_SEED_LEN - CIPHER_IV_LEN) +/* The length of the stream cipher key we will use for the PRNG, in bits. + */ +#define KEY_BITS (KEY_LEN * 8) + +/* Make sure that we have a key length we can actually use with AES. */ +CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256); + +struct crypto_fast_rng_t { + /** How many more fills does this buffer have before we should mix + * in the output of crypto_strongest_rand()? + * + * This value may be negative if unit tests are enabled. If so, it + * indicates that we should never mix in extra data from + * crypto_strongest_rand(). + */ + int16_t n_till_reseed; + /** How many bytes are remaining in cbuf.bytes? */ + uint16_t bytes_left; +#ifdef CHECK_PID + /** Which process owns this fast_rng? If this value is zero, we do not + * need to test the owner. */ + pid_t owner; +#endif + struct cbuf { + /** The seed (key and IV) that we will use the next time that we refill + * cbuf. */ + uint8_t seed[SEED_LEN]; + /** + * Bytes that we are yielding to the user. The next byte to be + * yielded is at bytes[BUFLEN-bytes_left]; all other bytes in this + * array are set to zero. + */ + uint8_t bytes[BUFLEN]; + } buf; +}; + +/* alignof(uint8_t) should be 1, so there shouldn't be any padding in cbuf. + */ +CTASSERT(sizeof(struct cbuf) == BUFLEN+SEED_LEN); +/* We're trying to fit all of the RNG state into a nice mmapable chunk. + */ +CTASSERT(sizeof(crypto_fast_rng_t) <= MAPLEN); + +/** + * Initialize and return a new fast PRNG, using a strong random seed. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +crypto_fast_rng_t * +crypto_fast_rng_new(void) +{ + uint8_t seed[SEED_LEN]; + crypto_strongest_rand(seed, sizeof(seed)); + crypto_fast_rng_t *result = crypto_fast_rng_new_from_seed(seed); + memwipe(seed, 0, sizeof(seed)); + return result; +} + +/** + * Initialize and return a new fast PRNG, using a seed value specified + * in <b>seed</b>. This value must be CRYPTO_FAST_RNG_SEED_LEN bytes + * long. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, you should probably look at get_thread_fast_rng(). Alternatively, + * use crypto_rand(), wrap this in a mutex. + **/ +crypto_fast_rng_t * +crypto_fast_rng_new_from_seed(const uint8_t *seed) +{ + unsigned inherit = INHERIT_RES_KEEP; + /* We try to allocate this object as securely as we can, to avoid + * having it get dumped, swapped, or shared after fork. + */ + crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result), + ANONMAP_PRIVATE | ANONMAP_NOINHERIT, + &inherit); + memcpy(result->buf.seed, seed, SEED_LEN); + /* Causes an immediate refill once the user asks for data. */ + result->bytes_left = 0; + result->n_till_reseed = RESEED_AFTER; +#ifdef CHECK_PID + if (inherit == INHERIT_RES_KEEP) { + /* This value will neither be dropped nor zeroed after fork, so we need to + * check our pid to make sure we are not sharing it across a fork. This + * can be expensive if the pid value isn't cached, sadly. + */ + result->owner = getpid(); + } +#elif defined(_WIN32) + /* Windows can't fork(), so there's no need to noinherit. */ +#else + /* We decided above that noinherit would always do _something_. Assert here + * that we were correct. */ + tor_assertf(inherit != INHERIT_RES_KEEP, + "We failed to create a non-inheritable memory region, even " + "though we believed such a failure to be impossible! This is " + "probably a bug in Tor support for your platform; please report " + "it."); +#endif /* defined(CHECK_PID) || ... */ + return result; +} + +#ifdef TOR_UNIT_TESTS +/** + * Unit tests only: prevent a crypto_fast_rng_t from ever mixing in more + * entropy. + */ +void +crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng) +{ + rng->n_till_reseed = -1; +} +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Helper: create a crypto_cipher_t object from SEED_LEN bytes of + * input. The first KEY_LEN bytes are used as the stream cipher's key, + * and the remaining CIPHER_IV_LEN bytes are used as its IV. + **/ +static inline crypto_cipher_t * +cipher_from_seed(const uint8_t *seed) +{ + return crypto_cipher_new_with_iv_and_bits(seed, seed+KEY_LEN, KEY_BITS); +} + +/** + * Helper: mix additional entropy into <b>rng</b> by using our XOF to mix the + * old value for the seed with some additional bytes from + * crypto_strongest_rand(). + **/ +static void +crypto_fast_rng_add_entopy(crypto_fast_rng_t *rng) +{ + crypto_xof_t *xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN); + { + uint8_t seedbuf[SEED_LEN]; + crypto_strongest_rand(seedbuf, SEED_LEN); + crypto_xof_add_bytes(xof, seedbuf, SEED_LEN); + memwipe(seedbuf, 0, SEED_LEN); + } + crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN); + crypto_xof_free(xof); +} + +/** + * Helper: refill the seed bytes and output buffer of <b>rng</b>, using + * the input seed bytes as input (key and IV) for the stream cipher. + * + * If the n_till_reseed counter has reached zero, mix more random bytes into + * the seed before refilling the buffer. + **/ +static void +crypto_fast_rng_refill(crypto_fast_rng_t *rng) +{ + rng->n_till_reseed--; + if (rng->n_till_reseed == 0) { + /* It's time to reseed the RNG. */ + crypto_fast_rng_add_entopy(rng); + rng->n_till_reseed = RESEED_AFTER; + } else if (rng->n_till_reseed < 0) { +#ifdef TOR_UNIT_TESTS + /* Reseeding is disabled for testing; never do it on this prng. */ + rng->n_till_reseed = -1; +#else + /* If testing is disabled, this shouldn't be able to become negative. */ + tor_assert_unreached(); +#endif /* defined(TOR_UNIT_TESTS) */ + } + /* Now fill rng->buf with output from our stream cipher, initialized from + * that seed value. */ + crypto_cipher_t *c = cipher_from_seed(rng->buf.seed); + memset(&rng->buf, 0, sizeof(rng->buf)); + crypto_cipher_crypt_inplace(c, (char*)&rng->buf, sizeof(rng->buf)); + crypto_cipher_free(c); + + rng->bytes_left = sizeof(rng->buf.bytes); +} + +/** + * Release all storage held by <b>rng</b>. + **/ +void +crypto_fast_rng_free_(crypto_fast_rng_t *rng) +{ + if (!rng) + return; + memwipe(rng, 0, sizeof(*rng)); + tor_munmap_anonymous(rng, sizeof(*rng)); +} + +/** + * Helper: extract bytes from the PRNG, refilling it as necessary. Does not + * optimize the case when the user has asked for a huge output. + **/ +static void +crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out, + const size_t n) +{ +#ifdef CHECK_PID + if (rng->owner) { + /* Note that we only need to do this check when we have owner set: that + * is, when our attempt to block inheriting failed, and the result was + * INHERIT_RES_KEEP. + * + * If the result was INHERIT_RES_DROP, then any attempt to access the rng + * memory after forking will crash. + * + * If the result was INHERIT_RES_ZERO, then forking will set the bytes_left + * and n_till_reseed fields to zero. This function will call + * crypto_fast_rng_refill(), which will in turn reseed the PRNG. + * + * So we only need to do this test in the case when mmap_anonymous() + * returned INHERIT_KEEP. We avoid doing it needlessly, since getpid() is + * often a system call, and that can be slow. + */ + tor_assert(rng->owner == getpid()); + } +#endif /* defined(CHECK_PID) */ + + size_t bytes_to_yield = n; + + while (bytes_to_yield) { + if (rng->bytes_left == 0) + crypto_fast_rng_refill(rng); + + const size_t to_copy = MIN(rng->bytes_left, bytes_to_yield); + + tor_assert(sizeof(rng->buf.bytes) >= rng->bytes_left); + uint8_t *copy_from = rng->buf.bytes + + (sizeof(rng->buf.bytes) - rng->bytes_left); + memcpy(out, copy_from, to_copy); + memset(copy_from, 0, to_copy); + + out += to_copy; + bytes_to_yield -= to_copy; + rng->bytes_left -= to_copy; + } +} + +/** + * Extract <b>n</b> bytes from <b>rng</b> into the buffer at <b>out</b>. + **/ +void +crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n) +{ + if (PREDICT_UNLIKELY(n > BUFLEN)) { + /* The user has asked for a lot of output; generate it from a stream + * cipher seeded by the PRNG rather than by pulling it out of the PRNG + * directly. + */ + uint8_t seed[SEED_LEN]; + crypto_fast_rng_getbytes_impl(rng, seed, SEED_LEN); + crypto_cipher_t *c = cipher_from_seed(seed); + memset(out, 0, n); + crypto_cipher_crypt_inplace(c, (char*)out, n); + crypto_cipher_free(c); + memwipe(seed, 0, sizeof(seed)); + return; + } + + crypto_fast_rng_getbytes_impl(rng, out, n); +} + +#if defined(TOR_UNIT_TESTS) +/** for white-box testing: return the number of bytes that are returned from + * the user for each invocation of the stream cipher in this RNG. */ +size_t +crypto_fast_rng_get_bytes_used_per_stream(void) +{ + return BUFLEN; +} +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Thread-local instance for our fast RNG. + **/ +static tor_threadlocal_t thread_rng; + +/** + * Return a per-thread fast RNG, initializing it if necessary. + * + * You do not need to free this yourself. + * + * It is NOT safe to share this value across threads. + **/ +crypto_fast_rng_t * +get_thread_fast_rng(void) +{ + crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng); + + if (PREDICT_UNLIKELY(rng == NULL)) { + rng = crypto_fast_rng_new(); + tor_threadlocal_set(&thread_rng, rng); + } + + return rng; +} + +/** + * Used when a thread is exiting: free the per-thread fast RNG if needed. + * Invoked from the crypto subsystem's thread-cleanup code. + **/ +void +destroy_thread_fast_rng(void) +{ + crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng); + if (!rng) + return; + crypto_fast_rng_free(rng); + tor_threadlocal_set(&thread_rng, NULL); +} + +#ifdef TOR_UNIT_TESTS +/** + * Replace the current thread's rng with <b>rng</b>. For use by the + * unit tests only. Returns the previous thread rng. + **/ +crypto_fast_rng_t * +crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng) +{ + crypto_fast_rng_t *old_rng = tor_threadlocal_get(&thread_rng); + tor_threadlocal_set(&thread_rng, rng); + return old_rng; +} +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Initialize the global thread-local key that will be used to keep track + * of per-thread fast RNG instances. Called from the crypto subsystem's + * initialization code. + **/ +void +crypto_rand_fast_init(void) +{ + tor_threadlocal_init(&thread_rng); +} + +/** + * Initialize the global thread-local key that will be used to keep track + * of per-thread fast RNG instances. Called from the crypto subsystem's + * shutdown code. + **/ +void +crypto_rand_fast_shutdown(void) +{ + destroy_thread_fast_rng(); + tor_threadlocal_destroy(&thread_rng); +} diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c new file mode 100644 index 0000000000..ffbfa2d56c --- /dev/null +++ b/src/lib/crypt_ops/crypto_rand_numeric.c @@ -0,0 +1,194 @@ +/** + * \file crypto_rand_numeric.c + * + * \brief Functions for retrieving uniformly distributed numbers + * from our PRNGs. + **/ + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/log/util_bug.h" + +/** + * Implementation macro: yields code that returns a uniform unbiased + * random number between 0 and limit. "type" is the type of the number to + * return; "maxval" is the largest possible value of "type"; and "fill_stmt" + * is a code snippet that fills an object named "val" with random bits. + **/ +#define IMPLEMENT_RAND_UNSIGNED(type, maxval, limit, fill_stmt) \ + do { \ + type val; \ + type cutoff; \ + tor_assert((limit) > 0); \ + \ + /* We ignore any values that are >= 'cutoff,' to avoid biasing */ \ + /* the distribution with clipping at the upper end of the type's */ \ + /* range. */ \ + cutoff = (maxval) - ((maxval)%(limit)); \ + while (1) { \ + fill_stmt; \ + if (val < cutoff) \ + return val % (limit); \ + } \ + } while (0) + +/** + * Return a pseudorandom integer chosen uniformly from the values between 0 + * and <b>limit</b>-1 inclusive. limit must be strictly between 0 and + * UINT_MAX. */ +unsigned +crypto_rand_uint(unsigned limit) +{ + tor_assert(limit < UINT_MAX); + IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit, + crypto_rand((char*)&val, sizeof(val))); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and + * INT_MAX+1, inclusive. + */ +int +crypto_rand_int(unsigned int max) +{ + /* We can't use IMPLEMENT_RAND_UNSIGNED directly, since we're trying + * to return a signed type. Instead we make sure that the range is + * reasonable for a nonnegative int, use crypto_rand_uint(), and cast. + */ + tor_assert(max <= ((unsigned int)INT_MAX)+1); + + return (int)crypto_rand_uint(max); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values i such + * that min <= i < max. + * + * <b>min</b> MUST be in range [0, <b>max</b>). + * <b>max</b> MUST be in range (min, INT_MAX]. + **/ +int +crypto_rand_int_range(unsigned int min, unsigned int max) +{ + tor_assert(min < max); + tor_assert(max <= INT_MAX); + + /* The overflow is avoided here because crypto_rand_int() returns a value + * between 0 and (max - min) inclusive. */ + return min + crypto_rand_int(max - min); +} + +/** + * As crypto_rand_int_range, but supports uint64_t. + **/ +uint64_t +crypto_rand_uint64_range(uint64_t min, uint64_t max) +{ + tor_assert(min < max); + return min + crypto_rand_uint64(max - min); +} + +/** + * As crypto_rand_int_range, but supports time_t. + **/ +time_t +crypto_rand_time_range(time_t min, time_t max) +{ + tor_assert(min < max); + return min + (time_t)crypto_rand_uint64(max - min); +} + +/** + * Return a pseudorandom 64-bit integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. + **/ +uint64_t +crypto_rand_uint64(uint64_t max) +{ + tor_assert(max < UINT64_MAX); + IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, max, + crypto_rand((char*)&val, sizeof(val))); +} + +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif /* SIZEOF_INT == 4 || ... */ + +/** + * Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + **/ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int u; + crypto_rand((char*)&u, sizeof(u)); + return ((double)u) / UINT_MAX_AS_DOUBLE; +} + +/** + * As crypto_rand_uint, but extract the result from a crypto_fast_rng_t + */ +unsigned +crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit) +{ + tor_assert(limit < UINT_MAX); + IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit, + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val))); +} + +/** + * As crypto_rand_uint64, but extract the result from a crypto_fast_rng_t. + */ +uint64_t +crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit) +{ + tor_assert(limit < UINT64_MAX); + IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, limit, + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val))); +} + +/** + * As crypto_rand_u32, but extract the result from a crypto_fast_rng_t. + */ +uint32_t +crypto_fast_rng_get_u32(crypto_fast_rng_t *rng) +{ + uint32_t val; + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)); + return val; +} + +/** + * As crypto_rand_uint64_range(), but extract the result from a + * crypto_fast_rng_t. + */ +uint64_t +crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng, + uint64_t min, uint64_t max) +{ + /* Handle corrupted input */ + if (BUG(min >= max)) { + return min; + } + + return min + crypto_fast_rng_get_uint64(rng, max - min); +} + +/** + * As crypto_rand_get_double() but extract the result from a crypto_fast_rng_t. + */ +double +crypto_fast_rng_get_double(crypto_fast_rng_t *rng) +{ + unsigned int u; + crypto_fast_rng_getbytes(rng, (void*)&u, sizeof(u)); + return ((double)u) / UINT_MAX_AS_DOUBLE; +} + diff --git a/src/lib/crypt_ops/crypto_rsa.c b/src/lib/crypt_ops/crypto_rsa.c index 8fd8a8aa7b..1289a82e62 100644 --- a/src/lib/crypt_ops/crypto_rsa.c +++ b/src/lib/crypt_ops/crypto_rsa.c @@ -59,7 +59,7 @@ crypto_get_rsa_padding(int padding) default: tor_assert(0); return -1; // LCOV_EXCL_LINE } } -#endif +#endif /* defined(ENABLE_OPENSSL) */ /** Compare the public-key components of a and b. Return non-zero iff * a==b. A NULL key is considered to be distinct from all non-NULL diff --git a/src/lib/crypt_ops/crypto_rsa.h b/src/lib/crypt_ops/crypto_rsa.h index 6d9cc8d30e..01e6d10906 100644 --- a/src/lib/crypt_ops/crypto_rsa.h +++ b/src/lib/crypt_ops/crypto_rsa.h @@ -122,7 +122,7 @@ struct rsa_st *crypto_pk_get_openssl_rsa_(crypto_pk_t *env); crypto_pk_t *crypto_new_pk_from_openssl_rsa_(struct rsa_st *rsa); MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_openssl_evp_pkey_,( crypto_pk_t *env,int private)); -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS struct SECKEYPublicKeyStr; @@ -132,7 +132,7 @@ const struct SECKEYPublicKeyStr *crypto_pk_get_nss_pubkey( const crypto_pk_t *key); const struct SECKEYPrivateKeyStr *crypto_pk_get_nss_privkey( const crypto_pk_t *key); -#endif +#endif /* defined(ENABLE_NSS) */ void crypto_pk_assign_public(crypto_pk_t *dest, const crypto_pk_t *src); void crypto_pk_assign_private(crypto_pk_t *dest, const crypto_pk_t *src); @@ -143,6 +143,6 @@ struct SECItemStr; STATIC int secitem_uint_cmp(const struct SECItemStr *a, const struct SECItemStr *b); #endif -#endif +#endif /* defined(TOR_UNIT_TESTS) */ -#endif +#endif /* !defined(TOR_CRYPTO_RSA_H) */ diff --git a/src/lib/crypt_ops/crypto_rsa_nss.c b/src/lib/crypt_ops/crypto_rsa_nss.c index 7abf6716f0..fd8fda486e 100644 --- a/src/lib/crypt_ops/crypto_rsa_nss.c +++ b/src/lib/crypt_ops/crypto_rsa_nss.c @@ -156,7 +156,7 @@ crypto_pk_get_openssl_evp_pkey_,(crypto_pk_t *pk, int private)) tor_free(buf); return result; } -#endif +#endif /* defined(ENABLE_OPENSSL) */ /** Allocate and return storage for a public key. The key itself will not yet * be set. diff --git a/src/lib/crypt_ops/crypto_rsa_openssl.c b/src/lib/crypt_ops/crypto_rsa_openssl.c index 17eae24cc2..41d936d25b 100644 --- a/src/lib/crypt_ops/crypto_rsa_openssl.c +++ b/src/lib/crypt_ops/crypto_rsa_openssl.c @@ -54,7 +54,7 @@ crypto_pk_key_is_private(const crypto_pk_t *k) const BIGNUM *p, *q; RSA_get0_factors(k->key, &p, &q); return p != NULL; /* XXX/yawning: Should we check q? */ -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ return k && k->key && k->key->p; #endif /* defined(OPENSSL_1_1_API) */ } @@ -288,7 +288,7 @@ crypto_pk_num_bits(crypto_pk_t *env) tor_assert(n != NULL); return RSA_bits(env->key); -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ tor_assert(env->key->n); return BN_num_bits(env->key->n); #endif /* defined(OPENSSL_1_1_API) */ diff --git a/src/lib/crypt_ops/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c index 42276597d4..361db18927 100644 --- a/src/lib/crypt_ops/crypto_s2k.c +++ b/src/lib/crypt_ops/crypto_s2k.c @@ -285,7 +285,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv < 0) return S2K_FAILED; return (int)key_out_len; -#else +#else /* !defined(ENABLE_OPENSSL) */ SECItem passItem = { .type = siBuffer, .data = (unsigned char *) secret, .len = (int)secret_len }; @@ -325,7 +325,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (alg) SECOID_DestroyAlgorithmID(alg, PR_TRUE); return rv; -#endif +#endif /* defined(ENABLE_OPENSSL) */ } case S2K_TYPE_SCRYPT: { @@ -348,7 +348,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv != 0) return S2K_FAILED; return (int)key_out_len; -#else /* !(defined(HAVE_SCRYPT)) */ +#else /* !defined(HAVE_SCRYPT) */ return S2K_NO_SCRYPT_SUPPORT; #endif /* defined(HAVE_SCRYPT) */ } diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h new file mode 100644 index 0000000000..894243b175 --- /dev/null +++ b/src/lib/crypt_ops/crypto_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_crypto.h + * \brief Declare subsystem object for the crypto module. + **/ + +#ifndef TOR_CRYPTO_SYS_H +#define TOR_CRYPTO_SYS_H + +extern const struct subsys_fns_t sys_crypto; + +#endif /* !defined(TOR_CRYPTO_SYS_H) */ diff --git a/src/lib/crypt_ops/crypto_util.c b/src/lib/crypt_ops/crypto_util.c index 67a1a9eb92..5e3f4a87a1 100644 --- a/src/lib/crypt_ops/crypto_util.c +++ b/src/lib/crypt_ops/crypto_util.c @@ -30,7 +30,7 @@ DISABLE_GCC_WARNING(redundant-decls) #include <openssl/err.h> #include <openssl/crypto.h> ENABLE_GCC_WARNING(redundant-decls) -#endif +#endif /* defined(ENABLE_OPENSSL) */ #include "lib/log/log.h" #include "lib/log/util_bug.h" diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c index 0dba64d595..c931b58369 100644 --- a/src/lib/crypt_ops/digestset.c +++ b/src/lib/crypt_ops/digestset.c @@ -11,7 +11,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/defs/digest_sizes.h" #include "lib/crypt_ops/digestset.h" -#include "siphash.h" +#include "ext/siphash.h" /* Wrap our hash function to have the signature that the bloom filter * needs. */ diff --git a/src/lib/crypt_ops/digestset.h b/src/lib/crypt_ops/digestset.h index 91d53a0542..7d6d687342 100644 --- a/src/lib/crypt_ops/digestset.h +++ b/src/lib/crypt_ops/digestset.h @@ -26,4 +26,4 @@ void digestset_add(digestset_t *set, const char *addr); int digestset_probably_contains(const digestset_t *set, const char *addr); -#endif +#endif /* !defined(TOR_DIGESTSET_H) */ diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index 1022096fdc..1f58a33d38 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-crypt-ops-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_crypt_ops_a_SOURCES = \ src/lib/crypt_ops/crypto_cipher.c \ src/lib/crypt_ops/crypto_curve25519.c \ @@ -17,6 +18,8 @@ src_lib_libtor_crypt_ops_a_SOURCES = \ src/lib/crypt_ops/crypto_ope.c \ src/lib/crypt_ops/crypto_pwbox.c \ src/lib/crypt_ops/crypto_rand.c \ + src/lib/crypt_ops/crypto_rand_fast.c \ + src/lib/crypt_ops/crypto_rand_numeric.c \ src/lib/crypt_ops/crypto_rsa.c \ src/lib/crypt_ops/crypto_s2k.c \ src/lib/crypt_ops/crypto_util.c \ @@ -25,12 +28,14 @@ src_lib_libtor_crypt_ops_a_SOURCES = \ if USE_NSS src_lib_libtor_crypt_ops_a_SOURCES += \ src/lib/crypt_ops/aes_nss.c \ + src/lib/crypt_ops/crypto_digest_nss.c \ src/lib/crypt_ops/crypto_dh_nss.c \ src/lib/crypt_ops/crypto_nss_mgt.c \ src/lib/crypt_ops/crypto_rsa_nss.c else src_lib_libtor_crypt_ops_a_SOURCES += \ src/lib/crypt_ops/aes_openssl.c \ + src/lib/crypt_ops/crypto_digest_openssl.c \ src/lib/crypt_ops/crypto_rsa_openssl.c endif @@ -48,6 +53,7 @@ src_lib_libtor_crypt_ops_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_crypt_ops_testing_a_CFLAGS = \ $(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/crypt_ops/aes.h \ src/lib/crypt_ops/compat_openssl.h \ @@ -66,5 +72,6 @@ noinst_HEADERS += \ src/lib/crypt_ops/crypto_rand.h \ src/lib/crypt_ops/crypto_rsa.h \ src/lib/crypt_ops/crypto_s2k.h \ + src/lib/crypt_ops/crypto_sys.h \ src/lib/crypt_ops/crypto_util.h \ src/lib/crypt_ops/digestset.h diff --git a/src/lib/ctime/include.am b/src/lib/ctime/include.am index b46c43ba0c..83942ca4e0 100644 --- a/src/lib/ctime/include.am +++ b/src/lib/ctime/include.am @@ -11,6 +11,7 @@ else mulodi4_source= endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_ctime_a_SOURCES = \ $(mulodi4_source) \ src/ext/csiphash.c \ @@ -21,5 +22,6 @@ src_lib_libtor_ctime_testing_a_SOURCES = \ src_lib_libtor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@ src_lib_libtor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/ctime/di_ops.h diff --git a/src/lib/defs/dh_sizes.h b/src/lib/defs/dh_sizes.h index a2ffbc51c2..b0d1eba0c5 100644 --- a/src/lib/defs/dh_sizes.h +++ b/src/lib/defs/dh_sizes.h @@ -19,4 +19,4 @@ /** Length of our legacy DH keys. */ #define DH1024_KEY_LEN (1024/8) -#endif +#endif /* !defined(TOR_DH_SIZES_H) */ diff --git a/src/lib/defs/digest_sizes.h b/src/lib/defs/digest_sizes.h index 525e5209d6..a0dd97a74d 100644 --- a/src/lib/defs/digest_sizes.h +++ b/src/lib/defs/digest_sizes.h @@ -24,4 +24,4 @@ /** Length of the output of our 64-bit optimized message digests (SHA512). */ #define DIGEST512_LEN 64 -#endif +#endif /* !defined(TOR_DIGEST_SIZES_H) */ diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am index 48ee7f29fc..84ee403771 100644 --- a/src/lib/defs/include.am +++ b/src/lib/defs/include.am @@ -1,5 +1,8 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/defs/dh_sizes.h \ src/lib/defs/digest_sizes.h \ + src/lib/defs/logging_types.h \ + src/lib/defs/time.h \ src/lib/defs/x25519_sizes.h diff --git a/src/lib/defs/logging_types.h b/src/lib/defs/logging_types.h new file mode 100644 index 0000000000..d3eacde464 --- /dev/null +++ b/src/lib/defs/logging_types.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file logging_types.h + * + * \brief Global definition for types used by logging systems. + **/ + +#ifndef TOR_LOGGING_TYPES_H +#define TOR_LOGGING_TYPES_H + +/* We define this here so that it can be used both by backtrace.h and + * log.h. + */ + +/** Mask of zero or more log domains, OR'd together. */ +typedef uint64_t log_domain_mask_t; + +#endif /* !defined(TOR_LOGGING_TYPES_H) */ diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h new file mode 100644 index 0000000000..459afbf42d --- /dev/null +++ b/src/lib/defs/time.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_TIME_DEFS_H +#define TOR_TIME_DEFS_H + +/** + * \file time.h + * + * \brief Definitions for timing-related constants. + **/ + +/** How many microseconds per second */ +#define TOR_USEC_PER_SEC (1000000) +/** How many nanoseconds per microsecond */ +#define TOR_NSEC_PER_USEC (1000) +/* How many nanoseconds per millisecond */ +#define TOR_NSEC_PER_MSEC (1000*1000) + +#endif /* !defined(TOR_TIME_DEFS_H) */ diff --git a/src/lib/defs/x25519_sizes.h b/src/lib/defs/x25519_sizes.h index 8933a8866b..6431f0a2dd 100644 --- a/src/lib/defs/x25519_sizes.h +++ b/src/lib/defs/x25519_sizes.h @@ -33,4 +33,4 @@ #define ED25519_BASE64_LEN 43 #define ED25519_SIG_BASE64_LEN 86 -#endif +#endif /* !defined(TOR_X25519_SIZES_H) */ diff --git a/src/lib/dispatch/.may_include b/src/lib/dispatch/.may_include new file mode 100644 index 0000000000..884f4c0dbc --- /dev/null +++ b/src/lib/dispatch/.may_include @@ -0,0 +1,11 @@ +orconfig.h + +ext/tor_queue.h + +lib/cc/*.h +lib/container/*.h +lib/dispatch/*.h +lib/intmath/*.h +lib/log/*.h +lib/malloc/*.h +lib/testsupport/*.h
\ No newline at end of file diff --git a/src/lib/dispatch/dispatch.h b/src/lib/dispatch/dispatch.h new file mode 100644 index 0000000000..a9e655409a --- /dev/null +++ b/src/lib/dispatch/dispatch.h @@ -0,0 +1,114 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DISPATCH_H +#define TOR_DISPATCH_H + +#include "lib/dispatch/msgtypes.h" + +/** + * \file dispatch.h + * \brief Low-level APIs for message-passing system. + * + * This module implements message dispatch based on a set of short integer + * identifiers. For a higher-level interface, see pubsub.h. + * + * Each message is represented as a generic msg_t object, and is discriminated + * by its message_id_t. Messages are delivered by a dispatch_t object, which + * delivers each message to its recipients by a configured "channel". + * + * A "channel" is a means of delivering messages. Every message_id_t must + * be associated with exactly one channel, identified by channel_id_t. + * When a channel receives messages, a callback is invoked to either process + * the messages immediately, or to cause them to be processed later. + * + * Every message_id_t has zero or more associated receiver functions set up in + * the dispatch_t object. Once the dispatch_t object is created, receivers + * can be enabled or disabled [TODO], but not added or removed. + * + * Every message_id_t has an associated datatype, identified by a + * msg_type_id_t. These datatypes can be associated with functions to + * (for example) free them, or format them for debugging. + * + * To setup a dispatch_t object, first create a dispatch_cfg_t object, and + * configure messages with their types, channels, and receivers. Then, use + * dispatch_new() with that dispatch_cfg_t to create the dispatch_t object. + * + * (We use a two-phase contruction procedure here to enable better static + * reasoning about publish/subscribe relationships.) + * + * Once you have a dispatch_t, you can queue messages on it with + * dispatch_send*(), and cause those messages to be delivered with + * dispatch_flush(). + **/ + +/** + * A "dispatcher" is the highest-level object; it handles making sure that + * messages are received and delivered properly. Only the mainloop + * should handle this type directly. + */ +typedef struct dispatch_t dispatch_t; + +struct dispatch_cfg_t; + +dispatch_t *dispatch_new(const struct dispatch_cfg_t *cfg); + +/** + * Free a dispatcher. Tor does this at exit. + */ +#define dispatch_free(d) \ + FREE_AND_NULL(dispatch_t, dispatch_free_, (d)) + +void dispatch_free_(dispatch_t *); + +int dispatch_send(dispatch_t *d, + subsys_id_t sender, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + msg_aux_data_t auxdata); + +int dispatch_send_msg(dispatch_t *d, msg_t *m); + +int dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m); + +/* Flush up to <b>max_msgs</b> currently pending messages from the + * dispatcher. Messages that are not pending when this function are + * called, are not flushed by this call. Return 0 on success, -1 on + * unrecoverable error. + */ +int dispatch_flush(dispatch_t *, channel_id_t chan, int max_msgs); + +/** + * Function callback type used to alert some other module when a channel's + * queue changes from empty to nonempty. + * + * Ex 1: To cause messages to be processed immediately on-stack, this callback + * should invoke dispatch_flush() directly. + * + * Ex 2: To cause messages to be processed very soon, from the event queue, + * this callback should schedule an event callback to run dispatch_flush(). + * + * Ex 3: To cause messages to be processed periodically, this function should + * do nothing, and a periodic event should invoke dispatch_flush(). + **/ +typedef void (*dispatch_alertfn_t)(struct dispatch_t *, + channel_id_t, void *); + +int dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan, + dispatch_alertfn_t fn, void *userdata); + +#define dispatch_free_msg(d,msg) \ + STMT_BEGIN { \ + msg_t **msg_tmp_ptr__ = &(msg); \ + dispatch_free_msg_((d), *msg_tmp_ptr__); \ + *msg_tmp_ptr__= NULL; \ + } STMT_END +void dispatch_free_msg_(const dispatch_t *d, msg_t *msg); + +char *dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg); + +#endif /* !defined(TOR_DISPATCH_H) */ diff --git a/src/lib/dispatch/dispatch_cfg.c b/src/lib/dispatch/dispatch_cfg.c new file mode 100644 index 0000000000..b3a72ec22f --- /dev/null +++ b/src/lib/dispatch/dispatch_cfg.c @@ -0,0 +1,141 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_cfg.c + * \brief Create and configure a dispatch_cfg_t. + * + * A dispatch_cfg_t object is used to configure a set of messages and + * associated information before creating a dispatch_t. + */ + +#define DISPATCH_PRIVATE + +#include "orconfig.h" +#include "lib/dispatch/dispatch_cfg.h" +#include "lib/dispatch/dispatch_cfg_st.h" +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" + +#include "lib/container/smartlist.h" +#include "lib/malloc/malloc.h" + +/** + * Create and return a new dispatch_cfg_t. + **/ +dispatch_cfg_t * +dcfg_new(void) +{ + dispatch_cfg_t *cfg = tor_malloc(sizeof(dispatch_cfg_t)); + cfg->type_by_msg = smartlist_new(); + cfg->chan_by_msg = smartlist_new(); + cfg->fns_by_type = smartlist_new(); + cfg->recv_by_msg = smartlist_new(); + return cfg; +} + +/** + * Associate a message with a datatype. Return 0 on success, -1 if a + * different type was previously associated with the message ID. + **/ +int +dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg, + msg_type_id_t type) +{ + smartlist_grow(cfg->type_by_msg, msg+1); + msg_type_id_t *oldval = smartlist_get(cfg->type_by_msg, msg); + if (oldval != NULL && *oldval != type) { + return -1; + } + if (!oldval) + smartlist_set(cfg->type_by_msg, msg, tor_memdup(&type, sizeof(type))); + return 0; +} + +/** + * Associate a message with a channel. Return 0 on success, -1 if a + * different channel was previously associated with the message ID. + **/ +int +dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg, + channel_id_t chan) +{ + smartlist_grow(cfg->chan_by_msg, msg+1); + channel_id_t *oldval = smartlist_get(cfg->chan_by_msg, msg); + if (oldval != NULL && *oldval != chan) { + return -1; + } + if (!oldval) + smartlist_set(cfg->chan_by_msg, msg, tor_memdup(&chan, sizeof(chan))); + return 0; +} + +/** + * Associate a set of functions with a datatype. Return 0 on success, -1 if + * different functions were previously associated with the type. + **/ +int +dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type, + const dispatch_typefns_t *fns) +{ + smartlist_grow(cfg->fns_by_type, type+1); + dispatch_typefns_t *oldfns = smartlist_get(cfg->fns_by_type, type); + if (oldfns && (oldfns->free_fn != fns->free_fn || + oldfns->fmt_fn != fns->fmt_fn)) + return -1; + if (!oldfns) + smartlist_set(cfg->fns_by_type, type, tor_memdup(fns, sizeof(*fns))); + return 0; +} + +/** + * Associate a receiver with a message ID. Multiple receivers may be + * associated with a single messasge ID. + * + * Return 0 on success, on failure. + **/ +int +dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg, + subsys_id_t sys, recv_fn_t fn) +{ + smartlist_grow(cfg->recv_by_msg, msg+1); + smartlist_t *receivers = smartlist_get(cfg->recv_by_msg, msg); + if (!receivers) { + receivers = smartlist_new(); + smartlist_set(cfg->recv_by_msg, msg, receivers); + } + + dispatch_rcv_t *rcv = tor_malloc(sizeof(dispatch_rcv_t)); + rcv->sys = sys; + rcv->enabled = true; + rcv->fn = fn; + smartlist_add(receivers, (void*)rcv); + return 0; +} + +/** Helper: release all storage held by <b>cfg</b>. */ +void +dcfg_free_(dispatch_cfg_t *cfg) +{ + if (!cfg) + return; + + SMARTLIST_FOREACH(cfg->type_by_msg, msg_type_id_t *, id, tor_free(id)); + SMARTLIST_FOREACH(cfg->chan_by_msg, channel_id_t *, id, tor_free(id)); + SMARTLIST_FOREACH(cfg->fns_by_type, dispatch_typefns_t *, f, tor_free(f)); + smartlist_free(cfg->type_by_msg); + smartlist_free(cfg->chan_by_msg); + smartlist_free(cfg->fns_by_type); + SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, receivers) { + if (!receivers) + continue; + SMARTLIST_FOREACH(receivers, dispatch_rcv_t *, rcv, tor_free(rcv)); + smartlist_free(receivers); + } SMARTLIST_FOREACH_END(receivers); + smartlist_free(cfg->recv_by_msg); + + tor_free(cfg); +} diff --git a/src/lib/dispatch/dispatch_cfg.h b/src/lib/dispatch/dispatch_cfg.h new file mode 100644 index 0000000000..348dce8d40 --- /dev/null +++ b/src/lib/dispatch/dispatch_cfg.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DISPATCH_CFG_H +#define TOR_DISPATCH_CFG_H + +#include "lib/dispatch/msgtypes.h" +#include "lib/testsupport/testsupport.h" + +/** + * A "dispatch_cfg" is the configuration used to set up a dispatcher. + * It is created and accessed with a set of dcfg_* functions, and then + * used with dispatcher_new() to make the dispatcher. + */ +typedef struct dispatch_cfg_t dispatch_cfg_t; + +dispatch_cfg_t *dcfg_new(void); + +int dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg, + msg_type_id_t type); + +int dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg, + channel_id_t chan); + +int dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type, + const dispatch_typefns_t *fns); + +int dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg, + subsys_id_t sys, recv_fn_t fn); + +/** Free a dispatch_cfg_t. */ +#define dcfg_free(cfg) \ + FREE_AND_NULL(dispatch_cfg_t, dcfg_free_, (cfg)) + +void dcfg_free_(dispatch_cfg_t *cfg); + +#ifdef DISPATCH_NEW_PRIVATE +struct smartlist_t; +STATIC int max_in_u16_sl(const struct smartlist_t *sl, int dflt); +#endif + +#endif /* !defined(TOR_DISPATCH_CFG_H) */ diff --git a/src/lib/dispatch/dispatch_cfg_st.h b/src/lib/dispatch/dispatch_cfg_st.h new file mode 100644 index 0000000000..57b6f0347f --- /dev/null +++ b/src/lib/dispatch/dispatch_cfg_st.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DISPATCH_CFG_ST_H +#define TOR_DISPATCH_CFG_ST_H + +struct smartlist_t; + +/* Information needed to create a dispatcher, but in a less efficient, more + * mutable format. */ +struct dispatch_cfg_t { + /** A list of msg_type_id_t (cast to void*), indexed by msg_t. */ + struct smartlist_t *type_by_msg; + /** A list of channel_id_t (cast to void*), indexed by msg_t. */ + struct smartlist_t *chan_by_msg; + /** A list of dispatch_rcv_t, indexed by msg_type_id_t. */ + struct smartlist_t *fns_by_type; + /** A list of dispatch_typefns_t, indexed by msg_t. */ + struct smartlist_t *recv_by_msg; +}; + +#endif /* !defined(TOR_DISPATCH_CFG_ST_H) */ diff --git a/src/lib/dispatch/dispatch_core.c b/src/lib/dispatch/dispatch_core.c new file mode 100644 index 0000000000..da54f9b437 --- /dev/null +++ b/src/lib/dispatch/dispatch_core.c @@ -0,0 +1,260 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_core.c + * \brief Core module for sending and receiving messages. + */ + +#define DISPATCH_PRIVATE +#include "orconfig.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" +#include "lib/dispatch/dispatch_naming.h" + +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +#include <string.h> + +/** + * Use <b>d</b> to drop all storage held for <b>msg</b>. + * + * (We need the dispatcher so we know how to free the auxiliary data.) + **/ +void +dispatch_free_msg_(const dispatch_t *d, msg_t *msg) +{ + if (!msg) + return; + + d->typefns[msg->type].free_fn(msg->aux_data__); + tor_free(msg); +} + +/** + * Format the auxiliary data held by msg. + **/ +char * +dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg) +{ + if (!msg) + return NULL; + + return d->typefns[msg->type].fmt_fn(msg->aux_data__); +} + +/** + * Release all storage held by <b>d</b>. + **/ +void +dispatch_free_(dispatch_t *d) +{ + if (d == NULL) + return; + + size_t n_queues = d->n_queues; + for (size_t i = 0; i < n_queues; ++i) { + msg_t *m, *mtmp; + TOR_SIMPLEQ_FOREACH_SAFE(m, &d->queues[i].queue, next, mtmp) { + dispatch_free_msg(d, m); + } + } + + size_t n_msgs = d->n_msgs; + + for (size_t i = 0; i < n_msgs; ++i) { + tor_free(d->table[i]); + } + tor_free(d->table); + tor_free(d->typefns); + tor_free(d->queues); + + // This is the only time we will treat d->cfg as non-const. + //dispatch_cfg_free_((dispatch_items_t *) d->cfg); + + tor_free(d); +} + +/** + * Tell the dispatcher to call <b>fn</b> with <b>userdata</b> whenever + * <b>chan</b> becomes nonempty. Return 0 on success, -1 on error. + **/ +int +dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan, + dispatch_alertfn_t fn, void *userdata) +{ + if (BUG(chan >= d->n_queues)) + return -1; + + dqueue_t *q = &d->queues[chan]; + q->alert_fn = fn; + q->alert_fn_arg = userdata; + return 0; +} + +/** + * Send a message on the appropriate channel notifying that channel if + * necessary. + * + * This function takes ownership of the auxiliary data; it can't be static or + * stack-allocated, and the caller is not allowed to use it afterwards. + * + * This function does not check the various vields of the message object for + * consistency. + **/ +int +dispatch_send(dispatch_t *d, + subsys_id_t sender, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + msg_aux_data_t auxdata) +{ + if (!d->table[msg]) { + /* Fast path: nobody wants this data. */ + + d->typefns[type].free_fn(auxdata); + return 0; + } + + msg_t *m = tor_malloc(sizeof(msg_t)); + + m->sender = sender; + m->channel = channel; + m->msg = msg; + m->type = type; + memcpy(&m->aux_data__, &auxdata, sizeof(msg_aux_data_t)); + + return dispatch_send_msg(d, m); +} + +int +dispatch_send_msg(dispatch_t *d, msg_t *m) +{ + if (BUG(!d)) + goto err; + if (BUG(!m)) + goto err; + if (BUG(m->channel >= d->n_queues)) + goto err; + if (BUG(m->msg >= d->n_msgs)) + goto err; + + dtbl_entry_t *ent = d->table[m->msg]; + if (ent) { + if (BUG(m->type != ent->type)) + goto err; + if (BUG(m->channel != ent->channel)) + goto err; + } + + return dispatch_send_msg_unchecked(d, m); + err: + /* Probably it isn't safe to free m, since type could be wrong. */ + return -1; +} + +/** + * Send a message on the appropriate queue, notifying that queue if necessary. + * + * This function takes ownership of the message object and its auxiliary data; + * it can't be static or stack-allocated, and the caller isn't allowed to use + * it afterwards. + * + * This function does not check the various fields of the message object for + * consistency, and can crash if they are out of range. Only functions that + * have already constructed the message in a safe way, or checked it for + * correctness themselves, should call this function. + **/ +int +dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m) +{ + /* Find the right queue. */ + dqueue_t *q = &d->queues[m->channel]; + bool was_empty = TOR_SIMPLEQ_EMPTY(&q->queue); + + /* Append the message. */ + TOR_SIMPLEQ_INSERT_TAIL(&q->queue, m, next); + + if (debug_logging_enabled()) { + char *arg = dispatch_fmt_msg_data(d, m); + log_debug(LD_MESG, + "Queued: %s (%s) from %s, on %s.", + get_message_id_name(m->msg), + arg, + get_subsys_id_name(m->sender), + get_channel_id_name(m->channel)); + tor_free(arg); + } + + /* If we just made the queue nonempty for the first time, call the alert + * function. */ + if (was_empty) { + q->alert_fn(d, m->channel, q->alert_fn_arg); + } + + return 0; +} + +/** + * Run all of the callbacks on <b>d</b> associated with <b>m</b>. + **/ +static void +dispatcher_run_msg_cbs(const dispatch_t *d, msg_t *m) +{ + tor_assert(m->msg <= d->n_msgs); + dtbl_entry_t *ent = d->table[m->msg]; + int n_fns = ent->n_fns; + + if (debug_logging_enabled()) { + char *arg = dispatch_fmt_msg_data(d, m); + log_debug(LD_MESG, + "Delivering: %s (%s) from %s, on %s:", + get_message_id_name(m->msg), + arg, + get_subsys_id_name(m->sender), + get_channel_id_name(m->channel)); + tor_free(arg); + } + + int i; + for (i=0; i < n_fns; ++i) { + if (ent->rcv[i].enabled) { + log_debug(LD_MESG, " Delivering to %s.", + get_subsys_id_name(ent->rcv[i].sys)); + ent->rcv[i].fn(m); + } + } +} + +/** + * Run up to <b>max_msgs</b> callbacks for messages on the channel <b>ch</b> + * on the given dispatcher. Return 0 on success or recoverable failure, + * -1 on unrecoverable error. + **/ +int +dispatch_flush(dispatch_t *d, channel_id_t ch, int max_msgs) +{ + if (BUG(ch >= d->n_queues)) + return 0; + + int n_flushed = 0; + dqueue_t *q = &d->queues[ch]; + + while (n_flushed < max_msgs) { + msg_t *m = TOR_SIMPLEQ_FIRST(&q->queue); + if (!m) + break; + TOR_SIMPLEQ_REMOVE_HEAD(&q->queue, next); + dispatcher_run_msg_cbs(d, m); + dispatch_free_msg(d, m); + ++n_flushed; + } + + return 0; +} diff --git a/src/lib/dispatch/dispatch_naming.c b/src/lib/dispatch/dispatch_naming.c new file mode 100644 index 0000000000..83d9a2d604 --- /dev/null +++ b/src/lib/dispatch/dispatch_naming.c @@ -0,0 +1,63 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#include "lib/cc/compat_compiler.h" + +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/msgtypes.h" + +#include "lib/container/namemap.h" +#include "lib/container/namemap_st.h" + +#include "lib/log/util_bug.h" +#include "lib/log/log.h" + +#include <stdlib.h> + +/** Global namemap for message IDs. */ +static namemap_t message_id_map = NAMEMAP_INIT(); +/** Global namemap for subsystem IDs. */ +static namemap_t subsys_id_map = NAMEMAP_INIT(); +/** Global namemap for channel IDs. */ +static namemap_t channel_id_map = NAMEMAP_INIT(); +/** Global namemap for message type IDs. */ +static namemap_t msg_type_id_map = NAMEMAP_INIT(); + +void +dispatch_naming_init(void) +{ +} + +/* Helper macro: declare functions to map IDs to and from names for a given + * type in a namemap_t. + */ +#define DECLARE_ID_MAP_FNS(type) \ + type##_id_t \ + get_##type##_id(const char *name) \ + { \ + unsigned u = namemap_get_or_create_id(&type##_id_map, name); \ + tor_assert(u != NAMEMAP_ERR); \ + tor_assert(u != ERROR_ID); \ + return (type##_id_t) u; \ + } \ + const char * \ + get_##type##_id_name(type##_id_t id) \ + { \ + return namemap_fmt_name(&type##_id_map, id); \ + } \ + size_t \ + get_num_##type##_ids(void) \ + { \ + return namemap_get_size(&type##_id_map); \ + } \ + EAT_SEMICOLON + +DECLARE_ID_MAP_FNS(message); +DECLARE_ID_MAP_FNS(channel); +DECLARE_ID_MAP_FNS(subsys); +DECLARE_ID_MAP_FNS(msg_type); diff --git a/src/lib/dispatch/dispatch_naming.h b/src/lib/dispatch/dispatch_naming.h new file mode 100644 index 0000000000..fd6c83cc12 --- /dev/null +++ b/src/lib/dispatch/dispatch_naming.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DISPATCH_NAMING_H +#define TOR_DISPATCH_NAMING_H + +#include "lib/dispatch/msgtypes.h" +#include <stddef.h> + +/** + * Return an existing channel ID by name, allocating the channel ID if + * if necessary. Returns ERROR_ID if we have run out of + * channels + */ +channel_id_t get_channel_id(const char *); +/** + * Return the name corresponding to a given channel ID. + **/ +const char *get_channel_id_name(channel_id_t); +/** + * Return the total number of _named_ channel IDs. + **/ +size_t get_num_channel_ids(void); + +/* As above, but for messages. */ +message_id_t get_message_id(const char *); +const char *get_message_id_name(message_id_t); +size_t get_num_message_ids(void); + +/* As above, but for subsystems */ +subsys_id_t get_subsys_id(const char *); +const char *get_subsys_id_name(subsys_id_t); +size_t get_num_subsys_ids(void); + +/* As above, but for types. Note that types additionally must be + * "defined", if any message is to use them. */ +msg_type_id_t get_msg_type_id(const char *); +const char *get_msg_type_id_name(msg_type_id_t); +size_t get_num_msg_type_ids(void); + +void dispatch_naming_init(void); + +#endif /* !defined(TOR_DISPATCH_NAMING_H) */ diff --git a/src/lib/dispatch/dispatch_new.c b/src/lib/dispatch/dispatch_new.c new file mode 100644 index 0000000000..d8e59d610a --- /dev/null +++ b/src/lib/dispatch/dispatch_new.c @@ -0,0 +1,176 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_new.c + * \brief Code to construct a dispatch_t from a dispatch_cfg_t. + **/ + +#define DISPATCH_NEW_PRIVATE +#define DISPATCH_PRIVATE +#include "orconfig.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" +#include "lib/dispatch/dispatch_cfg.h" +#include "lib/dispatch/dispatch_cfg_st.h" + +#include "lib/cc/ctassert.h" +#include "lib/intmath/cmp.h" +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +#include <string.h> + +/** Given a smartlist full of (possibly NULL) pointers to uint16_t values, + * return the largest value, or dflt if the list is empty. */ +STATIC int +max_in_u16_sl(const smartlist_t *sl, int dflt) +{ + uint16_t *maxptr = NULL; + SMARTLIST_FOREACH_BEGIN(sl, uint16_t *, u) { + if (!maxptr) + maxptr = u; + else if (u && *u > *maxptr) + maxptr = u; + } SMARTLIST_FOREACH_END(u); + + return maxptr ? *maxptr : dflt; +} + +/* The above function is only safe to call if we are sure that channel_id_t + * and msg_type_id_t are really uint16_t. They should be so defined in + * msgtypes.h, but let's be extra cautious. + */ +CTASSERT(sizeof(uint16_t) == sizeof(msg_type_id_t)); +CTASSERT(sizeof(uint16_t) == sizeof(channel_id_t)); + +/** Helper: Format an unformattable message auxiliary data item: just return a +* copy of the string <>. */ +static char * +type_fmt_nop(msg_aux_data_t arg) +{ + (void)arg; + return tor_strdup("<>"); +} + +/** Helper: Free an unfreeable message auxiliary data item: do nothing. */ +static void +type_free_nop(msg_aux_data_t arg) +{ + (void)arg; +} + +/** Type functions to use when no type functions are provided. */ +static dispatch_typefns_t nop_typefns = { + .free_fn = type_free_nop, + .fmt_fn = type_fmt_nop +}; + +/** + * Alert function to use when none is configured: do nothing. + **/ +static void +alert_fn_nop(dispatch_t *d, channel_id_t ch, void *arg) +{ + (void)d; + (void)ch; + (void)arg; +} + +/** + * Given a list of recvfn_t, create and return a new dtbl_entry_t mapping + * to each of those functions. + **/ +static dtbl_entry_t * +dtbl_entry_from_lst(smartlist_t *receivers) +{ + if (!receivers) + return NULL; + + size_t n_recv = smartlist_len(receivers); + dtbl_entry_t *ent; + ent = tor_malloc_zero(offsetof(dtbl_entry_t, rcv) + + sizeof(dispatch_rcv_t) * n_recv); + + ent->n_fns = n_recv; + + SMARTLIST_FOREACH_BEGIN(receivers, const dispatch_rcv_t *, rcv) { + memcpy(&ent->rcv[rcv_sl_idx], rcv, sizeof(*rcv)); + if (rcv->enabled) { + ++ent->n_enabled; + } + } SMARTLIST_FOREACH_END(rcv); + + return ent; +} + +/** Create and return a new dispatcher from a given dispatch_cfg_t. */ +dispatch_t * +dispatch_new(const dispatch_cfg_t *cfg) +{ + dispatch_t *d = tor_malloc_zero(sizeof(dispatch_t)); + + /* Any message that has a type or a receiver counts towards our messages */ + const size_t n_msgs = MAX(smartlist_len(cfg->type_by_msg), + smartlist_len(cfg->recv_by_msg)) + 1; + + /* Any channel that any message has counts towards the number of channels. */ + const size_t n_chans = (size_t) + MAX(1, max_in_u16_sl(cfg->chan_by_msg,0)) + 1; + + /* Any type that a message has, or that has functions, counts towards + * the number of types. */ + const size_t n_types = (size_t) MAX(max_in_u16_sl(cfg->type_by_msg,0), + smartlist_len(cfg->fns_by_type)) + 1; + + d->n_msgs = n_msgs; + d->n_queues = n_chans; + d->n_types = n_types; + + /* Initialize the array of type-functions. */ + d->typefns = tor_calloc(n_types, sizeof(dispatch_typefns_t)); + for (size_t i = 0; i < n_types; ++i) { + /* Default to no-op for everything... */ + memcpy(&d->typefns[i], &nop_typefns, sizeof(dispatch_typefns_t)); + } + SMARTLIST_FOREACH_BEGIN(cfg->fns_by_type, dispatch_typefns_t *, fns) { + /* Set the functions if they are provided. */ + if (fns) { + if (fns->free_fn) + d->typefns[fns_sl_idx].free_fn = fns->free_fn; + if (fns->fmt_fn) + d->typefns[fns_sl_idx].fmt_fn = fns->fmt_fn; + } + } SMARTLIST_FOREACH_END(fns); + + /* Initialize the message queues: one for each channel. */ + d->queues = tor_calloc(d->n_queues, sizeof(dqueue_t)); + for (size_t i = 0; i < d->n_queues; ++i) { + TOR_SIMPLEQ_INIT(&d->queues[i].queue); + d->queues[i].alert_fn = alert_fn_nop; + } + + /* Build the dispatch tables mapping message IDs to receivers. */ + d->table = tor_calloc(d->n_msgs, sizeof(dtbl_entry_t *)); + SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, rcv) { + d->table[rcv_sl_idx] = dtbl_entry_from_lst(rcv); + } SMARTLIST_FOREACH_END(rcv); + + /* Fill in the empty entries in the dispatch tables: + * types and channels for each message. */ + SMARTLIST_FOREACH_BEGIN(cfg->type_by_msg, msg_type_id_t *, type) { + if (d->table[type_sl_idx]) + d->table[type_sl_idx]->type = *type; + } SMARTLIST_FOREACH_END(type); + + SMARTLIST_FOREACH_BEGIN(cfg->chan_by_msg, channel_id_t *, chan) { + if (d->table[chan_sl_idx]) + d->table[chan_sl_idx]->channel = *chan; + } SMARTLIST_FOREACH_END(chan); + + return d; +} diff --git a/src/lib/dispatch/dispatch_st.h b/src/lib/dispatch/dispatch_st.h new file mode 100644 index 0000000000..ee42518b5a --- /dev/null +++ b/src/lib/dispatch/dispatch_st.h @@ -0,0 +1,108 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_st.h + * + * \brief private structures used for the dispatcher module + */ + +#ifndef TOR_DISPATCH_ST_H +#define TOR_DISPATCH_ST_H + +#ifdef DISPATCH_PRIVATE + +#include "lib/container/smartlist.h" + +/** + * Information about the recipient of a message. + **/ +typedef struct dispatch_rcv_t { + /** The subsystem receiving a message. */ + subsys_id_t sys; + /** True iff this recipient is enabled. */ + bool enabled; + /** The function that will handle the message. */ + recv_fn_t fn; +} dispatch_rcv_t; + +/** + * Information used by a dispatcher to handle and dispatch a single message + * ID. It maps that message ID to its type, channel, and list of receiver + * functions. + * + * This structure is used when the dispatcher is running. + **/ +typedef struct dtbl_entry_t { + /** The number of enabled non-stub subscribers for this message. + * + * Note that for now, this will be the same as <b>n_fns</b>, since there is + * no way to turn these subscribers on an off yet. */ + uint16_t n_enabled; + /** The channel that handles this message. */ + channel_id_t channel; + /** The associated C type for this message. */ + msg_type_id_t type; + /** + * The number of functions pointers for subscribers that receive this + * message, in rcv. */ + uint16_t n_fns; + /** + * The recipients for this message. + */ + dispatch_rcv_t rcv[FLEXIBLE_ARRAY_MEMBER]; +} dtbl_entry_t; + +/** + * A queue of messages for a given channel, used by a live dispatcher. + */ +typedef struct dqueue_t { + /** The queue of messages itself. */ + TOR_SIMPLEQ_HEAD( , msg_t) queue; + /** A function to be called when the queue becomes nonempty. */ + dispatch_alertfn_t alert_fn; + /** An argument for the alert_fn. */ + void *alert_fn_arg; +} dqueue_t ; + +/** + * A single dispatcher for cross-module messages. + */ +struct dispatch_t { + /** + * The length of <b>table</b>: the number of message IDs that this + * dispatcher can handle. + */ + size_t n_msgs; + /** + * The length of <b>queues</b>: the number of channels that this dispatcher + * has configured. + */ + size_t n_queues; + /** + * The length of <b>typefns</b>: the number of C type IDs that this + * dispatcher has configured. + */ + size_t n_types; + /** + * An array of message queues, indexed by channel ID. + */ + dqueue_t *queues; + /** + * An array of entries about how to handle particular message types, indexed + * by message ID. + */ + dtbl_entry_t **table; + /** + * An array of function tables for manipulating types, index by message + * type ID. + **/ + dispatch_typefns_t *typefns; +}; + +#endif /* defined(DISPATCH_PRIVATE) */ + +#endif /* !defined(TOR_DISPATCH_ST_H) */ diff --git a/src/lib/dispatch/include.am b/src/lib/dispatch/include.am new file mode 100644 index 0000000000..4a0e0dfd90 --- /dev/null +++ b/src/lib/dispatch/include.am @@ -0,0 +1,27 @@ + +noinst_LIBRARIES += src/lib/libtor-dispatch.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-dispatch-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_dispatch_a_SOURCES = \ + src/lib/dispatch/dispatch_cfg.c \ + src/lib/dispatch/dispatch_core.c \ + src/lib/dispatch/dispatch_naming.c \ + src/lib/dispatch/dispatch_new.c + +src_lib_libtor_dispatch_testing_a_SOURCES = \ + $(src_lib_libtor_dispatch_a_SOURCES) +src_lib_libtor_dispatch_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_dispatch_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/dispatch/dispatch.h \ + src/lib/dispatch/dispatch_cfg.h \ + src/lib/dispatch/dispatch_cfg_st.h \ + src/lib/dispatch/dispatch_naming.h \ + src/lib/dispatch/dispatch_st.h \ + src/lib/dispatch/msgtypes.h diff --git a/src/lib/dispatch/msgtypes.h b/src/lib/dispatch/msgtypes.h new file mode 100644 index 0000000000..b4c4a10248 --- /dev/null +++ b/src/lib/dispatch/msgtypes.h @@ -0,0 +1,80 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file msgtypes.h + * \brief Types used for messages in the dispatcher code. + **/ + +#ifndef TOR_DISPATCH_MSGTYPES_H +#define TOR_DISPATCH_MSGTYPES_H + +#include <stdint.h> + +#include "ext/tor_queue.h" + +/** + * These types are aliases for subsystems, channels, and message IDs. + **/ +typedef uint16_t subsys_id_t; +typedef uint16_t channel_id_t; +typedef uint16_t message_id_t; + +/** + * This identifies a C type that can be sent along with a message. + **/ +typedef uint16_t msg_type_id_t; + +/** + * An ID value returned for *_type_t when none exists. + */ +#define ERROR_ID 65535 + +/** + * Auxiliary (untyped) data sent along with a message. + * + * We define this as a union of a pointer and a u64, so that the integer + * types will have the same range across platforms. + **/ +typedef union { + void *ptr; + uint64_t u64; +} msg_aux_data_t; + +/** + * Structure of a received message. + **/ +typedef struct msg_t { + TOR_SIMPLEQ_ENTRY(msg_t) next; + subsys_id_t sender; + channel_id_t channel; + message_id_t msg; + /** We could omit this field, since it is implicit in the message type, but + * IMO let's leave it in for safety. */ + msg_type_id_t type; + /** Untyped auxiliary data. You shouldn't have to mess with this + * directly. */ + msg_aux_data_t aux_data__; +} msg_t; + +/** + * A function that a subscriber uses to receive a message. + **/ +typedef void (*recv_fn_t)(const msg_t *m); + +/** + * Table of functions to use for a given C type. Any omitted (NULL) functions + * will be treated as no-ops. + **/ +typedef struct dispatch_typefns_t { + /** Release storage held for the auxiliary data of this type. */ + void (*free_fn)(msg_aux_data_t); + /** Format and return a newly allocated string describing the contents + * of this data element. */ + char *(*fmt_fn)(msg_aux_data_t); +} dispatch_typefns_t; + +#endif /* !defined(TOR_DISPATCH_MSGTYPES_H) */ diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include index 7c2ef36929..c9bf4b1786 100644 --- a/src/lib/encoding/.may_include +++ b/src/lib/encoding/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/cc/*.h +lib/container/*.h lib/ctime/*.h lib/encoding/*.h lib/intmath/*.h diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c index bd063440d6..fc64e014e7 100644 --- a/src/lib/encoding/binascii.c +++ b/src/lib/encoding/binascii.c @@ -84,7 +84,7 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) } /** Implements base32 decoding as in RFC 4648. - * Returns 0 if successful, -1 otherwise. + * Return the number of bytes decoded if successful; -1 otherwise. */ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) @@ -147,7 +147,7 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) memset(tmp, 0, srclen); /* on the heap, this should be safe */ tor_free(tmp); tmp = NULL; - return 0; + return i; } #define BASE64_OPENSSL_LINELEN 64 @@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags) return enclen; } +/** Return an upper bound on the number of bytes that might be needed to hold + * the data from decoding the base64 string <b>srclen</b>. This is only an + * upper bound, since some part of the base64 string might be padding or + * space. */ +size_t +base64_decode_maxsize(size_t srclen) +{ + tor_assert(srclen < INT_MAX / 3); + + return CEIL_DIV(srclen * 3, 4); +} + /** Internal table mapping 6 bit values to the Base64 alphabet. */ static const char base64_encode_table[64] = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', @@ -309,8 +321,10 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, return (int) enclen; } -/** As base64_encode, but do not add any internal spaces or external padding - * to the output stream. */ +/** As base64_encode, but do not add any internal spaces, and remove external + * padding from the output stream. + * dest must be at least base64_encode_size(srclen, 0), including space for + * the removed external padding. */ int base64_encode_nopad(char *dest, size_t destlen, const uint8_t *src, size_t srclen) diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h index 7e3cc04f09..40c5593b11 100644 --- a/src/lib/encoding/binascii.h +++ b/src/lib/encoding/binascii.h @@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen); #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); +size_t base64_decode_maxsize(size_t srclen); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int flags); int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); @@ -57,4 +58,4 @@ size_t base32_encoded_size(size_t srclen); void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen); -#endif /* !defined(TOR_UTIL_FORMAT_H) */ +#endif /* !defined(TOR_BINASCII_H) */ diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c index 8110f3dd9c..0d8384db13 100644 --- a/src/lib/encoding/confline.c +++ b/src/lib/encoding/confline.c @@ -82,6 +82,19 @@ config_line_find(const config_line_t *lines, return NULL; } +/** As config_line_find(), but perform a case-insensitive comparison. */ +const config_line_t * +config_line_find_case(const config_line_t *lines, + const char *key) +{ + const config_line_t *cl; + for (cl = lines; cl; cl = cl->next) { + if (!strcasecmp(cl->key, key)) + return cl; + } + return NULL; +} + /** Auxiliary function that does all the work of config_get_lines. * <b>recursion_level</b> is the count of how many nested %includes we have. * <b>opened_lst</b> will have a list of opened files if provided. @@ -243,7 +256,7 @@ config_lines_dup_and_filter(const config_line_t *inp, /** Return true iff a and b contain identical keys and values in identical * order. */ int -config_lines_eq(config_line_t *a, config_line_t *b) +config_lines_eq(const config_line_t *a, const config_line_t *b) { while (a && b) { if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h index 3d9ae8a662..12c554c6e7 100644 --- a/src/lib/encoding/confline.h +++ b/src/lib/encoding/confline.h @@ -48,7 +48,9 @@ config_line_t *config_lines_dup_and_filter(const config_line_t *inp, const char *key); const config_line_t *config_line_find(const config_line_t *lines, const char *key); -int config_lines_eq(config_line_t *a, config_line_t *b); +const config_line_t *config_line_find_case(const config_line_t *lines, + const char *key); +int config_lines_eq(const config_line_t *a, const config_line_t *b); int config_count_key(const config_line_t *a, const char *key); void config_free_lines_(config_line_t *front); #define config_free_lines(front) \ diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am index 2d2aa3988a..48d0120bfc 100644 --- a/src/lib/encoding/include.am +++ b/src/lib/encoding/include.am @@ -4,12 +4,15 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-encoding-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_encoding_a_SOURCES = \ src/lib/encoding/binascii.c \ src/lib/encoding/confline.c \ src/lib/encoding/cstring.c \ src/lib/encoding/keyval.c \ + src/lib/encoding/kvline.c \ src/lib/encoding/pem.c \ + src/lib/encoding/qstring.c \ src/lib/encoding/time_fmt.c src_lib_libtor_encoding_testing_a_SOURCES = \ @@ -17,10 +20,13 @@ src_lib_libtor_encoding_testing_a_SOURCES = \ src_lib_libtor_encoding_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_encoding_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/encoding/binascii.h \ src/lib/encoding/confline.h \ src/lib/encoding/cstring.h \ src/lib/encoding/keyval.h \ + src/lib/encoding/kvline.h \ src/lib/encoding/pem.h \ + src/lib/encoding/qstring.h \ src/lib/encoding/time_fmt.h diff --git a/src/lib/encoding/keyval.h b/src/lib/encoding/keyval.h index cd327b7a82..dcddfa3396 100644 --- a/src/lib/encoding/keyval.h +++ b/src/lib/encoding/keyval.h @@ -14,4 +14,4 @@ int string_is_key_value(int severity, const char *string); -#endif +#endif /* !defined(TOR_KEYVAL_H) */ diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c new file mode 100644 index 0000000000..d4a8f510ba --- /dev/null +++ b/src/lib/encoding/kvline.c @@ -0,0 +1,289 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.c + * + * \brief Manipulating lines of key-value pairs. + **/ + +#include "orconfig.h" + +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/encoding/qstring.h" +#include "lib/malloc/malloc.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" + +#include <stdbool.h> +#include <stddef.h> +#include <string.h> + +/** Return true iff we need to quote and escape the string <b>s</b> to encode + * it. */ +static bool +needs_escape(const char *s, bool as_keyless_val) +{ + if (as_keyless_val && *s == 0) + return true; + + for (; *s; ++s) { + if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) || + *s == '\'' || *s == '\"') { + return true; + } + } + return false; +} + +/** + * Return true iff the key in <b>line</b> is not set. + **/ +static bool +line_has_no_key(const config_line_t *line) +{ + return line->key == NULL || strlen(line->key) == 0; +} + +/** + * Return true iff the value in <b>line</b> is not set. + **/ +static bool +line_has_no_val(const config_line_t *line) +{ + return line->value == NULL || strlen(line->value) == 0; +} + +/** + * Return true iff the all the lines in <b>line</b> can be encoded + * using <b>flags</b>. + **/ +static bool +kvline_can_encode_lines(const config_line_t *line, unsigned flags) +{ + for ( ; line; line = line->next) { + const bool keyless = line_has_no_key(line); + if (keyless) { + if (! (flags & KV_OMIT_KEYS)) { + /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */ + return false; + } + if (strchr(line->value, '=') && !( flags & KV_QUOTED)) { + /* We can't have a keyless value with = without quoting it. */ + return false; + } + } + + if (needs_escape(line->value, keyless) && ! (flags & KV_QUOTED)) { + /* If KV_QUOTED is false, we can't encode a value that needs quotes. */ + return false; + } + if (line->key && strlen(line->key) && + (needs_escape(line->key, false) || strchr(line->key, '='))) { + /* We can't handle keys that need quoting. */ + return false; + } + } + return true; +} + +/** + * Encode a linked list of lines in <b>line</b> as a series of 'Key=Value' + * pairs, using the provided <b>flags</b> to encode it. Return a newly + * allocated string on success, or NULL on failure. + * + * If KV_QUOTED is set in <b>flags</b>, then all values that contain + * spaces or unusual characters are escaped and quoted. Otherwise, such + * values are not allowed. + * + * If KV_OMIT_KEYS is set in <b>flags</b>, then pairs with empty keys are + * allowed, and are encoded as 'Value'. Otherwise, such pairs are not + * allowed. + * + * If KV_OMIT_VALS is set in <b>flags</b>, then an empty value is + * encoded as 'Key', not as 'Key=' or 'Key=""'. Mutually exclusive with + * KV_OMIT_KEYS. + * + * KV_QUOTED_QSTRING is not supported. + */ +char * +kvline_encode(const config_line_t *line, + unsigned flags) +{ + tor_assert(! (flags & KV_QUOTED_QSTRING)); + + if (!kvline_can_encode_lines(line, flags)) + return NULL; + + tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) != + (KV_OMIT_KEYS|KV_OMIT_VALS)); + + smartlist_t *elements = smartlist_new(); + + for (; line; line = line->next) { + + const char *k = ""; + const char *eq = "="; + const char *v = ""; + const bool keyless = line_has_no_key(line); + bool esc = needs_escape(line->value, keyless); + char *tmp = NULL; + + if (! keyless) { + k = line->key; + } else { + eq = ""; + if (strchr(line->value, '=')) { + esc = true; + } + } + + if ((flags & KV_OMIT_VALS) && line_has_no_val(line)) { + eq = ""; + v = ""; + } else if (esc) { + tmp = esc_for_log(line->value); + v = tmp; + } else { + v = line->value; + } + + smartlist_add_asprintf(elements, "%s%s%s", k, eq, v); + tor_free(tmp); + } + + char *result = smartlist_join_strings(elements, " ", 0, NULL); + + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + + return result; +} + +/** + * Decode a <b>line</b> containing a series of space-separated 'Key=Value' + * pairs, using the provided <b>flags</b> to decode it. Return a newly + * allocated list of pairs on success, or NULL on failure. + * + * If KV_QUOTED is set in <b>flags</b>, then (double-)quoted values are + * allowed and handled as C strings. Otherwise, such values are not allowed. + * + * If KV_OMIT_KEYS is set in <b>flags</b>, then values without keys are + * allowed. Otherwise, such values are not allowed. + * + * If KV_OMIT_VALS is set in <b>flags</b>, then keys without values are + * allowed. Otherwise, such keys are not allowed. Mutually exclusive with + * KV_OMIT_KEYS. + * + * If KV_QUOTED_QSTRING is set in <b>flags</b>, then double-quoted values + * are allowed and handled as QuotedStrings per qstring.c. Do not add + * new users of this flag. + */ +config_line_t * +kvline_parse(const char *line, unsigned flags) +{ + tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) != + (KV_OMIT_KEYS|KV_OMIT_VALS)); + + const char *cp = line, *cplast = NULL; + const bool omit_keys = (flags & KV_OMIT_KEYS) != 0; + const bool omit_vals = (flags & KV_OMIT_VALS) != 0; + const bool quoted = (flags & (KV_QUOTED|KV_QUOTED_QSTRING)) != 0; + const bool c_quoted = (flags & (KV_QUOTED)) != 0; + + config_line_t *result = NULL; + config_line_t **next_line = &result; + + char *key = NULL; + char *val = NULL; + + while (*cp) { + key = val = NULL; + /* skip all spaces */ + { + size_t idx = strspn(cp, " \t\r\v\n"); + cp += idx; + } + if (BUG(cp == cplast)) { + /* If we didn't parse anything since the last loop, this code is + * broken. */ + goto err; // LCOV_EXCL_LINE + } + cplast = cp; + if (! *cp) + break; /* End of string; we're done. */ + + /* Possible formats are K=V, K="V", K, V, and "V", depending on flags. */ + + /* Find where the key ends */ + if (*cp != '\"') { + size_t idx = strcspn(cp, " \t\r\v\n="); + + if (cp[idx] == '=') { + key = tor_memdup_nulterm(cp, idx); + cp += idx + 1; + } else if (omit_vals) { + key = tor_memdup_nulterm(cp, idx); + cp += idx; + goto commit; + } else { + if (!omit_keys) + goto err; + } + } + + if (*cp == '\"') { + /* The type is "V". */ + if (!quoted) + goto err; + size_t len=0; + if (c_quoted) { + cp = unescape_string(cp, &val, &len); + } else { + cp = decode_qstring(cp, strlen(cp), &val, &len); + } + if (cp == NULL || len != strlen(val)) { + // The string contains a NUL or is badly coded. + goto err; + } + } else { + size_t idx = strcspn(cp, " \t\r\v\n"); + val = tor_memdup_nulterm(cp, idx); + cp += idx; + } + + commit: + if (key && strlen(key) == 0) { + /* We don't allow empty keys. */ + goto err; + } + + *next_line = tor_malloc_zero(sizeof(config_line_t)); + (*next_line)->key = key ? key : tor_strdup(""); + (*next_line)->value = val ? val : tor_strdup(""); + next_line = &(*next_line)->next; + key = val = NULL; + } + + if (! (flags & KV_QUOTED_QSTRING)) { + if (!kvline_can_encode_lines(result, flags)) { + goto err; + } + } + return result; + + err: + tor_free(key); + tor_free(val); + config_free_lines(result); + return NULL; +} diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h new file mode 100644 index 0000000000..dea2ce1809 --- /dev/null +++ b/src/lib/encoding/kvline.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.h + * + * \brief Header for kvline.c + **/ + +#ifndef TOR_KVLINE_H +#define TOR_KVLINE_H + +struct config_line_t; + +#define KV_QUOTED (1u<<0) +#define KV_OMIT_KEYS (1u<<1) +#define KV_OMIT_VALS (1u<<2) +#define KV_QUOTED_QSTRING (1u<<3) + +struct config_line_t *kvline_parse(const char *line, unsigned flags); +char *kvline_encode(const struct config_line_t *line, unsigned flags); + +#endif /* !defined(TOR_KVLINE_H) */ diff --git a/src/lib/encoding/pem.h b/src/lib/encoding/pem.h index 0bbb06a794..6b20350aa8 100644 --- a/src/lib/encoding/pem.h +++ b/src/lib/encoding/pem.h @@ -23,4 +23,4 @@ int pem_encode(char *dest, size_t destlen, const uint8_t *src, size_t srclen, int pem_decode(uint8_t *dest, size_t destlen, const char *src, size_t srclen, const char *objtype); -#endif +#endif /* !defined(TOR_PEM_H) */ diff --git a/src/lib/encoding/qstring.c b/src/lib/encoding/qstring.c new file mode 100644 index 0000000000..a92d28c706 --- /dev/null +++ b/src/lib/encoding/qstring.c @@ -0,0 +1,90 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file qstring.c + * \brief Implement QuotedString parsing. + * + * Note that this is only used for controller authentication; do not + * create new users for this. Instead, prefer the cstring.c functions. + **/ + +#include "orconfig.h" +#include "lib/encoding/qstring.h" +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +/** If the first <b>in_len_max</b> characters in <b>start</b> contain a + * QuotedString, return the length of that + * string (as encoded, including quotes). Otherwise return -1. */ +static inline int +get_qstring_length(const char *start, size_t in_len_max, + int *chars_out) +{ + const char *cp, *end; + int chars = 0; + + if (*start != '\"') + return -1; + + cp = start+1; + end = start+in_len_max; + + /* Calculate length. */ + while (1) { + if (cp >= end) { + return -1; /* Too long. */ + } else if (*cp == '\\') { + if (++cp == end) + return -1; /* Can't escape EOS. */ + ++cp; + ++chars; + } else if (*cp == '\"') { + break; + } else { + ++cp; + ++chars; + } + } + if (chars_out) + *chars_out = chars; + return (int)(cp - start+1); +} + +/** Given a pointer to a string starting at <b>start</b> containing + * <b>in_len_max</b> characters, decode a string beginning with one double + * quote, containing any number of non-quote characters or characters escaped + * with a backslash, and ending with a final double quote. Place the resulting + * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>; + * store its length in <b>out_len</b>. On success, return a pointer to the + * character immediately following the escaped string. On failure, return + * NULL. */ +const char * +decode_qstring(const char *start, size_t in_len_max, + char **out, size_t *out_len) +{ + const char *cp, *end; + char *outp; + int len, n_chars = 0; + + len = get_qstring_length(start, in_len_max, &n_chars); + if (len<0) + return NULL; + + end = start+len-1; /* Index of last quote. */ + tor_assert(*end == '\"'); + outp = *out = tor_malloc(len+1); + *out_len = n_chars; + + cp = start+1; + while (cp < end) { + if (*cp == '\\') + ++cp; + *outp++ = *cp++; + } + *outp = '\0'; + tor_assert((outp - *out) == (int)*out_len); + + return end+1; +} diff --git a/src/lib/encoding/qstring.h b/src/lib/encoding/qstring.h new file mode 100644 index 0000000000..840e1044ce --- /dev/null +++ b/src/lib/encoding/qstring.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file qstring.h + * \brief Header for qstring.c + */ + +#ifndef TOR_ENCODING_QSTRING_H +#define TOR_ENCODING_QSTRING_H + +#include <stddef.h> + +const char *decode_qstring(const char *start, size_t in_len_max, + char **out, size_t *out_len); + +#endif /* !defined(TOR_ENCODING_QSTRING_H) */ diff --git a/src/lib/encoding/time_fmt.h b/src/lib/encoding/time_fmt.h index 0ddeca57fc..d14bc1f902 100644 --- a/src/lib/encoding/time_fmt.h +++ b/src/lib/encoding/time_fmt.h @@ -41,4 +41,4 @@ int parse_iso_time_nospace(const char *cp, time_t *t); int parse_http_time(const char *buf, struct tm *tm); int format_time_interval(char *out, size_t out_len, long interval); -#endif +#endif /* !defined(TOR_TIME_FMT_H) */ diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include index 48cc0ef088..314424545e 100644 --- a/src/lib/err/.may_include +++ b/src/lib/err/.may_include @@ -1,3 +1,6 @@ orconfig.h lib/cc/*.h +lib/defs/*.h lib/err/*.h +lib/subsys/*.h +lib/version/*.h diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c index 8606f42177..ce8ddcd7c0 100644 --- a/src/lib/err/backtrace.c +++ b/src/lib/err/backtrace.c @@ -52,12 +52,14 @@ #include <pthread.h> #endif +#include "lib/cc/ctassert.h" + #define EXPOSE_CLEAN_BACKTRACE #include "lib/err/backtrace.h" -#include "lib/err/torerr.h" #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ - defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) + defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) && \ + defined(HAVE_PTHREAD_H) #define USE_BACKTRACE #endif @@ -72,15 +74,40 @@ static char bt_version[128] = ""; #ifdef USE_BACKTRACE + /** Largest stack depth to try to dump. */ #define MAX_DEPTH 256 -/** Static allocation of stack to dump. This is static so we avoid stack - * pressure. */ -static void *cb_buf[MAX_DEPTH]; +/** The size of the callback buffer, so we can clear it in unlock_cb_buf(). */ +#define SIZEOF_CB_BUF (MAX_DEPTH * sizeof(void *)) /** Protects cb_buf from concurrent access. Pthreads, since this code * is Unix-only, and since this code needs to be lowest-level. */ static pthread_mutex_t cb_buf_mutex = PTHREAD_MUTEX_INITIALIZER; +/** Lock and return a static stack pointer buffer that can hold up to + * MAX_DEPTH function pointers. */ +static void ** +lock_cb_buf(void) +{ + /* Lock the mutex first, before even declaring the buffer. */ + pthread_mutex_lock(&cb_buf_mutex); + + /** Static allocation of stack to dump. This is static so we avoid stack + * pressure. */ + static void *cb_buf[MAX_DEPTH]; + CTASSERT(SIZEOF_CB_BUF == sizeof(cb_buf)); + memset(cb_buf, 0, SIZEOF_CB_BUF); + + return cb_buf; +} + +/** Unlock the static stack pointer buffer. */ +static void +unlock_cb_buf(void **cb_buf) +{ + memset(cb_buf, 0, SIZEOF_CB_BUF); + pthread_mutex_unlock(&cb_buf_mutex); +} + /** Change a stacktrace in <b>stack</b> of depth <b>depth</b> so that it will * log the correct function from which a signal was received with context * <b>ctx</b>. (When we get a signal, the current function will not have @@ -104,7 +131,7 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx) return; stack[n] = (void*) ctx->PC_FROM_UCONTEXT; -#else /* !(defined(PC_FROM_UCONTEXT)) */ +#else /* !defined(PC_FROM_UCONTEXT) */ (void) depth; (void) ctx; (void) stack; @@ -115,14 +142,14 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx) * that with a backtrace log. Send messages via the tor_log function at * logger". */ void -log_backtrace_impl(int severity, int domain, const char *msg, +log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg, tor_log_fn logger) { size_t depth; char **symbols; size_t i; - pthread_mutex_lock(&cb_buf_mutex); + void **cb_buf = lock_cb_buf(); depth = backtrace(cb_buf, MAX_DEPTH); symbols = backtrace_symbols(cb_buf, (int)depth); @@ -140,7 +167,7 @@ log_backtrace_impl(int severity, int domain, const char *msg, raw_free(symbols); done: - pthread_mutex_unlock(&cb_buf_mutex); + unlock_cb_buf(cb_buf); } static void crash_handler(int sig, siginfo_t *si, void *ctx_) @@ -156,6 +183,8 @@ crash_handler(int sig, siginfo_t *si, void *ctx_) int n_fds, i; const int *fds = NULL; + void **cb_buf = lock_cb_buf(); + (void) si; depth = backtrace(cb_buf, MAX_DEPTH); @@ -172,7 +201,9 @@ crash_handler(int sig, siginfo_t *si, void *ctx_) for (i=0; i < n_fds; ++i) backtrace_symbols_fd(cb_buf, (int)depth, fds[i]); - abort(); + unlock_cb_buf(cb_buf); + + tor_raw_abort_(); } /** Write a backtrace to all of the emergency-error fds. */ @@ -183,20 +214,26 @@ dump_stack_symbols_to_error_fds(void) const int *fds = NULL; size_t depth; + void **cb_buf = lock_cb_buf(); + depth = backtrace(cb_buf, MAX_DEPTH); n_fds = tor_log_get_sigsafe_err_fds(&fds); for (i=0; i < n_fds; ++i) backtrace_symbols_fd(cb_buf, (int)depth, fds[i]); + + unlock_cb_buf(cb_buf); } +/* The signals that we want our backtrace handler to trap */ +static int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, + SIGIO, -1 }; + /** Install signal handlers as needed so that when we crash, we produce a * useful stack trace. Return 0 on success, -errno on failure. */ static int install_bt_handler(void) { - int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, - SIGIO, -1 }; int i, rv=0; struct sigaction sa; @@ -219,10 +256,12 @@ install_bt_handler(void) * libc has pre-loaded the symbols we need to dump things, so that later * reads won't be denied by the sandbox code */ char **symbols; + void **cb_buf = lock_cb_buf(); size_t depth = backtrace(cb_buf, MAX_DEPTH); symbols = backtrace_symbols(cb_buf, (int) depth); if (symbols) raw_free(symbols); + unlock_cb_buf(cb_buf); } return rv; @@ -232,12 +271,29 @@ install_bt_handler(void) static void remove_bt_handler(void) { + int i; + + struct sigaction sa; + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = SIG_DFL; + sigfillset(&sa.sa_mask); + + for (i = 0; trap_signals[i] >= 0; ++i) { + /* remove_bt_handler() is called on shutdown, from low-level code. + * It's not a fatal error, so we just ignore it. */ + (void)sigaction(trap_signals[i], &sa, NULL); + } + + /* cb_buf_mutex is statically initialised, so we can not destroy it. + * If we destroy it, and then re-initialise tor, all our backtraces will + * fail. */ } #endif /* defined(USE_BACKTRACE) */ #ifdef NO_BACKTRACE_IMPL void -log_backtrace_impl(int severity, int domain, const char *msg, +log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg, tor_log_fn logger) { logger(severity, domain, "%s: %s. (Stack trace not available)", diff --git a/src/lib/err/backtrace.h b/src/lib/err/backtrace.h index 48b41fca02..7e09a0a5a7 100644 --- a/src/lib/err/backtrace.h +++ b/src/lib/err/backtrace.h @@ -12,11 +12,14 @@ #include "orconfig.h" #include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" +#include "lib/defs/logging_types.h" -typedef void (*tor_log_fn)(int, unsigned, const char *fmt, ...) +typedef void (*tor_log_fn)(int, log_domain_mask_t, const char *fmt, ...) CHECK_PRINTF(3,4); -void log_backtrace_impl(int severity, int domain, const char *msg, +void log_backtrace_impl(int severity, log_domain_mask_t domain, + const char *msg, tor_log_fn logger); int configure_backtrace_handler(const char *tor_version); void clean_up_backtrace_handler(void); diff --git a/src/lib/err/include.am b/src/lib/err/include.am index f2a409c51e..883ac91511 100644 --- a/src/lib/err/include.am +++ b/src/lib/err/include.am @@ -5,15 +5,19 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-err-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_err_a_SOURCES = \ - src/lib/err/backtrace.c \ - src/lib/err/torerr.c + src/lib/err/backtrace.c \ + src/lib/err/torerr.c \ + src/lib/err/torerr_sys.c src_lib_libtor_err_testing_a_SOURCES = \ $(src_lib_libtor_err_a_SOURCES) src_lib_libtor_err_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/err/backtrace.h \ - src/lib/err/torerr.h + src/lib/err/torerr.h \ + src/lib/err/torerr_sys.h diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c index 6b5224273a..a5c00ca389 100644 --- a/src/lib/err/torerr.c +++ b/src/lib/err/torerr.c @@ -110,6 +110,14 @@ tor_log_get_sigsafe_err_fds(const int **out) * Update the list of fds that get errors from inside a signal handler or * other emergency condition. Ignore any beyond the first * TOR_SIGSAFE_LOG_MAX_FDS. + * + * These fds must remain open even after the log module has shut down. (And + * they should remain open even while logs are being reconfigured.) Therefore, + * any fds closed by the log module should be dup()ed, and the duplicate fd + * should be given to the err module in fds. In particular, the log module + * closes the file log fds, but does not close the stdio log fds. + * + * If fds is NULL or n is 0, clears the list of error fds. */ void tor_log_set_sigsafe_err_fds(const int *fds, int n) @@ -118,8 +126,28 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n) n = TOR_SIGSAFE_LOG_MAX_FDS; } - memcpy(sigsafe_log_fds, fds, n * sizeof(int)); - n_sigsafe_log_fds = n; + /* Clear the entire array. This code mitigates against some race conditions, + * but there are still some races here: + * - err logs are disabled while the array is cleared, and + * - a thread can read the old value of n_sigsafe_log_fds, then read a + * partially written array. + * We could fix these races using atomics, but atomics use the err module. */ + n_sigsafe_log_fds = 0; + memset(sigsafe_log_fds, 0, sizeof(sigsafe_log_fds)); + if (fds && n > 0) { + memcpy(sigsafe_log_fds, fds, n * sizeof(int)); + n_sigsafe_log_fds = n; + } +} + +/** + * Reset the list of emergency error fds to its default. + */ +void +tor_log_reset_sigsafe_err_fds(void) +{ + int fds[] = { STDERR_FILENO }; + tor_log_set_sigsafe_err_fds(fds, 1); } /** @@ -161,6 +189,17 @@ tor_raw_assertion_failed_msg_(const char *file, int line, const char *expr, tor_log_err_sigsafe_write("\n"); } +/** + * Call the abort() function to kill the current process with a fatal + * error. This is a separate function, so that log users don't have to include + * the header for abort(). + **/ +void +tor_raw_abort_(void) +{ + abort(); +} + /* As format_{hex,dex}_number_sigsafe, but takes a <b>radix</b> argument * in range 2..16 inclusive. */ static int @@ -195,7 +234,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len, unsigned digit = (unsigned) (x % radix); if (cp <= buf) { /* Not tor_assert(); see above. */ - abort(); + tor_raw_abort_(); } --cp; *cp = "0123456789ABCDEF"[digit]; @@ -204,7 +243,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len, /* NOT tor_assert; see above. */ if (cp != buf) { - abort(); // LCOV_EXCL_LINE + tor_raw_abort_(); // LCOV_EXCL_LINE } return len; @@ -224,8 +263,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len, * does not guarantee that an int is wider than a char (an int must be at * least 16 bits but it is permitted for a char to be that wide as well), we * can't assume a signed int is sufficient to accommodate an unsigned char. - * Thus, format_helper_exit_status() will still need to emit any require '-' - * on its own. + * Thus, callers will still need to add any required '-' to the final string. * * For most purposes, you'd want to use tor_snprintf("%x") instead of this * function; it's designed to be used in code paths where you can't call diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h index 6ae91fbe85..a1822d9c9a 100644 --- a/src/lib/err/torerr.h +++ b/src/lib/err/torerr.h @@ -20,13 +20,13 @@ #define raw_assert(expr) STMT_BEGIN \ if (!(expr)) { \ tor_raw_assertion_failed_msg_(__FILE__, __LINE__, #expr, NULL); \ - abort(); \ + tor_raw_abort_(); \ } \ STMT_END #define raw_assert_unreached(expr) raw_assert(0) #define raw_assert_unreached_msg(msg) STMT_BEGIN \ tor_raw_assertion_failed_msg_(__FILE__, __LINE__, "0", (msg)); \ - abort(); \ + tor_raw_abort_(); \ STMT_END void tor_raw_assertion_failed_msg_(const char *file, int line, @@ -39,9 +39,12 @@ void tor_raw_assertion_failed_msg_(const char *file, int line, void tor_log_err_sigsafe(const char *m, ...); int tor_log_get_sigsafe_err_fds(const int **out); void tor_log_set_sigsafe_err_fds(const int *fds, int n); +void tor_log_reset_sigsafe_err_fds(void); void tor_log_sigsafe_err_set_granularity(int ms); +void tor_raw_abort_(void) ATTR_NORETURN; + int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len); int format_dec_number_sigsafe(unsigned long x, char *buf, int max_len); -#endif /* !defined(TOR_TORLOG_H) */ +#endif /* !defined(TOR_TORERR_H) */ diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c new file mode 100644 index 0000000000..c42e958093 --- /dev/null +++ b/src/lib/err/torerr_sys.c @@ -0,0 +1,43 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.c + * \brief Subsystem object for the error handling subsystem. + **/ + +#include "orconfig.h" +#include "lib/err/backtrace.h" +#include "lib/err/torerr.h" +#include "lib/err/torerr_sys.h" +#include "lib/subsys/subsys.h" +#include "lib/version/torversion.h" + +#include <stddef.h> + +static int +subsys_torerr_initialize(void) +{ + if (configure_backtrace_handler(get_version()) < 0) + return -1; + tor_log_reset_sigsafe_err_fds(); + + return 0; +} +static void +subsys_torerr_shutdown(void) +{ + /* Stop handling signals with backtraces. */ + clean_up_backtrace_handler(); +} + +const subsys_fns_t sys_torerr = { + .name = "err", + /* Low-level error handling is a diagnostic feature, we want it to init + * right after windows process security, and shutdown last. + * (Security never shuts down.) */ + .level = -99, + .supported = true, + .initialize = subsys_torerr_initialize, + .shutdown = subsys_torerr_shutdown +}; diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h new file mode 100644 index 0000000000..c947695689 --- /dev/null +++ b/src/lib/err/torerr_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.h + * \brief Declare subsystem object for torerr.c + **/ + +#ifndef TOR_TORERR_SYS_H +#define TOR_TORERR_SYS_H + +extern const struct subsys_fns_t sys_torerr; + +#endif /* !defined(TOR_TORERR_SYS_H) */ diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include index 30af508914..54aa75fbff 100644 --- a/src/lib/evloop/.may_include +++ b/src/lib/evloop/.may_include @@ -8,9 +8,10 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h lib/time/*.h -src/ext/timeouts/timeout.c -tor_queue.h
\ No newline at end of file +ext/timeouts/timeout.c +ext/tor_queue.h
\ No newline at end of file diff --git a/src/lib/evloop/evloop_sys.c b/src/lib/evloop/evloop_sys.c new file mode 100644 index 0000000000..56641a3175 --- /dev/null +++ b/src/lib/evloop/evloop_sys.c @@ -0,0 +1,49 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file evloop_sys.c + * @brief Subsystem definition for the event loop module + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/evloop/evloop_sys.h" +#include "lib/log/log.h" + +static int +subsys_evloop_initialize(void) +{ + if (tor_init_libevent_rng() < 0) { + log_warn(LD_NET, "Problem initializing libevent RNG."); + return -1; + } + return 0; +} + +static void +subsys_evloop_postfork(void) +{ +#ifdef TOR_UNIT_TESTS + tor_libevent_postfork(); +#endif +} + +static void +subsys_evloop_shutdown(void) +{ + tor_libevent_free_all(); +} + +const struct subsys_fns_t sys_evloop = { + .name = "evloop", + .supported = true, + .level = -20, + .initialize = subsys_evloop_initialize, + .shutdown = subsys_evloop_shutdown, + .postfork = subsys_evloop_postfork, +}; diff --git a/src/lib/evloop/evloop_sys.h b/src/lib/evloop/evloop_sys.h new file mode 100644 index 0000000000..e6155c25b0 --- /dev/null +++ b/src/lib/evloop/evloop_sys.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file evloop_sys.h + * @brief Declare subsystem object for the event loop module. + **/ + +#ifndef TOR_LIB_EVLOOP_EVLOOP_SYS_H +#define TOR_LIB_EVLOOP_EVLOOP_SYS_H + +extern const struct subsys_fns_t sys_evloop; + +#endif /* !defined(TOR_LIB_EVLOOP_EVLOOP_SYS_H) */ diff --git a/src/lib/evloop/include.am b/src/lib/evloop/include.am index 6b0076272a..41cd2f45c5 100644 --- a/src/lib/evloop/include.am +++ b/src/lib/evloop/include.am @@ -5,21 +5,24 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-evloop-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_evloop_a_SOURCES = \ src/lib/evloop/compat_libevent.c \ + src/lib/evloop/evloop_sys.c \ src/lib/evloop/procmon.c \ src/lib/evloop/timers.c \ src/lib/evloop/token_bucket.c \ src/lib/evloop/workqueue.c - src_lib_libtor_evloop_testing_a_SOURCES = \ $(src_lib_libtor_evloop_a_SOURCES) src_lib_libtor_evloop_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_evloop_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/evloop/compat_libevent.h \ + src/lib/evloop/evloop_sys.h \ src/lib/evloop/procmon.h \ src/lib/evloop/timers.h \ src/lib/evloop/token_bucket.h \ diff --git a/src/lib/evloop/procmon.c b/src/lib/evloop/procmon.c index 52469fa5fc..b2d81fc14b 100644 --- a/src/lib/evloop/procmon.c +++ b/src/lib/evloop/procmon.c @@ -303,7 +303,7 @@ tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_) tor_free(errmsg); } } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /* Unix makes this part easy, if a bit racy. */ its_dead_jim = kill(procmon->pid, 0); its_dead_jim = its_dead_jim && (errno == ESRCH); diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c index e46d2635a8..4b2a96ef7d 100644 --- a/src/lib/evloop/timers.c +++ b/src/lib/evloop/timers.c @@ -80,7 +80,8 @@ struct timeout_cb { * use 32-bit math. */ #define WHEEL_BIT 5 #endif -#include "src/ext/timeouts/timeout.c" + +#include "ext/timeouts/timeout.c" static struct timeouts *global_timeouts = NULL; static struct mainloop_event_t *global_timer_event = NULL; diff --git a/src/lib/evloop/token_bucket.c b/src/lib/evloop/token_bucket.c index ee6d631e3b..ec62d1b018 100644 --- a/src/lib/evloop/token_bucket.c +++ b/src/lib/evloop/token_bucket.c @@ -256,3 +256,55 @@ token_bucket_rw_dec(token_bucket_rw_t *bucket, flags |= TB_WRITE; return flags; } + +/** Initialize a token bucket in <b>bucket</b>, set up to allow <b>rate</b> + * per second, with a maximum burst of <b>burst</b>. The bucket is created + * such that <b>now_ts</b> is the current timestamp. The bucket starts out + * full. */ +void +token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst, uint32_t now_ts) +{ + memset(bucket, 0, sizeof(token_bucket_ctr_t)); + token_bucket_ctr_adjust(bucket, rate, burst); + token_bucket_ctr_reset(bucket, now_ts); +} + +/** Change the configured rate and burst of the given token bucket object in + * <b>bucket</b>. */ +void +token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst) +{ + token_bucket_cfg_init(&bucket->cfg, rate, burst); + token_bucket_raw_adjust(&bucket->counter, &bucket->cfg); +} + +/** Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>. */ +void +token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts) +{ + token_bucket_raw_reset(&bucket->counter, &bucket->cfg); + bucket->last_refilled_at_timestamp = now_ts; +} + +/** Refill <b>bucket</b> as appropriate, given that the current timestamp is + * <b>now_ts</b>. */ +void +token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts) +{ + const uint32_t elapsed_ticks = + (now_ts - bucket->last_refilled_at_timestamp); + if (elapsed_ticks > UINT32_MAX-(300*1000)) { + /* Either about 48 days have passed since the last refill, or the + * monotonic clock has somehow moved backwards. (We're looking at you, + * Windows.). We accept up to a 5 minute jump backwards as + * "unremarkable". + */ + return; + } + + token_bucket_raw_refill_steps(&bucket->counter, &bucket->cfg, + elapsed_ticks); + bucket->last_refilled_at_timestamp = now_ts; +} diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h index 9398d2baa3..dde9bd65a4 100644 --- a/src/lib/evloop/token_bucket.h +++ b/src/lib/evloop/token_bucket.h @@ -103,6 +103,35 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket) return token_bucket_raw_get(&bucket->write_bucket); } +/** + * A specialized bucket containing a single counter. + */ + +typedef struct token_bucket_ctr_t { + token_bucket_cfg_t cfg; + token_bucket_raw_t counter; + uint32_t last_refilled_at_timestamp; +} token_bucket_ctr_t; + +void token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst, uint32_t now_ts); +void token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst); +void token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts); +void token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts); + +static inline bool +token_bucket_ctr_dec(token_bucket_ctr_t *bucket, ssize_t n) +{ + return token_bucket_raw_dec(&bucket->counter, n); +} + +static inline size_t +token_bucket_ctr_get(const token_bucket_ctr_t *bucket) +{ + return token_bucket_raw_get(&bucket->counter); +} + #ifdef TOKEN_BUCKET_PRIVATE /* To avoid making the rates too small, we consider units of "steps", @@ -112,6 +141,6 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket) STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate); -#endif +#endif /* defined(TOKEN_BUCKET_PRIVATE) */ -#endif /* TOR_TOKEN_BUCKET_H */ +#endif /* !defined(TOR_TOKEN_BUCKET_H) */ diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 931f65e710..015b694290 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -15,7 +15,7 @@ * * The main thread informs the worker threads of pending work by using a * condition variable. The workers inform the main process of completed work - * by using an alert_sockets_t object, as implemented in compat_threads.c. + * by using an alert_sockets_t object, as implemented in net/alertsock.c. * * The main thread can also queue an "update" that will be handled by all the * workers. This is useful for updating state that all the workers share. @@ -36,7 +36,7 @@ #include "lib/net/socket.h" #include "lib/thread/threads.h" -#include "tor_queue.h" +#include "ext/tor_queue.h" #include <event2/event.h> #include <string.h> @@ -59,9 +59,6 @@ struct threadpool_s { * <b>p</b> is work[p]. */ work_tailq_t work[WORKQUEUE_N_PRIORITIES]; - /** Weak RNG, used to decide when to ignore priority. */ - tor_weak_rng_t weak_rng; - /** The current 'update generation' of the threadpool. Any thread that is * at an earlier generation needs to run the update function. */ unsigned generation; @@ -238,7 +235,7 @@ worker_thread_extract_next_work(workerthread_t *thread) this_queue = &pool->work[i]; if (!TOR_TAILQ_EMPTY(this_queue)) { queue = this_queue; - if (! tor_weak_random_one_in_n(&pool->weak_rng, + if (! crypto_fast_rng_one_in_n(get_thread_fast_rng(), thread->lower_priority_chance)) { /* Usually we'll just break now, so that we can get out of the loop * and use the queue where we found work. But with a small @@ -555,11 +552,6 @@ threadpool_new(int n_threads, for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { TOR_TAILQ_INIT(&pool->work[i]); } - { - unsigned seed; - crypto_rand((void*)&seed, sizeof(seed)); - tor_init_weak_random(&pool->weak_rng, seed); - } pool->new_thread_state_fn = new_thread_state_fn; pool->new_thread_state_arg = arg; @@ -622,8 +614,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg) tp->reply_cb(tp); } -/** Register the threadpool <b>tp</b>'s reply queue with the libevent - * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after +/** Register the threadpool <b>tp</b>'s reply queue with Tor's global + * libevent mainloop. If <b>cb</b> is provided, it is run after * each time there is work to process from the reply queue. Return 0 on * success, -1 on failure. */ diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h index 333a3f6dde..d0ee8f2be2 100644 --- a/src/lib/evloop/workqueue.h +++ b/src/lib/evloop/workqueue.h @@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp); replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); void replyqueue_process(replyqueue_t *queue); -struct event_base; int threadpool_register_reply_event(threadpool_t *tp, void (*cb)(threadpool_t *tp)); diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c index d723d04d2a..2d0864cc19 100644 --- a/src/lib/fdio/fdio.c +++ b/src/lib/fdio/fdio.c @@ -17,12 +17,16 @@ #ifdef _WIN32 #include <windows.h> #endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif #include "lib/fdio/fdio.h" #include "lib/cc/torint.h" #include "lib/err/torerr.h" #include <stdlib.h> +#include <stdio.h> /** @{ */ /** Some old versions of Unix didn't define constants for these values, diff --git a/src/lib/fdio/include.am b/src/lib/fdio/include.am index 6c18f00a0d..545bbc929e 100644 --- a/src/lib/fdio/include.am +++ b/src/lib/fdio/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-fdio-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_fdio_a_SOURCES = \ src/lib/fdio/fdio.c @@ -13,5 +14,6 @@ src_lib_libtor_fdio_testing_a_SOURCES = \ src_lib_libtor_fdio_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_fdio_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/fdio/fdio.h diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include index b1e49fc891..c192e6181c 100644 --- a/src/lib/fs/.may_include +++ b/src/lib/fs/.may_include @@ -13,4 +13,4 @@ lib/malloc/*.h lib/memarea/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/fs/conffile.h b/src/lib/fs/conffile.h index 7af9119dbb..29115e1085 100644 --- a/src/lib/fs/conffile.h +++ b/src/lib/fs/conffile.h @@ -20,4 +20,4 @@ int config_get_lines_include(const char *string, struct config_line_t **result, int extended, int *has_include, struct smartlist_t *opened_lst); -#endif /* !defined(TOR_CONFLINE_H) */ +#endif /* !defined(TOR_CONFFILE_H) */ diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c index 3c31e00d99..390836b048 100644 --- a/src/lib/fs/dir.c +++ b/src/lib/fs/dir.c @@ -262,7 +262,7 @@ check_private_dir,(const char *dirname, cpd_check_t check, } } close(fd); -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ /* Win32 case: we can't open() a directory. */ (void)effective_user; @@ -347,7 +347,7 @@ tor_listdir, (const char *dirname)) } FindClose(handle); tor_free(pattern); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ const char *prot_dname = sandbox_intern_string(dirname); DIR *d; struct dirent *de; diff --git a/src/lib/fs/dir.h b/src/lib/fs/dir.h index 826bc2dfc5..9ff81faa42 100644 --- a/src/lib/fs/dir.h +++ b/src/lib/fs/dir.h @@ -30,4 +30,4 @@ MOCK_DECL(int, check_private_dir, (const char *dirname, cpd_check_t check, MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname)); -#endif +#endif /* !defined(TOR_DIR_H) */ diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h index 52c94c914f..ed983f3b3c 100644 --- a/src/lib/fs/files.h +++ b/src/lib/fs/files.h @@ -27,7 +27,7 @@ #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif -#endif +#endif /* defined(_WIN32) */ #ifndef O_BINARY #define O_BINARY 0 @@ -108,7 +108,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, * Tor is built for unit tests, or when Tor is built on an operating system * without its own getdelim(). */ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); -#endif +#endif /* !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS) */ #ifdef HAVE_GETDELIM /** @@ -123,10 +123,10 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); */ #define tor_getdelim(lineptr, n, delim, stream) \ getdelim((lineptr), (n), (delim), (stream)) -#else +#else /* !defined(HAVE_GETDELIM) */ #define tor_getdelim(lineptr, n, delim, stream) \ compat_getdelim_((lineptr), (n), (delim), (stream)) -#endif +#endif /* defined(HAVE_GETDELIM) */ #ifdef HAVE_GETLINE /** @@ -137,9 +137,9 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); */ #define tor_getline(lineptr, n, stream) \ getline((lineptr), (n), (stream)) -#else +#else /* !defined(HAVE_GETLINE) */ #define tor_getline(lineptr, n, stream) \ tor_getdelim((lineptr), (n), '\n', (stream)) -#endif +#endif /* defined(HAVE_GETLINE) */ -#endif +#endif /* !defined(TOR_FS_H) */ diff --git a/src/lib/fs/include.am b/src/lib/fs/include.am index f33e4d6430..493db8f044 100644 --- a/src/lib/fs/include.am +++ b/src/lib/fs/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-fs-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_fs_a_SOURCES = \ src/lib/fs/conffile.c \ src/lib/fs/dir.c \ @@ -25,6 +26,7 @@ src_lib_libtor_fs_testing_a_SOURCES = \ src_lib_libtor_fs_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_fs_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/fs/conffile.h \ src/lib/fs/dir.h \ diff --git a/src/lib/fs/lockfile.h b/src/lib/fs/lockfile.h index 8aeee4cc7f..fc0281e253 100644 --- a/src/lib/fs/lockfile.h +++ b/src/lib/fs/lockfile.h @@ -17,4 +17,4 @@ tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking, int *locked_out); void tor_lockfile_unlock(tor_lockfile_t *lockfile); -#endif +#endif /* !defined(TOR_LOCKFILE_H) */ diff --git a/src/lib/fs/mmap.c b/src/lib/fs/mmap.c index daaee1f9b1..f71c0cff7a 100644 --- a/src/lib/fs/mmap.c +++ b/src/lib/fs/mmap.c @@ -237,4 +237,4 @@ tor_munmap_file(tor_mmap_t *handle) } #else #error "cannot implement tor_mmap_file" -#endif /* defined(HAVE_MMAP) || ... || ... */ +#endif /* defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN) || ... */ diff --git a/src/lib/fs/mmap.h b/src/lib/fs/mmap.h index 18fb18a13c..61aad544b2 100644 --- a/src/lib/fs/mmap.h +++ b/src/lib/fs/mmap.h @@ -38,4 +38,4 @@ typedef struct tor_mmap_t { tor_mmap_t *tor_mmap_file(const char *filename); int tor_munmap_file(tor_mmap_t *handle); -#endif +#endif /* !defined(TOR_MMAP_H) */ diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c index b3ef61979d..28dde62aea 100644 --- a/src/lib/fs/path.c +++ b/src/lib/fs/path.c @@ -72,7 +72,7 @@ expand_filename(const char *filename) * Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/ */ return tor_strdup(filename); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ if (*filename == '~') { char *home, *result=NULL; const char *rest; @@ -102,7 +102,7 @@ expand_filename(const char *filename) } tor_free(username); rest = slash ? (slash+1) : ""; -#else /* !(defined(HAVE_PWD_H)) */ +#else /* !defined(HAVE_PWD_H) */ log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h"); return tor_strdup(filename); #endif /* defined(HAVE_PWD_H) */ @@ -153,7 +153,7 @@ clean_fname_for_stat(char *name) return; name[len-1]='\0'; } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ (void)name; #endif /* defined(_WIN32) */ } @@ -233,7 +233,7 @@ alloc_getcwd(void) raw_free(cwd); // alias for free to avoid tripping check-spaces. } return result; -#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */ +#else /* !defined(HAVE_GET_CURRENT_DIR_NAME) */ size_t size = 1024; char *buf = NULL; char *ptr = NULL; @@ -268,7 +268,7 @@ make_path_absolute(char *fname) if (absfname_malloced) raw_free(absfname_malloced); return absfname; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ char *absfname = NULL, *path = NULL; tor_assert(fname); diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h index 4675ac84e8..28a1838b88 100644 --- a/src/lib/fs/path.h +++ b/src/lib/fs/path.h @@ -27,4 +27,4 @@ void clean_fname_for_stat(char *name); int get_parent_directory(char *fname); char *make_path_absolute(char *fname); -#endif +#endif /* !defined(TOR_PATH_H) */ diff --git a/src/lib/fs/userdb.h b/src/lib/fs/userdb.h index 5c39794873..5e5ddb89a3 100644 --- a/src/lib/fs/userdb.h +++ b/src/lib/fs/userdb.h @@ -21,6 +21,6 @@ struct passwd; const struct passwd *tor_getpwnam(const char *username); const struct passwd *tor_getpwuid(uid_t uid); char *get_user_homedir(const char *username); -#endif +#endif /* !defined(_WIN32) */ -#endif +#endif /* !defined(TOR_USERDB_H) */ diff --git a/src/lib/fs/winlib.h b/src/lib/fs/winlib.h index 64a22439e5..7237226c76 100644 --- a/src/lib/fs/winlib.h +++ b/src/lib/fs/winlib.h @@ -17,6 +17,6 @@ #include <tchar.h> HANDLE load_windows_system_library(const TCHAR *library_name); -#endif +#endif /* defined(_WIN32) */ -#endif +#endif /* !defined(TOR_WINLIB_H) */ diff --git a/src/lib/geoip/country.h b/src/lib/geoip/country.h index 9a8911d494..a24a1c4c0d 100644 --- a/src/lib/geoip/country.h +++ b/src/lib/geoip/country.h @@ -13,4 +13,4 @@ typedef int16_t country_t; #define COUNTRY_MAX INT16_MAX -#endif +#endif /* !defined(TOR_COUNTRY_H) */ diff --git a/src/lib/geoip/include.am b/src/lib/geoip/include.am index 9710d75ac7..ea426d14bc 100644 --- a/src/lib/geoip/include.am +++ b/src/lib/geoip/include.am @@ -4,6 +4,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-geoip-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_geoip_a_SOURCES = \ src/lib/geoip/geoip.c @@ -12,6 +13,7 @@ src_lib_libtor_geoip_testing_a_SOURCES = \ src_lib_libtor_geoip_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_geoip_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/geoip/geoip.h \ src/lib/geoip/country.h diff --git a/src/lib/intmath/addsub.h b/src/lib/intmath/addsub.h index 83efa82919..3f745d457d 100644 --- a/src/lib/intmath/addsub.h +++ b/src/lib/intmath/addsub.h @@ -16,4 +16,4 @@ uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b); -#endif /* !defined(TOR_INTMATH_MULDIV_H) */ +#endif /* !defined(TOR_INTMATH_ADDSUB_H) */ diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h index d0b0e8b954..67a738861b 100644 --- a/src/lib/intmath/cmp.h +++ b/src/lib/intmath/cmp.h @@ -36,4 +36,7 @@ ((v) > (max)) ? (max) : \ (v) ) +/** Give the absolute value of <b>x</b>, independent of its type. */ +#define ABS(x) ( ((x)<0) ? -(x) : (x) ) + #endif /* !defined(TOR_INTMATH_CMP_H) */ diff --git a/src/lib/intmath/include.am b/src/lib/intmath/include.am index 45ee3bd53b..155ffa145a 100644 --- a/src/lib/intmath/include.am +++ b/src/lib/intmath/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-intmath-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_intmath_a_SOURCES = \ src/lib/intmath/addsub.c \ src/lib/intmath/bits.c \ @@ -16,6 +17,7 @@ src_lib_libtor_intmath_testing_a_SOURCES = \ src_lib_libtor_intmath_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_intmath_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/intmath/addsub.h \ src/lib/intmath/cmp.h \ diff --git a/src/lib/intmath/logic.h b/src/lib/intmath/logic.h index a4cecd69cc..b2f77462e1 100644 --- a/src/lib/intmath/logic.h +++ b/src/lib/intmath/logic.h @@ -17,4 +17,4 @@ /** Macro: true if two values have different boolean values. */ #define bool_neq(a,b) (!(a)!=!(b)) -#endif +#endif /* !defined(HAVE_TOR_LOGIC_H) */ diff --git a/src/lib/intmath/weakrng.h b/src/lib/intmath/weakrng.h index e26bf58cbb..40941e59b2 100644 --- a/src/lib/intmath/weakrng.h +++ b/src/lib/intmath/weakrng.h @@ -28,4 +28,4 @@ int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top); * <b>n</b> */ #define tor_weak_random_one_in_n(rng, n) (0==tor_weak_random_range((rng),(n))) -#endif +#endif /* !defined(TOR_WEAKRNG_H) */ diff --git a/src/lib/lock/compat_mutex.c b/src/lib/lock/compat_mutex.c index 4ad5929715..670bd0174c 100644 --- a/src/lib/lock/compat_mutex.c +++ b/src/lib/lock/compat_mutex.c @@ -29,7 +29,15 @@ tor_mutex_new_nonrecursive(void) tor_mutex_init_nonrecursive(m); return m; } -/** Release all storage and system resources held by <b>m</b>. */ +/** Release all storage and system resources held by <b>m</b>. + * + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ void tor_mutex_free_(tor_mutex_t *m) { diff --git a/src/lib/lock/compat_mutex.h b/src/lib/lock/compat_mutex.h index b63ce24024..e0c3d7cb78 100644 --- a/src/lib/lock/compat_mutex.h +++ b/src/lib/lock/compat_mutex.h @@ -48,7 +48,7 @@ typedef struct tor_mutex_t { #else /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */ int _unused; -#endif /* defined(USE_WIN32_MUTEX) || ... */ +#endif /* defined(USE_WIN32_THREADS) || ... */ } tor_mutex_t; tor_mutex_t *tor_mutex_new(void); diff --git a/src/lib/lock/compat_mutex_pthreads.c b/src/lib/lock/compat_mutex_pthreads.c index ee5f520cd0..f82ad9f0e8 100644 --- a/src/lib/lock/compat_mutex_pthreads.c +++ b/src/lib/lock/compat_mutex_pthreads.c @@ -88,12 +88,26 @@ tor_mutex_release(tor_mutex_t *m) } /** Clean up the mutex <b>m</b> so that it no longer uses any system * resources. Does not free <b>m</b>. This function must only be called on - * mutexes from tor_mutex_init(). */ + * mutexes from tor_mutex_init(). + * + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ void tor_mutex_uninit(tor_mutex_t *m) { int err; raw_assert(m); + /* If the mutex is already locked, wait until after it is unlocked to destroy + * it. Locking and releasing the mutex makes undefined behaviour less likely, + * but does not prevent it. Another thread can lock the mutex between release + * and destroy. */ + tor_mutex_acquire(m); + tor_mutex_release(m); err = pthread_mutex_destroy(&m->mutex); if (PREDICT_UNLIKELY(err)) { // LCOV_EXCL_START diff --git a/src/lib/lock/include.am b/src/lib/lock/include.am index 4e6f444347..1475b9911b 100644 --- a/src/lib/lock/include.am +++ b/src/lib/lock/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-lock-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_lock_a_SOURCES = \ src/lib/lock/compat_mutex.c @@ -20,5 +21,6 @@ src_lib_libtor_lock_testing_a_SOURCES = \ src_lib_libtor_lock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_lock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/lock/compat_mutex.h diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include index 852173aab3..54d96324db 100644 --- a/src/lib/log/.may_include +++ b/src/lib/log/.may_include @@ -1,6 +1,7 @@ orconfig.h lib/cc/*.h +lib/defs/*.h lib/smartlist_core/*.h lib/err/*.h lib/fdio/*.h @@ -9,7 +10,7 @@ lib/lock/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h +lib/version/*.h lib/wallclock/*.h - -micro-revision.i
\ No newline at end of file diff --git a/src/lib/log/escape.h b/src/lib/log/escape.h index 2f726186c5..0b9fc3406b 100644 --- a/src/lib/log/escape.h +++ b/src/lib/log/escape.h @@ -20,4 +20,4 @@ char *esc_for_log(const char *string) ATTR_MALLOC; char *esc_for_log_len(const char *chars, size_t n) ATTR_MALLOC; const char *escaped(const char *string); -#endif /* !defined(TOR_TORLOG_H) */ +#endif /* !defined(TOR_ESCAPE_H) */ diff --git a/src/lib/log/include.am b/src/lib/log/include.am index 4a6c9b3686..5b9f7113ba 100644 --- a/src/lib/log/include.am +++ b/src/lib/log/include.am @@ -5,11 +5,12 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-log-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_log_a_SOURCES = \ src/lib/log/escape.c \ - src/lib/log/git_revision.c \ src/lib/log/ratelim.c \ src/lib/log/log.c \ + src/lib/log/log_sys.c \ src/lib/log/util_bug.c if WIN32 @@ -21,16 +22,11 @@ src_lib_libtor_log_testing_a_SOURCES = \ src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) -# Declare that these object files depend on micro-revision.i. Without this -# rule, we could try to build them before micro-revision.i was created. -src/lib/log/git_revision.$(OBJEXT) \ - src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \ - micro-revision.i - +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/log/escape.h \ - src/lib/log/git_revision.h \ src/lib/log/ratelim.h \ src/lib/log/log.h \ + src/lib/log/log_sys.h \ src/lib/log/util_bug.h \ src/lib/log/win32err.h diff --git a/src/lib/log/log.c b/src/lib/log/log.c index a9ad38fb25..c53db46a8c 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -32,7 +32,8 @@ #define LOG_PRIVATE #include "lib/log/log.h" -#include "lib/log/git_revision.h" +#include "lib/log/log_sys.h" +#include "lib/version/git_revision.h" #include "lib/log/ratelim.h" #include "lib/lock/compat_mutex.h" #include "lib/smartlist_core/smartlist_core.h" @@ -48,15 +49,12 @@ #include "lib/wallclock/approx_time.h" #include "lib/wallclock/time_to_tm.h" #include "lib/fdio/fdio.h" +#include "lib/cc/ctassert.h" #ifdef HAVE_ANDROID_LOG_H #include <android/log.h> #endif // HAVE_ANDROID_LOG_H. -/** Given a severity, yields an index into log_severity_list_t.masks to use - * for that severity. */ -#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR) - /** @{ */ /** The string we stick at the end of a log message when it is too long, * and its length. */ @@ -153,7 +151,7 @@ severity_to_android_log_priority(int severity) // LCOV_EXCL_STOP } } -#endif // HAVE_ANDROID_LOG_H. +#endif /* defined(HAVE_ANDROID_LOG_H) */ /** A mutex to guard changes to logfiles and logging. */ static tor_mutex_t log_mutex; @@ -223,6 +221,7 @@ int log_global_min_severity_ = LOG_NOTICE; static void delete_log(logfile_t *victim); static void close_log(logfile_t *victim); +static void close_log_sigsafe(logfile_t *victim); static char *domain_to_string(log_domain_mask_t domain, char *buf, size_t buflen); @@ -533,7 +532,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len, if (m != msg_after_prefix) { tor_free(m); } -#else /* !(defined(MAXLINE)) */ +#else /* !defined(MAXLINE) */ /* We have syslog but not MAXLINE. That's promising! */ syslog(severity, "%s", msg_after_prefix); #endif /* defined(MAXLINE) */ @@ -585,8 +584,7 @@ logv,(int severity, log_domain_mask_t domain, const char *funcname, /* check that severity is sane. Overrunning the masks array leads to * interesting and hard to diagnose effects */ raw_assert(severity >= LOG_ERR && severity <= LOG_DEBUG); - /* check that we've initialised the log mutex before we try to lock it */ - raw_assert(log_mutex_initialized); + LOCK_LOGS(); if ((! (domain & LD_NOCB)) && pending_cb_messages @@ -630,6 +628,10 @@ void tor_log(int severity, log_domain_mask_t domain, const char *format, ...) { va_list ap; + + /* check that domain is composed of known domains and flags */ + raw_assert((domain & (LD_ALL_DOMAINS|LD_ALL_FLAGS)) == domain); + if (severity > log_global_min_severity_) return; va_start(ap,format); @@ -663,18 +665,26 @@ tor_log_update_sigsafe_err_fds(void) const logfile_t *lf; int found_real_stderr = 0; - int fds[TOR_SIGSAFE_LOG_MAX_FDS]; + /* log_fds and err_fds contain matching entries: log_fds are the fds used by + * the log module, and err_fds are the fds used by the err module. + * For stdio logs, the log_fd and err_fd values are identical. + * For file logs, the err_fd is a dup() of the log_fd. + */ + int log_fds[TOR_SIGSAFE_LOG_MAX_FDS]; + int err_fds[TOR_SIGSAFE_LOG_MAX_FDS]; int n_fds; LOCK_LOGS(); /* Reserve the first one for stderr. This is safe because when we daemonize, - * we dup2 /dev/null to stderr, */ - fds[0] = STDERR_FILENO; + * we dup2 /dev/null to stderr. + * For stderr, log_fds and err_fds are the same. */ + log_fds[0] = err_fds[0] = STDERR_FILENO; n_fds = 1; for (lf = logfiles; lf; lf = lf->next) { - /* Don't try callback to the control port, or syslogs: We can't - * do them from a signal handler. Don't try stdout: we always do stderr. + /* Don't try callback to the control port, syslogs, android logs, or any + * other non-file descriptor log: We can't call arbitrary functions from a + * signal handler. */ if (lf->is_temporary || logfile_is_external(lf) || lf->seems_dead || lf->fd < 0) @@ -683,25 +693,40 @@ tor_log_update_sigsafe_err_fds(void) (LD_BUG|LD_GENERAL)) { if (lf->fd == STDERR_FILENO) found_real_stderr = 1; - /* Avoid duplicates */ - if (int_array_contains(fds, n_fds, lf->fd)) + /* Avoid duplicates by checking the log module fd against log_fds */ + if (int_array_contains(log_fds, n_fds, lf->fd)) continue; - fds[n_fds++] = lf->fd; + /* Update log_fds using the log module's fd */ + log_fds[n_fds] = lf->fd; + if (lf->needs_close) { + /* File log fds are duplicated, because close_log() closes the log + * module's fd. Both refer to the same file. */ + err_fds[n_fds] = dup(lf->fd); + } else { + /* stdio log fds are not closed by the log module. */ + err_fds[n_fds] = lf->fd; + } + n_fds++; if (n_fds == TOR_SIGSAFE_LOG_MAX_FDS) break; } } if (!found_real_stderr && - int_array_contains(fds, n_fds, STDOUT_FILENO)) { - /* Don't use a virtual stderr when we're also logging to stdout. */ + int_array_contains(log_fds, n_fds, STDOUT_FILENO)) { + /* Don't use a virtual stderr when we're also logging to stdout. + * If we reached max_fds logs, we'll now have (max_fds - 1) logs. + * That's ok, max_fds is large enough that most tor instances don't exceed + * it. */ raw_assert(n_fds >= 2); /* Don't tor_assert inside log fns */ - fds[0] = fds[--n_fds]; + --n_fds; + log_fds[0] = log_fds[n_fds]; + err_fds[0] = err_fds[n_fds]; } UNLOCK_LOGS(); - tor_log_set_sigsafe_err_fds(fds, n_fds); + tor_log_set_sigsafe_err_fds(err_fds, n_fds); } /** Add to <b>out</b> a copy of every currently configured log file name. Used @@ -727,7 +752,7 @@ tor_log_get_logfile_names(smartlist_t *out) /** Implementation of the log_fn backend, used when we have * variadic macros. All arguments are as for log_fn, except for - * <b>fn</b>, which is the name of the calling functions. */ + * <b>fn</b>, which is the name of the calling function. */ void log_fn_(int severity, log_domain_mask_t domain, const char *fn, const char *format, ...) @@ -804,16 +829,16 @@ logs_free_all(void) } /* We _could_ destroy the log mutex here, but that would screw up any logs - * that happened between here and the end of execution. */ + * that happened between here and the end of execution. + * If tor is re-initialized, log_mutex_initialized will still be 1. So we + * won't trigger any undefined behaviour by trying to re-initialize the + * log mutex. */ } /** Remove and free the log entry <b>victim</b> from the linked-list * logfiles (it is probably present, but it might not be due to thread * racing issues). After this function is called, the caller shouldn't * refer to <b>victim</b> anymore. - * - * Long-term, we need to do something about races in the log subsystem - * in general. See bug 222 for more details. */ static void delete_log(logfile_t *victim) @@ -833,13 +858,26 @@ delete_log(logfile_t *victim) } /** Helper: release system resources (but not memory) held by a single - * logfile_t. */ + * signal-safe logfile_t. If the log's resources can not be released in + * a signal handler, does nothing. */ static void -close_log(logfile_t *victim) +close_log_sigsafe(logfile_t *victim) { if (victim->needs_close && victim->fd >= 0) { + /* We can't do anything useful here if close() fails: we're shutting + * down logging, and the err module only does fatal errors. */ close(victim->fd); victim->fd = -1; + } +} + +/** Helper: release system resources (but not memory) held by a single + * logfile_t. */ +static void +close_log(logfile_t *victim) +{ + if (victim->needs_close) { + close_log_sigsafe(victim); } else if (victim->is_syslog) { #ifdef HAVE_SYSLOG_H if (--syslog_count == 0) { @@ -863,7 +901,7 @@ set_log_severity_config(int loglevelMin, int loglevelMax, raw_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG); memset(severity_out, 0, sizeof(log_severity_list_t)); for (i = loglevelMin; i >= loglevelMax; --i) { - severity_out->masks[SEVERITY_MASK_IDX(i)] = ~0u; + severity_out->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS; } } @@ -1020,7 +1058,7 @@ flush_pending_log_callbacks(void) do { SMARTLIST_FOREACH_BEGIN(messages, pending_log_message_t *, msg) { const int severity = msg->severity; - const int domain = msg->domain; + const log_domain_mask_t domain = msg->domain; for (lf = logfiles; lf; lf = lf->next) { if (! lf->callback || lf->seems_dead || ! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) { @@ -1231,7 +1269,7 @@ add_android_log(const log_severity_list_t *severity, UNLOCK_LOGS(); return 0; } -#endif // HAVE_ANDROID_LOG_H. +#endif /* defined(HAVE_ANDROID_LOG_H) */ /** If <b>level</b> is a valid log severity, return the corresponding * numeric value. Otherwise, return -1. */ @@ -1267,9 +1305,16 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", "MESG", + NULL }; +CTASSERT(ARRAY_LENGTH(domain_list) == N_LOGGING_DOMAINS + 1); + +CTASSERT(HIGHEST_RESERVED_LD_DOMAIN_ < LD_ALL_DOMAINS); +CTASSERT(LD_ALL_DOMAINS < LOWEST_RESERVED_LD_FLAG_); +CTASSERT(LOWEST_RESERVED_LD_FLAG_ < LD_ALL_FLAGS); + /** Return a bitmask for the log domain for which <b>domain</b> is the name, * or 0 if there is no such name. */ static log_domain_mask_t @@ -1278,7 +1323,7 @@ parse_log_domain(const char *domain) int i; for (i=0; domain_list[i]; ++i) { if (!strcasecmp(domain, domain_list[i])) - return (1u<<i); + return (UINT64_C(1)<<i); } return 0; } @@ -1350,7 +1395,7 @@ parse_log_severity_config(const char **cfg_ptr, const char *dash, *space; char *sev_lo, *sev_hi; int low, high, i; - log_domain_mask_t domains = ~0u; + log_domain_mask_t domains = LD_ALL_DOMAINS; if (*cfg == '[') { int err = 0; @@ -1368,9 +1413,9 @@ parse_log_severity_config(const char **cfg_ptr, tor_free(domains_str); SMARTLIST_FOREACH_BEGIN(domains_list, const char *, domain) { if (!strcmp(domain, "*")) { - domains = ~0u; + domains = LD_ALL_DOMAINS; } else { - int d; + log_domain_mask_t d; int negate=0; if (*domain == '~') { negate = 1; @@ -1464,7 +1509,7 @@ switch_logs_debug(void) LOCK_LOGS(); for (lf = logfiles; lf; lf=lf->next) { for (i = LOG_DEBUG; i >= LOG_ERR; --i) - lf->severities->masks[SEVERITY_MASK_IDX(i)] = ~0u; + lf->severities->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS; } log_global_min_severity_ = get_min_log_level(); UNLOCK_LOGS(); diff --git a/src/lib/log/log.h b/src/lib/log/log.h index d7a5070610..f09c63cdd9 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -11,10 +11,12 @@ **/ #ifndef TOR_TORLOG_H +#define TOR_TORLOG_H #include <stdarg.h> #include "lib/cc/torint.h" #include "lib/cc/compat_compiler.h" +#include "lib/defs/logging_types.h" #include "lib/testsupport/testsupport.h" #ifdef HAVE_SYSLOG_H @@ -24,7 +26,7 @@ #error "Your syslog.h thinks high numbers are more important. " \ "We aren't prepared to deal with that." #endif -#else /* !(defined(HAVE_SYSLOG_H)) */ +#else /* !defined(HAVE_SYSLOG_H) */ /* Note: Syslog's logging code refers to priorities, with 0 being the most * important. Thus, all our comparisons needed to be reversed when we added * syslog support. @@ -55,75 +57,92 @@ /* Logging domains */ /** Catch-all for miscellaneous events and fatal errors. */ -#define LD_GENERAL (1u<<0) +#define LD_GENERAL (UINT64_C(1)<<0) /** The cryptography subsystem. */ -#define LD_CRYPTO (1u<<1) +#define LD_CRYPTO (UINT64_C(1)<<1) /** Networking. */ -#define LD_NET (1u<<2) +#define LD_NET (UINT64_C(1)<<2) /** Parsing and acting on our configuration. */ -#define LD_CONFIG (1u<<3) +#define LD_CONFIG (UINT64_C(1)<<3) /** Reading and writing from the filesystem. */ -#define LD_FS (1u<<4) +#define LD_FS (UINT64_C(1)<<4) /** Other servers' (non)compliance with the Tor protocol. */ -#define LD_PROTOCOL (1u<<5) +#define LD_PROTOCOL (UINT64_C(1)<<5) /** Memory management. */ -#define LD_MM (1u<<6) +#define LD_MM (UINT64_C(1)<<6) /** HTTP implementation. */ -#define LD_HTTP (1u<<7) +#define LD_HTTP (UINT64_C(1)<<7) /** Application (socks) requests. */ -#define LD_APP (1u<<8) +#define LD_APP (UINT64_C(1)<<8) /** Communication via the controller protocol. */ -#define LD_CONTROL (1u<<9) +#define LD_CONTROL (UINT64_C(1)<<9) /** Building, using, and managing circuits. */ -#define LD_CIRC (1u<<10) +#define LD_CIRC (UINT64_C(1)<<10) /** Hidden services. */ -#define LD_REND (1u<<11) +#define LD_REND (UINT64_C(1)<<11) /** Internal errors in this Tor process. */ -#define LD_BUG (1u<<12) +#define LD_BUG (UINT64_C(1)<<12) /** Learning and using information about Tor servers. */ -#define LD_DIR (1u<<13) +#define LD_DIR (UINT64_C(1)<<13) /** Learning and using information about Tor servers. */ -#define LD_DIRSERV (1u<<14) +#define LD_DIRSERV (UINT64_C(1)<<14) /** Onion routing protocol. */ -#define LD_OR (1u<<15) +#define LD_OR (UINT64_C(1)<<15) /** Generic edge-connection functionality. */ -#define LD_EDGE (1u<<16) +#define LD_EDGE (UINT64_C(1)<<16) #define LD_EXIT LD_EDGE /** Bandwidth accounting. */ -#define LD_ACCT (1u<<17) +#define LD_ACCT (UINT64_C(1)<<17) /** Router history */ -#define LD_HIST (1u<<18) +#define LD_HIST (UINT64_C(1)<<18) /** OR handshaking */ -#define LD_HANDSHAKE (1u<<19) +#define LD_HANDSHAKE (UINT64_C(1)<<19) /** Heartbeat messages */ -#define LD_HEARTBEAT (1u<<20) +#define LD_HEARTBEAT (UINT64_C(1)<<20) /** Abstract channel_t code */ -#define LD_CHANNEL (1u<<21) +#define LD_CHANNEL (UINT64_C(1)<<21) /** Scheduler */ -#define LD_SCHED (1u<<22) +#define LD_SCHED (UINT64_C(1)<<22) /** Guard nodes */ -#define LD_GUARD (1u<<23) +#define LD_GUARD (UINT64_C(1)<<23) /** Generation and application of consensus diffs. */ -#define LD_CONSDIFF (1u<<24) +#define LD_CONSDIFF (UINT64_C(1)<<24) /** Denial of Service mitigation. */ -#define LD_DOS (1u<<25) -/** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 26 +#define LD_DOS (UINT64_C(1)<<25) +/** Processes */ +#define LD_PROCESS (UINT64_C(1)<<26) +/** Pluggable Transports. */ +#define LD_PT (UINT64_C(1)<<27) +/** Bootstrap tracker. */ +#define LD_BTRACK (UINT64_C(1)<<28) +/** Message-passing backend. */ +#define LD_MESG (UINT64_C(1)<<29) -/** This log message is not safe to send to a callback-based logger - * immediately. Used as a flag, not a log domain. */ -#define LD_NOCB (1u<<31) -/** This log message should not include a function name, even if it otherwise - * would. Used as a flag, not a log domain. */ -#define LD_NOFUNCNAME (1u<<30) +/** The number of log domains. */ +#define N_LOGGING_DOMAINS 30 +/** The highest log domain */ +#define HIGHEST_RESERVED_LD_DOMAIN_ (UINT64_C(1)<<(N_LOGGING_DOMAINS - 1)) +/** All log domains. */ +#define LD_ALL_DOMAINS ((~(UINT64_C(0)))>>(64 - N_LOGGING_DOMAINS)) + +/** The number of log flags. */ +#define N_LOGGING_FLAGS 3 +/** First bit that is reserved in log_domain_mask_t for non-domain flags. */ +#define LOWEST_RESERVED_LD_FLAG_ (UINT64_C(1)<<(64 - N_LOGGING_FLAGS)) +/** All log flags. */ +#define LD_ALL_FLAGS ((~(UINT64_C(0)))<<(64 - N_LOGGING_FLAGS)) #ifdef TOR_UNIT_TESTS /** This log message should not be intercepted by mock_saving_logv */ -#define LD_NO_MOCK (1u<<29) +#define LD_NO_MOCK (UINT64_C(1)<<61) #endif -/** Mask of zero or more log domains, OR'd together. */ -typedef uint32_t log_domain_mask_t; +/** This log message is not safe to send to a callback-based logger + * immediately. Used as a flag, not a log domain. */ +#define LD_NOCB (UINT64_C(1)<<62) +/** This log message should not include a function name, even if it otherwise + * would. Used as a flag, not a log domain. */ +#define LD_NOFUNCNAME (UINT64_C(1)<<63) /** Configures which severities are logged for each logging domain for a given * log target. */ @@ -134,7 +153,8 @@ typedef struct log_severity_list_t { } log_severity_list_t; /** Callback type used for add_callback_log. */ -typedef void (*log_callback)(int severity, uint32_t domain, const char *msg); +typedef void (*log_callback)(int severity, log_domain_mask_t domain, + const char *msg); void init_logging(int disable_startup_queue); int parse_log_level(const char *level); @@ -186,6 +206,21 @@ void tor_log_get_logfile_names(struct smartlist_t *out); extern int log_global_min_severity_; +#ifdef TOR_COVERAGE +/* For coverage builds, we try to avoid our log_debug optimization, since it + * can have weird effects on internal macro coverage. */ +#define debug_logging_enabled() (1) +#else +static inline bool debug_logging_enabled(void); +/** + * Return true iff debug logging is enabled for at least one domain. + */ +static inline bool debug_logging_enabled(void) +{ + return PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG); +} +#endif /* defined(TOR_COVERAGE) */ + void log_fn_(int severity, log_domain_mask_t domain, const char *funcname, const char *format, ...) CHECK_PRINTF(4,5); @@ -215,8 +250,8 @@ void tor_log_string(int severity, log_domain_mask_t domain, log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, args) #define log_debug(domain, args...) \ STMT_BEGIN \ - if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG)) \ - log_fn_(LOG_DEBUG, domain, __FUNCTION__, args); \ + if (debug_logging_enabled()) \ + log_fn_(LOG_DEBUG, domain, __FUNCTION__, args); \ STMT_END #define log_info(domain, args...) \ log_fn_(LOG_INFO, domain, __FUNCTION__, args) @@ -233,8 +268,8 @@ void tor_log_string(int severity, log_domain_mask_t domain, #define log_debug(domain, args, ...) \ STMT_BEGIN \ - if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG)) \ - log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__); \ + if (debug_logging_enabled()) \ + log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__); \ STMT_END #define log_info(domain, args,...) \ log_fn_(LOG_INFO, domain, __FUNCTION__, args, ##__VA_ARGS__) @@ -272,5 +307,10 @@ MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, va_list ap) CHECK_PRINTF(5,0)); #endif -# define TOR_TORLOG_H +#if defined(LOG_PRIVATE) || defined(TOR_UNIT_TESTS) +/** Given a severity, yields an index into log_severity_list_t.masks to use + * for that severity. */ +#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR) +#endif + #endif /* !defined(TOR_TORLOG_H) */ diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c new file mode 100644 index 0000000000..826358546a --- /dev/null +++ b/src/lib/log/log_sys.c @@ -0,0 +1,37 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.c + * \brief Setup and tear down the logging module. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/log_sys.h" + +static int +subsys_logging_initialize(void) +{ + init_logging(0); + return 0; +} + +static void +subsys_logging_shutdown(void) +{ + logs_free_all(); + escaped(NULL); +} + +const subsys_fns_t sys_logging = { + .name = "log", + .supported = true, + /* Logging depends on threads, approx time, raw logging, and security. + * Most other lib modules depend on logging. */ + .level = -90, + .initialize = subsys_logging_initialize, + .shutdown = subsys_logging_shutdown, +}; diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h new file mode 100644 index 0000000000..7043253066 --- /dev/null +++ b/src/lib/log/log_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.h + * \brief Declare subsystem object for the logging module. + **/ + +#ifndef TOR_LOG_SYS_H +#define TOR_LOG_SYS_H + +extern const struct subsys_fns_t sys_logging; + +#endif /* !defined(TOR_LOG_SYS_H) */ diff --git a/src/lib/log/ratelim.h b/src/lib/log/ratelim.h index 48edd7c849..1db54ba726 100644 --- a/src/lib/log/ratelim.h +++ b/src/lib/log/ratelim.h @@ -50,4 +50,4 @@ typedef struct ratelim_t { char *rate_limit_log(ratelim_t *lim, time_t now); -#endif +#endif /* !defined(TOR_RATELIM_H) */ diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index c65a91ae9e..af1ac40732 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -11,6 +11,7 @@ #include "lib/log/util_bug.h" #include "lib/log/log.h" #include "lib/err/backtrace.h" +#include "lib/err/torerr.h" #ifdef TOR_UNIT_TESTS #include "lib/smartlist_core/smartlist_core.h" #include "lib/smartlist_core/smartlist_foreach.h" @@ -63,32 +64,52 @@ tor_set_failed_assertion_callback(void (*fn)(void)) { failed_assertion_cb = fn; } -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ #define capturing_bugs() (0) #define add_captured_bug(s) do { } while (0) #endif /* defined(TOR_UNIT_TESTS) */ /** Helper for tor_assert: report the assertion failure. */ void +CHECK_PRINTF(5, 6) tor_assertion_failed_(const char *fname, unsigned int line, - const char *func, const char *expr) + const char *func, const char *expr, + const char *fmt, ...) { - char buf[256]; + char *buf = NULL; + char *extra = NULL; + va_list ap; + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" +#endif + if (fmt) { + va_start(ap,fmt); + tor_vasprintf(&extra, fmt, ap); + va_end(ap); + } +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.", fname, line, func, expr); - tor_snprintf(buf, sizeof(buf), - "Assertion %s failed in %s at %s:%u", - expr, func, fname, line); + tor_asprintf(&buf, "Assertion %s failed in %s at %s:%u: %s", + expr, func, fname, line, extra ? extra : ""); + tor_free(extra); log_backtrace(LOG_ERR, LD_BUG, buf); + tor_free(buf); } /** Helper for tor_assert_nonfatal: report the assertion failure. */ void +CHECK_PRINTF(6, 7) tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, - int once) + int once, const char *fmt, ...) { - char buf[256]; + char *buf = NULL; const char *once_str = once ? " (Future instances of this warning will be silenced.)": ""; if (! expr) { @@ -98,7 +119,7 @@ tor_bug_occurred_(const char *fname, unsigned int line, } log_warn(LD_BUG, "%s:%u: %s: This line should not have been reached.%s", fname, line, func, once_str); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "Line unexpectedly reached at %s at %s:%u", func, fname, line); } else { @@ -106,13 +127,32 @@ tor_bug_occurred_(const char *fname, unsigned int line, add_captured_bug(expr); return; } + + va_list ap; + char *extra = NULL; + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" +#endif + if (fmt) { + va_start(ap,fmt); + tor_vasprintf(&extra, fmt, ap); + va_end(ap); + } +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s", fname, line, func, expr, once_str); - tor_snprintf(buf, sizeof(buf), - "Non-fatal assertion %s failed in %s at %s:%u", - expr, func, fname, line); + tor_asprintf(&buf, "Non-fatal assertion %s failed in %s at %s:%u%s%s", + expr, func, fname, line, fmt ? " : " : "", + extra ? extra : ""); + tor_free(extra); } log_backtrace(LOG_WARN, LD_BUG, buf); + tor_free(buf); #ifdef TOR_UNIT_TESTS if (failed_assertion_cb) { @@ -122,16 +162,16 @@ tor_bug_occurred_(const char *fname, unsigned int line, } /** - * Call the abort() function to kill the current process with a fatal - * error. + * Call the tor_raw_abort_() function to close raw logs, then kill the current + * process with a fatal error. * * (This is a separate function so that we declare it in util_bug.h without - * including stdlib in all the users of util_bug.h) + * including torerr.h in all the users of util_bug.h) **/ void tor_abort_(void) { - abort(); + tor_raw_abort_(); } #ifdef _WIN32 diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index 2a4d68127e..7eaa6173c8 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -80,10 +80,10 @@ tor__assert_tmp_value__; \ } ) #define ASSERT_PREDICT_LIKELY_(e) ASSERT_PREDICT_UNLIKELY_(e) -#else +#else /* !(defined(TOR_UNIT_TESTS) && defined(__GNUC__)) */ #define ASSERT_PREDICT_UNLIKELY_(e) PREDICT_UNLIKELY(e) #define ASSERT_PREDICT_LIKELY_(e) PREDICT_LIKELY(e) -#endif +#endif /* defined(TOR_UNIT_TESTS) && defined(__GNUC__) */ /* Sometimes we don't want to use assertions during branch coverage tests; it * leads to tons of unreached branches which in reality are only assertions we @@ -92,21 +92,28 @@ #define tor_assert(a) STMT_BEGIN \ (void)(a); \ STMT_END -#else +#define tor_assertf(a, fmt, ...) STMT_BEGIN \ + (void)(a); \ + (void)(fmt); \ + STMT_END +#else /* !(defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_T...)) */ /** Like assert(3), but send assertion failures to the log as well as to * stderr. */ -#define tor_assert(expr) STMT_BEGIN \ +#define tor_assert(expr) tor_assertf(expr, NULL) + +#define tor_assertf(expr, fmt, ...) STMT_BEGIN \ if (ASSERT_PREDICT_LIKELY_(expr)) { \ } else { \ - tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr); \ - tor_abort_(); \ + tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr, \ + fmt, ##__VA_ARGS__); \ + tor_abort_(); \ } STMT_END #endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */ #define tor_assert_unreached() \ STMT_BEGIN { \ tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, \ - "line should be unreached"); \ + "line should be unreached", NULL); \ tor_abort_(); \ } STMT_END @@ -136,34 +143,47 @@ #ifdef ALL_BUGS_ARE_FATAL #define tor_assert_nonfatal_unreached() tor_assert(0) #define tor_assert_nonfatal(cond) tor_assert((cond)) +#define tor_assertf_nonfatal(cond, fmt, ...) \ + tor_assertf(cond, fmt, ##__VA_ARGS__) #define tor_assert_nonfatal_unreached_once() tor_assert(0) #define tor_assert_nonfatal_once(cond) tor_assert((cond)) #define BUG(cond) \ (ASSERT_PREDICT_UNLIKELY_(cond) ? \ - (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")"), \ + (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",NULL), \ tor_abort_(), 1) \ : 0) #elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) #define tor_assert_nonfatal_unreached() STMT_NIL #define tor_assert_nonfatal(cond) ((void)(cond)) +#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN \ + (void)cond; \ + (void)fmt; \ + STMT_END #define tor_assert_nonfatal_unreached_once() STMT_NIL #define tor_assert_nonfatal_once(cond) ((void)(cond)) #define BUG(cond) (ASSERT_PREDICT_UNLIKELY_(cond) ? 1 : 0) #else /* Normal case, !ALL_BUGS_ARE_FATAL, !DISABLE_ASSERTS_IN_UNIT_TESTS */ #define tor_assert_nonfatal_unreached() STMT_BEGIN \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0, NULL); \ STMT_END #define tor_assert_nonfatal(cond) STMT_BEGIN \ if (ASSERT_PREDICT_LIKELY_(cond)) { \ } else { \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0, NULL);\ + } \ + STMT_END +#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN \ + if (ASSERT_PREDICT_UNLIKELY_(cond)) { \ + } else { \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0, \ + fmt, ##__VA_ARGS__); \ } \ STMT_END #define tor_assert_nonfatal_unreached_once() STMT_BEGIN \ static int warning_logged__ = 0; \ if (!warning_logged__) { \ warning_logged__ = 1; \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1, NULL); \ } \ STMT_END #define tor_assert_nonfatal_once(cond) STMT_BEGIN \ @@ -171,12 +191,12 @@ if (ASSERT_PREDICT_LIKELY_(cond)) { \ } else if (!warning_logged__) { \ warning_logged__ = 1; \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1, NULL);\ } \ STMT_END #define BUG(cond) \ (ASSERT_PREDICT_UNLIKELY_(cond) ? \ - (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \ + (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0,NULL),1) \ : 0) #endif /* defined(ALL_BUGS_ARE_FATAL) || ... */ @@ -188,17 +208,17 @@ if (bool_result && !var) { \ var = 1; \ tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, \ - "!("#cond")", 1); \ + "!("#cond")", 1, NULL); \ } \ bool_result; } )) -#else /* !(defined(__GNUC__)) */ +#else /* !defined(__GNUC__) */ #define IF_BUG_ONCE__(cond,var) \ static int var = 0; \ if ((cond) ? \ (var ? 1 : \ (var=1, \ tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, \ - "!("#cond")", 1), \ + "!("#cond")", 1, NULL), \ 1)) \ : 0) #endif /* defined(__GNUC__) */ @@ -221,10 +241,13 @@ #define tor_fragile_assert() tor_assert_nonfatal_unreached_once() void tor_assertion_failed_(const char *fname, unsigned int line, - const char *func, const char *expr); + const char *func, const char *expr, + const char *fmt, ...) + CHECK_PRINTF(5,6); void tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, - int once); + int once, const char *fmt, ...) + CHECK_PRINTF(6,7); void tor_abort_(void) ATTR_NORETURN; diff --git a/src/lib/log/win32err.c b/src/lib/log/win32err.c index dc45cb4c3d..03d5c9fad2 100644 --- a/src/lib/log/win32err.c +++ b/src/lib/log/win32err.c @@ -47,7 +47,7 @@ format_win32_error(DWORD err) result = tor_malloc(len); wcstombs(result,str,len); result[len-1] = '\0'; -#else /* !(defined(UNICODE)) */ +#else /* !defined(UNICODE) */ result = tor_strdup(str); #endif /* defined(UNICODE) */ } else { diff --git a/src/lib/log/win32err.h b/src/lib/log/win32err.h index 33413dfd15..ecfa88792d 100644 --- a/src/lib/log/win32err.h +++ b/src/lib/log/win32err.h @@ -19,4 +19,4 @@ char *format_win32_error(DWORD err); #endif -#endif +#endif /* !defined(TOR_WIN32ERR_H) */ diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include index cc62bb1013..7686bf862a 100644 --- a/src/lib/malloc/.may_include +++ b/src/lib/malloc/.may_include @@ -3,4 +3,4 @@ orconfig.h lib/cc/*.h lib/err/*.h lib/malloc/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am index 502cc1c6b7..b74292bc6e 100644 --- a/src/lib/malloc/include.am +++ b/src/lib/malloc/include.am @@ -5,8 +5,10 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-malloc-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_malloc_a_SOURCES = \ - src/lib/malloc/malloc.c + src/lib/malloc/malloc.c \ + src/lib/malloc/map_anon.c if USE_OPENBSD_MALLOC src_lib_libtor_malloc_a_SOURCES += src/ext/OpenBSD_malloc_Linux.c @@ -17,5 +19,7 @@ src_lib_libtor_malloc_testing_a_SOURCES = \ src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ - src/lib/malloc/malloc.h + src/lib/malloc/malloc.h \ + src/lib/malloc/map_anon.h diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h index ef6b509ca4..39a45901a1 100644 --- a/src/lib/malloc/malloc.h +++ b/src/lib/malloc/malloc.h @@ -48,12 +48,12 @@ void tor_free_(void *mem); raw_free(*tor_free__tmpvar); \ *tor_free__tmpvar=NULL; \ STMT_END -#else +#else /* !defined(__GNUC__) */ #define tor_free(p) STMT_BEGIN \ raw_free(p); \ (p)=NULL; \ STMT_END -#endif +#endif /* defined(__GNUC__) */ #define tor_malloc(size) tor_malloc_(size) #define tor_malloc_zero(size) tor_malloc_zero_(size) diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c new file mode 100644 index 0000000000..9559cbe2d4 --- /dev/null +++ b/src/lib/malloc/map_anon.c @@ -0,0 +1,271 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file map_anon.c + * \brief Manage anonymous mappings. + **/ + +#include "orconfig.h" +#include "lib/malloc/map_anon.h" +#include "lib/malloc/malloc.h" +#include "lib/err/torerr.h" + +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_MACH_VM_INHERIT_H +#include <mach/vm_inherit.h> +#endif + +#ifdef _WIN32 +#include <windows.h> +#endif + +#include <string.h> +#include <errno.h> + +/** + * Macro to get the high bytes of a size_t, if there are high bytes. + * Windows needs this; other operating systems define a size_t that does + * what it should. + */ +#if SIZEOF_SIZE_T > 4 +#define HIGH_SIZE_T_BYTES(sz) ((sz) >> 32) +#else +#define HIGH_SIZE_T_BYTES(sz) (0) +#endif + +/* Here we define a MINHERIT macro that is minherit() or madvise(), depending + * on what we actually want. + * + * If there's a flag that sets pages to zero after fork, we define FLAG_ZERO + * to be that flag. If there's a flag unmaps pages after fork, we define + * FLAG_NOINHERIT to be that flag. + */ +#if defined(HAVE_MINHERIT) +#define MINHERIT minherit + +#ifdef INHERIT_ZERO +#define FLAG_ZERO INHERIT_ZERO +#elif defined(MAP_INHERIT_ZERO) +#define FLAG_ZERO MAP_INHERIT_ZERO +#endif +#ifdef INHERIT_NONE +#define FLAG_NOINHERIT INHERIT_NONE +#elif defined(VM_INHERIT_NONE) +#define FLAG_NOINHERIT VM_INHERIT_NONE +#elif defined(MAP_INHERIT_NONE) +#define FLAG_NOINHERIT MAP_INHERIT_NONE +#endif /* defined(INHERIT_NONE) || ... */ + +#elif defined(HAVE_MADVISE) + +#define MINHERIT madvise + +#ifdef MADV_WIPEONFORK +#define FLAG_ZERO MADV_WIPEONFORK +#endif +#ifdef MADV_DONTFORK +#define FLAG_NOINHERIT MADV_DONTFORK +#endif + +#endif /* defined(HAVE_MINHERIT) || ... */ + +#if defined(HAVE_MINHERIT) && !defined(FLAG_ZERO) && !defined(FLAG_NOINHERIT) +#warn "minherit() is defined, but we couldn't find the right flag for it." +#warn "This is probably a bug in Tor's support for this platform." +#endif + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being swapped + * to disk. Return 0 on success or if the facility is not available on this + * OS; return -1 on failure. + */ +static int +lock_mem(void *mem, size_t sz) +{ +#ifdef _WIN32 + return VirtualLock(mem, sz) ? 0 : -1; +#elif defined(HAVE_MLOCK) + return mlock(mem, sz); +#else + (void) mem; + (void) sz; + + return 0; +#endif /* defined(_WIN32) || ... */ +} + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from appearing in + * a core dump. Return 0 on success or if the facility is not available on + * this OS; return -1 on failure. + */ +static int +nodump_mem(void *mem, size_t sz) +{ +#if defined(MADV_DONTDUMP) + int rv = madvise(mem, sz, MADV_DONTDUMP); + if (rv == 0) { + return 0; + } else if (errno == ENOSYS || errno == EINVAL) { + return 0; // syscall not supported, or flag not supported. + } else { + tor_log_err_sigsafe("Unexpected error from madvise: ", + strerror(errno), + NULL); + return -1; + } +#else /* !defined(MADV_DONTDUMP) */ + (void) mem; + (void) sz; + return 0; +#endif /* defined(MADV_DONTDUMP) */ +} + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being + * accessible in child processes -- ideally by having them set to 0 after a + * fork, and if that doesn't work, by having them unmapped after a fork. + * Return 0 on success or if the facility is not available on this OS; return + * -1 on failure. + * + * If we successfully make the memory uninheritable, adjust the value of + * *<b>inherit_result_out</b>. + */ +static int +noinherit_mem(void *mem, size_t sz, inherit_res_t *inherit_result_out) +{ +#ifdef FLAG_ZERO + int r = MINHERIT(mem, sz, FLAG_ZERO); + if (r == 0) { + *inherit_result_out = INHERIT_RES_ZERO; + return 0; + } +#endif /* defined(FLAG_ZERO) */ + +#ifdef FLAG_NOINHERIT + int r2 = MINHERIT(mem, sz, FLAG_NOINHERIT); + if (r2 == 0) { + *inherit_result_out = INHERIT_RES_DROP; + return 0; + } +#endif /* defined(FLAG_NOINHERIT) */ + +#if defined(FLAG_ZERO) || defined(FLAG_NOINHERIT) + /* At least one operation was tried, and neither succeeded. */ + + if (errno == ENOSYS || errno == EINVAL) { + /* Syscall not supported, or flag not supported. */ + return 0; + } else { + tor_log_err_sigsafe("Unexpected error from minherit: ", + strerror(errno), + NULL); + return -1; + } +#else /* !(defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)) */ + (void)inherit_result_out; + (void)mem; + (void)sz; + return 0; +#endif /* defined(FLAG_ZERO) || defined(FLAG_NOINHERIT) */ +} + +/** + * Return a new anonymous memory mapping that holds <b>sz</b> bytes. + * + * Memory mappings are unlike the results from malloc() in that they are + * handled separately by the operating system, and as such can have different + * kernel-level flags set on them. + * + * The "flags" argument may be zero or more of ANONMAP_PRIVATE and + * ANONMAP_NOINHERIT. + * + * Memory returned from this function must be released with + * tor_munmap_anonymous(). + * + * If <b>inherit_result_out</b> is non-NULL, set it to one of + * INHERIT_RES_KEEP, INHERIT_RES_DROP, or INHERIT_RES_ZERO, depending on the + * properties of the returned memory. + * + * [Note: OS people use the word "anonymous" here to mean that the memory + * isn't associated with any file. This has *nothing* to do with the kind of + * anonymity that Tor is trying to provide.] + */ +void * +tor_mmap_anonymous(size_t sz, unsigned flags, + inherit_res_t *inherit_result_out) +{ + void *ptr; + inherit_res_t itmp=0; + if (inherit_result_out == NULL) { + inherit_result_out = &itmp; + } + *inherit_result_out = INHERIT_RES_KEEP; + +#if defined(_WIN32) + HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE, + NULL, /*attributes*/ + PAGE_READWRITE, + HIGH_SIZE_T_BYTES(sz), + sz & 0xffffffff, + NULL /* name */); + raw_assert(mapping != NULL); + ptr = MapViewOfFile(mapping, FILE_MAP_WRITE, + 0, 0, /* Offset */ + 0 /* Extend to end of mapping */); + raw_assert(ptr); + CloseHandle(mapping); /* mapped view holds a reference */ +#elif defined(HAVE_SYS_MMAN_H) + ptr = mmap(NULL, sz, + PROT_READ|PROT_WRITE, + MAP_ANON|MAP_PRIVATE, + -1, 0); + raw_assert(ptr != MAP_FAILED); + raw_assert(ptr != NULL); +#else + ptr = tor_malloc_zero(sz); +#endif /* defined(_WIN32) || ... */ + + if (flags & ANONMAP_PRIVATE) { + int lock_result = lock_mem(ptr, sz); + raw_assert(lock_result == 0); + int nodump_result = nodump_mem(ptr, sz); + raw_assert(nodump_result == 0); + } + + if (flags & ANONMAP_NOINHERIT) { + int noinherit_result = noinherit_mem(ptr, sz, inherit_result_out); + raw_assert(noinherit_result == 0); + } + + return ptr; +} + +/** + * Release <b>sz</b> bytes of memory that were previously mapped at + * <b>mapping</b> by tor_mmap_anonymous(). + **/ +void +tor_munmap_anonymous(void *mapping, size_t sz) +{ + if (!mapping) + return; + +#if defined(_WIN32) + (void)sz; + UnmapViewOfFile(mapping); +#elif defined(HAVE_SYS_MMAN_H) + munmap(mapping, sz); +#else + (void)sz; + tor_free(mapping); +#endif /* defined(_WIN32) || ... */ +} diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h new file mode 100644 index 0000000000..4c4690e12f --- /dev/null +++ b/src/lib/malloc/map_anon.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file map_anon.h + * \brief Headers for map_anon.c + **/ + +#ifndef TOR_MAP_ANON_H +#define TOR_MAP_ANON_H + +#include "lib/malloc/malloc.h" +#include <stddef.h> + +/** + * When this flag is specified, try to prevent the mapping from being + * swapped or dumped. + * + * In some operating systems, this flag is not implemented. + */ +#define ANONMAP_PRIVATE (1u<<0) +/** + * When this flag is specified, try to prevent the mapping from being + * inherited after a fork(). In some operating systems, trying to access it + * afterwards will cause its contents to be zero. In others, trying to access + * it afterwards will cause a crash. + * + * In some operating systems, this flag is not implemented at all. + */ +#define ANONMAP_NOINHERIT (1u<<1) + +typedef enum { + /** Possible value for inherit_result_out: the memory will be kept + * by any child process. */ + INHERIT_RES_KEEP=0, + /** Possible value for inherit_result_out: the memory will be dropped in the + * child process. Attempting to access it will likely cause a segfault. */ + INHERIT_RES_DROP, + /** Possible value for inherit_result_out: the memory will be cleared in + * the child process. */ + INHERIT_RES_ZERO +} inherit_res_t; + +/* Here we define the NOINHERIT_CAN_FAIL macro if and only if + * it's possible that ANONMAP_NOINHERIT might yield inheritable memory. + */ +#ifdef _WIN32 +/* Windows can't fork, so NOINHERIT is never needed. */ +#elif defined(HAVE_MINHERIT) +/* minherit() will always have a working MAP_INHERIT_NONE or MAP_INHERIT_ZERO. + * NOINHERIT should always work. + */ +#elif defined(HAVE_MADVISE) +/* madvise() sometimes has neither MADV_DONTFORK and MADV_WIPEONFORK. + * We need to be ready for the possibility it failed. + * + * (Linux added DONTFORK in 2.6.16 and WIPEONFORK in 4.14. If we someday + * require 2.6.16 or later, we can assume that DONTFORK will work.) + */ +#define NOINHERIT_CAN_FAIL +#else +#define NOINHERIT_CAN_FAIL +#endif /* defined(_WIN32) || ... */ + +void *tor_mmap_anonymous(size_t sz, unsigned flags, + inherit_res_t *inherit_result_out); +void tor_munmap_anonymous(void *mapping, size_t sz); + +#endif /* !defined(TOR_MAP_ANON_H) */ diff --git a/src/lib/math/.may_include b/src/lib/math/.may_include index 1fd26864dc..f8bc264a5f 100644 --- a/src/lib/math/.may_include +++ b/src/lib/math/.may_include @@ -3,3 +3,5 @@ orconfig.h lib/cc/*.h lib/log/*.h lib/math/*.h +lib/testsupport/*.h +lib/crypt_ops/*.h diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c index eafad358c3..49a2a6a2ca 100644 --- a/src/lib/math/fp.c +++ b/src/lib/math/fp.c @@ -75,7 +75,7 @@ clamp_double_to_int64(double number) */ #define PROBLEMATIC_FLOAT_CONVERSION_WARNING DISABLE_GCC_WARNING(float-conversion) -#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */ +#endif /* (defined(MINGW_ANY)||defined(__FreeBSD__)) && GCC_VERSION >= 409 */ /* With clang 4.0 we apparently run into "double promotion" warnings here, @@ -121,3 +121,23 @@ ENABLE_GCC_WARNING(double-promotion) ENABLE_GCC_WARNING(float-conversion) #endif } + +/* isinf() wrapper for tor */ +int +tor_isinf(double x) +{ + /* Same as above, work around the "double promotion" warnings */ +#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING +DISABLE_GCC_WARNING(float-conversion) +#endif +#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING +DISABLE_GCC_WARNING(double-promotion) +#endif + return isinf(x); +#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING +ENABLE_GCC_WARNING(double-promotion) +#endif +#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING +ENABLE_GCC_WARNING(float-conversion) +#endif +} diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h index 6f07152e92..a73789c945 100644 --- a/src/lib/math/fp.h +++ b/src/lib/math/fp.h @@ -19,5 +19,6 @@ double tor_mathlog(double d) ATTR_CONST; long tor_lround(double d) ATTR_CONST; int64_t tor_llround(double d) ATTR_CONST; int64_t clamp_double_to_int64(double number); +int tor_isinf(double x); -#endif +#endif /* !defined(TOR_FP_H) */ diff --git a/src/lib/math/include.am b/src/lib/math/include.am index b088b3f3cc..b2ca280f47 100644 --- a/src/lib/math/include.am +++ b/src/lib/math/include.am @@ -5,16 +5,19 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-math-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_math_a_SOURCES = \ src/lib/math/fp.c \ - src/lib/math/laplace.c - + src/lib/math/laplace.c \ + src/lib/math/prob_distr.c src_lib_libtor_math_testing_a_SOURCES = \ $(src_lib_libtor_math_a_SOURCES) src_lib_libtor_math_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/math/fp.h \ - src/lib/math/laplace.h + src/lib/math/laplace.h \ + src/lib/math/prob_distr.h diff --git a/src/lib/math/laplace.h b/src/lib/math/laplace.h index e8651e5197..02b0e890f0 100644 --- a/src/lib/math/laplace.h +++ b/src/lib/math/laplace.h @@ -19,4 +19,4 @@ int64_t sample_laplace_distribution(double mu, double b, double p); int64_t add_laplace_noise(int64_t signal, double random, double delta_f, double epsilon); -#endif +#endif /* !defined(TOR_LAPLACE_H) */ diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c new file mode 100644 index 0000000000..d44dc28265 --- /dev/null +++ b/src/lib/math/prob_distr.c @@ -0,0 +1,1688 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file prob_distr.c + * + * \brief + * Implements various probability distributions. + * Almost all code is courtesy of Riastradh. + * + * \details + * Here are some details that might help you understand this file: + * + * - Throughout this file, `eps' means the largest relative error of a + * correctly rounded floating-point operation, which in binary64 + * floating-point arithmetic is 2^-53. Here the relative error of a + * true value x from a computed value y is |x - y|/|x|. This + * definition of epsilon is conventional for numerical analysts when + * writing error analyses. (If your libm doesn't provide correctly + * rounded exp and log, their relative error is usually below 2*2^-53 + * and probably closer to 1.1*2^-53 instead.) + * + * The C constant DBL_EPSILON is actually twice this, and should + * perhaps rather be named ulp(1) -- that is, it is the distance from + * 1 to the next greater floating-point number, which is usually of + * more interest to programmers and hardware engineers. + * + * Since this file is concerned mainly with error bounds rather than + * with low-level bit-hacking of floating-point numbers, we adopt the + * numerical analysts' definition in the comments, though we do use + * DBL_EPSILON in a handful of places where it is convenient to use + * some function of eps = DBL_EPSILON/2 in a case analysis. + * + * - In various functions (e.g. sample_log_logistic()) we jump through hoops so + * that we can use reals closer to 0 than closer to 1, since we achieve much + * greater accuracy for floating point numbers near 0. In particular, we can + * represent differences as small as 10^-300 for numbers near 0, but of no + * less than 10^-16 for numbers near 1. + **/ + +#define PROB_DISTR_PRIVATE + +#include "orconfig.h" + +#include "lib/math/prob_distr.h" + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/cc/ctassert.h" +#include "lib/log/util_bug.h" + +#include <float.h> +#include <math.h> +#include <stddef.h> + +/** Declare a function that downcasts from a generic dist struct to the actual + * subtype probablity distribution it represents. */ +#define DECLARE_PROB_DISTR_DOWNCAST_FN(name) \ + static inline \ + const struct name * \ + dist_to_const_##name(const struct dist *obj) { \ + tor_assert(obj->ops == &name##_ops); \ + return SUBTYPE_P(obj, struct name, base); \ + } +DECLARE_PROB_DISTR_DOWNCAST_FN(uniform) +DECLARE_PROB_DISTR_DOWNCAST_FN(geometric) +DECLARE_PROB_DISTR_DOWNCAST_FN(logistic) +DECLARE_PROB_DISTR_DOWNCAST_FN(log_logistic) +DECLARE_PROB_DISTR_DOWNCAST_FN(genpareto) +DECLARE_PROB_DISTR_DOWNCAST_FN(weibull) + +/** + * Count number of one bits in 32-bit word. + */ +static unsigned +bitcount32(uint32_t x) +{ + + /* Count two-bit groups. */ + x -= (x >> 1) & UINT32_C(0x55555555); + + /* Count four-bit groups. */ + x = ((x >> 2) & UINT32_C(0x33333333)) + (x & UINT32_C(0x33333333)); + + /* Count eight-bit groups. */ + x = (x + (x >> 4)) & UINT32_C(0x0f0f0f0f); + + /* Sum all eight-bit groups, and extract the sum. */ + return (x * UINT32_C(0x01010101)) >> 24; +} + +/** + * Count leading zeros in 32-bit word. + */ +static unsigned +clz32(uint32_t x) +{ + + /* Round up to a power of two. */ + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + + /* Subtract count of one bits from 32. */ + return (32 - bitcount32(x)); +} + +/* + * Some lemmas that will be used throughout this file to prove various error + * bounds: + * + * Lemma 1. If |d| <= 1/2, then 1/(1 + d) <= 2. + * + * Proof. If 0 <= d <= 1/2, then 1 + d >= 1, so that 1/(1 + d) <= 1. + * If -1/2 <= d <= 0, then 1 + d >= 1/2, so that 1/(1 + d) <= 2. QED. + * + * Lemma 2. If b = a*(1 + d)/(1 + d') for |d'| < 1/2 and nonzero a, b, + * then b = a*(1 + e) for |e| <= 2|d' - d|. + * + * Proof. |a - b|/|a| + * = |a - a*(1 + d)/(1 + d')|/|a| + * = |1 - (1 + d)/(1 + d')| + * = |(1 + d' - 1 - d)/(1 + d')| + * = |(d' - d)/(1 + d')| + * <= 2|d' - d|, by Lemma 1, + * + * QED. + * + * Lemma 3. For |d|, |d'| < 1/4, + * + * |log((1 + d)/(1 + d'))| <= 4|d - d'|. + * + * Proof. Write + * + * log((1 + d)/(1 + d')) + * = log(1 + (1 + d)/(1 + d') - 1) + * = log(1 + (1 + d - 1 - d')/(1 + d') + * = log(1 + (d - d')/(1 + d')). + * + * By Lemma 1, |(d - d')/(1 + d')| < 2|d' - d| < 1, so the Taylor + * series of log(1 + x) converges absolutely for (d - d')/(1 + d'), + * and thus we have + * + * |log(1 + (d - d')/(1 + d'))| + * = |\sum_{n=1}^\infty ((d - d')/(1 + d'))^n/n| + * <= \sum_{n=1}^\infty |(d - d')/(1 + d')|^n/n + * <= \sum_{n=1}^\infty |2(d' - d)|^n/n + * <= \sum_{n=1}^\infty |2(d' - d)|^n + * = 1/(1 - |2(d' - d)|) + * <= 4|d' - d|, + * + * QED. + * + * Lemma 4. If 1/e <= 1 + x <= e, then + * + * log(1 + (1 + d) x) = (1 + d') log(1 + x) + * + * for |d'| < 8|d|. + * + * Proof. Write + * + * log(1 + (1 + d) x) + * = log(1 + x + x*d) + * = log((1 + x) (1 + x + x*d)/(1 + x)) + * = log(1 + x) + log((1 + x + x*d)/(1 + x)) + * = log(1 + x) (1 + log((1 + x + x*d)/(1 + x))/log(1 + x)). + * + * The relative error is bounded by + * + * |log((1 + x + x*d)/(1 + x))/log(1 + x)| + * <= 4|x + x*d - x|/|log(1 + x)|, by Lemma 3, + * = 4|x*d|/|log(1 + x)| + * < 8|d|, + * + * since in this range 0 < 1 - 1/e < x/log(1 + x) <= e - 1 < 2. QED. + */ + +/** + * Compute the logistic function: f(x) = 1/(1 + e^{-x}) = e^x/(1 + e^x). + * Maps a log-odds-space probability in [-\infty, +\infty] into a direct-space + * probability in [0,1]. Inverse of logit. + * + * Ill-conditioned for large x; the identity logistic(-x) = 1 - + * logistic(x) and the function logistichalf(x) = logistic(x) - 1/2 may + * help to rearrange a computation. + * + * This implementation gives relative error bounded by 7 eps. + */ +STATIC double +logistic(double x) +{ + if (x <= log(DBL_EPSILON/2)) { + /* + * If x <= log(DBL_EPSILON/2) = log(eps), then e^x <= eps. In this case + * we will approximate the logistic() function with e^x because the + * relative error is less than eps. Here is a calculation of the + * relative error between the logistic() function and e^x and a proof + * that it's less than eps: + * + * |e^x - e^x/(1 + e^x)|/|e^x/(1 + e^x)| + * <= |1 - 1/(1 + e^x)|*|1 + e^x| + * = |e^x/(1 + e^x)|*|1 + e^x| + * = |e^x| + * <= eps. + */ + return exp(x); /* return e^x */ + } else if (x <= -log(DBL_EPSILON/2)) { + /* + * e^{-x} > 0, so 1 + e^{-x} > 1, and 0 < 1/(1 + + * e^{-x}) < 1; further, since e^{-x} < 1 + e^{-x}, we + * also have 0 < 1/(1 + e^{-x}) < 1. Thus, if exp has + * relative error d0, + has relative error d1, and / + * has relative error d2, then we get + * + * (1 + d2)/[(1 + (1 + d0) e^{-x})(1 + d1)] + * = (1 + d0)/[1 + e^{-x} + d0 e^{-x} + * + d1 + d1 e^{-x} + d0 d1 e^{-x}] + * = (1 + d0)/[(1 + e^{-x}) + * * (1 + d0 e^{-x}/(1 + e^{-x}) + * + d1/(1 + e^{-x}) + * + d0 d1 e^{-x}/(1 + e^{-x}))]. + * = (1 + d0)/[(1 + e^{-x})(1 + d')] + * = [1/(1 + e^{-x})] (1 + d0)/(1 + d') + * + * where + * + * d' = d0 e^{-x}/(1 + e^{-x}) + * + d1/(1 + e^{-x}) + * + d0 d1 e^{-x}/(1 + e^{-x}). + * + * By Lemma 2 this relative error is bounded by + * + * 2|d0 - d'| + * = 2|d0 - d0 e^{-x}/(1 + e^{-x}) + * - d1/(1 + e^{-x}) + * - d0 d1 e^{-x}/(1 + e^{-x})| + * <= 2|d0| + 2|d0 e^{-x}/(1 + e^{-x})| + * + 2|d1/(1 + e^{-x})| + * + 2|d0 d1 e^{-x}/(1 + e^{-x})| + * <= 2|d0| + 2|d0| + 2|d1| + 2|d0 d1| + * <= 4|d0| + 2|d1| + 2|d0 d1| + * <= 6 eps + 2 eps^2. + */ + return 1/(1 + exp(-x)); + } else { + /* + * e^{-x} <= eps, so the relative error of 1 from 1/(1 + * + e^{-x}) is + * + * |1/(1 + e^{-x}) - 1|/|1/(1 + e^{-x})| + * = |e^{-x}/(1 + e^{-x})|/|1/(1 + e^{-x})| + * = |e^{-x}| + * <= eps. + * + * This computation avoids an intermediate overflow + * exception, although the effect on the result is + * harmless. + * + * XXX Should maybe raise inexact here. + */ + return 1; + } +} + +/** + * Compute the logit function: log p/(1 - p). Defined on [0,1]. Maps + * a direct-space probability in [0,1] to a log-odds-space probability + * in [-\infty, +\infty]. Inverse of logistic. + * + * Ill-conditioned near 1/2 and 1; the identity logit(1 - p) = + * -logit(p) and the function logithalf(p0) = logit(1/2 + p0) may help + * to rearrange a computation for p in [1/(1 + e), 1 - 1/(1 + e)]. + * + * This implementation gives relative error bounded by 10 eps. + */ +STATIC double +logit(double p) +{ + + /* logistic(-1) <= p <= logistic(+1) */ + if (1/(1 + exp(1)) <= p && p <= 1/(1 + exp(-1))) { + /* + * For inputs near 1/2, we want to compute log1p(near + * 0) rather than log(near 1), so write this as: + * + * log(p/(1 - p)) = -log((1 - p)/p) + * = -log(1 + (1 - p)/p - 1) + * = -log(1 + (1 - p - p)/p) + * = -log(1 + (1 - 2p)/p). + * + * Since p = 2p/2 <= 1 <= 2*2p = 4p, the floating-point + * evaluation of 1 - 2p is exact; the only error arises + * from division and log1p. First, note that if + * logistic(-1) <= p <= logistic(+1), (1 - 2p)/p lies + * in the bounds of Lemma 4. + * + * If division has relative error d0 and log1p has + * relative error d1, the outcome is + * + * -(1 + d1) log(1 + (1 - 2p) (1 + d0)/p) + * = -(1 + d1) (1 + d') log(1 + (1 - 2p)/p) + * = -(1 + d1 + d' + d1 d') log(1 + (1 - 2p)/p). + * + * where |d'| < 8|d0| by Lemma 4. The relative error + * is then bounded by + * + * |d1 + d' + d1 d'| + * <= |d1| + 8|d0| + 8|d1 d0| + * <= 9 eps + 8 eps^2. + */ + return -log1p((1 - 2*p)/p); + } else { + /* + * For inputs near 0, although 1 - p may be rounded to + * 1, it doesn't matter much because the magnitude of + * the result is so much larger. For inputs near 1, we + * can compute 1 - p exactly, although the precision on + * the input is limited so we won't ever get more than + * about 700 for the output. + * + * If - has relative error d0, / has relative error d1, + * and log has relative error d2, then + * + * (1 + d2) log((1 + d0) p/[(1 - p)(1 + d1)]) + * = (1 + d2) [log(p/(1 - p)) + log((1 + d0)/(1 + d1))] + * = log(p/(1 - p)) + d2 log(p/(1 - p)) + * + (1 + d2) log((1 + d0)/(1 + d1)) + * = log(p/(1 - p))*[1 + d2 + + * + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))] + * + * Since 0 <= p < logistic(-1) or logistic(+1) < p <= + * 1, we have |log(p/(1 - p))| > 1. Hence this error + * is bounded by + * + * |d2 + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))| + * <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1)) + * / log(p/(1 - p))| + * <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))| + * <= |d2| + 4|(1 + d2) (d0 - d1)|, by Lemma 3, + * <= |d2| + 4|d0 - d1 + d2 d0 - d1 d0| + * <= |d2| + 4|d0| + 4|d1| + 4|d2 d0| + 4|d1 d0| + * <= 9 eps + 8 eps^2. + */ + return log(p/(1 - p)); + } +} + +/** + * Compute the logit function, translated in input by 1/2: logithalf(p) + * = logit(1/2 + p). Defined on [-1/2, 1/2]. Inverse of logistichalf. + * + * Ill-conditioned near +/-1/2. If |p0| > 1/2 - 1/(1 + e), it may be + * better to compute 1/2 + p0 or -1/2 - p0 and to use logit instead. + * This implementation gives relative error bounded by 34 eps. + */ +STATIC double +logithalf(double p0) +{ + + if (fabs(p0) <= 0.5 - 1/(1 + exp(1))) { + /* + * logit(1/2 + p0) + * = log((1/2 + p0)/(1 - (1/2 + p0))) + * = log((1/2 + p0)/(1/2 - p0)) + * = log(1 + (1/2 + p0)/(1/2 - p0) - 1) + * = log(1 + (1/2 + p0 - (1/2 - p0))/(1/2 - p0)) + * = log(1 + (1/2 + p0 - 1/2 + p0)/(1/2 - p0)) + * = log(1 + 2 p0/(1/2 - p0)) + * + * If the error of subtraction is d0, the error of + * division is d1, and the error of log1p is d2, then + * what we compute is + * + * (1 + d2) log(1 + (1 + d1) 2 p0/[(1 + d0) (1/2 - p0)]) + * = (1 + d2) log(1 + (1 + d') 2 p0/(1/2 - p0)) + * = (1 + d2) (1 + d'') log(1 + 2 p0/(1/2 - p0)) + * = (1 + d2 + d'' + d2 d'') log(1 + 2 p0/(1/2 - p0)), + * + * where |d'| < 2|d0 - d1| <= 4 eps by Lemma 2, and + * |d''| < 8|d'| < 32 eps by Lemma 4 since + * + * 1/e <= 1 + 2*p0/(1/2 - p0) <= e + * + * when |p0| <= 1/2 - 1/(1 + e). Hence the relative + * error is bounded by + * + * |d2 + d'' + d2 d''| + * <= |d2| + |d''| + |d2 d''| + * <= |d1| + 32 |d0| + 32 |d1 d0| + * <= 33 eps + 32 eps^2. + */ + return log1p(2*p0/(0.5 - p0)); + } else { + /* + * We have a choice of computing logit(1/2 + p0) or + * -logit(1 - (1/2 + p0)) = -logit(1/2 - p0). It + * doesn't matter which way we do this: either way, + * since 1/2 p0 <= 1/2 <= 2 p0, the sum and difference + * are computed exactly. So let's do the one that + * skips the final negation. + * + * The result is + * + * (1 + d1) log((1 + d0) (1/2 + p0)/[(1 + d2) (1/2 - p0)]) + * = (1 + d1) (1 + log((1 + d0)/(1 + d2)) + * / log((1/2 + p0)/(1/2 - p0))) + * * log((1/2 + p0)/(1/2 - p0)) + * = (1 + d') log((1/2 + p0)/(1/2 - p0)) + * = (1 + d') logit(1/2 + p0) + * + * where + * + * d' = d1 + log((1 + d0)/(1 + d2))/logit(1/2 + p0) + * + d1 log((1 + d0)/(1 + d2))/logit(1/2 + p0). + * + * For |p| > 1/2 - 1/(1 + e), logit(1/2 + p0) > 1. + * Provided |d0|, |d2| < 1/4, by Lemma 3 we have + * + * |log((1 + d0)/(1 + d2))| <= 4|d0 - d2|. + * + * Hence the relative error is bounded by + * + * |d'| <= |d1| + 4|d0 - d2| + 4|d1| |d0 - d2| + * <= |d1| + 4|d0| + 4|d2| + 4|d1 d0| + 4|d1 d2| + * <= 9 eps + 8 eps^2. + */ + return log((0.5 + p0)/(0.5 - p0)); + } +} + +/* + * The following random_uniform_01 is tailored for IEEE 754 binary64 + * floating-point or smaller. It can be adapted to larger + * floating-point formats like i387 80-bit or IEEE 754 binary128, but + * it may require sampling more bits. + */ +CTASSERT(FLT_RADIX == 2); +CTASSERT(-DBL_MIN_EXP <= 1021); +CTASSERT(DBL_MANT_DIG <= 53); + +/** + * Draw a floating-point number in [0, 1] with uniform distribution. + * + * Note that the probability of returning 0 is less than 2^-1074, so + * callers need not check for it. However, callers that cannot handle + * rounding to 1 must deal with that, because it occurs with + * probability 2^-54, which is small but nonnegligible. + */ +STATIC double +random_uniform_01(void) +{ + uint32_t z, x, hi, lo; + double s; + + /* + * Draw an exponent, geometrically distributed, but give up if + * we get a run of more than 1088 zeros, which really means the + * system is broken. + */ + z = 0; + while ((x = crypto_fast_rng_get_u32(get_thread_fast_rng())) == 0) { + if (z >= 1088) + /* Your bit sampler is broken. Go home. */ + return 0; + z += 32; + } + z += clz32(x); + + /* + * Pick 32-bit halves of an odd normalized significand. + * Picking it odd breaks ties in the subsequent rounding, which + * occur only with measure zero in the uniform distribution on + * [0, 1]. + */ + hi = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x80000000); + lo = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x00000001); + + /* Round to nearest scaled significand in [2^63, 2^64]. */ + s = hi*(double)4294967296 + lo; + + /* Rescale into [1/2, 1] and apply exponent in one swell foop. */ + return s * ldexp(1, -(64 + z)); +} + +/*******************************************************************/ + +/* Functions for specific probability distributions start here: */ + +/* + * Logistic(mu, sigma) distribution, supported on (-\infty,+\infty) + * + * This is the uniform distribution on [0,1] mapped into log-odds + * space, scaled by sigma and translated by mu. + * + * pdf(x) = e^{-(x - mu)/sigma} sigma (1 + e^{-(x - mu)/sigma})^2 + * cdf(x) = 1/(1 + e^{-(x - mu)/sigma}) = logistic((x - mu)/sigma) + * sf(x) = 1 - cdf(x) = 1 - logistic((x - mu)/sigma = logistic(-(x - mu)/sigma) + * icdf(p) = mu + sigma log p/(1 - p) = mu + sigma logit(p) + * isf(p) = mu + sigma log (1 - p)/p = mu - sigma logit(p) + */ + +/** + * Compute the CDF of the Logistic(mu, sigma) distribution: the + * logistic function. Well-conditioned for negative inputs and small + * positive inputs; ill-conditioned for large positive inputs. + */ +STATIC double +cdf_logistic(double x, double mu, double sigma) +{ + return logistic((x - mu)/sigma); +} + +/** + * Compute the SF of the Logistic(mu, sigma) distribution: the logistic + * function reflected over the y axis. Well-conditioned for positive + * inputs and small negative inputs; ill-conditioned for large negative + * inputs. + */ +STATIC double +sf_logistic(double x, double mu, double sigma) +{ + return logistic(-(x - mu)/sigma); +} + +/** + * Compute the inverse of the CDF of the Logistic(mu, sigma) + * distribution: the logit function. Well-conditioned near 0; + * ill-conditioned near 1/2 and 1. + */ +STATIC double +icdf_logistic(double p, double mu, double sigma) +{ + return mu + sigma*logit(p); +} + +/** + * Compute the inverse of the SF of the Logistic(mu, sigma) + * distribution: the -logit function. Well-conditioned near 0; + * ill-conditioned near 1/2 and 1. + */ +STATIC double +isf_logistic(double p, double mu, double sigma) +{ + return mu - sigma*logit(p); +} + +/* + * LogLogistic(alpha, beta) distribution, supported on (0, +\infty). + * + * This is the uniform distribution on [0,1] mapped into odds space, + * scaled by positive alpha and shaped by positive beta. + * + * Equivalent to computing exp of a Logistic(log alpha, 1/beta) sample. + * (Name arises because the pdf has LogLogistic(x; alpha, beta) = + * Logistic(log x; log alpha, 1/beta) and mathematicians got their + * covariance contravariant.) + * + * pdf(x) = (beta/alpha) (x/alpha)^{beta - 1}/(1 + (x/alpha)^beta)^2 + * = (1/e^mu sigma) (x/e^mu)^{1/sigma - 1} / + * (1 + (x/e^mu)^{1/sigma})^2 + * cdf(x) = 1/(1 + (x/alpha)^-beta) = 1/(1 + (x/e^mu)^{-1/sigma}) + * = 1/(1 + (e^{log x}/e^mu)^{-1/sigma}) + * = 1/(1 + (e^{log x - mu})^{-1/sigma}) + * = 1/(1 + e^{-(log x - mu)/sigma}) + * = logistic((log x - mu)/sigma) + * = logistic((log x - log alpha)/(1/beta)) + * sf(x) = 1 - 1/(1 + (x/alpha)^-beta) + * = (x/alpha)^-beta/(1 + (x/alpha)^-beta) + * = 1/((x/alpha)^beta + 1) + * = 1/(1 + (x/alpha)^beta) + * icdf(p) = alpha (p/(1 - p))^{1/beta} + * = alpha e^{logit(p)/beta} + * = e^{mu + sigma logit(p)} + * isf(p) = alpha ((1 - p)/p)^{1/beta} + * = alpha e^{-logit(p)/beta} + * = e^{mu - sigma logit(p)} + */ + +/** + * Compute the CDF of the LogLogistic(alpha, beta) distribution. + * Well-conditioned for all x and alpha, and the condition number + * + * -beta/[1 + (x/alpha)^{-beta}] + * + * grows linearly with beta. + * + * Loosely, the relative error of this implementation is bounded by + * + * 4 eps + 2 eps^2 + O(beta eps), + * + * so don't bother trying this for beta anywhere near as large as + * 1/eps, around which point it levels off at 1. + */ +STATIC double +cdf_log_logistic(double x, double alpha, double beta) +{ + /* + * Let d0 be the error of x/alpha; d1, of pow; d2, of +; and + * d3, of the final quotient. The exponentiation gives + * + * ((1 + d0) x/alpha)^{-beta} + * = (x/alpha)^{-beta} (1 + d0)^{-beta} + * = (x/alpha)^{-beta} (1 + (1 + d0)^{-beta} - 1) + * = (x/alpha)^{-beta} (1 + d') + * + * where d' = (1 + d0)^{-beta} - 1. If y = (x/alpha)^{-beta}, + * the denominator is + * + * (1 + d2) (1 + (1 + d1) (1 + d') y) + * = (1 + d2) (1 + y + (d1 + d' + d1 d') y) + * = 1 + y + (1 + d2) (d1 + d' + d1 d') y + * = (1 + y) (1 + (1 + d2) (d1 + d' + d1 d') y/(1 + y)) + * = (1 + y) (1 + d''), + * + * where d'' = (1 + d2) (d1 + d' + d1 d') y/(1 + y). The + * final result is + * + * (1 + d3) / [(1 + d2) (1 + d'') (1 + y)] + * = (1 + d''') / (1 + y) + * + * for |d'''| <= 2|d3 - d''| by Lemma 2 as long as |d''| < 1/2 + * (which may not be the case for very large beta). This + * relative error is therefore bounded by + * + * |d'''| + * <= 2|d3 - d''| + * <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d') y/(1 + y)| + * <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d')| + * = 2|d3| + 2|d1 + d' + d1 d' + d2 d1 + d2 d' + d2 d1 d'| + * <= 2|d3| + 2|d1| + 2|d'| + 2|d1 d'| + 2|d2 d1| + 2|d2 d'| + * + 2|d2 d1 d'| + * <= 4 eps + 2 eps^2 + (2 + 2 eps + 2 eps^2) |d'|. + * + * Roughly, |d'| = |(1 + d0)^{-beta} - 1| grows like beta eps, + * until it levels off at 1. + */ + return 1/(1 + pow(x/alpha, -beta)); +} + +/** + * Compute the SF of the LogLogistic(alpha, beta) distribution. + * Well-conditioned for all x and alpha, and the condition number + * + * beta/[1 + (x/alpha)^beta] + * + * grows linearly with beta. + * + * Loosely, the relative error of this implementation is bounded by + * + * 4 eps + 2 eps^2 + O(beta eps) + * + * so don't bother trying this for beta anywhere near as large as + * 1/eps, beyond which point it grows unbounded. + */ +STATIC double +sf_log_logistic(double x, double alpha, double beta) +{ + /* + * The error analysis here is essentially the same as in + * cdf_log_logistic, except that rather than levelling off at + * 1, |(1 + d0)^beta - 1| grows unbounded. + */ + return 1/(1 + pow(x/alpha, beta)); +} + +/** + * Compute the inverse of the CDF of the LogLogistic(alpha, beta) + * distribution. Ill-conditioned for p near 1 and beta near 0 with + * condition number 1/[beta (1 - p)]. + */ +STATIC double +icdf_log_logistic(double p, double alpha, double beta) +{ + return alpha*pow(p/(1 - p), 1/beta); +} + +/** + * Compute the inverse of the SF of the LogLogistic(alpha, beta) + * distribution. Ill-conditioned for p near 1 and for large beta, with + * condition number -1/[beta (1 - p)]. + */ +STATIC double +isf_log_logistic(double p, double alpha, double beta) +{ + return alpha*pow((1 - p)/p, 1/beta); +} + +/* + * Weibull(lambda, k) distribution, supported on (0, +\infty). + * + * pdf(x) = (k/lambda) (x/lambda)^{k - 1} e^{-(x/lambda)^k} + * cdf(x) = 1 - e^{-(x/lambda)^k} + * icdf(p) = lambda * (-log (1 - p))^{1/k} + * sf(x) = e^{-(x/lambda)^k} + * isf(p) = lambda * (-log p)^{1/k} + */ + +/** + * Compute the CDF of the Weibull(lambda, k) distribution. + * Well-conditioned for small x and k, and for large lambda -- + * condition number + * + * -k (x/lambda)^k exp(-(x/lambda)^k)/[exp(-(x/lambda)^k) - 1] + * + * grows linearly with k, x^k, and lambda^{-k}. + */ +STATIC double +cdf_weibull(double x, double lambda, double k) +{ + return -expm1(-pow(x/lambda, k)); +} + +/** + * Compute the SF of the Weibull(lambda, k) distribution. + * Well-conditioned for small x and k, and for large lambda -- + * condition number + * + * -k (x/lambda)^k + * + * grows linearly with k, x^k, and lambda^{-k}. + */ +STATIC double +sf_weibull(double x, double lambda, double k) +{ + return exp(-pow(x/lambda, k)); +} + +/** + * Compute the inverse of the CDF of the Weibull(lambda, k) + * distribution. Ill-conditioned for p near 1, and for k near 0; + * condition number is + * + * (p/(1 - p))/(k log(1 - p)). + */ +STATIC double +icdf_weibull(double p, double lambda, double k) +{ + return lambda*pow(-log1p(-p), 1/k); +} + +/** + * Compute the inverse of the SF of the Weibull(lambda, k) + * distribution. Ill-conditioned for p near 0, and for k near 0; + * condition number is + * + * 1/(k log(p)). + */ +STATIC double +isf_weibull(double p, double lambda, double k) +{ + return lambda*pow(-log(p), 1/k); +} + +/* + * GeneralizedPareto(mu, sigma, xi), supported on (mu, +\infty) for + * nonnegative xi, or (mu, mu - sigma/xi) for negative xi. + * + * Samples: + * = mu - sigma log U, if xi = 0; + * = mu + sigma (U^{-xi} - 1)/xi = mu + sigma*expm1(-xi log U)/xi, if xi =/= 0, + * where U is uniform on (0,1]. + * = mu + sigma (e^{xi X} - 1)/xi, + * where X has standard exponential distribution. + * + * pdf(x) = sigma^{-1} (1 + xi (x - mu)/sigma)^{-(1 + 1/xi)} + * cdf(x) = 1 - (1 + xi (x - mu)/sigma)^{-1/xi} + * = 1 - e^{-log(1 + xi (x - mu)/sigma)/xi} + * --> 1 - e^{-(x - mu)/sigma} as xi --> 0 + * sf(x) = (1 + xi (x - mu)/sigma)^{-1/xi} + * --> e^{-(x - mu)/sigma} as xi --> 0 + * icdf(p) = mu + sigma*(p^{-xi} - 1)/xi + * = mu + sigma*expm1(-xi log p)/xi + * --> mu + sigma*log p as xi --> 0 + * isf(p) = mu + sigma*((1 - p)^{xi} - 1)/xi + * = mu + sigma*expm1(-xi log1p(-p))/xi + * --> mu + sigma*log1p(-p) as xi --> 0 + */ + +/** + * Compute the CDF of the GeneralizedPareto(mu, sigma, xi) + * distribution. Well-conditioned everywhere. For standard + * distribution (mu=0, sigma=1), condition number + * + * (x/(1 + x xi)) / ((1 + x xi)^{1/xi} - 1) + * + * is bounded by 1, attained only at x = 0. + */ +STATIC double +cdf_genpareto(double x, double mu, double sigma, double xi) +{ + double x_0 = (x - mu)/sigma; + + /* + * log(1 + xi x_0)/xi + * = (-1/xi) \sum_{n=1}^\infty (-xi x_0)^n/n + * = (-1/xi) (-xi x_0 + \sum_{n=2}^\infty (-xi x_0)^n/n) + * = x_0 - (1/xi) \sum_{n=2}^\infty (-xi x_0)^n/n + * = x_0 - x_0 \sum_{n=2}^\infty (-xi x_0)^{n-1}/n + * = x_0 (1 - d), + * + * where d = \sum_{n=2}^\infty (-xi x_0)^{n-1}/n. If |xi| < + * eps/4|x_0|, then + * + * |d| <= \sum_{n=2}^\infty (eps/4)^{n-1}/n + * <= \sum_{n=2}^\infty (eps/4)^{n-1} + * = \sum_{n=1}^\infty (eps/4)^n + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2 + * + * for any 0 < eps < 2. Thus, the relative error of x_0 from + * log(1 + xi x_0)/xi is bounded by eps. + */ + if (fabs(xi) < 1e-17/x_0) + return -expm1(-x_0); + else + return -expm1(-log1p(xi*x_0)/xi); +} + +/** + * Compute the SF of the GeneralizedPareto(mu, sigma, xi) distribution. + * For standard distribution (mu=0, sigma=1), ill-conditioned for xi + * near 0; condition number + * + * -x (1 + x xi)^{(-1 - xi)/xi}/(1 + x xi)^{-1/xi} + * = -x (1 + x xi)^{-1/xi - 1}/(1 + x xi)^{-1/xi} + * = -(x/(1 + x xi)) (1 + x xi)^{-1/xi}/(1 + x xi)^{-1/xi} + * = -x/(1 + x xi) + * + * is bounded by 1/xi. + */ +STATIC double +sf_genpareto(double x, double mu, double sigma, double xi) +{ + double x_0 = (x - mu)/sigma; + + if (fabs(xi) < 1e-17/x_0) + return exp(-x_0); + else + return exp(-log1p(xi*x_0)/xi); +} + +/** + * Compute the inverse of the CDF of the GeneralizedPareto(mu, sigma, + * xi) distribution. Ill-conditioned for p near 1; condition number is + * + * xi (p/(1 - p))/(1 - (1 - p)^xi) + */ +STATIC double +icdf_genpareto(double p, double mu, double sigma, double xi) +{ + /* + * To compute f(xi) = (U^{-xi} - 1)/xi = (e^{-xi log U} - 1)/xi + * for xi near zero (note f(xi) --> -log U as xi --> 0), write + * the absolutely convergent Taylor expansion + * + * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^\infty (-xi log U)^n/n! + * = -log U + (1/xi)*\sum_{n=2}^\infty (-xi log U)^n/n! + * = -log U + \sum_{n=2}^\infty xi^{n-1} (-log U)^n/n! + * = -log U - log U \sum_{n=2}^\infty (-xi log U)^{n-1}/n! + * = -log U (1 + \sum_{n=2}^\infty (-xi log U)^{n-1}/n!). + * + * Let d = \sum_{n=2}^\infty (-xi log U)^{n-1}/n!. What do we + * lose if we discard it and use -log U as an approximation to + * f(xi)? If |xi| < eps/-4log U, then + * + * |d| <= \sum_{n=2}^\infty |xi log U|^{n-1}/n! + * <= \sum_{n=2}^\infty (eps/4)^{n-1}/n! + * <= \sum_{n=1}^\infty (eps/4)^n + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2, + * + * for any 0 < eps < 2. Hence, as long as |xi| < eps/-2log U, + * f(xi) = -log U (1 + d) for |d| <= eps/2. |d| is the + * relative error of f(xi) from -log U; from this bound, the + * relative error of -log U from f(xi) is at most (eps/2)/(1 - + * eps/2) = eps/2 + (eps/2)^2 + (eps/2)^3 + ... < eps for 0 < + * eps < 1. Since -log U < 1000 for all U in (0, 1] in + * binary64 floating-point, we can safely cut xi off at 1e-20 < + * eps/4000 and attain <1ulp error from series truncation. + */ + if (fabs(xi) <= 1e-20) + return mu - sigma*log1p(-p); + else + return mu + sigma*expm1(-xi*log1p(-p))/xi; +} + +/** + * Compute the inverse of the SF of the GeneralizedPareto(mu, sigma, + * xi) distribution. Ill-conditioned for p near 1; conditon number is + * + * -xi/(1 - p^{-xi}) + */ +STATIC double +isf_genpareto(double p, double mu, double sigma, double xi) +{ + if (fabs(xi) <= 1e-20) + return mu - sigma*log(p); + else + return mu + sigma*expm1(-xi*log(p))/xi; +} + +/*******************************************************************/ + +/** + * Deterministic samplers, parametrized by uniform integer and (0,1] + * samples. No guarantees are made about _which_ mapping from the + * integer and (0,1] samples these use; all that is guaranteed is the + * distribution of the outputs conditioned on a uniform distribution on + * the inputs. The automatic tests in test_prob_distr.c double-check + * the particular mappings we use. + * + * Beware: Unlike random_uniform_01(), these are not guaranteed to be + * supported on all possible outputs. See Ilya Mironov, `On the + * Significance of the Least Significant Bits for Differential + * Privacy', for an example of what can go wrong if you try to use + * these to conceal information from an adversary but you expose the + * specific full-precision floating-point values. + * + * Note: None of these samplers use rejection sampling; they are all + * essentially inverse-CDF transforms with tweaks. If you were to add, + * say, a Gamma sampler with the Marsaglia-Tsang method, you would have + * to parametrize it by a potentially infinite stream of uniform (and + * perhaps normal) samples rather than a fixed number, which doesn't + * make for quite as nice automatic testing as for these. + */ + +/** + * Deterministically sample from the interval [a, b], indexed by a + * uniform random floating-point number p0 in (0, 1]. + * + * Note that even if p0 is nonzero, the result may be equal to a, if + * ulp(a)/2 is nonnegligible, e.g. if a = 1. For maximum resolution, + * arrange |a| <= |b|. + */ +STATIC double +sample_uniform_interval(double p0, double a, double b) +{ + /* + * XXX Prove that the distribution is, in fact, uniform on + * [a,b], particularly around p0 = 1, or at least has very + * small deviation from uniform, quantified appropriately + * (e.g., like in Monahan 1984, or by KL divergence). It + * almost certainly does but it would be nice to quantify the + * error. + */ + if ((a <= 0 && 0 <= b) || (b <= 0 && 0 <= a)) { + /* + * When ab < 0, (1 - t) a + t b is monotonic, since for + * a <= b it is a sum of nondecreasing functions of t, + * and for b <= a, of nonincreasing functions of t. + * Further, clearly at 0 and 1 it attains a and b, + * respectively. Hence it is bounded within [a, b]. + */ + return (1 - p0)*a + p0*b; + } else { + /* + * a + (b - a) t is monotonic -- it is obviously a + * nondecreasing function of t for a <= b. Further, it + * attains a at 0, and while it may overshoot b at 1, + * we have a + * + * Theorem. If 0 <= t < 1, then the floating-point + * evaluation of a + (b - a) t is bounded in [a, b]. + * + * Lemma 1. If 0 <= t < 1 is a floating-point number, + * then for any normal floating-point number x except + * the smallest in magnitude, |round(x*t)| < |x|. + * + * Proof. WLOG, assume x >= 0. Since the rounding + * function and t |---> x*t are nondecreasing, their + * composition t |---> round(x*t) is also + * nondecreasing, so it suffices to consider the + * largest floating-point number below 1, in particular + * t = 1 - ulp(1)/2. + * + * Case I: If x is a power of two, then the next + * floating-point number below x is x - ulp(x)/2 = x - + * x*ulp(1)/2 = x*(1 - ulp(1)/2) = x*t, so, since x*t + * is a floating-point number, multiplication is exact, + * and thus round(x*t) = x*t < x. + * + * Case II: If x is not a power of two, then the + * greatest lower bound of real numbers rounded to x is + * x - ulp(x)/2 = x - ulp(T(x))/2 = x - T(x)*ulp(1)/2, + * where T(X) is the largest power of two below x. + * Anything below this bound is rounded to a + * floating-point number smaller than x, and x*t = x*(1 + * - ulp(1)/2) = x - x*ulp(1)/2 < x - T(x)*ulp(1)/2 + * since T(x) < x, so round(x*t) < x*t < x. QED. + * + * Lemma 2. If x and y are subnormal, then round(x + + * y) = x + y. + * + * Proof. It is a matter of adding the significands, + * since if we treat subnormals as having an implicit + * zero bit before the `binary' point, their exponents + * are all the same. There is at most one carry/borrow + * bit, which can always be acommodated either in a + * subnormal, or, at largest, in the implicit one bit + * of a normal. + * + * Lemma 3. Let x and y be floating-point numbers. If + * round(x - y) is subnormal or zero, then it is equal + * to x - y. + * + * Proof. Case I (equal): round(x - y) = 0 iff x = y; + * hence if round(x - y) = 0, then round(x - y) = 0 = x + * - y. + * + * Case II (subnormal/subnormal): If x and y are both + * subnormal, this follows directly from Lemma 2. + * + * Case IIIa (normal/subnormal): If x is normal and y + * is subnormal, then x and y must share sign, or else + * x - y would be larger than x and thus rounded to + * normal. If s is the smallest normal positive + * floating-point number, |x| < 2s since by + * construction 2s - |y| is normal for all subnormal y. + * This means that x and y must have the same exponent, + * so the difference is the difference of significands, + * which is exact. + * + * Case IIIb (subnormal/normal): Same as case IIIa for + * -(y - x). + * + * Case IV (normal/normal): If x and y are both normal, + * then they must share sign, or else x - y would be + * larger than x and thus rounded to normal. Note that + * |y| < 2|x|, for if |y| >= 2|x|, then |x| - |y| <= + * -|x| but -|x| is normal like x. Also, |x|/2 < |y|: + * if |x|/2 is subnormal, it must hold because y is + * normal; if |x|/2 is normal, then |x|/2 >= s, so + * since |x| - |y| < s, + * + * |x|/2 = |x| - |x|/2 <= |x| - s <= |y|; + * + * that is, |x|/2 < |y| < 2|x|, so by the Sterbenz + * lemma, round(x - y) = x - y. QED. + * + * Proof of theorem. WLOG, assume 0 <= a <= b. Since + * round(a + round(round(b - a)*t) is nondecreasing in + * t and attains a at 0, the lower end of the bound is + * trivial; we must show the upper end of the bound + * strictly. It suffices to show this for the largest + * floating-point number below 1, namely 1 - ulp(1)/2. + * + * Case I: round(b - a) is normal. Then it is at most + * the smallest floating-point number above b - a. By + * Lemma 1, round(round(b - a)*t) < round(b - a). + * Since the inequality is strict, and since + * round(round(b - a)*t) is a floating-point number + * below round(b - a), and since there are no + * floating-point numbers between b - a and round(b - + * a), we must have round(round(b - a)*t) < b - a. + * Then since y |---> round(a + y) is nondecreasing, we + * must have + * + * round(a + round(round(b - a)*t)) + * <= round(a + (b - a)) + * = round(b) = b. + * + * Case II: round(b - a) is subnormal. In this case, + * Lemma 1 falls apart -- we are not guaranteed the + * strict inequality. However, by Lemma 3, the + * difference is exact: round(b - a) = b - a. Thus, + * + * round(a + round(round(b - a)*t)) + * <= round(a + round((b - a)*t)) + * <= round(a + (b - a)) + * = round(b) + * = b, + * + * QED. + */ + + /* p0 is restricted to [0,1], but we use >= to silence -Wfloat-equal. */ + if (p0 >= 1) + return b; + return a + (b - a)*p0; + } +} + +/** + * Deterministically sample from the standard logistic distribution, + * indexed by a uniform random 32-bit integer s and uniform random + * floating-point numbers t and p0 in (0, 1]. + */ +STATIC double +sample_logistic(uint32_t s, double t, double p0) +{ + double sign = (s & 1) ? -1 : +1; + double r; + + /* + * We carve up the interval (0, 1) into subregions to compute + * the inverse CDF precisely: + * + * A = (0, 1/(1 + e)] ---> (-\infty, -1] + * B = [1/(1 + e), 1/2] ---> [-1, 0] + * C = [1/2, 1 - 1/(1 + e)] ---> [0, 1] + * D = [1 - 1/(1 + e), 1) ---> [1, +\infty) + * + * Cases D and C are mirror images of cases A and B, + * respectively, so we choose between them by the sign chosen + * by a fair coin toss. We choose between cases A and B by a + * coin toss weighted by + * + * 2/(1 + e) = 1 - [1/2 - 1/(1 + e)]/(1/2): + * + * if it comes up heads, scale p0 into a uniform (0, 1/(1 + e)] + * sample p; if it comes up tails, scale p0 into a uniform (0, + * 1/2 - 1/(1 + e)] sample and compute the inverse CDF of p = + * 1/2 - p0. + */ + if (t <= 2/(1 + exp(1))) { + /* p uniform in (0, 1/(1 + e)], represented by p. */ + p0 /= 1 + exp(1); + r = logit(p0); + } else { + /* + * p uniform in [1/(1 + e), 1/2), actually represented + * by p0 = 1/2 - p uniform in (0, 1/2 - 1/(1 + e)], so + * that p = 1/2 - p. + */ + p0 *= 0.5 - 1/(1 + exp(1)); + r = logithalf(p0); + } + + /* + * We have chosen from the negative half of the standard + * logistic distribution, which is symmetric with the positive + * half. Now use the sign to choose uniformly between them. + */ + return sign*r; +} + +/** + * Deterministically sample from the logistic distribution scaled by + * sigma and translated by mu. + */ +static double +sample_logistic_locscale(uint32_t s, double t, double p0, double mu, + double sigma) +{ + + return mu + sigma*sample_logistic(s, t, p0); +} + +/** + * Deterministically sample from the standard log-logistic + * distribution, indexed by a uniform random 32-bit integer s and a + * uniform random floating-point number p0 in (0, 1]. + */ +STATIC double +sample_log_logistic(uint32_t s, double p0) +{ + + /* + * Carve up the interval (0, 1) into (0, 1/2] and [1/2, 1); the + * condition numbers of the icdf and the isf coincide at 1/2. + */ + p0 *= 0.5; + if ((s & 1) == 0) { + /* p = p0 in (0, 1/2] */ + return p0/(1 - p0); + } else { + /* p = 1 - p0 in [1/2, 1) */ + return (1 - p0)/p0; + } +} + +/** + * Deterministically sample from the log-logistic distribution with + * scale alpha and shape beta. + */ +static double +sample_log_logistic_scaleshape(uint32_t s, double p0, double alpha, + double beta) +{ + double x = sample_log_logistic(s, p0); + + return alpha*pow(x, 1/beta); +} + +/** + * Deterministically sample from the standard exponential distribution, + * indexed by a uniform random 32-bit integer s and a uniform random + * floating-point number p0 in (0, 1]. + */ +static double +sample_exponential(uint32_t s, double p0) +{ + /* + * We would like to evaluate log(p) for p near 0, and log1p(-p) + * for p near 1. Simply carve the interval into (0, 1/2] and + * [1/2, 1) by a fair coin toss. + */ + p0 *= 0.5; + if ((s & 1) == 0) + /* p = p0 in (0, 1/2] */ + return -log(p0); + else + /* p = 1 - p0 in [1/2, 1) */ + return -log1p(-p0); +} + +/** + * Deterministically sample from a Weibull distribution with scale + * lambda and shape k -- just an exponential with a shape parameter in + * addition to a scale parameter. (Yes, lambda really is the scale, + * _not_ the rate.) + */ +STATIC double +sample_weibull(uint32_t s, double p0, double lambda, double k) +{ + + return lambda*pow(sample_exponential(s, p0), 1/k); +} + +/** + * Deterministically sample from the generalized Pareto distribution + * with shape xi, indexed by a uniform random 32-bit integer s and a + * uniform random floating-point number p0 in (0, 1]. + */ +STATIC double +sample_genpareto(uint32_t s, double p0, double xi) +{ + double x = sample_exponential(s, p0); + + /* + * Write f(xi) = (e^{xi x} - 1)/xi for xi near zero as the + * absolutely convergent Taylor series + * + * f(x) = (1/xi) (xi x + \sum_{n=2}^\infty (xi x)^n/n!) + * = x + (1/xi) \sum_{n=2}^\inty (xi x)^n/n! + * = x + \sum_{n=2}^\infty xi^{n-1} x^n/n! + * = x + x \sum_{n=2}^\infty (xi x)^{n-1}/n! + * = x (1 + \sum_{n=2}^\infty (xi x)^{n-1}/n!). + * + * d = \sum_{n=2}^\infty (xi x)^{n-1}/n! is the relative error + * of f(x) from x. If |xi| < eps/4x, then + * + * |d| <= \sum_{n=2}^\infty |xi x|^{n-1}/n! + * <= \sum_{n=2}^\infty (eps/4)^{n-1}/n! + * <= \sum_{n=1}^\infty (eps/4) + * = (eps/4) \sum_{n=0}^\infty (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2, + * + * for any 0 < eps < 2. Hence, as long as |xi| < eps/2x, f(xi) + * = x (1 + d) for |d| <= eps/2, so x = f(xi) (1 + d') for |d'| + * <= eps. What bound should we use for x? + * + * - If x is exponentially distributed, x > 200 with + * probability below e^{-200} << 2^{-256}, i.e. never. + * + * - If x is computed by -log(U) for U in (0, 1], x is + * guaranteed to be below 1000 in IEEE 754 binary64 + * floating-point. + * + * We can safely cut xi off at 1e-20 < eps/4000 and attain an + * error bounded by 0.5 ulp for this expression. + */ + return (fabs(xi) < 1e-20 ? x : expm1(xi*x)/xi); +} + +/** + * Deterministically sample from a generalized Pareto distribution with + * shape xi, scaled by sigma and translated by mu. + */ +static double +sample_genpareto_locscale(uint32_t s, double p0, double mu, double sigma, + double xi) +{ + + return mu + sigma*sample_genpareto(s, p0, xi); +} + +/** + * Deterministically sample from the geometric distribution with + * per-trial success probability p. + * + * XXX Quantify the error (KL divergence?) of this + * ceiling-of-exponential sampler from a true geometric distribution, + * which we could get by rejection sampling. Relevant papers: + * + * John F. Monahan, `Accuracy in Random Number Generation', + * Mathematics of Computation 45(172), October 1984, pp. 559--568. +*https://pdfs.semanticscholar.org/aca6/74b96da1df77b2224e8cfc5dd6d61a471632.pdf + * + * Karl Bringmann and Tobias Friedrich, `Exact and Efficient + * Generation of Geometric Random Variates and Random Graphs', in + * Proceedings of the 40th International Colloaquium on Automata, + * Languages, and Programming -- ICALP 2013, Springer LNCS 7965, + * pp.267--278. + * https://doi.org/10.1007/978-3-642-39206-1_23 + * https://people.mpi-inf.mpg.de/~kbringma/paper/2013ICALP-1.pdf + */ +static double +sample_geometric(uint32_t s, double p0, double p) +{ + double x = sample_exponential(s, p0); + + /* This is actually a check against 1, but we do >= so that the compiler + does not raise a -Wfloat-equal */ + if (p >= 1) + return 1; + + return ceil(-x/log1p(-p)); +} + +/*******************************************************************/ + +/** Public API for probability distributions: + * + * These are wrapper functions on top of the various probability distribution + * operations using the generic <b>dist</b> structure. + + * These are the functions that should be used by consumers of this API. + */ + +/** Returns the name of the distribution in <b>dist</b>. */ +const char * +dist_name(const struct dist *dist) +{ + return dist->ops->name; +} + +/* Sample a value from <b>dist</b> and return it. */ +double +dist_sample(const struct dist *dist) +{ + return dist->ops->sample(dist); +} + +/** Compute the CDF of <b>dist</b> at <b>x</b>. */ +double +dist_cdf(const struct dist *dist, double x) +{ + return dist->ops->cdf(dist, x); +} + +/** Compute the SF (Survival function) of <b>dist</b> at <b>x</b>. */ +double +dist_sf(const struct dist *dist, double x) +{ + return dist->ops->sf(dist, x); +} + +/** Compute the iCDF (Inverse CDF) of <b>dist</b> at <b>x</b>. */ +double +dist_icdf(const struct dist *dist, double p) +{ + return dist->ops->icdf(dist, p); +} + +/** Compute the iSF (Inverse Survival function) of <b>dist</b> at <b>x</b>. */ +double +dist_isf(const struct dist *dist, double p) +{ + return dist->ops->isf(dist, p); +} + +/** Functions for uniform distribution */ + +static double +uniform_sample(const struct dist *dist) +{ + const struct uniform *U = dist_to_const_uniform(dist); + double p0 = random_uniform_01(); + + return sample_uniform_interval(p0, U->a, U->b); +} + +static double +uniform_cdf(const struct dist *dist, double x) +{ + const struct uniform *U = dist_to_const_uniform(dist); + if (x < U->a) + return 0; + else if (x < U->b) + return (x - U->a)/(U->b - U->a); + else + return 1; +} + +static double +uniform_sf(const struct dist *dist, double x) +{ + const struct uniform *U = dist_to_const_uniform(dist); + + if (x > U->b) + return 0; + else if (x > U->a) + return (U->b - x)/(U->b - U->a); + else + return 1; +} + +static double +uniform_icdf(const struct dist *dist, double p) +{ + const struct uniform *U = dist_to_const_uniform(dist); + double w = U->b - U->a; + + return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p))); +} + +static double +uniform_isf(const struct dist *dist, double p) +{ + const struct uniform *U = dist_to_const_uniform(dist); + double w = U->b - U->a; + + return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p))); +} + +const struct dist_ops uniform_ops = { + .name = "uniform", + .sample = uniform_sample, + .cdf = uniform_cdf, + .sf = uniform_sf, + .icdf = uniform_icdf, + .isf = uniform_isf, +}; + +/*******************************************************************/ + +/** Private functions for each probability distribution. */ + +/** Functions for logistic distribution: */ + +static double +logistic_sample(const struct dist *dist) +{ + const struct logistic *L = dist_to_const_logistic(dist); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); + double t = random_uniform_01(); + double p0 = random_uniform_01(); + + return sample_logistic_locscale(s, t, p0, L->mu, L->sigma); +} + +static double +logistic_cdf(const struct dist *dist, double x) +{ + const struct logistic *L = dist_to_const_logistic(dist); + return cdf_logistic(x, L->mu, L->sigma); +} + +static double +logistic_sf(const struct dist *dist, double x) +{ + const struct logistic *L = dist_to_const_logistic(dist); + return sf_logistic(x, L->mu, L->sigma); +} + +static double +logistic_icdf(const struct dist *dist, double p) +{ + const struct logistic *L = dist_to_const_logistic(dist); + return icdf_logistic(p, L->mu, L->sigma); +} + +static double +logistic_isf(const struct dist *dist, double p) +{ + const struct logistic *L = dist_to_const_logistic(dist); + return isf_logistic(p, L->mu, L->sigma); +} + +const struct dist_ops logistic_ops = { + .name = "logistic", + .sample = logistic_sample, + .cdf = logistic_cdf, + .sf = logistic_sf, + .icdf = logistic_icdf, + .isf = logistic_isf, +}; + +/** Functions for log-logistic distribution: */ + +static double +log_logistic_sample(const struct dist *dist) +{ + const struct log_logistic *LL = dist_to_const_log_logistic(dist); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); + double p0 = random_uniform_01(); + + return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta); +} + +static double +log_logistic_cdf(const struct dist *dist, double x) +{ + const struct log_logistic *LL = dist_to_const_log_logistic(dist); + return cdf_log_logistic(x, LL->alpha, LL->beta); +} + +static double +log_logistic_sf(const struct dist *dist, double x) +{ + const struct log_logistic *LL = dist_to_const_log_logistic(dist); + return sf_log_logistic(x, LL->alpha, LL->beta); +} + +static double +log_logistic_icdf(const struct dist *dist, double p) +{ + const struct log_logistic *LL = dist_to_const_log_logistic(dist); + return icdf_log_logistic(p, LL->alpha, LL->beta); +} + +static double +log_logistic_isf(const struct dist *dist, double p) +{ + const struct log_logistic *LL = dist_to_const_log_logistic(dist); + return isf_log_logistic(p, LL->alpha, LL->beta); +} + +const struct dist_ops log_logistic_ops = { + .name = "log logistic", + .sample = log_logistic_sample, + .cdf = log_logistic_cdf, + .sf = log_logistic_sf, + .icdf = log_logistic_icdf, + .isf = log_logistic_isf, +}; + +/** Functions for Weibull distribution */ + +static double +weibull_sample(const struct dist *dist) +{ + const struct weibull *W = dist_to_const_weibull(dist); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); + double p0 = random_uniform_01(); + + return sample_weibull(s, p0, W->lambda, W->k); +} + +static double +weibull_cdf(const struct dist *dist, double x) +{ + const struct weibull *W = dist_to_const_weibull(dist); + return cdf_weibull(x, W->lambda, W->k); +} + +static double +weibull_sf(const struct dist *dist, double x) +{ + const struct weibull *W = dist_to_const_weibull(dist); + return sf_weibull(x, W->lambda, W->k); +} + +static double +weibull_icdf(const struct dist *dist, double p) +{ + const struct weibull *W = dist_to_const_weibull(dist); + return icdf_weibull(p, W->lambda, W->k); +} + +static double +weibull_isf(const struct dist *dist, double p) +{ + const struct weibull *W = dist_to_const_weibull(dist); + return isf_weibull(p, W->lambda, W->k); +} + +const struct dist_ops weibull_ops = { + .name = "Weibull", + .sample = weibull_sample, + .cdf = weibull_cdf, + .sf = weibull_sf, + .icdf = weibull_icdf, + .isf = weibull_isf, +}; + +/** Functions for generalized Pareto distributions */ + +static double +genpareto_sample(const struct dist *dist) +{ + const struct genpareto *GP = dist_to_const_genpareto(dist); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); + double p0 = random_uniform_01(); + + return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_cdf(const struct dist *dist, double x) +{ + const struct genpareto *GP = dist_to_const_genpareto(dist); + return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_sf(const struct dist *dist, double x) +{ + const struct genpareto *GP = dist_to_const_genpareto(dist); + return sf_genpareto(x, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_icdf(const struct dist *dist, double p) +{ + const struct genpareto *GP = dist_to_const_genpareto(dist); + return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_isf(const struct dist *dist, double p) +{ + const struct genpareto *GP = dist_to_const_genpareto(dist); + return isf_genpareto(p, GP->mu, GP->sigma, GP->xi); +} + +const struct dist_ops genpareto_ops = { + .name = "generalized Pareto", + .sample = genpareto_sample, + .cdf = genpareto_cdf, + .sf = genpareto_sf, + .icdf = genpareto_icdf, + .isf = genpareto_isf, +}; + +/** Functions for geometric distribution on number of trials before success */ + +static double +geometric_sample(const struct dist *dist) +{ + const struct geometric *G = dist_to_const_geometric(dist); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); + double p0 = random_uniform_01(); + + return sample_geometric(s, p0, G->p); +} + +static double +geometric_cdf(const struct dist *dist, double x) +{ + const struct geometric *G = dist_to_const_geometric(dist); + + if (x < 1) + return 0; + /* 1 - (1 - p)^floor(x) = 1 - e^{floor(x) log(1 - p)} */ + return -expm1(floor(x)*log1p(-G->p)); +} + +static double +geometric_sf(const struct dist *dist, double x) +{ + const struct geometric *G = dist_to_const_geometric(dist); + + if (x < 1) + return 0; + /* (1 - p)^floor(x) = e^{ceil(x) log(1 - p)} */ + return exp(floor(x)*log1p(-G->p)); +} + +static double +geometric_icdf(const struct dist *dist, double p) +{ + const struct geometric *G = dist_to_const_geometric(dist); + + return log1p(-p)/log1p(-G->p); +} + +static double +geometric_isf(const struct dist *dist, double p) +{ + const struct geometric *G = dist_to_const_geometric(dist); + + return log(p)/log1p(-G->p); +} + +const struct dist_ops geometric_ops = { + .name = "geometric (1-based)", + .sample = geometric_sample, + .cdf = geometric_cdf, + .sf = geometric_sf, + .icdf = geometric_icdf, + .isf = geometric_isf, +}; diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h new file mode 100644 index 0000000000..7254dc8623 --- /dev/null +++ b/src/lib/math/prob_distr.h @@ -0,0 +1,253 @@ + +/** + * \file prob_distr.h + * + * \brief Header for prob_distr.c + **/ + +#ifndef TOR_PROB_DISTR_H +#define TOR_PROB_DISTR_H + +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" +#include "lib/testsupport/testsupport.h" + +/** + * Container for distribution parameters for sampling, CDF, &c. + */ +struct dist { + const struct dist_ops *ops; +}; + +/** + * Untyped initializer element for struct dist using the specified + * struct dist_ops pointer. Don't actually use this directly -- use + * the type-specific macro built out of DIST_BASE_TYPED below -- but if + * you did use this directly, it would be something like: + * + * struct weibull mydist = { + * DIST_BASE(&weibull_ops), + * .lambda = ..., + * .k = ..., + * }; + * + * Note there is NO COMPILER FEEDBACK if you accidentally do something + * like + * + * struct geometric mydist = { + * DIST_BASE(&weibull_ops), + * ... + * }; + */ +#define DIST_BASE(OPS) { .ops = (OPS) } + +/** A compile-time type-checking macro for use with DIST_BASE_TYPED. + * + * This macro works by checking that &OBJ is a pointer type that is the same + * type (except for qualifiers) as (const TYPE *)&OBJ. It's a C constraint + * violation (which requires a diagnostic) if two pointers are different types + * and are subtracted. The sizeof() forces compile-time evaluation, and the + * multiplication by zero is to discard the result of the sizeof() from the + * expression. + * + * We define this conditionally to suppress false positives from + * Coverity, which gets confused by the sizeof business. + */ +#ifdef __COVERITY__ +#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) 0 +#else +#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \ + (0*sizeof(&(OBJ) - (const TYPE *)&(OBJ))) +#endif /* defined(__COVERITY__) */ + +/** +* Typed initializer element for struct dist using the specified struct +* dist_ops pointer. Don't actually use this directly -- use a +* type-specific macro built out of it -- but if you did use this +* directly, it would be something like: +* +* struct weibull mydist = { +* DIST_BASE_TYPED(&weibull_ops, mydist, struct weibull), +* .lambda = ..., +* .k = ..., +* }; +* +* If you want to define a distribution type, define a canonical set of +* operations and define a type-specific initializer element like so: +* +* struct foo { +* struct dist base; +* int omega; +* double tau; +* double phi; +* }; +* +* struct dist_ops foo_ops = ...; +* +* #define FOO(OBJ) DIST_BASE_TYPED(&foo_ops, OBJ, struct foo) +* +* Then users can do: +* +* struct foo mydist = { +* FOO(mydist), +* .omega = ..., +* .tau = ..., +* .phi = ..., +* }; +* +* If you accidentally write +* +* struct bar mydist = { +* FOO(mydist), +* ... +* }; +* +* then the compiler will report a type mismatch in the sizeof +* expression, which otherwise evaporates at runtime. +*/ +#define DIST_BASE_TYPED(OPS, OBJ, TYPE) \ + DIST_BASE((OPS) + TYPE_CHECK_OBJ(OPS,OBJ,TYPE)) + +/** + * Generic operations on distributions. These simply defer to the + * corresponding dist_ops function. In the parlance of C++, these call + * virtual member functions. + */ +const char *dist_name(const struct dist *); +double dist_sample(const struct dist *); +double dist_cdf(const struct dist *, double x); +double dist_sf(const struct dist *, double x); +double dist_icdf(const struct dist *, double p); +double dist_isf(const struct dist *, double p); + +/** + * Set of operations on a potentially parametric family of + * distributions. In the parlance of C++, this would be called a + * `vtable' and the members are virtual member functions. + */ +struct dist_ops { + const char *name; + double (*sample)(const struct dist *); + double (*cdf)(const struct dist *, double x); + double (*sf)(const struct dist *, double x); + double (*icdf)(const struct dist *, double p); + double (*isf)(const struct dist *, double p); +}; + +/* Geometric distribution on positive number of trials before first success */ + +struct geometric { + struct dist base; + double p; /* success probability */ +}; + +extern const struct dist_ops geometric_ops; + +#define GEOMETRIC(OBJ) \ + DIST_BASE_TYPED(&geometric_ops, OBJ, struct geometric) + +/* Pareto distribution */ + +struct genpareto { + struct dist base; + double mu; + double sigma; + double xi; +}; + +extern const struct dist_ops genpareto_ops; + +#define GENPARETO(OBJ) \ + DIST_BASE_TYPED(&genpareto_ops, OBJ, struct genpareto) + +/* Weibull distribution */ + +struct weibull { + struct dist base; + double lambda; + double k; +}; + +extern const struct dist_ops weibull_ops; + +#define WEIBULL(OBJ) \ + DIST_BASE_TYPED(&weibull_ops, OBJ, struct weibull) + +/* Log-logistic distribution */ + +struct log_logistic { + struct dist base; + double alpha; + double beta; +}; + +extern const struct dist_ops log_logistic_ops; + +#define LOG_LOGISTIC(OBJ) \ + DIST_BASE_TYPED(&log_logistic_ops, OBJ, struct log_logistic) + +/* Logistic distribution */ + +struct logistic { + struct dist base; + double mu; + double sigma; +}; + +extern const struct dist_ops logistic_ops; + +#define LOGISTIC(OBJ) \ + DIST_BASE_TYPED(&logistic_ops, OBJ, struct logistic) + +/* Uniform distribution */ + +struct uniform { + struct dist base; + double a; + double b; +}; + +extern const struct dist_ops uniform_ops; + +#define UNIFORM(OBJ) \ + DIST_BASE_TYPED(&uniform_ops, OBJ, struct uniform) + +/** Only by unittests */ + +#ifdef PROB_DISTR_PRIVATE + +STATIC double logithalf(double p0); +STATIC double logit(double p); + +STATIC double random_uniform_01(void); + +STATIC double logistic(double x); +STATIC double cdf_logistic(double x, double mu, double sigma); +STATIC double sf_logistic(double x, double mu, double sigma); +STATIC double icdf_logistic(double p, double mu, double sigma); +STATIC double isf_logistic(double p, double mu, double sigma); +STATIC double sample_logistic(uint32_t s, double t, double p0); + +STATIC double cdf_log_logistic(double x, double alpha, double beta); +STATIC double sf_log_logistic(double x, double alpha, double beta); +STATIC double icdf_log_logistic(double p, double alpha, double beta); +STATIC double isf_log_logistic(double p, double alpha, double beta); +STATIC double sample_log_logistic(uint32_t s, double p0); + +STATIC double cdf_weibull(double x, double lambda, double k); +STATIC double sf_weibull(double x, double lambda, double k); +STATIC double icdf_weibull(double p, double lambda, double k); +STATIC double isf_weibull(double p, double lambda, double k); +STATIC double sample_weibull(uint32_t s, double p0, double lambda, double k); + +STATIC double sample_uniform_interval(double p0, double a, double b); + +STATIC double cdf_genpareto(double x, double mu, double sigma, double xi); +STATIC double sf_genpareto(double x, double mu, double sigma, double xi); +STATIC double icdf_genpareto(double p, double mu, double sigma, double xi); +STATIC double isf_genpareto(double p, double mu, double sigma, double xi); +STATIC double sample_genpareto(uint32_t s, double p0, double xi); + +#endif /* defined(PROB_DISTR_PRIVATE) */ + +#endif /* !defined(TOR_PROB_DISTR_H) */ diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include index 814652a93c..a1edaf2231 100644 --- a/src/lib/memarea/.may_include +++ b/src/lib/memarea/.may_include @@ -1,7 +1,7 @@ orconfig.h lib/arch/*.h lib/cc/*.h -lib/container/*.h lib/log/*.h lib/malloc/*.h lib/memarea/*.h +lib/smartlist_core/*.h
\ No newline at end of file diff --git a/src/lib/memarea/include.am b/src/lib/memarea/include.am index 94343dcead..83fb99ec73 100644 --- a/src/lib/memarea/include.am +++ b/src/lib/memarea/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-memarea-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_memarea_a_SOURCES = \ src/lib/memarea/memarea.c @@ -13,5 +14,6 @@ src_lib_libtor_memarea_testing_a_SOURCES = \ src_lib_libtor_memarea_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_memarea_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/memarea/memarea.h diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index 486673116c..f3bb79a1e2 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -16,7 +16,8 @@ #include "lib/arch/bytes.h" #include "lib/cc/torint.h" -#include "lib/container/smartlist.h" +#include "lib/smartlist_core/smartlist_core.h" +#include "lib/smartlist_core/smartlist_foreach.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" @@ -67,7 +68,7 @@ uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]); \ tor_assert(sent_val == SENTINEL_VAL); \ STMT_END -#else /* !(defined(USE_SENTINELS)) */ +#else /* !defined(USE_SENTINELS) */ #define SENTINEL_LEN 0 #define SET_SENTINEL(chunk) STMT_NIL #define CHECK_SENTINEL(chunk) STMT_NIL @@ -314,7 +315,7 @@ memarea_assert_ok(memarea_t *area) } } -#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */ +#else /* defined(DISABLE_MEMORY_SENTINELS) */ struct memarea_t { smartlist_t *pieces; diff --git a/src/lib/meminfo/include.am b/src/lib/meminfo/include.am index d1fdde6313..12c1bff72d 100644 --- a/src/lib/meminfo/include.am +++ b/src/lib/meminfo/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-meminfo-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_meminfo_a_SOURCES = \ src/lib/meminfo/meminfo.c @@ -13,5 +14,6 @@ src_lib_libtor_meminfo_testing_a_SOURCES = \ src_lib_libtor_meminfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_meminfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/meminfo/meminfo.h diff --git a/src/lib/meminfo/meminfo.c b/src/lib/meminfo/meminfo.c index f4fa45167e..bff71c2f05 100644 --- a/src/lib/meminfo/meminfo.c +++ b/src/lib/meminfo/meminfo.c @@ -54,7 +54,7 @@ tor_log_mallinfo(int severity) mi.arena, mi.ordblks, mi.smblks, mi.hblks, mi.hblkhd, mi.usmblks, mi.fsmblks, mi.uordblks, mi.fordblks, mi.keepcost); -#else /* !(defined(HAVE_MALLINFO)) */ +#else /* !defined(HAVE_MALLINFO) */ (void)severity; #endif /* defined(HAVE_MALLINFO) */ } diff --git a/src/lib/meminfo/meminfo.h b/src/lib/meminfo/meminfo.h index 2d64e1ab06..9580640f4d 100644 --- a/src/lib/meminfo/meminfo.h +++ b/src/lib/meminfo/meminfo.h @@ -18,4 +18,4 @@ void tor_log_mallinfo(int severity); MOCK_DECL(int, get_total_system_memory, (size_t *mem_out)); -#endif +#endif /* !defined(TOR_MEMINFO_H) */ diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index 13b209bbed..e4368f799b 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -1,8 +1,9 @@ orconfig.h -siphash.h -ht.h +ext/siphash.h +ext/ht.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h @@ -11,5 +12,6 @@ lib/lock/*.h lib/log/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/malloc/*.h
\ No newline at end of file diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 69004ddb0e..41017935e1 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -40,6 +40,7 @@ #include "lib/net/address.h" #include "lib/net/socket.h" +#include "lib/cc/ctassert.h" #include "lib/container/smartlist.h" #include "lib/ctime/di_ops.h" #include "lib/log/log.h" @@ -52,7 +53,7 @@ #include "lib/string/printf.h" #include "lib/string/util_string.h" -#include "siphash.h" +#include "ext/siphash.h" #ifdef HAVE_SYS_TIME_H #include <sys/time.h> @@ -98,6 +99,7 @@ #if AF_UNSPEC != 0 #error We rely on AF_UNSPEC being 0. Let us know about your platform, please! #endif +CTASSERT(AF_UNSPEC == 0); /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a * sockaddr object in *<b>sa_out</b> of object size <b>len</b>. If not enough @@ -337,7 +339,7 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate) break; case AF_INET6: /* Shortest addr [ :: ] + \0 */ - if (len < (3 + (decorate ? 2u : 0u))) + if (len < (3u + (decorate ? 2 : 0))) return NULL; if (decorate) @@ -371,7 +373,8 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate) * * If <b>accept_regular</b> is set and the address is in neither recognized * reverse lookup hostname format, try parsing the address as a regular - * IPv4 or IPv6 address too. + * IPv4 or IPv6 address too. This mode will accept IPv6 addresses with or + * without square brackets. */ int tor_addr_parse_PTR_name(tor_addr_t *result, const char *address, @@ -1186,37 +1189,75 @@ fmt_addr32(uint32_t addr) } /** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>. The string - * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by - * square brackets. + * may be an IPv4 address, or an IPv6 address surrounded by square brackets. * - * Return an address family on success, or -1 if an invalid address string is - * provided. */ -int -tor_addr_parse(tor_addr_t *addr, const char *src) + * If <b>allow_ipv6_without_brackets</b> is true, also allow IPv6 addresses + * without brackets. + * + * Always rejects IPv4 addresses with brackets. + * + * Returns an address family on success, or -1 if an invalid address string is + * provided. */ +static int +tor_addr_parse_impl(tor_addr_t *addr, const char *src, + bool allow_ipv6_without_brackets) { /* Holds substring of IPv6 address after removing square brackets */ char *tmp = NULL; - int result; + int result = -1; struct in_addr in_tmp; struct in6_addr in6_tmp; + int brackets_detected = 0; + tor_assert(addr && src); - if (src[0] == '[' && src[1]) + + size_t len = strlen(src); + + if (len && src[0] == '[' && src[len - 1] == ']') { + brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); + } - if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { - result = AF_INET6; - tor_addr_from_in6(addr, &in6_tmp); - } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { - result = AF_INET; - tor_addr_from_in(addr, &in_tmp); - } else { - result = -1; + /* Try to parse an IPv6 address if it has brackets, or if IPv6 addresses + * without brackets are allowed */ + if (brackets_detected || allow_ipv6_without_brackets) { + if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { + result = AF_INET6; + tor_addr_from_in6(addr, &in6_tmp); + } + } + + /* Try to parse an IPv4 address without brackets */ + if (!brackets_detected) { + if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { + result = AF_INET; + tor_addr_from_in(addr, &in_tmp); + } + } + + /* Clear the address on error, to avoid returning uninitialised or partly + * parsed data. + */ + if (result == -1) { + memset(addr, 0, sizeof(tor_addr_t)); } tor_free(tmp); return result; } +/** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>. The string + * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by + * square brackets. + * + * Returns an address family on success, or -1 if an invalid address string is + * provided. */ +int +tor_addr_parse(tor_addr_t *addr, const char *src) +{ + return tor_addr_parse_impl(addr, src, 1); +} + #ifdef HAVE_IFADDRS_TO_SMARTLIST /* * Convert a linked list consisting of <b>ifaddrs</b> structures @@ -1709,6 +1750,11 @@ get_interface_address6_list,(int severity, * form "ip" or "ip:0". Otherwise, accept those forms, and set * *<b>port_out</b> to <b>default_port</b>. * + * This function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * * Return 0 on success, -1 on failure. */ int tor_addr_port_parse(int severity, const char *addrport, @@ -1718,6 +1764,7 @@ tor_addr_port_parse(int severity, const char *addrport, int retval = -1; int r; char *addr_tmp = NULL; + bool has_port; tor_assert(addrport); tor_assert(address_out); @@ -1727,28 +1774,47 @@ tor_addr_port_parse(int severity, const char *addrport, if (r < 0) goto done; - if (!*port_out) { + has_port = !! *port_out; + /* If there's no port, use the default port, or fail if there is no default + */ + if (!has_port) { if (default_port >= 0) *port_out = default_port; else goto done; } - /* make sure that address_out is an IP address */ - if (tor_addr_parse(address_out, addr_tmp) < 0) + /* Make sure that address_out is an IP address. + * If there is no port in addrport, allow IPv6 addresses without brackets. */ + if (tor_addr_parse_impl(address_out, addr_tmp, !has_port) < 0) goto done; retval = 0; done: + /* Clear the address and port on error, to avoid returning uninitialised or + * partly parsed data. + */ + if (retval == -1) { + memset(address_out, 0, sizeof(tor_addr_t)); + *port_out = 0; + } tor_free(addr_tmp); return retval; } /** Given an address of the form "host[:port]", try to divide it into its host - * and port portions, setting *<b>address_out</b> to a newly allocated string - * holding the address portion and *<b>port_out</b> to the port (or 0 if no - * port is given). Return 0 on success, -1 on failure. */ + * and port portions. + * + * Like tor_addr_port_parse(), this function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * + * Sets *<b>address_out</b> to a newly allocated string holding the address + * portion, and *<b>port_out</b> to the port (or 0 if no port is given). + * + * Return 0 on success, -1 on failure. */ int tor_addr_port_split(int severity, const char *addrport, char **address_out, uint16_t *port_out) @@ -1757,8 +1823,11 @@ tor_addr_port_split(int severity, const char *addrport, tor_assert(addrport); tor_assert(address_out); tor_assert(port_out); + /* We need to check for IPv6 manually because the logic below doesn't - * do a good job on IPv6 addresses that lack a port. */ + * do a good job on IPv6 addresses that lack a port. + * If an IPv6 address without square brackets is ambiguous, it gets parsed + * here as an address, rather than address:port. */ if (tor_addr_parse(&a_tmp, addrport) == AF_INET6) { *port_out = 0; *address_out = tor_strdup(addrport); @@ -1798,8 +1867,7 @@ tor_addr_port_split(int severity, const char *addrport, tor_free(address_); } - if (port_out) - *port_out = ok ? ((uint16_t) port_) : 0; + *port_out = ok ? ((uint16_t) port_) : 0; return ok ? 0 : -1; } @@ -2018,8 +2086,12 @@ string_is_valid_nonrfc_hostname(const char *string) smartlist_split_string(components,string,".",0,0); - if (BUG(smartlist_len(components) == 0)) - return 0; // LCOV_EXCL_LINE should be impossible given the earlier checks. + if (BUG(smartlist_len(components) == 0)) { + // LCOV_EXCL_START should be impossible given the earlier checks. + smartlist_free(components); + return 0; + // LCOV_EXCL_STOP + } /* Allow a single terminating '.' used rarely to indicate domains * are FQDNs rather than relative. */ diff --git a/src/lib/net/alertsock.h b/src/lib/net/alertsock.h index c45f42be81..4d0d0dd57c 100644 --- a/src/lib/net/alertsock.h +++ b/src/lib/net/alertsock.h @@ -42,4 +42,4 @@ typedef struct alert_sockets_t { int alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags); void alert_sockets_close(alert_sockets_t *socks); -#endif +#endif /* !defined(TOR_ALERTSOCK_H) */ diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 3eb0a033d5..cfe1a7dc26 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/net/buffers_net.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/net/nettypes.h" @@ -22,6 +22,10 @@ #include <stdlib.h> +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + #ifdef PARANOIA /** Helper: If PARANOIA is defined, assert that the buffer in local variable * <b>buf</b> is well-formed. */ @@ -30,27 +34,36 @@ #define check() STMT_NIL #endif /* defined(PARANOIA) */ -/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into +/** Read up to <b>at_most</b> bytes from the file descriptor <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set - * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, - * and the number of bytes read otherwise. */ + * *<b>reached_eof</b> to 1. Uses <b>tor_socket_recv()</b> iff <b>is_socket</b> + * is true, otherwise it uses <b>read()</b>. Return -1 on error (and sets + * *<b>error</b> to errno), 0 on eof or blocking, and the number of bytes read + * otherwise. */ static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, - int *reached_eof, int *socket_error) + int *reached_eof, int *error, bool is_socket) { ssize_t read_result; if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) at_most = CHUNK_REMAINING_CAPACITY(chunk); - read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (is_socket) + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + else + read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most); if (read_result < 0) { - int e = tor_socket_errno(fd); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) - log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); + log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?", + is_socket ? "recv" : "read"); #endif - *socket_error = e; + if (error) + *error = e; return -1; } return 0; /* would block. */ @@ -68,16 +81,17 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, } } -/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most - * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 - * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on - * error; else return the number of bytes read. +/** Read from file descriptor <b>fd</b>, writing onto end of <b>buf</b>. Read + * at most <b>at_most</b> bytes, growing the buffer as necessary. If recv() + * returns 0 (because of EOF), set *<b>reached_eof</b> to 1 and return 0. + * Return -1 on error; else return the number of bytes read. */ /* XXXX indicate "read blocked" somehow? */ -int -buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, - int *reached_eof, - int *socket_error) +static int +buf_read_from_fd(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error, + bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. @@ -87,7 +101,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, check(); tor_assert(reached_eof); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(buf->datalen >= INT_MAX)) return -1; @@ -108,7 +122,8 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + r = read_to_chunk(buf, chunk, fd, readlen, + reached_eof, socket_error, is_socket); check(); if (r < 0) return r; /* Error */ @@ -122,22 +137,27 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, } /** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk - * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. On success, deduct - * the bytes written from *<b>buf_flushlen</b>. Return the number of bytes - * written on success, 0 on blocking, -1 on failure. + * <b>chunk</b> of buffer <b>buf</b> onto file descriptor <b>fd</b>. On + * success, deduct the bytes written from *<b>buf_flushlen</b>. Return the + * number of bytes written on success, 0 on blocking, -1 on failure. */ static inline int -flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, - size_t *buf_flushlen) +flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen, bool is_socket) { ssize_t write_result; if (sz > chunk->datalen) sz = chunk->datalen; - write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (is_socket) + write_result = tor_socket_send(fd, chunk->data, sz, 0); + else + write_result = write(fd, chunk->data, sz); if (write_result < 0) { - int e = tor_socket_errno(s); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) @@ -155,15 +175,15 @@ flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, } } -/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most +/** Write data from <b>buf</b> to the file descriptor <b>fd</b>. Write at most * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by * the number of bytes actually written, and remove the written bytes * from the buffer. Return the number of bytes written on success, * -1 on failure. Return 0 if write() would block. */ -int -buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, - size_t *buf_flushlen) +static int +buf_flush_to_fd(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen, bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. @@ -171,7 +191,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, int r; size_t flushed = 0; tor_assert(buf_flushlen); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(*buf_flushlen > buf->datalen)) { *buf_flushlen = buf->datalen; } @@ -188,7 +208,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket); check(); if (r < 0) return r; @@ -200,3 +220,55 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, tor_assert(flushed < INT_MAX); return (int)flushed; } + +/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); +} + +/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); +} + +/** Write data from <b>buf</b> to the pipe <b>fd</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_pipe(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false); +} + +/** Read from pipe <b>fd</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If read() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_pipe(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false); +} diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h index 5f69bebedf..5058dd0a26 100644 --- a/src/lib/net/buffers_net.h +++ b/src/lib/net/buffers_net.h @@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most, int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz, size_t *buf_flushlen); -#endif /* !defined(TOR_BUFFERS_H) */ +int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error); + +int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen); + +#endif /* !defined(TOR_BUFFERS_NET_H) */ diff --git a/src/lib/net/gethostname.h b/src/lib/net/gethostname.h index 69b0528bc0..b3b77b0589 100644 --- a/src/lib/net/gethostname.h +++ b/src/lib/net/gethostname.h @@ -16,4 +16,4 @@ MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen)); -#endif +#endif /* !defined(TOR_GETHOSTNAME_H) */ diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c index 1a2406ce5f..d9ae7cd562 100644 --- a/src/lib/net/inaddr.c +++ b/src/lib/net/inaddr.c @@ -168,6 +168,13 @@ tor_inet_pton(int af, const char *src, void *dst) if (af == AF_INET) { return tor_inet_aton(src, dst); } else if (af == AF_INET6) { + ssize_t len = strlen(src); + + /* Reject if src has needless trailing ':'. */ + if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { + return 0; + } + struct in6_addr *out = dst; uint16_t words[8]; int gapPos = -1, i, setWords=0; @@ -207,7 +214,6 @@ tor_inet_pton(int af, const char *src, void *dst) return 0; if (TOR_ISXDIGIT(*src)) { char *next; - ssize_t len; long r = strtol(src, &next, 16); if (next == NULL || next == src) { /* The 'next == src' error case can happen on versions of openbsd diff --git a/src/lib/net/inaddr.h b/src/lib/net/inaddr.h index 36352b65ea..602573944c 100644 --- a/src/lib/net/inaddr.h +++ b/src/lib/net/inaddr.h @@ -24,4 +24,4 @@ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len); const char *tor_inet_ntop(int af, const void *src, char *dst, size_t len); int tor_inet_pton(int af, const char *src, void *dst); -#endif +#endif /* !defined(TOR_INADDR_H) */ diff --git a/src/lib/net/inaddr_st.h b/src/lib/net/inaddr_st.h index 806f2c096a..230f29a63a 100644 --- a/src/lib/net/inaddr_st.h +++ b/src/lib/net/inaddr_st.h @@ -104,4 +104,4 @@ struct sockaddr_in6 { }; #endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */ -#endif /* TOR_INADDR_ST_H */ +#endif /* !defined(TOR_INADDR_ST_H) */ diff --git a/src/lib/net/include.am b/src/lib/net/include.am index ff0967e786..485019f4b7 100644 --- a/src/lib/net/include.am +++ b/src/lib/net/include.am @@ -5,12 +5,14 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-net-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_net_a_SOURCES = \ src/lib/net/address.c \ src/lib/net/alertsock.c \ src/lib/net/buffers_net.c \ src/lib/net/gethostname.c \ src/lib/net/inaddr.c \ + src/lib/net/network_sys.c \ src/lib/net/resolve.c \ src/lib/net/socket.c \ src/lib/net/socketpair.c @@ -20,6 +22,7 @@ src_lib_libtor_net_testing_a_SOURCES = \ src_lib_libtor_net_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_net_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/net/address.h \ src/lib/net/alertsock.h \ @@ -28,6 +31,7 @@ noinst_HEADERS += \ src/lib/net/inaddr.h \ src/lib/net/inaddr_st.h \ src/lib/net/nettypes.h \ + src/lib/net/network_sys.h \ src/lib/net/resolve.h \ src/lib/net/socket.h \ src/lib/net/socketpair.h \ diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h index 6209bbe18a..60039bac09 100644 --- a/src/lib/net/nettypes.h +++ b/src/lib/net/nettypes.h @@ -31,7 +31,7 @@ typedef int socklen_t; #define TOR_SOCKET_T_FORMAT "%"PRIuPTR #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET) #define TOR_INVALID_SOCKET INVALID_SOCKET -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /** Type used for a network socket. */ #define tor_socket_t int #define TOR_SOCKET_T_FORMAT "%d" @@ -41,4 +41,4 @@ typedef int socklen_t; #define TOR_INVALID_SOCKET (-1) #endif /* defined(_WIN32) */ -#endif +#endif /* !defined(TOR_NET_TYPES_H) */ diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c new file mode 100644 index 0000000000..e0a2625d73 --- /dev/null +++ b/src/lib/net/network_sys.c @@ -0,0 +1,46 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file network_sys.c + * \brief Subsystem object for networking setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/net/network_sys.h" +#include "lib/net/resolve.h" +#include "lib/net/socket.h" + +#ifdef _WIN32 +#include <winsock2.h> +#include <windows.h> +#endif + +static int +subsys_network_initialize(void) +{ + if (network_init() < 0) + return -1; + + return 0; +} + +static void +subsys_network_shutdown(void) +{ +#ifdef _WIN32 + WSACleanup(); +#endif + tor_free_getaddrinfo_cache(); +} + +const subsys_fns_t sys_network = { + .name = "network", + /* Network depends on logging, and a lot of other modules depend on network. + */ + .level = -80, + .supported = true, + .initialize = subsys_network_initialize, + .shutdown = subsys_network_shutdown, +}; diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h new file mode 100644 index 0000000000..43e62592ca --- /dev/null +++ b/src/lib/net/network_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_network.h + * \brief Declare subsystem object for the network module. + **/ + +#ifndef TOR_NETWORK_SYS_H +#define TOR_NETWORK_SYS_H + +extern const struct subsys_fns_t sys_network; + +#endif /* !defined(TOR_NETWORK_SYS_H) */ diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 8cee29df37..442bc4a6b3 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -8,6 +8,7 @@ * \brief Use the libc DNS resolver to convert hostnames into addresses. **/ +#define RESOLVE_PRIVATE #include "lib/net/resolve.h" #include "lib/net/address.h" @@ -16,8 +17,8 @@ #include "lib/string/parse_int.h" #include "lib/string/util_string.h" -#include "siphash.h" -#include "ht.h" +#include "ext/siphash.h" +#include "ext/ht.h" #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> @@ -35,6 +36,8 @@ * *<b>addr</b> to the proper IP address, in host byte order. Returns 0 * on success, -1 on failure; 1 on transient failure. * + * This function only accepts IPv4 addresses. + * * (This function exists because standard windows gethostbyname * doesn't treat raw IP addresses properly.) */ @@ -45,6 +48,11 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) tor_addr_t myaddr; int ret; + if (BUG(!addr)) + return -1; + + *addr = 0; + if ((ret = tor_addr_lookup(name, AF_INET, &myaddr))) return ret; @@ -56,183 +64,257 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) return -1; } -/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set - * *<b>addr</b> to the proper IP address and family. The <b>family</b> - * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a - * <i>preferred</i> family, though another one may be returned if only one - * family is implemented for this address. +#ifdef HAVE_GETADDRINFO + +/* Host lookup helper for tor_addr_lookup(), when getaddrinfo() is + * available on this system. * - * Return 0 on success, -1 on failure; 1 on transient failure. + * See tor_addr_lookup() for details. */ -MOCK_IMPL(int, -tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) +MOCK_IMPL(STATIC int, +tor_addr_lookup_host_impl,(const char *name, + uint16_t family, + tor_addr_t *addr)) { - /* Perhaps eventually this should be replaced by a tor_getaddrinfo or - * something. - */ - struct in_addr iaddr; - struct in6_addr iaddr6; - tor_assert(name); - tor_assert(addr); - tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC); - if (!*name) { - /* Empty address is an error. */ - return -1; - } else if (tor_inet_pton(AF_INET, name, &iaddr)) { - /* It's an IPv4 IP. */ - if (family == AF_INET6) - return -1; - tor_addr_from_in(addr, &iaddr); - return 0; - } else if (tor_inet_pton(AF_INET6, name, &iaddr6)) { - if (family == AF_INET) - return -1; - tor_addr_from_in6(addr, &iaddr6); - return 0; - } else { -#ifdef HAVE_GETADDRINFO - int err; - struct addrinfo *res=NULL, *res_p; - struct addrinfo *best=NULL; - struct addrinfo hints; - int result = -1; - memset(&hints, 0, sizeof(hints)); - hints.ai_family = family; - hints.ai_socktype = SOCK_STREAM; - err = tor_getaddrinfo(name, NULL, &hints, &res); - /* The check for 'res' here shouldn't be necessary, but it makes static - * analysis tools happy. */ - if (!err && res) { - best = NULL; - for (res_p = res; res_p; res_p = res_p->ai_next) { - if (family == AF_UNSPEC) { - if (res_p->ai_family == AF_INET) { - best = res_p; - break; - } else if (res_p->ai_family == AF_INET6 && !best) { - best = res_p; - } - } else if (family == res_p->ai_family) { + int err; + struct addrinfo *res=NULL, *res_p; + struct addrinfo *best=NULL; + struct addrinfo hints; + int result = -1; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = family; + hints.ai_socktype = SOCK_STREAM; + err = tor_getaddrinfo(name, NULL, &hints, &res); + /* The check for 'res' here shouldn't be necessary, but it makes static + * analysis tools happy. */ + if (!err && res) { + best = NULL; + for (res_p = res; res_p; res_p = res_p->ai_next) { + if (family == AF_UNSPEC) { + if (res_p->ai_family == AF_INET) { best = res_p; break; + } else if (res_p->ai_family == AF_INET6 && !best) { + best = res_p; } + } else if (family == res_p->ai_family) { + best = res_p; + break; } - if (!best) - best = res; - if (best->ai_family == AF_INET) { - tor_addr_from_in(addr, - &((struct sockaddr_in*)best->ai_addr)->sin_addr); - result = 0; - } else if (best->ai_family == AF_INET6) { - tor_addr_from_in6(addr, - &((struct sockaddr_in6*)best->ai_addr)->sin6_addr); - result = 0; - } - tor_freeaddrinfo(res); - return result; } - return (err == EAI_AGAIN) ? 1 : -1; -#else /* !(defined(HAVE_GETADDRINFO)) */ - struct hostent *ent; - int err; + if (!best) + best = res; + if (best->ai_family == AF_INET) { + tor_addr_from_in(addr, + &((struct sockaddr_in*)best->ai_addr)->sin_addr); + result = 0; + } else if (best->ai_family == AF_INET6) { + tor_addr_from_in6(addr, + &((struct sockaddr_in6*)best->ai_addr)->sin6_addr); + result = 0; + } + tor_freeaddrinfo(res); + return result; + } + return (err == EAI_AGAIN) ? 1 : -1; +} + +#else /* !defined(HAVE_GETADDRINFO) */ + +/* Host lookup helper for tor_addr_lookup(), which calls gethostbyname(). + * Used when getaddrinfo() is not available on this system. + * + * See tor_addr_lookup() for details. + */ +MOCK_IMPL(STATIC int, +tor_addr_lookup_host_impl,(const char *name, + uint16_t family, + tor_addr_t *addr)) +{ + (void) family; + struct hostent *ent; + int err; #ifdef HAVE_GETHOSTBYNAME_R_6_ARG - char buf[2048]; - struct hostent hostent; - int r; - r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err); + char buf[2048]; + struct hostent hostent; + int r; + r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err); #elif defined(HAVE_GETHOSTBYNAME_R_5_ARG) - char buf[2048]; - struct hostent hostent; - ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err); + char buf[2048]; + struct hostent hostent; + ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err); #elif defined(HAVE_GETHOSTBYNAME_R_3_ARG) - struct hostent_data data; - struct hostent hent; - memset(&data, 0, sizeof(data)); - err = gethostbyname_r(name, &hent, &data); - ent = err ? NULL : &hent; + struct hostent_data data; + struct hostent hent; + memset(&data, 0, sizeof(data)); + err = gethostbyname_r(name, &hent, &data); + ent = err ? NULL : &hent; #else - ent = gethostbyname(name); + ent = gethostbyname(name); #ifdef _WIN32 - err = WSAGetLastError(); + err = WSAGetLastError(); #else - err = h_errno; -#endif + err = h_errno; +#endif /* defined(_WIN32) */ #endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */ - if (ent) { - if (ent->h_addrtype == AF_INET) { - tor_addr_from_in(addr, (struct in_addr*) ent->h_addr); - } else if (ent->h_addrtype == AF_INET6) { - tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr); - } else { - tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type - } - return 0; + if (ent) { + if (ent->h_addrtype == AF_INET) { + tor_addr_from_in(addr, (struct in_addr*) ent->h_addr); + } else if (ent->h_addrtype == AF_INET6) { + tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr); + } else { + tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type } + return 0; + } #ifdef _WIN32 - return (err == WSATRY_AGAIN) ? 1 : -1; + return (err == WSATRY_AGAIN) ? 1 : -1; #else - return (err == TRY_AGAIN) ? 1 : -1; + return (err == TRY_AGAIN) ? 1 : -1; #endif +} #endif /* defined(HAVE_GETADDRINFO) */ + +/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set + * *<b>addr</b> to the proper IP address and family. The <b>family</b> + * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a + * <i>preferred</i> family, though another one may be returned if only one + * family is implemented for this address. + * + * Like tor_addr_parse(), this function accepts IPv6 addresses with or without + * square brackets. + * + * Return 0 on success, -1 on failure; 1 on transient failure. + */ +MOCK_IMPL(int, +tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) +{ + /* Perhaps eventually this should be replaced by a tor_getaddrinfo or + * something. + */ + int parsed_family = 0; + int result = -1; + + tor_assert(name); + tor_assert(addr); + tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC); + + if (!*name) { + /* Empty address is an error. */ + goto permfail; + } + + /* Is it an IP address? */ + parsed_family = tor_addr_parse(addr, name); + + if (parsed_family >= 0) { + /* If the IP address family matches, or was unspecified */ + if (parsed_family == family || family == AF_UNSPEC) { + goto success; + } else { + goto permfail; + } + } else { + /* Clear the address after a failed tor_addr_parse(). */ + memset(addr, 0, sizeof(tor_addr_t)); + result = tor_addr_lookup_host_impl(name, family, addr); + goto done; } + + /* If we weren't successful, and haven't already set the result, + * assume it's a permanent failure */ + permfail: + result = -1; + goto done; + success: + result = 0; + + /* We have set the result, now it's time to clean up */ + done: + if (result) { + /* Clear the address on error */ + memset(addr, 0, sizeof(tor_addr_t)); + } + return result; } /** Parse an address or address-port combination from <b>s</b>, resolve the * address as needed, and put the result in <b>addr_out</b> and (optionally) - * <b>port_out</b>. Return 0 on success, negative on failure. */ + * <b>port_out</b>. + * + * Like tor_addr_port_parse(), this function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * + * Return 0 on success, negative on failure. */ int tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out) { - const char *port; tor_addr_t addr; - uint16_t portval; + uint16_t portval = 0; char *tmp = NULL; + int rv = 0; + int result; tor_assert(s); tor_assert(addr_out); s = eat_whitespace(s); - if (*s == '[') { - port = strstr(s, "]"); - if (!port) - goto err; - tmp = tor_strndup(s+1, port-(s+1)); - port = port+1; - if (*port == ':') - port++; - else - port = NULL; - } else { - port = strchr(s, ':'); - if (port) - tmp = tor_strndup(s, port-s); - else - tmp = tor_strdup(s); - if (port) - ++port; + /* Try parsing s as an address:port first, so we don't have to duplicate + * the logic that rejects IPv6:Port with no square brackets. */ + rv = tor_addr_port_parse(LOG_WARN, s, &addr, &portval, 0); + /* That was easy, no DNS required. */ + if (rv == 0) + goto success; + + /* Now let's check for malformed IPv6 addresses and ports: + * tor_addr_port_parse() requires squared brackes if there is a port, + * and we want tor_addr_port_lookup() to have the same requirement. + * But we strip the port using tor_addr_port_split(), so tor_addr_lookup() + * only sees the address, and will accept it without square brackets. */ + int family = tor_addr_parse(&addr, s); + /* If tor_addr_parse() succeeds where tor_addr_port_parse() failed, we need + * to reject this address as malformed. */ + if (family >= 0) { + /* Double-check it's an IPv6 address. If not, we have a parsing bug. + */ + tor_assertf_nonfatal(family == AF_INET6, + "Wrong family: %d (should be IPv6: %d) which " + "failed IP:port parsing, but passed IP parsing. " + "input string: '%s'; parsed address: '%s'.", + family, AF_INET6, s, fmt_addr(&addr)); + goto err; } - if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0) + /* Now we have a hostname. Let's split off the port, if any. */ + rv = tor_addr_port_split(LOG_WARN, s, &tmp, &portval); + if (rv < 0) goto err; - tor_free(tmp); - if (port) { - portval = (int) tor_parse_long(port, 10, 1, 65535, NULL, NULL); - if (!portval) - goto err; - } else { - portval = 0; - } + /* And feed the hostname to the lookup function. */ + if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0) + goto err; + success: if (port_out) *port_out = portval; tor_addr_copy(addr_out, &addr); + result = 0; + goto done; - return 0; err: + /* Clear the address and port on error */ + memset(addr_out, 0, sizeof(tor_addr_t)); + if (port_out) + *port_out = 0; + result = -1; + + /* We have set the result, now it's time to clean up */ + done: tor_free(tmp); - return -1; + return result; } #ifdef USE_SANDBOX_GETADDRINFO @@ -421,4 +503,13 @@ tor_make_getaddrinfo_cache_active(void) { sandbox_getaddrinfo_is_active = 1; } -#endif +#else /* !defined(USE_SANDBOX_GETADDRINFO) */ +void +sandbox_disable_getaddrinfo_cache(void) +{ +} +void +tor_make_getaddrinfo_cache_active(void) +{ +} +#endif /* defined(USE_SANDBOX_GETADDRINFO) */ diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index 47a283c81c..b979b2fb41 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -24,12 +24,18 @@ struct tor_addr_t; +/* + * Primary lookup functions. + */ MOCK_DECL(int, tor_lookup_hostname,(const char *name, uint32_t *addr)); MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family, struct tor_addr_t *addr_out)); int tor_addr_port_lookup(const char *s, struct tor_addr_t *addr_out, uint16_t *port_out); +/* + * Sandbox helpers + */ struct addrinfo; #ifdef USE_SANDBOX_GETADDRINFO /** Pre-calls getaddrinfo in order to pre-record result. */ @@ -42,8 +48,7 @@ int tor_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void tor_freeaddrinfo(struct addrinfo *addrinfo); void tor_free_getaddrinfo_cache(void); -void tor_make_getaddrinfo_cache_active(void); -#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ +#else /* !defined(USE_SANDBOX_GETADDRINFO) */ #define tor_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) #define tor_add_addrinfo(name) \ @@ -54,5 +59,15 @@ void tor_make_getaddrinfo_cache_active(void); #endif /* defined(USE_SANDBOX_GETADDRINFO) */ void sandbox_disable_getaddrinfo_cache(void); +void tor_make_getaddrinfo_cache_active(void); +/* + * Internal resolver wrapper; exposed for mocking. + */ +#ifdef RESOLVE_PRIVATE +MOCK_DECL(STATIC int, tor_addr_lookup_host_impl, (const char *name, + uint16_t family, + struct tor_addr_t *addr)); #endif + +#endif /* !defined(TOR_RESOLVE_H) */ diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index fba90b7506..e1b82251ed 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -31,6 +31,9 @@ #endif #include <stddef.h> #include <string.h> +#ifdef __FreeBSD__ +#include <sys/sysctl.h> +#endif /** Called before we make any calls to network-related functions. * (Some operating systems require their network libraries to be @@ -60,6 +63,32 @@ network_init(void) return 0; } +/** + * Warn the user if any system network parameters should be changed. + */ +void +check_network_configuration(bool server_mode) +{ +#ifdef __FreeBSD__ + if (server_mode) { + int random_id_state; + size_t state_size = sizeof(random_id_state); + + if (sysctlbyname("net.inet.ip.random_id", &random_id_state, + &state_size, NULL, 0)) { + log_warn(LD_CONFIG, + "Failed to figure out if IP ids are randomized."); + } else if (random_id_state == 0) { + log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " + "Please consider setting the net.inet.ip.random_id sysctl, " + "so your relay makes it harder to figure out how busy it is."); + } + } +#else /* !defined(__FreeBSD__) */ + (void) server_mode; +#endif /* defined(__FreeBSD__) */ +} + /* When set_max_file_sockets() is called, update this with the max file * descriptor value so we can use it to check the limit when opening a new * socket. Default value is what Debian sets as the default hard limit. */ @@ -177,7 +206,7 @@ mark_socket_closed(tor_socket_t s) bitarray_clear(open_sockets, s); } } -#else /* !(defined(DEBUG_SOCKET_COUNTING)) */ +#else /* !defined(DEBUG_SOCKET_COUNTING) */ #define mark_socket_open(s) ((void) (s)) #define mark_socket_closed(s) ((void) (s)) #endif /* defined(DEBUG_SOCKET_COUNTING) */ @@ -279,7 +308,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol, return TOR_INVALID_SOCKET; } } -#else /* !(defined(FD_CLOEXEC)) */ +#else /* !defined(FD_CLOEXEC) */ (void)cloexec; #endif /* defined(FD_CLOEXEC) */ @@ -389,7 +418,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, return TOR_INVALID_SOCKET; } } -#else /* !(defined(FD_CLOEXEC)) */ +#else /* !defined(FD_CLOEXEC) */ (void)cloexec; #endif /* defined(FD_CLOEXEC) */ @@ -429,7 +458,9 @@ get_n_open_sockets(void) * localhost is inaccessible (for example, if the networking * stack is down). And even if it succeeds, the socket pair will not * be able to read while localhost is down later (the socket pair may - * even close, depending on OS-specific timeouts). + * even close, depending on OS-specific timeouts). The socket pair + * should work on IPv4-only, IPv6-only, and dual-stack systems, as long + * as they have the standard localhost addresses. * * Returns 0 on success and -errno on failure; do not rely on the value * of errno or WSAGetLastError(). @@ -456,11 +487,11 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) r = socketpair(family, type, protocol, fd); if (r < 0) return -errno; -#else +#else /* !(defined(HAVE_SOCKETPAIR) && !defined(_WIN32)) */ r = tor_ersatz_socketpair(family, type, protocol, fd); if (r < 0) return -r; -#endif +#endif /* defined(HAVE_SOCKETPAIR) && !defined(_WIN32) */ #if defined(FD_CLOEXEC) if (SOCKET_OK(fd[0])) { diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 0909619510..53a9f1bb92 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock); int set_socket_nonblocking(tor_socket_t socket); int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); int network_init(void); +void check_network_configuration(bool server_mode); int get_max_sockets(void); void set_max_sockets(int); @@ -91,7 +92,7 @@ ssize_t read_all_from_socket(tor_socket_t fd, char *buf, size_t count); #define ERRNO_IS_EINTR(e) ((e) == WSAEINTR || 0) int tor_socket_errno(tor_socket_t sock); const char *tor_socket_strerror(int e); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ #define SOCK_ERRNO(e) e #if EAGAIN == EWOULDBLOCK /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */ @@ -115,4 +116,4 @@ const char *tor_socket_strerror(int e); #define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b #endif -#endif +#endif /* !defined(TOR_SOCKET_H) */ diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c index 10eb749735..f3a0c3770a 100644 --- a/src/lib/net/socketpair.c +++ b/src/lib/net/socketpair.c @@ -22,11 +22,11 @@ #include <windows.h> #define socket_errno() (WSAGetLastError()) #define SOCKET_EPROTONOSUPPORT WSAEPROTONOSUPPORT -#else +#else /* !defined(_WIN32) */ #define closesocket(x) close(x) #define socket_errno() (errno) #define SOCKET_EPROTONOSUPPORT EPROTONOSUPPORT -#endif +#endif /* defined(_WIN32) */ #ifdef NEED_ERSATZ_SOCKETPAIR @@ -105,7 +105,12 @@ sockaddr_eq(struct sockaddr *sa1, struct sockaddr *sa2) /** * Helper used to implement socketpair on systems that lack it, by * making a direct connection to localhost. - */ + * + * See tor_socketpair() for details. + * + * The direct connection defaults to IPv4, but falls back to IPv6 if + * IPv4 is not supported. + **/ int tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) { diff --git a/src/lib/net/socketpair.h b/src/lib/net/socketpair.h index 6be0803881..5820606973 100644 --- a/src/lib/net/socketpair.h +++ b/src/lib/net/socketpair.h @@ -16,4 +16,4 @@ int tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); #endif -#endif +#endif /* !defined(TOR_SOCKETPAIR_H) */ diff --git a/src/lib/net/socks5_status.h b/src/lib/net/socks5_status.h index e55242ce66..e55119e0b0 100644 --- a/src/lib/net/socks5_status.h +++ b/src/lib/net/socks5_status.h @@ -29,4 +29,4 @@ typedef enum { SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08, } socks5_reply_status_t; -#endif +#endif /* !defined(TOR_SOCKS5_STATUS_H) */ diff --git a/src/lib/osinfo/include.am b/src/lib/osinfo/include.am index 16c5812604..84bd7feb00 100644 --- a/src/lib/osinfo/include.am +++ b/src/lib/osinfo/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-osinfo-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_osinfo_a_SOURCES = \ src/lib/osinfo/uname.c @@ -13,5 +14,6 @@ src_lib_libtor_osinfo_testing_a_SOURCES = \ src_lib_libtor_osinfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_osinfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/osinfo/uname.h diff --git a/src/lib/osinfo/uname.c b/src/lib/osinfo/uname.c index 2b37ff136c..34860c407a 100644 --- a/src/lib/osinfo/uname.c +++ b/src/lib/osinfo/uname.c @@ -137,7 +137,7 @@ get_uname,(void)) if (!is_server && !is_client) { strlcat(uname_result, " [client or server]", sizeof(uname_result)); } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /* LCOV_EXCL_START -- can't provoke uname failure */ strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); /* LCOV_EXCL_STOP */ diff --git a/src/lib/osinfo/uname.h b/src/lib/osinfo/uname.h index fcce629074..443a30d358 100644 --- a/src/lib/osinfo/uname.h +++ b/src/lib/osinfo/uname.h @@ -15,4 +15,4 @@ MOCK_DECL(const char *, get_uname,(void)); -#endif +#endif /* !defined(HAVE_TOR_UNAME_H) */ diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index 05414d2a96..ce1b6ecf59 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -1,17 +1,20 @@ orconfig.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h lib/err/*.h -lib/intmath/*.h +lib/evloop/*.h lib/fs/*.h +lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/net/*.h lib/process/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h -ht.h
\ No newline at end of file +ext/ht.h diff --git a/src/lib/process/daemon.c b/src/lib/process/daemon.c index 3b90bef671..ae34b5bcb8 100644 --- a/src/lib/process/daemon.c +++ b/src/lib/process/daemon.c @@ -165,7 +165,7 @@ finish_daemon(const char *desired_cwd) return 0; } -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ /* defined(_WIN32) */ int start_daemon(void) diff --git a/src/lib/process/daemon.h b/src/lib/process/daemon.h index 20920e0aae..423c498837 100644 --- a/src/lib/process/daemon.h +++ b/src/lib/process/daemon.h @@ -18,4 +18,4 @@ int finish_daemon(const char *desired_cwd); bool start_daemon_has_been_called(void); -#endif +#endif /* !defined(TOR_DAEMON_H) */ diff --git a/src/lib/process/env.c b/src/lib/process/env.c index 0060200ba1..3912ade197 100644 --- a/src/lib/process/env.c +++ b/src/lib/process/env.c @@ -47,7 +47,7 @@ get_environment(void) * when we do a mostly-static build on OSX 10.7, the resulting binary won't * work on OSX 10.6. */ return *_NSGetEnviron(); -#else /* !(defined(HAVE__NSGETENVIRON)) */ +#else /* !defined(HAVE__NSGETENVIRON) */ return environ; #endif /* defined(HAVE__NSGETENVIRON) */ } diff --git a/src/lib/process/env.h b/src/lib/process/env.h index 15d59351e0..19c2235970 100644 --- a/src/lib/process/env.h +++ b/src/lib/process/env.h @@ -38,4 +38,4 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars, const char *new_var, void (*free_old)(void*), int free_p); -#endif +#endif /* !defined(TOR_ENV_H) */ diff --git a/src/lib/process/include.am b/src/lib/process/include.am index c6cc3a6699..af5f99617b 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -5,25 +5,35 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-process-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_process_a_SOURCES = \ src/lib/process/daemon.c \ src/lib/process/env.c \ src/lib/process/pidfile.c \ + src/lib/process/process.c \ + src/lib/process/process_sys.c \ + src/lib/process/process_unix.c \ + src/lib/process/process_win32.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ - src/lib/process/subprocess.c \ - src/lib/process/waitpid.c + src/lib/process/waitpid.c \ + src/lib/process/winprocess_sys.c src_lib_libtor_process_testing_a_SOURCES = \ $(src_lib_libtor_process_a_SOURCES) src_lib_libtor_process_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_process_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/process/daemon.h \ src/lib/process/env.h \ src/lib/process/pidfile.h \ + src/lib/process/process.h \ + src/lib/process/process_sys.h \ + src/lib/process/process_unix.h \ + src/lib/process/process_win32.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ - src/lib/process/subprocess.h \ - src/lib/process/waitpid.h + src/lib/process/waitpid.h \ + src/lib/process/winprocess_sys.h diff --git a/src/lib/process/pidfile.h b/src/lib/process/pidfile.h index dfeb39e046..af59041f80 100644 --- a/src/lib/process/pidfile.h +++ b/src/lib/process/pidfile.h @@ -13,4 +13,4 @@ int write_pidfile(const char *filename); -#endif +#endif /* !defined(TOR_PIDFILE_H) */ diff --git a/src/lib/process/process.c b/src/lib/process/process.c new file mode 100644 index 0000000000..2194a603ff --- /dev/null +++ b/src/lib/process/process.c @@ -0,0 +1,797 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.c + * \brief Module for working with other processes. + **/ + +#define PROCESS_PRIVATE +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/process_win32.h" +#include "lib/process/env.h" + +#ifdef HAVE_STDDEF_H +#include <stddef.h> +#endif + +/** A list of all <b>process_t</b> instances currently allocated. */ +static smartlist_t *processes; + +/** + * Boolean. If true, then Tor may call execve or CreateProcess via + * tor_spawn_background. + **/ +static int may_spawn_background_process = 1; + +/** Structure to represent a child process. */ +struct process_t { + /** Process status. */ + process_status_t status; + + /** Which protocol is the process using? */ + process_protocol_t protocol; + + /** Which function to call when we have data ready from stdout? */ + process_read_callback_t stdout_read_callback; + + /** Which function to call when we have data ready from stderr? */ + process_read_callback_t stderr_read_callback; + + /** Which function call when our process terminated? */ + process_exit_callback_t exit_callback; + + /** Our exit code when the process have terminated. */ + process_exit_code_t exit_code; + + /** Name of the command we want to execute (for example: /bin/ls). */ + char *command; + + /** The arguments used for the new process. The format here is one argument + * per element of the smartlist_t. On Windows these arguments are combined + * together using the <b>tor_join_win_cmdline</b> function. On Unix the + * process name (argv[0]) and the trailing NULL is added automatically before + * the process is executed. */ + smartlist_t *arguments; + + /** The environment used for the new process. */ + smartlist_t *environment; + + /** Buffer to store data from stdout when it is read. */ + buf_t *stdout_buffer; + + /** Buffer to store data from stderr when it is read. */ + buf_t *stderr_buffer; + + /** Buffer to store data to stdin before it is written. */ + buf_t *stdin_buffer; + + /** Do we need to store some custom data with the process? */ + void *data; + +#ifndef _WIN32 + /** Our Unix process handle. */ + process_unix_t *unix_process; +#else + /** Our Win32 process handle. */ + process_win32_t *win32_process; +#endif /* !defined(_WIN32) */ +}; + +/** Convert a given process status in <b>status</b> to its string + * representation. */ +const char * +process_status_to_string(process_status_t status) +{ + switch (status) { + case PROCESS_STATUS_NOT_RUNNING: + return "not running"; + case PROCESS_STATUS_RUNNING: + return "running"; + case PROCESS_STATUS_ERROR: + return "error"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Convert a given process protocol in <b>protocol</b> to its string + * representation. */ +const char * +process_protocol_to_string(process_protocol_t protocol) +{ + switch (protocol) { + case PROCESS_PROTOCOL_LINE: + return "Line"; + case PROCESS_PROTOCOL_RAW: + return "Raw"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** + * Turn off may_spawn_background_process, so that all future calls to + * tor_spawn_background are guaranteed to fail. + **/ +void +tor_disable_spawning_background_processes(void) +{ + may_spawn_background_process = 0; +} + +/** Initialize the Process subsystem. This function initializes the Process + * subsystem's global state. For cleaning up, <b>process_free_all()</b> should + * be called. */ +void +process_init(void) +{ + processes = smartlist_new(); + +#ifdef _WIN32 + process_win32_init(); +#endif +} + +/** Free up all resources that is handled by the Process subsystem. Note that + * this call does not terminate already running processes. */ +void +process_free_all(void) +{ +#ifdef _WIN32 + process_win32_deinit(); +#endif + + SMARTLIST_FOREACH(processes, process_t *, x, process_free(x)); + smartlist_free(processes); +} + +/** Get a list of all processes. This function returns a smartlist of + * <b>process_t</b> containing all the currently allocated processes. */ +const smartlist_t * +process_get_all_processes(void) +{ + return processes; +} + +/** Allocate and initialize a new process. This function returns a newly + * allocated and initialized process data, which can be used to configure and + * later run a subprocess of Tor. Use the various <b>process_set_*()</b> + * methods to configure it and run the process using <b>process_exec()</b>. Use + * <b>command</b> to specify the path to the command to run. You can either + * specify an absolute path to the command or relative where Tor will use the + * underlying operating system's functionality for finding the command to run. + * */ +process_t * +process_new(const char *command) +{ + tor_assert(command); + + process_t *process; + process = tor_malloc_zero(sizeof(process_t)); + + /* Set our command. */ + process->command = tor_strdup(command); + + /* By default we are not running. */ + process->status = PROCESS_STATUS_NOT_RUNNING; + + /* Prepare process environment. */ + process->arguments = smartlist_new(); + process->environment = smartlist_new(); + + /* Prepare the buffers. */ + process->stdout_buffer = buf_new(); + process->stderr_buffer = buf_new(); + process->stdin_buffer = buf_new(); + +#ifndef _WIN32 + /* Prepare our Unix process handle. */ + process->unix_process = process_unix_new(); +#else + /* Prepare our Win32 process handle. */ + process->win32_process = process_win32_new(); +#endif /* !defined(_WIN32) */ + + smartlist_add(processes, process); + + return process; +} + +/** Deallocate the given process in <b>process</b>. */ +void +process_free_(process_t *process) +{ + if (! process) + return; + + /* Cleanup parameters. */ + tor_free(process->command); + + /* Cleanup arguments and environment. */ + SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x)); + smartlist_free(process->arguments); + + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + + /* Cleanup the buffers. */ + buf_free(process->stdout_buffer); + buf_free(process->stderr_buffer); + buf_free(process->stdin_buffer); + +#ifndef _WIN32 + /* Cleanup our Unix process handle. */ + process_unix_free(process->unix_process); +#else + /* Cleanup our Win32 process handle. */ + process_win32_free(process->win32_process); +#endif /* !defined(_WIN32) */ + + smartlist_remove(processes, process); + + tor_free(process); +} + +/** Execute the given process. This function executes the given process as a + * subprocess of Tor. Returns <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_exec(process_t *process) +{ + tor_assert(process); + + if (BUG(may_spawn_background_process == 0)) + return PROCESS_STATUS_ERROR; + + process_status_t status = PROCESS_STATUS_NOT_RUNNING; + + log_info(LD_PROCESS, "Starting new process: %s", process->command); + +#ifndef _WIN32 + status = process_unix_exec(process); +#else + status = process_win32_exec(process); +#endif + + /* Update our state. */ + process_set_status(process, status); + + if (status != PROCESS_STATUS_RUNNING) { + log_warn(LD_PROCESS, "Failed to start process: %s", + process_get_command(process)); + } + + return status; +} + +/** Terminate the given process. Returns true on success, + * otherwise false. */ +bool +process_terminate(process_t *process) +{ + tor_assert(process); + + /* Terminating a non-running process isn't going to work. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + log_debug(LD_PROCESS, "Terminating process"); + +#ifndef _WIN32 + return process_unix_terminate(process); +#else + return process_win32_terminate(process); +#endif +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_get_pid(process_t *process) +{ + tor_assert(process); + +#ifndef _WIN32 + return process_unix_get_pid(process); +#else + return process_win32_get_pid(process); +#endif +} + +/** Set the callback function for output from the child process's standard out + * handle. This function sets the callback function which is called every time + * the child process have written output to its standard out file handle. + * + * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stdout_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stdout_read_callback = callback; +} + +/** Set the callback function for output from the child process's standard + * error handle. This function sets the callback function which is called + * every time the child process have written output to its standard error file + * handle. + * + * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stderr_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stderr_read_callback = callback; +} + +/** Set the callback function for process exit notification. The + * <b>callback</b> function will be called every time your child process have + * terminated. */ +void +process_set_exit_callback(process_t *process, + process_exit_callback_t callback) +{ + tor_assert(process); + process->exit_callback = callback; +} + +/** Get the current command of the given process. */ +const char * +process_get_command(const process_t *process) +{ + tor_assert(process); + return process->command; +} + +void +process_set_protocol(process_t *process, process_protocol_t protocol) +{ + tor_assert(process); + process->protocol = protocol; +} + +/** Get the currently used protocol of the given process. */ +process_protocol_t +process_get_protocol(const process_t *process) +{ + tor_assert(process); + return process->protocol; +} + +/** Set opague pointer to data. This function allows you to store a pointer to + * your own data in the given process. Use <b>process_get_data()</b> in the + * various callback functions to retrieve the data again. + * + * Note that the given process does NOT take ownership of the data and you are + * responsible for freeing up any resources allocated by the given data. + * */ +void +process_set_data(process_t *process, void *data) +{ + tor_assert(process); + process->data = data; +} + +/** Get the opaque pointer to callback data from the given process. This + * function allows you get the data you stored with <b>process_set_data()</b> + * in the different callback functions. */ +void * +process_get_data(const process_t *process) +{ + tor_assert(process); + return process->data; +} + +/** Set the status of a given process. */ +void +process_set_status(process_t *process, process_status_t status) +{ + tor_assert(process); + process->status = status; +} + +/** Get the status of the given process. */ +process_status_t +process_get_status(const process_t *process) +{ + tor_assert(process); + return process->status; +} + +/** Append an argument to the list of arguments in the given process. */ +void +process_append_argument(process_t *process, const char *argument) +{ + tor_assert(process); + tor_assert(argument); + + smartlist_add(process->arguments, tor_strdup(argument)); +} + +/** Returns a list of arguments (excluding the command itself) from the + * given process. */ +const smartlist_t * +process_get_arguments(const process_t *process) +{ + tor_assert(process); + return process->arguments; +} + +/** Returns a newly allocated Unix style argument vector. Use <b>tor_free()</b> + * to deallocate it after use. */ +char ** +process_get_argv(const process_t *process) +{ + tor_assert(process); + + /** Generate a Unix style process argument vector from our process's + * arguments smartlist_t. */ + char **argv = NULL; + + char *filename = process->command; + const smartlist_t *arguments = process->arguments; + const size_t size = smartlist_len(arguments); + + /* Make space for the process filename as argv[0] and a trailing NULL. */ + argv = tor_malloc_zero(sizeof(char *) * (size + 2)); + + /* Set our filename as first argument. */ + argv[0] = filename; + + /* Put in the rest of the values from arguments. */ + SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) { + tor_assert(arg_val != NULL); + + argv[arg_val_sl_idx + 1] = arg_val; + } SMARTLIST_FOREACH_END(arg_val); + + return argv; +} + +/** This function clears the internal environment and copies over every string + * from <b>env</b> as the new environment. */ +void +process_reset_environment(process_t *process, const smartlist_t *env) +{ + tor_assert(process); + tor_assert(env); + + /* Cleanup old environment. */ + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + process->environment = smartlist_new(); + + SMARTLIST_FOREACH(env, char *, x, + smartlist_add(process->environment, tor_strdup(x))); +} + +/** Set the given <b>key</b>/<b>value</b> pair as environment variable in the + * given process. */ +void +process_set_environment(process_t *process, + const char *key, + const char *value) +{ + tor_assert(process); + tor_assert(key); + tor_assert(value); + + smartlist_add_asprintf(process->environment, "%s=%s", key, value); +} + +/** Returns a newly allocated <b>process_environment_t</b> containing the + * environment variables for the given process. */ +process_environment_t * +process_get_environment(const process_t *process) +{ + tor_assert(process); + return process_environment_make(process->environment); +} + +#ifndef _WIN32 +/** Get the internal handle for the Unix backend. */ +process_unix_t * +process_get_unix_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->unix_process); + return process->unix_process; +} +#else /* defined(_WIN32) */ +/** Get the internal handle for Windows backend. */ +process_win32_t * +process_get_win32_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->win32_process); + return process->win32_process; +} +#endif /* !defined(_WIN32) */ + +/** Write <b>size</b> bytes of <b>data</b> to the given process's standard + * input. */ +void +process_write(process_t *process, + const uint8_t *data, size_t size) +{ + tor_assert(process); + tor_assert(data); + + buf_add(process->stdin_buffer, (char *)data, size); + process_write_stdin(process, process->stdin_buffer); +} + +/** As tor_vsnprintf(), but write the data to the given process's standard + * input. */ +void +process_vprintf(process_t *process, + const char *format, va_list args) +{ + tor_assert(process); + tor_assert(format); + + int size; + char *data; + + size = tor_vasprintf(&data, format, args); + process_write(process, (uint8_t *)data, size); + tor_free(data); +} + +/** As tor_snprintf(), but write the data to the given process's standard + * input. */ +void +process_printf(process_t *process, + const char *format, ...) +{ + tor_assert(process); + tor_assert(format); + + va_list ap; + va_start(ap, format); + process_vprintf(process, format, ap); + va_end(ap); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard output + * handle. */ +void +process_notify_event_stdout(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stdout(process, process->stdout_buffer); + + if (ret > 0) + process_read_data(process, + process->stdout_buffer, + process->stdout_read_callback); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard error + * handle. */ +void +process_notify_event_stderr(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stderr(process, process->stderr_buffer); + + if (ret > 0) + process_read_data(process, + process->stderr_buffer, + process->stderr_read_callback); +} + +/** This function is called by the Process backend when a given process is + * allowed to begin writing data to the standard input of the child process. */ +void +process_notify_event_stdin(process_t *process) +{ + tor_assert(process); + + process_write_stdin(process, process->stdin_buffer); +} + +/** This function is called by the Process backend when a given process have + * terminated. The exit status code is passed in <b>exit_code</b>. We mark the + * process as no longer running and calls the <b>exit_callback</b> with + * information about the process termination. The given <b>process</b> is + * free'd iff the exit_callback returns true. */ +void +process_notify_event_exit(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_debug(LD_PROCESS, + "Process terminated with exit code: %"PRIu64, exit_code); + + /* Update our state. */ + process_set_status(process, PROCESS_STATUS_NOT_RUNNING); + process->exit_code = exit_code; + + /* Call our exit callback, if it exists. */ + bool free_process_handle = false; + + /* The exit callback will tell us if we should process_free() our handle. */ + if (process->exit_callback) + free_process_handle = process->exit_callback(process, exit_code); + + if (free_process_handle) + process_free(process); +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard out handle. Returns the number of + * bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + return process_unix_read_stdout(process, buffer); +#else + return process_win32_read_stdout(process, buffer); +#endif +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard error handle. Returns the number + * of bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + return process_unix_read_stderr(process, buffer); +#else + return process_win32_read_stderr(process, buffer); +#endif +} + +/** This function calls the backend function for the given process whenever + * there is data to be written to the backends' file handles. */ +MOCK_IMPL(STATIC void, process_write_stdin, + (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + process_unix_write(process, buffer); +#else + process_win32_write(process, buffer); +#endif +} + +/** This function calls the protocol handlers based on the value of + * <b>process_get_protocol(process)</b>. Currently we call + * <b>process_read_buffer()</b> for <b>PROCESS_PROTOCOL_RAW</b> and + * <b>process_read_lines()</b> for <b>PROCESS_PROTOCOL_LINE</b>. */ +STATIC void +process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + switch (process_get_protocol(process)) { + case PROCESS_PROTOCOL_RAW: + process_read_buffer(process, buffer, callback); + break; + case PROCESS_PROTOCOL_LINE: + process_read_lines(process, buffer, callback); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return; + /* LCOV_EXCL_STOP */ + } +} + +/** This function takes the content of the given <b>buffer</b> and passes it to + * the given <b>callback</b> function, but ensures that an additional zero byte + * is added to the end of the data such that the given callback implementation + * can threat the content as a ASCIIZ string. */ +STATIC void +process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer); + + /* We allocate an extra byte for the zero byte in the end. */ + char *data = tor_malloc_zero(size + 1); + + buf_get_bytes(buffer, data, size); + log_debug(LD_PROCESS, "Read data from process"); + + if (callback) + callback(process, data, size); + + tor_free(data); +} + +/** This function tries to extract complete lines from the given <b>buffer</b> + * and calls the given <b>callback</b> function whenever it has a complete + * line. Before calling <b>callback</b> we remove the trailing "\n" or "\r\n" + * from the line. If we are unable to extract a complete line we leave the data + * in the buffer for next call. */ +STATIC void +process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer) + 1; + size_t line_size = 0; + char *data = tor_malloc_zero(size); + int ret; + + while (true) { + line_size = size; + ret = buf_get_line(buffer, data, &line_size); + + /* A complete line should always be smaller than the size of our + * buffer. */ + tor_assert(ret != -1); + + /* Remove \n from the end of the line. */ + if (line_size >= 1 && data[line_size - 1] == '\n') { + data[line_size - 1] = '\0'; + --line_size; + } + + /* Remove \r from the end of the line. */ + if (line_size >= 1 && data[line_size - 1] == '\r') { + data[line_size - 1] = '\0'; + --line_size; + } + + if (ret == 1) { + log_debug(LD_PROCESS, "Read line from process: \"%s\"", data); + + if (callback) + callback(process, data, line_size); + + /* We have read a whole line, let's see if there is more lines to read. + * */ + continue; + } + + /* No complete line for us to read. We are done for now. */ + tor_assert_nonfatal(ret == 0); + break; + } + + tor_free(data); +} diff --git a/src/lib/process/process.h b/src/lib/process/process.h new file mode 100644 index 0000000000..05c091a5bf --- /dev/null +++ b/src/lib/process/process.h @@ -0,0 +1,145 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.h + * \brief Header for process.c + **/ + +#ifndef TOR_PROCESS_H +#define TOR_PROCESS_H + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +/** Maximum number of bytes to write to a process' stdin. */ +#define PROCESS_MAX_WRITE (1024) + +/** Maximum number of bytes to read from a process' stdout/stderr. */ +#define PROCESS_MAX_READ (1024) + +typedef enum { + /** The process is not running. */ + PROCESS_STATUS_NOT_RUNNING, + + /** The process is running. */ + PROCESS_STATUS_RUNNING, + + /** The process is in an erroneous state. */ + PROCESS_STATUS_ERROR +} process_status_t; + +const char *process_status_to_string(process_status_t status); + +typedef enum { + /** Pass complete \n-terminated lines to the + * callback (with the \n or \r\n removed). */ + PROCESS_PROTOCOL_LINE, + + /** Pass the raw response from read() to the callback. */ + PROCESS_PROTOCOL_RAW +} process_protocol_t; + +const char *process_protocol_to_string(process_protocol_t protocol); + +void tor_disable_spawning_background_processes(void); + +struct smartlist_t; + +struct process_t; +typedef struct process_t process_t; + +typedef uint64_t process_exit_code_t; +typedef uint64_t process_pid_t; + +typedef void (*process_read_callback_t)(process_t *, + const char *, + size_t); +typedef bool +(*process_exit_callback_t)(process_t *, process_exit_code_t); + +void process_init(void); +void process_free_all(void); +const struct smartlist_t *process_get_all_processes(void); + +process_t *process_new(const char *command); +void process_free_(process_t *process); +#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s)) + +process_status_t process_exec(process_t *process); +bool process_terminate(process_t *process); + +process_pid_t process_get_pid(process_t *process); + +void process_set_stdout_read_callback(process_t *, + process_read_callback_t); +void process_set_stderr_read_callback(process_t *, + process_read_callback_t); +void process_set_exit_callback(process_t *, + process_exit_callback_t); + +const char *process_get_command(const process_t *process); + +void process_append_argument(process_t *process, const char *argument); +const struct smartlist_t *process_get_arguments(const process_t *process); +char **process_get_argv(const process_t *process); + +void process_reset_environment(process_t *process, + const struct smartlist_t *env); +void process_set_environment(process_t *process, + const char *key, + const char *value); + +struct process_environment_t; +struct process_environment_t *process_get_environment(const process_t *); + +void process_set_protocol(process_t *process, process_protocol_t protocol); +process_protocol_t process_get_protocol(const process_t *process); + +void process_set_data(process_t *process, void *data); +void *process_get_data(const process_t *process); + +void process_set_status(process_t *process, process_status_t status); +process_status_t process_get_status(const process_t *process); + +#ifndef _WIN32 +struct process_unix_t; +struct process_unix_t *process_get_unix_process(const process_t *process); +#else +struct process_win32_t; +struct process_win32_t *process_get_win32_process(const process_t *process); +#endif /* !defined(_WIN32) */ + +void process_write(process_t *process, + const uint8_t *data, size_t size); +void process_vprintf(process_t *process, + const char *format, va_list args) CHECK_PRINTF(2, 0); +void process_printf(process_t *process, + const char *format, ...) CHECK_PRINTF(2, 3); + +void process_notify_event_stdout(process_t *process); +void process_notify_event_stderr(process_t *process); +void process_notify_event_stdin(process_t *process); +void process_notify_event_exit(process_t *process, + process_exit_code_t); + +#ifdef PROCESS_PRIVATE +MOCK_DECL(STATIC int, process_read_stdout, (process_t *, buf_t *)); +MOCK_DECL(STATIC int, process_read_stderr, (process_t *, buf_t *)); +MOCK_DECL(STATIC void, process_write_stdin, (process_t *, buf_t *)); + +STATIC void process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +#endif /* defined(PROCESS_PRIVATE) */ + +#endif /* !defined(TOR_PROCESS_H) */ diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c new file mode 100644 index 0000000000..3c809a00e8 --- /dev/null +++ b/src/lib/process/process_sys.c @@ -0,0 +1,33 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.c + * \brief Subsystem object for process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/process_sys.h" +#include "lib/process/process.h" + +static int +subsys_process_initialize(void) +{ + process_init(); + return 0; +} + +static void +subsys_process_shutdown(void) +{ + process_free_all(); +} + +const subsys_fns_t sys_process = { + .name = "process", + .level = -35, + .supported = true, + .initialize = subsys_process_initialize, + .shutdown = subsys_process_shutdown +}; diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h new file mode 100644 index 0000000000..b7a116d838 --- /dev/null +++ b/src/lib/process/process_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.h + * \brief Declare subsystem object for the process module. + **/ + +#ifndef TOR_PROCESS_SYS_H +#define TOR_PROCESS_SYS_H + +extern const struct subsys_fns_t sys_process; + +#endif /* !defined(TOR_PROCESS_SYS_H) */ diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c new file mode 100644 index 0000000000..8191bdc1f0 --- /dev/null +++ b/src/lib/process/process_unix.c @@ -0,0 +1,698 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.c + * \brief Module for working with Unix processes. + **/ + +#define PROCESS_UNIX_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/waitpid.h" +#include "lib/process/env.h" + +#include <stdio.h> + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif + +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) +#include <sys/prctl.h> +#endif + +#if HAVE_SIGNAL_H +#include <signal.h> +#endif + +#ifndef _WIN32 + +/** Maximum number of file descriptors, if we cannot get it via sysconf() */ +#define DEFAULT_MAX_FD 256 + +/** Internal state for Unix handles. */ +struct process_unix_handle_t { + /** Unix File Descriptor. */ + int fd; + + /** Have we reached end of file? */ + bool reached_eof; + + /** Event structure for libevent. */ + struct event *event; + + /** Are we writing? */ + bool is_writing; +}; + +/** Internal state for our Unix process. */ +struct process_unix_t { + /** Standard in handle. */ + process_unix_handle_t stdin_handle; + + /** Standard out handle. */ + process_unix_handle_t stdout_handle; + + /** Standard error handle. */ + process_unix_handle_t stderr_handle; + + /** The process identifier of our process. */ + pid_t pid; + + /** Waitpid Callback structure. */ + waitpid_callback_t *waitpid; +}; + +/** Returns a newly allocated <b>process_unix_t</b>. */ +process_unix_t * +process_unix_new(void) +{ + process_unix_t *unix_process; + unix_process = tor_malloc_zero(sizeof(process_unix_t)); + + unix_process->stdin_handle.fd = -1; + unix_process->stderr_handle.fd = -1; + unix_process->stdout_handle.fd = -1; + + return unix_process; +} + +/** Deallocates the given <b>unix_process</b>. */ +void +process_unix_free_(process_unix_t *unix_process) +{ + if (! unix_process) + return; + + /* Clean up our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + + /* FIXME(ahf): Refactor waitpid code? */ + unix_process->waitpid = NULL; + + /* Close all our file descriptors. */ + process_unix_close_file_descriptors(unix_process); + + tor_event_free(unix_process->stdout_handle.event); + tor_event_free(unix_process->stderr_handle.event); + tor_event_free(unix_process->stdin_handle.event); + + tor_free(unix_process); +} + +/** Executes the given process as a child process of Tor. This function is + * responsible for setting up the child process and run it. This includes + * setting up pipes for interprocess communication, initialize the waitpid + * callbacks, and finally run fork() followed by execve(). Returns + * <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_unix_exec(process_t *process) +{ + static int max_fd = -1; + + process_unix_t *unix_process; + pid_t pid; + int stdin_pipe[2]; + int stdout_pipe[2]; + int stderr_pipe[2]; + int retval, fd; + + unix_process = process_get_unix_process(process); + + /* Create standard in pipe. */ + retval = pipe(stdin_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdin " + "communication with process: %s", + strerror(errno)); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard out pipe. */ + retval = pipe(stdout_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdout " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard error pipe. */ + retval = pipe(stderr_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stderr " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdout_pipe[0]); + close(stdout_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + +#ifdef _SC_OPEN_MAX + if (-1 == max_fd) { + max_fd = (int)sysconf(_SC_OPEN_MAX); + + if (max_fd == -1) { + max_fd = DEFAULT_MAX_FD; + log_warn(LD_PROCESS, + "Cannot find maximum file descriptor, assuming: %d", max_fd); + } + } +#else /* !defined(_SC_OPEN_MAX) */ + max_fd = DEFAULT_MAX_FD; +#endif /* defined(_SC_OPEN_MAX) */ + + pid = fork(); + + if (0 == pid) { + /* This code is running in the child process context. */ + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) + /* Attempt to have the kernel issue a SIGTERM if the parent + * goes away. Certain attributes of the binary being execve()ed + * will clear this during the execve() call, but it's better + * than nothing. + */ + prctl(PR_SET_PDEATHSIG, SIGTERM); +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ + + /* Link process stdout to the write end of the pipe. */ + retval = dup2(stdout_pipe[1], STDOUT_FILENO); + if (-1 == retval) + goto error; + + /* Link process stderr to the write end of the pipe. */ + retval = dup2(stderr_pipe[1], STDERR_FILENO); + if (-1 == retval) + goto error; + + /* Link process stdin to the read end of the pipe */ + retval = dup2(stdin_pipe[0], STDIN_FILENO); + if (-1 == retval) + goto error; + + /* Close our pipes now after they have been dup2()'ed. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + close(stdout_pipe[0]); + close(stdout_pipe[1]); + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /* Close all other fds, including the read end of the pipe. XXX: We should + * now be doing enough FD_CLOEXEC setting to make this needless. + */ + for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) + close(fd); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Call the requested program. */ + execve(argv[0], argv, env->unixoid_environment_block); + + /* If we made it here it is because execve failed :-( */ + tor_free(argv); + process_environment_free(env); + + error: + fprintf(stderr, "Error from child process: %s", strerror(errno)); + _exit(1); + } + + /* We are in the parent process. */ + if (-1 == pid) { + log_warn(LD_PROCESS, + "Failed to create child process: %s", strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdout_pipe[0]); + close(stdout_pipe[1]); + + /** Cleanup standard error pipe. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Register our PID. */ + unix_process->pid = pid; + + /* Setup waitpid callbacks. */ + unix_process->waitpid = set_waitpid_callback(pid, + process_unix_waitpid_callback, + process); + + /* Handle standard out. */ + unix_process->stdout_handle.fd = stdout_pipe[0]; + retval = close(stdout_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s", + strerror(errno)); + } + + /* Handle standard error. */ + unix_process->stderr_handle.fd = stderr_pipe[0]; + retval = close(stderr_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Failed to close write end of standard error pipe: %s", + strerror(errno)); + } + + /* Handle standard in. */ + unix_process->stdin_handle.fd = stdin_pipe[1]; + retval = close(stdin_pipe[0]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s", + strerror(errno)); + } + + /* Setup our handles. */ + process_unix_setup_handle(process, + &unix_process->stdout_handle, + EV_READ|EV_PERSIST, + stdout_read_callback); + + process_unix_setup_handle(process, + &unix_process->stderr_handle, + EV_READ|EV_PERSIST, + stderr_read_callback); + + process_unix_setup_handle(process, + &unix_process->stdin_handle, + EV_WRITE|EV_PERSIST, + stdin_write_callback); + + /* Start reading from standard out and standard error. */ + process_unix_start_reading(&unix_process->stdout_handle); + process_unix_start_reading(&unix_process->stderr_handle); + + return PROCESS_STATUS_RUNNING; +} + +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_unix_terminate(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + + /* All running processes should have a waitpid. */ + if (BUG(unix_process->waitpid == NULL)) + return false; + + bool success = true; + + /* Send a SIGTERM to our child process. */ + int ret; + + ret = kill(unix_process->pid, SIGTERM); + + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to terminate process: %s", + strerror(errno)); + success = false; + } + + /* Close all our FD's. */ + if (! process_unix_close_file_descriptors(unix_process)) + success = false; + + return success; +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_unix_get_pid(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + return (process_pid_t)unix_process->pid; +} + +/** Write the given <b>buffer</b> as input to the given <b>process</b>'s + * standard input. Returns the number of bytes written. */ +int +process_unix_write(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + size_t buffer_flush_len = buf_datalen(buffer); + const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len); + + /* If we have data to write (when buffer_flush_len > 0) and we are not + * currently getting file descriptor events from the kernel, we tell the + * kernel to start notifying us about when we can write to our file + * descriptor and return. */ + if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) { + process_unix_start_writing(&unix_process->stdin_handle); + return 0; + } + + /* We don't have any data to write, but the kernel is currently notifying us + * about whether we are able to write or not. Tell the kernel to stop + * notifying us until we have data to write. */ + if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) { + process_unix_stop_writing(&unix_process->stdin_handle); + return 0; + } + + /* We have data to write and the kernel have told us to write it. */ + return buf_flush_to_pipe(buffer, + process_get_unix_process(process)->stdin_handle.fd, + max_to_write, &buffer_flush_len); +} + +/** Read data from the given process's standard output and put it into + * <b>buffer</b>. Returns the number of bytes read. */ +int +process_unix_read_stdout(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stdout_handle, + buffer); +} + +/** Read data from the given process's standard error and put it into + * <b>buffer</b>. Returns the number of bytes read. */ +int +process_unix_read_stderr(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stderr_handle, + buffer); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard output. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stdout_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdout(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard error. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stderr_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stderr(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * written the child process's standard input. We notify the Process subsystem, + * which is then responsible for calling back to us for doing the actual write + * of the data. */ +STATIC void +stdin_write_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdin(process); +} + +/** This function tells libevent that we are interested in receiving read + * events from the given <b>handle</b>. */ +STATIC void +process_unix_start_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); +} + +/** This function tells libevent that we are no longer interested in receiving + * read events from the given <b>handle</b>. */ +STATIC void +process_unix_stop_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); +} + +/** This function tells libevent that we are interested in receiving write + * events from the given <b>handle</b>. */ +STATIC void +process_unix_start_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); + + handle->is_writing = true; +} + +/** This function tells libevent that we are no longer interested in receiving + * write events from the given <b>handle</b>. */ +STATIC void +process_unix_stop_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); + + handle->is_writing = false; +} + +/** This function is called when the waitpid system have detected that our + * process have terminated. We disable the waitpid system and notify the + * Process subsystem that we have terminated. */ +STATIC void +process_unix_waitpid_callback(int status, void *data) +{ + tor_assert(data); + + process_t *process = data; + process_unix_t *unix_process = process_get_unix_process(process); + + /* Remove our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + unix_process->waitpid = NULL; + + /* Notify our process. */ + process_notify_event_exit(process, status); + + /* Make sure you don't modify the process after we have called + * process_notify_event_exit() on it, to allow users to process_free() it in + * the exit callback. */ +} + +/** This function sets the file descriptor in the <b>handle</b> as non-blocking + * and configures the libevent event structure based on the given <b>flags</b> + * to ensure that <b>callback</b> is called whenever we have events on the + * given <b>handle</b>. */ +STATIC void +process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(callback); + + /* Put our file descriptor into non-blocking mode. */ + if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) { + log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s", + strerror(errno)); + } + + /* Setup libevent event. */ + handle->event = tor_event_new(tor_libevent_get_base(), + handle->fd, + flags, + callback, + process); +} + +/** This function reads data from the given <b>handle</b> and puts it into + * <b>buffer</b>. Returns the number of bytes read this way. */ +STATIC int +process_unix_read_handle(process_t *process, + process_unix_handle_t *handle, + buf_t *buffer) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(buffer); + + int ret = 0; + int eof = 0; + int error = 0; + + ret = buf_read_from_pipe(buffer, + handle->fd, + PROCESS_MAX_READ, + &eof, + &error); + + if (error) + log_warn(LD_PROCESS, + "Unable to read data: %s", strerror(error)); + + if (eof) { + handle->reached_eof = true; + process_unix_stop_reading(handle); + } + + return ret; +} + +/** Close the standard in, out, and error handles of the given + * <b>unix_process</b>. */ +STATIC bool +process_unix_close_file_descriptors(process_unix_t *unix_process) +{ + tor_assert(unix_process); + + int ret; + bool success = true; + + /* Stop reading and writing before we close() our + * file descriptors. */ + if (! unix_process->stdout_handle.reached_eof) + process_unix_stop_reading(&unix_process->stdout_handle); + + if (! unix_process->stderr_handle.reached_eof) + process_unix_stop_reading(&unix_process->stderr_handle); + + if (unix_process->stdin_handle.is_writing) + process_unix_stop_writing(&unix_process->stdin_handle); + + if (unix_process->stdin_handle.fd != -1) { + ret = close(unix_process->stdin_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard in"); + success = false; + } + + unix_process->stdin_handle.fd = -1; + } + + if (unix_process->stdout_handle.fd != -1) { + ret = close(unix_process->stdout_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard out"); + success = false; + } + + unix_process->stdout_handle.fd = -1; + } + + if (unix_process->stderr_handle.fd != -1) { + ret = close(unix_process->stderr_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard error"); + success = false; + } + + unix_process->stderr_handle.fd = -1; + } + + return success; +} + +#endif /* !defined(_WIN32) */ diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h new file mode 100644 index 0000000000..da40b3e567 --- /dev/null +++ b/src/lib/process/process_unix.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.h + * \brief Header for process_unix.c + **/ + +#ifndef TOR_PROCESS_UNIX_H +#define TOR_PROCESS_UNIX_H + +#ifndef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" + +#include <event2/event.h> + +struct process_t; + +struct process_unix_t; +typedef struct process_unix_t process_unix_t; + +process_unix_t *process_unix_new(void); +void process_unix_free_(process_unix_t *unix_process); +#define process_unix_free(s) \ + FREE_AND_NULL(process_unix_t, process_unix_free_, (s)) + +process_status_t process_unix_exec(struct process_t *process); +bool process_unix_terminate(struct process_t *process); + +process_pid_t process_unix_get_pid(struct process_t *process); + +int process_unix_write(struct process_t *process, buf_t *buffer); +int process_unix_read_stdout(struct process_t *process, buf_t *buffer); +int process_unix_read_stderr(struct process_t *process, buf_t *buffer); + +#ifdef PROCESS_UNIX_PRIVATE +struct process_unix_handle_t; +typedef struct process_unix_handle_t process_unix_handle_t; + +STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data); + +STATIC void process_unix_start_reading(process_unix_handle_t *); +STATIC void process_unix_stop_reading(process_unix_handle_t *); + +STATIC void process_unix_start_writing(process_unix_handle_t *); +STATIC void process_unix_stop_writing(process_unix_handle_t *); + +STATIC void process_unix_waitpid_callback(int status, void *data); + +STATIC void process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback); +STATIC int process_unix_read_handle(process_t *, + process_unix_handle_t *, + buf_t *); +STATIC bool process_unix_close_file_descriptors(process_unix_t *); +#endif /* defined(PROCESS_UNIX_PRIVATE) */ + +#endif /* !defined(_WIN32) */ + +#endif /* !defined(TOR_PROCESS_UNIX_H) */ diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c new file mode 100644 index 0000000000..7e4082ad13 --- /dev/null +++ b/src/lib/process/process_win32.c @@ -0,0 +1,1105 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.c + * \brief Module for working with Windows processes. + **/ + +#define PROCESS_WIN32_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/log/win32err.h" +#include "lib/process/process.h" +#include "lib/process/process_win32.h" +#include "lib/process/env.h" + +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef _WIN32 + +/** The size of our intermediate buffers. */ +#define BUFFER_SIZE (1024) + +/** Timer that ticks once a second and calls the process_win32_timer_callback() + * function. */ +static periodic_timer_t *periodic_timer; + +/** Structure to represent the state around the pipe HANDLE. + * + * This structure is used to store state about a given HANDLE, including + * whether we have reached end of file, its intermediate buffers, and how much + * data that is available in the intermediate buffer. */ +struct process_win32_handle_t { + /** Standard out pipe handle. */ + HANDLE pipe; + + /** True iff we have reached EOF from the pipe. */ + bool reached_eof; + + /** How much data is available in buffer. */ + size_t data_available; + + /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */ + char buffer[BUFFER_SIZE]; + + /** Overlapped structure for ReadFileEx() and WriteFileEx(). */ + OVERLAPPED overlapped; + + /** Are we waiting for another I/O operation to complete? */ + bool busy; +}; + +/** Structure to represent the Windows specific implementation details of this + * Process backend. + * + * This structure is attached to <b>process_t</b> (see process.h) and is + * reachable from <b>process_t</b> via the <b>process_get_win32_process()</b> + * method. */ +struct process_win32_t { + /** Standard in state. */ + process_win32_handle_t stdin_handle; + + /** Standard out state. */ + process_win32_handle_t stdout_handle; + + /** Standard error state. */ + process_win32_handle_t stderr_handle; + + /** Process Information. */ + PROCESS_INFORMATION process_information; +}; + +/** Create a new <b>process_win32_t</b>. + * + * This function constructs a new <b>process_win32_t</b> and initializes the + * default values. */ +process_win32_t * +process_win32_new(void) +{ + process_win32_t *win32_process; + win32_process = tor_malloc_zero(sizeof(process_win32_t)); + + win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE; + + return win32_process; +} + +/** Free a given <b>process_win32_t</b>. + * + * This function deinitializes and frees up the resources allocated for the + * given <b>process_win32_t</b>. */ +void +process_win32_free_(process_win32_t *win32_process) +{ + if (! win32_process) + return; + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + tor_free(win32_process); +} + +/** Initialize the Windows backend of the Process subsystem. */ +void +process_win32_init(void) +{ + /* We don't start the periodic timer here because it makes no sense to have + * the timer running until we have some processes that benefits from the + * timer timer ticks. */ +} + +/** Deinitialize the Windows backend of the Process subsystem. */ +void +process_win32_deinit(void) +{ + /* Stop our timer, but only if it's running. */ + if (process_win32_timer_running()) + process_win32_timer_stop(); +} + +/** Execute the given process. This function is responsible for setting up + * named pipes for I/O between the child process and the Tor process. Returns + * <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_win32_exec(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + HANDLE stdout_pipe_read = NULL; + HANDLE stdout_pipe_write = NULL; + HANDLE stderr_pipe_read = NULL; + HANDLE stderr_pipe_write = NULL; + HANDLE stdin_pipe_read = NULL; + HANDLE stdin_pipe_write = NULL; + BOOL ret = FALSE; + + /* Setup our security attributes. */ + SECURITY_ATTRIBUTES security_attributes; + security_attributes.nLength = sizeof(security_attributes); + security_attributes.bInheritHandle = TRUE; + /* FIXME: should we set explicit security attributes? + * (See Ticket #2046, comment 5) */ + security_attributes.lpSecurityDescriptor = NULL; + + /* Create our standard out pipe. */ + if (! process_win32_create_pipe(&stdout_pipe_read, + &stdout_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create our standard error pipe. */ + if (! process_win32_create_pipe(&stderr_pipe_read, + &stderr_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create out standard in pipe. */ + if (! process_win32_create_pipe(&stdin_pipe_read, + &stdin_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_WRITER)) { + return PROCESS_STATUS_ERROR; + } + + /* Configure startup info for our child process. */ + STARTUPINFOA startup_info; + + memset(&startup_info, 0, sizeof(startup_info)); + startup_info.cb = sizeof(startup_info); + startup_info.hStdError = stderr_pipe_write; + startup_info.hStdOutput = stdout_pipe_write; + startup_info.hStdInput = stdin_pipe_read; + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Windows expects argv to be a whitespace delimited string, so join argv up + */ + char *joined_argv = tor_join_win_cmdline((const char **)argv); + + /* Create the child process */ + ret = CreateProcessA(NULL, + joined_argv, + NULL, + NULL, + TRUE, + CREATE_NO_WINDOW, + env->windows_environment_block[0] == '\0' ? + NULL : env->windows_environment_block, + NULL, + &startup_info, + &win32_process->process_information); + + tor_free(argv); + tor_free(joined_argv); + process_environment_free(env); + + if (! ret) { + log_warn(LD_PROCESS, "CreateProcessA() failed: %s", + format_win32_error(GetLastError())); + + /* Cleanup our handles. */ + CloseHandle(stdout_pipe_read); + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_read); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + CloseHandle(stdin_pipe_write); + + /* In the Unix backend, we do not get an error in the Tor process when a + * child process fails to spawn its target executable since we need to + * first do the fork() call in the Tor process and then the child process + * is responsible for doing the call to execve(). + * + * This means that the user of the process_exec() API must check for + * whether it returns PROCESS_STATUS_ERROR, which will rarely happen on + * Unix, but will happen for error cases on Windows where it does not + * happen on Unix. For example: when the target executable does not exist + * on the file system. + * + * To have somewhat feature compatibility between the Unix and the Windows + * backend, we here notify the process_t owner that the process have exited + * (even though it never managed to run) to ensure that the exit callback + * is executed. + */ + process_notify_event_exit(process, 0); + + return PROCESS_STATUS_ERROR; + } + + /* TODO: Should we close hProcess and hThread in + * process_handle->process_information? */ + win32_process->stdout_handle.pipe = stdout_pipe_read; + win32_process->stderr_handle.pipe = stderr_pipe_read; + win32_process->stdin_handle.pipe = stdin_pipe_write; + + /* Close our ends of the pipes that is now owned by the child process. */ + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + + /* Used by the callback functions from ReadFileEx() and WriteFileEx() such + * that we can figure out which process_t that was responsible for the event. + * + * Warning, here be dragons: + * + * MSDN says that the hEvent member of the overlapped structure is unused + * for ReadFileEx() and WriteFileEx, which allows us to store a pointer to + * our process state there. + */ + win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process; + + /* Start our timer if it is not already running. */ + if (! process_win32_timer_running()) + process_win32_timer_start(); + + /* We use Windows Extended I/O functions, so our completion callbacks are + * called automatically for us when there is data to read. Because of this + * we start the read of standard out and error right away. */ + process_notify_event_stdout(process); + process_notify_event_stderr(process); + + return PROCESS_STATUS_RUNNING; +} + +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_win32_terminate(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + /* Terminate our process. */ + BOOL ret; + + ret = TerminateProcess(win32_process->process_information.hProcess, 0); + + if (! ret) { + log_warn(LD_PROCESS, "TerminateProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + return true; +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_win32_get_pid(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + return (process_pid_t)win32_process->process_information.dwProcessId; +} + +/** Schedule an async write of the data found in <b>buffer</b> for the given + * process. This function runs an async write operation of the content of + * buffer, if we are not already waiting for a pending I/O request. Returns the + * number of bytes that Windows will hopefully write for us in the background. + * */ +int +process_win32_write(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD error_code = 0; + const size_t buffer_size = buf_datalen(buffer); + + /* Windows is still writing our buffer. */ + if (win32_process->stdin_handle.busy) + return 0; + + /* Nothing for us to do right now. */ + if (buffer_size == 0) + return 0; + + /* We have reached end of file already? */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return 0; + + /* Figure out how much data we should read. */ + const size_t write_size = MIN(buffer_size, + sizeof(win32_process->stdin_handle.buffer)); + + /* Read data from the process_t buffer into our intermediate buffer. */ + buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size); + + /* Because of the slightly weird API for WriteFileEx() we must set this to 0 + * before we call WriteFileEx() because WriteFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + + /* Schedule our write. */ + ret = WriteFileEx(win32_process->stdin_handle.pipe, + win32_process->stdin_handle.buffer, + write_size, + &win32_process->stdin_handle.overlapped, + process_win32_stdin_write_done); + + if (! ret) { + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "WriteFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "WriteFileEx() failed: %s", + format_win32_error(error_code)); + } + + win32_process->stdin_handle.reached_eof = true; + return 0; + } + + /* Here be dragons: According to MSDN's documentation for WriteFileEx() we + * should check GetLastError() after a call to WriteFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "WriteFileEx() failed after returning success: %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + return 0; + /* LCOV_EXCL_STOP */ + } + + /* This cast should be safe since our buffer can maximum be BUFFER_SIZE + * large. */ + return (int)write_size; +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard output of the child process + * and asks Windows to call process_win32_stdout_read_done() when that + * operation is finished. Returns the number of bytes moved into <b>buffer</b>. + * */ +int +process_win32_read_stdout(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stdout_handle, + buffer, + process_win32_stdout_read_done); +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard error of the child process + * and asks Windows to call process_win32_stderr_read_done() when that + * operation is finished. Returns the number of bytes moved into <b>buffer</b>. + * */ +int +process_win32_read_stderr(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stderr_handle, + buffer, + process_win32_stderr_read_done); +} + +/** This function is responsible for moving the Tor process into what Microsoft + * calls an "alertable" state. Once the process is in an alertable state the + * Windows kernel will notify us when our background I/O requests have finished + * and the callbacks will be executed. */ +void +process_win32_trigger_completion_callbacks(void) +{ + DWORD ret; + + /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep + * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause + * the process to enter alertable state, where the Windows kernel will notify + * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which + * will cause our completion callbacks to be executed. + * + * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION + * if one or more I/O callbacks were executed. */ + ret = SleepEx(0, TRUE); + + /* Warn us if the function returned something we did not anticipate. */ + if (ret != 0 && ret != WAIT_IO_COMPLETION) { + log_warn(LD_PROCESS, "SleepEx() returned %lu", ret); + } +} + +/** Start the periodic timer which is reponsible for checking whether processes + * are still alive and to make sure that the Tor process is periodically being + * moved into an alertable state. */ +void +process_win32_timer_start(void) +{ + /* Make sure we never start our timer if it's already running. */ + if (BUG(process_win32_timer_running())) + return; + + /* Wake up once a second. */ + static const struct timeval interval = {1, 0}; + + log_info(LD_PROCESS, "Starting Windows Process I/O timer"); + periodic_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + process_win32_timer_callback, + NULL); +} + +/** Stops the periodic timer. */ +void +process_win32_timer_stop(void) +{ + if (BUG(periodic_timer == NULL)) + return; + + log_info(LD_PROCESS, "Stopping Windows Process I/O timer"); + periodic_timer_free(periodic_timer); +} + +/** Returns true iff the periodic timer is running. */ +bool +process_win32_timer_running(void) +{ + return periodic_timer != NULL; +} + +/** This function is called whenever the periodic_timer ticks. The function is + * responsible for moving the Tor process into an alertable state once a second + * and checking for whether our child processes have terminated since the last + * tick. */ +STATIC void +process_win32_timer_callback(periodic_timer_t *timer, void *data) +{ + tor_assert(timer == periodic_timer); + tor_assert(data == NULL); + + /* Move the process into an alertable state. */ + process_win32_trigger_completion_callbacks(); + + /* Check if our processes are still alive. */ + + /* Since the call to process_win32_timer_test_process() might call + * process_notify_event_exit() which again might call process_free() which + * updates the list of processes returned by process_get_all_processes() it + * is important here that we make sure to not touch the list of processes if + * the call to process_win32_timer_test_process() returns true. */ + bool done; + + do { + const smartlist_t *processes = process_get_all_processes(); + done = true; + + SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) { + /* If process_win32_timer_test_process() returns true, it means that + * smartlist_remove() might have been called on the list returned by + * process_get_all_processes(). We start the loop over again until we + * have a succesful run over the entire list where the list was not + * modified. */ + if (process_win32_timer_test_process(process)) { + done = false; + break; + } + } SMARTLIST_FOREACH_END(process); + } while (! done); +} + +/** Test whether a given process is still alive. Notify the Process subsystem + * if our process have died. Returns true iff the given process have + * terminated. */ +STATIC bool +process_win32_timer_test_process(process_t *process) +{ + tor_assert(process); + + /* No need to look at processes that don't claim they are running. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD exit_code = 0; + + /* Sometimes the Windows kernel wont give us the EOF/Broken Pipe error + * message until some time after the process have actually terminated. We + * make sure that our ReadFileEx() calls for the process have *all* returned + * and both standard out and error have been marked as EOF before we try to + * see if the process terminated. + * + * This ensures that we *never* call the exit callback of the `process_t`, + * which potentially ends up calling `process_free()` on our `process_t`, + * before all data have been received from the process. + * + * We do NOT have a check here for whether standard in reached EOF since + * standard in's WriteFileEx() function is only called on-demand when we have + * something to write and is thus usually not awaiting to finish any + * operations. If we WriteFileEx() to a file that has terminated we'll simply + * get an error from ReadFileEx() or its completion routine and move on with + * life. */ + if (! win32_process->stdout_handle.reached_eof) + return false; + + if (! win32_process->stderr_handle.reached_eof) + return false; + + /* We start by testing whether our process is still running. */ + ret = GetExitCodeProcess(win32_process->process_information.hProcess, + &exit_code); + + if (! ret) { + log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Notify our process_t that our process have terminated. Since our + * exit_callback might decide to process_free() our process handle it is very + * important that we do not touch the process_t after the call to + * process_notify_event_exit(). */ + if (exit_code != STILL_ACTIVE) { + process_notify_event_exit(process, exit_code); + return true; + } + + return false; +} + +/** Create a new overlapped named pipe. This function creates a new connected, + * named, pipe in <b>*read_pipe</b> and <b>*write_pipe</b> if the function is + * succesful. Returns true on sucess, false on failure. */ +STATIC bool +process_win32_create_pipe(HANDLE *read_pipe, + HANDLE *write_pipe, + SECURITY_ATTRIBUTES *attributes, + process_win32_pipe_type_t pipe_type) +{ + tor_assert(read_pipe); + tor_assert(write_pipe); + tor_assert(attributes); + + BOOL ret = FALSE; + + /* Buffer size. */ + const size_t size = 4096; + + /* Our additional read/write modes that depends on which pipe type we are + * creating. */ + DWORD read_mode = 0; + DWORD write_mode = 0; + + /* Generate the unique pipe name. */ + char pipe_name[MAX_PATH]; + static DWORD process_id = 0; + static DWORD counter = 0; + + if (process_id == 0) + process_id = GetCurrentProcessId(); + + tor_snprintf(pipe_name, sizeof(pipe_name), + "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu", + process_id, counter++); + + /* Only one of our handles can be overlapped. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + read_mode = FILE_FLAG_OVERLAPPED; + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + write_mode = FILE_FLAG_OVERLAPPED; + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + /* Setup our read and write handles. */ + HANDLE read_handle; + HANDLE write_handle; + + /* Create our named pipe. */ + read_handle = CreateNamedPipeA(pipe_name, + (PIPE_ACCESS_INBOUND|read_mode), + (PIPE_TYPE_BYTE|PIPE_WAIT), + 1, + size, + size, + 1000, + attributes); + + if (read_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Create our file in the pipe namespace. */ + write_handle = CreateFileA(pipe_name, + GENERIC_WRITE, + 0, + attributes, + OPEN_EXISTING, + (FILE_ATTRIBUTE_NORMAL|write_mode), + NULL); + + if (write_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateFileA() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + + return false; + } + + /* Set the inherit flag for our pipe. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0); + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + if (! ret) { + log_warn(LD_PROCESS, "SetHandleInformation() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + CloseHandle(write_handle); + + return false; + } + + /* Everything is good. */ + *read_pipe = read_handle; + *write_pipe = write_handle; + + return true; +} + +/** Cleanup a given <b>handle</b>. */ +STATIC void +process_win32_cleanup_handle(process_win32_handle_t *handle) +{ + tor_assert(handle); + +#if 0 + BOOL ret; + DWORD error_code; + + /* Cancel any pending I/O requests: This means that instead of getting + * ERROR_BROKEN_PIPE we get ERROR_OPERATION_ABORTED, but it doesn't seem + * like this is needed. */ + ret = CancelIo(handle->pipe); + + if (! ret) { + error_code = GetLastError(); + + /* There was no pending I/O requests for our handle. */ + if (error_code != ERROR_NOT_FOUND) { + log_warn(LD_PROCESS, "CancelIo() failed: %s", + format_win32_error(error_code)); + } + } +#endif /* 0 */ + + /* Close our handle. */ + if (handle->pipe != INVALID_HANDLE_VALUE) { + CloseHandle(handle->pipe); + handle->pipe = INVALID_HANDLE_VALUE; + handle->reached_eof = true; + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard output. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stdout_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stdout_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stdout(process); + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard error. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stderr_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stderr_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stderr(process); + } +} + +/** This function is called when WriteFileEx() completes its background write + * to the child process's standard input. We notify the Process subsystem that + * it can write data to us again. */ +STATIC VOID WINAPI +process_win32_stdin_write_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + (void)byte_count; + + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + /* Mark our handle as not having any outstanding I/O requests. */ + win32_process->stdin_handle.busy = false; + + /* Check if we have been asked to write to the handle that have been marked + * as having reached EOF. */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return; + + if (error_code == 0) { + /** Our data have been succesfully written. Clear our state and schedule + * the next write. */ + win32_process->stdin_handle.data_available = 0; + memset(win32_process->stdin_handle.buffer, 0, + sizeof(win32_process->stdin_handle.buffer)); + + /* Schedule the next write. */ + process_notify_event_stdin(process); + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our WriteFileEx() call was succesful, but we reached the end of our + * file. We mark our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + win32_process->stdin_handle.reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from WriteFileEx(): %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + } +} + +/** This function reads data from the given <b>handle</b>'s internal buffer and + * moves it into the given <b>buffer</b>. Additionally, we start the next + * ReadFileEx() background operation with the given <b>callback</b> as + * completion callback. Returns the number of bytes written to the buffer. */ +STATIC int +process_win32_read_from_handle(process_win32_handle_t *handle, + buf_t *buffer, + LPOVERLAPPED_COMPLETION_ROUTINE callback) +{ + tor_assert(handle); + tor_assert(buffer); + tor_assert(callback); + + BOOL ret = FALSE; + int bytes_available = 0; + DWORD error_code = 0; + + /* We already have a request to read data that isn't complete yet. */ + if (BUG(handle->busy)) + return 0; + + /* Check if we have been asked to read from a handle that have already told + * us that we have reached the end of the file. */ + if (BUG(handle->reached_eof)) + return 0; + + /* This cast should be safe since our buffer can be at maximum up to + * BUFFER_SIZE in size. */ + bytes_available = (int)handle->data_available; + + if (handle->data_available > 0) { + /* Read data from our intermediate buffer into the process_t buffer. */ + buf_add(buffer, handle->buffer, handle->data_available); + + /* Reset our read state. */ + handle->data_available = 0; + memset(handle->buffer, 0, sizeof(handle->buffer)); + } + + /* Because of the slightly weird API for ReadFileEx() we must set this to 0 + * before we call ReadFileEx() because ReadFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + + /* Ask the Windows kernel to read data from our pipe into our buffer and call + * the callback function when it is done. */ + ret = ReadFileEx(handle->pipe, + handle->buffer, + sizeof(handle->buffer), + &handle->overlapped, + callback); + + if (! ret) { + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "ReadFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "ReadFileEx() failed: %s", + format_win32_error(error_code)); + } + + handle->reached_eof = true; + return bytes_available; + } + + /* Here be dragons: According to MSDN's documentation for ReadFileEx() we + * should check GetLastError() after a call to ReadFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "ReadFileEx() failed after returning success: %s", + format_win32_error(error_code)); + handle->reached_eof = true; + return bytes_available; + /* LCOV_EXCL_STOP */ + } + + /* We mark our handle as having a pending I/O request. */ + handle->busy = true; + + return bytes_available; +} + +/** This function checks the callback values from ReadFileEx() in + * <b>error_code</b> and <b>byte_count</b> if we have read data. Returns true + * iff our caller should request more data from ReadFileEx(). */ +STATIC bool +process_win32_handle_read_completion(process_win32_handle_t *handle, + DWORD error_code, + DWORD byte_count) +{ + tor_assert(handle); + + /* Mark our handle as not having any outstanding I/O requests. */ + handle->busy = false; + + if (error_code == 0) { + /* Our ReadFileEx() call was succesful and there is data for us. */ + + /* This cast should be safe since byte_count should never be larger than + * BUFFER_SIZE. */ + tor_assert(byte_count <= BUFFER_SIZE); + handle->data_available = (size_t)byte_count; + + /* Tell our caller to schedule the next read. */ + return true; + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our ReadFileEx() finished, but we reached the end of our file. We mark + * our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + handle->reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from ReadFileEx(): %s", + format_win32_error(error_code)); + + handle->reached_eof = true; + } + + /* Our caller should NOT schedule the next read. */ + return false; +} + +/** Format a single argument for being put on a Windows command line. + * Returns a newly allocated string */ +STATIC char * +format_win_cmdline_argument(const char *arg) +{ + char *formatted_arg; + char need_quotes; + const char *c; + int i; + int bs_counter = 0; + /* Backslash we can point to when one is inserted into the string */ + const char backslash = '\\'; + + /* Smartlist of *char */ + smartlist_t *arg_chars; + arg_chars = smartlist_new(); + + /* Quote string if it contains whitespace or is empty */ + need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); + + /* Build up smartlist of *chars */ + for (c=arg; *c != '\0'; c++) { + if ('"' == *c) { + /* Double up backslashes preceding a quote */ + for (i=0; i<(bs_counter*2); i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + /* Escape the quote */ + smartlist_add(arg_chars, (void*)&backslash); + smartlist_add(arg_chars, (void*)c); + } else if ('\\' == *c) { + /* Count backslashes until we know whether to double up */ + bs_counter++; + } else { + /* Don't double up slashes preceding a non-quote */ + for (i=0; i<bs_counter; i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + smartlist_add(arg_chars, (void*)c); + } + } + /* Don't double up trailing backslashes */ + for (i=0; i<bs_counter; i++) + smartlist_add(arg_chars, (void*)&backslash); + + /* Allocate space for argument, quotes (if needed), and terminator */ + const size_t formatted_arg_len = smartlist_len(arg_chars) + + (need_quotes ? 2 : 0) + 1; + formatted_arg = tor_malloc_zero(formatted_arg_len); + + /* Add leading quote */ + i=0; + if (need_quotes) + formatted_arg[i++] = '"'; + + /* Add characters */ + SMARTLIST_FOREACH(arg_chars, char*, ch, + { + formatted_arg[i++] = *ch; + }); + + /* Add trailing quote */ + if (need_quotes) + formatted_arg[i++] = '"'; + formatted_arg[i] = '\0'; + + smartlist_free(arg_chars); + return formatted_arg; +} + +/** Format a command line for use on Windows, which takes the command as a + * string rather than string array. Follows the rules from "Parsing C++ + * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the + * Python subprocess module. Returns a newly allocated string */ +STATIC char * +tor_join_win_cmdline(const char *argv[]) +{ + smartlist_t *argv_list; + char *joined_argv; + int i; + + /* Format each argument and put the result in a smartlist */ + argv_list = smartlist_new(); + for (i=0; argv[i] != NULL; i++) { + smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i])); + } + + /* Join the arguments with whitespace */ + joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL); + + /* Free the newly allocated arguments, and the smartlist */ + SMARTLIST_FOREACH(argv_list, char *, arg, + { + tor_free(arg); + }); + smartlist_free(argv_list); + + return joined_argv; +} + +#endif /* defined(_WIN32) */ diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h new file mode 100644 index 0000000000..a50d86df5b --- /dev/null +++ b/src/lib/process/process_win32.h @@ -0,0 +1,97 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.h + * \brief Header for process_win32.c + **/ + +#ifndef TOR_PROCESS_WIN32_H +#define TOR_PROCESS_WIN32_H + +#ifdef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/evloop/compat_libevent.h" + +#include <windows.h> + +struct process_t; + +struct process_win32_t; +typedef struct process_win32_t process_win32_t; + +process_win32_t *process_win32_new(void); +void process_win32_free_(process_win32_t *win32_process); +#define process_win32_free(s) \ + FREE_AND_NULL(process_win32_t, process_win32_free_, (s)) + +void process_win32_init(void); +void process_win32_deinit(void); + +process_status_t process_win32_exec(struct process_t *process); +bool process_win32_terminate(struct process_t *process); + +process_pid_t process_win32_get_pid(struct process_t *process); + +int process_win32_write(struct process_t *process, buf_t *buffer); +int process_win32_read_stdout(struct process_t *process, buf_t *buffer); +int process_win32_read_stderr(struct process_t *process, buf_t *buffer); + +void process_win32_trigger_completion_callbacks(void); + +/* Timer handling. */ +void process_win32_timer_start(void); +void process_win32_timer_stop(void); +bool process_win32_timer_running(void); + +#ifdef PROCESS_WIN32_PRIVATE +STATIC void process_win32_timer_callback(periodic_timer_t *, void *); +STATIC bool process_win32_timer_test_process(process_t *); + +/* I/O pipe handling. */ +struct process_win32_handle_t; +typedef struct process_win32_handle_t process_win32_handle_t; + +typedef enum process_win32_pipe_type_t { + /** This pipe is used for reading. */ + PROCESS_WIN32_PIPE_TYPE_READER, + + /** This pipe is used for writing. */ + PROCESS_WIN32_PIPE_TYPE_WRITER +} process_win32_pipe_type_t; + +STATIC bool process_win32_create_pipe(HANDLE *, + HANDLE *, + SECURITY_ATTRIBUTES *, + process_win32_pipe_type_t); + +STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle); + +STATIC VOID WINAPI process_win32_stdout_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stderr_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stdin_write_done(DWORD, + DWORD, + LPOVERLAPPED); + +STATIC int process_win32_read_from_handle(process_win32_handle_t *, + buf_t *, + LPOVERLAPPED_COMPLETION_ROUTINE); +STATIC bool process_win32_handle_read_completion(process_win32_handle_t *, + DWORD, + DWORD); + +STATIC char *format_win_cmdline_argument(const char *arg); +STATIC char *tor_join_win_cmdline(const char *argv[]); +#endif /* defined(PROCESS_WIN32_PRIVATE) */ + +#endif /* defined(_WIN32) */ + +#endif /* !defined(TOR_PROCESS_WIN32_H) */ diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c index 534b39d101..fda284f3d9 100644 --- a/src/lib/process/restrict.c +++ b/src/lib/process/restrict.c @@ -152,7 +152,7 @@ tor_mlockall(void) "pages: %s", strerror(errno)); return -1; } -#else /* !(defined(HAVE_UNIX_MLOCKALL)) */ +#else /* !defined(HAVE_UNIX_MLOCKALL) */ log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); return -1; #endif /* defined(HAVE_UNIX_MLOCKALL) */ @@ -214,7 +214,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) return -1; } limit = MAX_CONNECTIONS; -#else /* !(!defined(HAVE_GETRLIMIT)) */ +#else /* defined(HAVE_GETRLIMIT) */ struct rlimit rlim; if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) { diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c index 6e8258f279..3c94ce4bac 100644 --- a/src/lib/process/setuid.c +++ b/src/lib/process/setuid.c @@ -72,7 +72,7 @@ log_credential_status(void) "UID is %u (real), %u (effective), %u (saved)", (unsigned)ruid, (unsigned)euid, (unsigned)suid); } -#else /* !(defined(HAVE_GETRESUID)) */ +#else /* !defined(HAVE_GETRESUID) */ /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */ ruid = getuid(); euid = geteuid(); @@ -93,7 +93,7 @@ log_credential_status(void) "GID is %u (real), %u (effective), %u (saved)", (unsigned)rgid, (unsigned)egid, (unsigned)sgid); } -#else /* !(defined(HAVE_GETRESGID)) */ +#else /* !defined(HAVE_GETRESGID) */ /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */ rgid = getgid(); egid = getegid(); @@ -154,7 +154,7 @@ have_capability_support(void) return 0; cap_free(caps); return 1; -#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ +#else /* !defined(HAVE_LINUX_CAPABILITIES) */ return 0; #endif /* defined(HAVE_LINUX_CAPABILITIES) */ } @@ -265,7 +265,7 @@ switch_id(const char *user, const unsigned flags) if (drop_capabilities(1)) return -1; } -#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ +#else /* !defined(HAVE_LINUX_CAPABILITIES) */ (void) keep_bindlow; if (warn_if_no_caps) { log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support " @@ -376,7 +376,7 @@ switch_id(const char *user, const unsigned flags) #endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */ return 0; -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ (void)user; (void)flags; diff --git a/src/lib/process/setuid.h b/src/lib/process/setuid.h index 7d03e1f025..a2125d2d06 100644 --- a/src/lib/process/setuid.h +++ b/src/lib/process/setuid.h @@ -19,4 +19,4 @@ int have_capability_support(void); #define SWITCH_ID_WARN_IF_NO_CAPS (1<<1) int switch_id(const char *user, unsigned flags); -#endif +#endif /* !defined(TOR_SETUID_H) */ diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c deleted file mode 100644 index f4429d7f76..0000000000 --- a/src/lib/process/subprocess.c +++ /dev/null @@ -1,1236 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file subprocess.c - * \brief Launch and monitor other processes. - **/ - -#define SUBPROCESS_PRIVATE -#include "lib/process/subprocess.h" - -#include "lib/container/smartlist.h" -#include "lib/err/torerr.h" -#include "lib/log/log.h" -#include "lib/log/util_bug.h" -#include "lib/log/win32err.h" -#include "lib/malloc/malloc.h" -#include "lib/process/env.h" -#include "lib/process/waitpid.h" -#include "lib/string/compat_ctype.h" - -#ifdef HAVE_SYS_TYPES_H -#include <sys/types.h> -#endif -#ifdef HAVE_SYS_PRCTL_H -#include <sys/prctl.h> -#endif -#ifdef HAVE_UNISTD_H -#include <unistd.h> -#endif -#ifdef HAVE_SIGNAL_H -#include <signal.h> -#endif -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif -#ifdef HAVE_SYS_WAIT_H -#include <sys/wait.h> -#endif -#include <errno.h> -#include <string.h> - -/** Format a single argument for being put on a Windows command line. - * Returns a newly allocated string */ -static char * -format_win_cmdline_argument(const char *arg) -{ - char *formatted_arg; - char need_quotes; - const char *c; - int i; - int bs_counter = 0; - /* Backslash we can point to when one is inserted into the string */ - const char backslash = '\\'; - - /* Smartlist of *char */ - smartlist_t *arg_chars; - arg_chars = smartlist_new(); - - /* Quote string if it contains whitespace or is empty */ - need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); - - /* Build up smartlist of *chars */ - for (c=arg; *c != '\0'; c++) { - if ('"' == *c) { - /* Double up backslashes preceding a quote */ - for (i=0; i<(bs_counter*2); i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - /* Escape the quote */ - smartlist_add(arg_chars, (void*)&backslash); - smartlist_add(arg_chars, (void*)c); - } else if ('\\' == *c) { - /* Count backslashes until we know whether to double up */ - bs_counter++; - } else { - /* Don't double up slashes preceding a non-quote */ - for (i=0; i<bs_counter; i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - smartlist_add(arg_chars, (void*)c); - } - } - /* Don't double up trailing backslashes */ - for (i=0; i<bs_counter; i++) - smartlist_add(arg_chars, (void*)&backslash); - - /* Allocate space for argument, quotes (if needed), and terminator */ - const size_t formatted_arg_len = smartlist_len(arg_chars) + - (need_quotes ? 2 : 0) + 1; - formatted_arg = tor_malloc_zero(formatted_arg_len); - - /* Add leading quote */ - i=0; - if (need_quotes) - formatted_arg[i++] = '"'; - - /* Add characters */ - SMARTLIST_FOREACH(arg_chars, char*, ch, - { - formatted_arg[i++] = *ch; - }); - - /* Add trailing quote */ - if (need_quotes) - formatted_arg[i++] = '"'; - formatted_arg[i] = '\0'; - - smartlist_free(arg_chars); - return formatted_arg; -} - -/** Format a command line for use on Windows, which takes the command as a - * string rather than string array. Follows the rules from "Parsing C++ - * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the - * Python subprocess module. Returns a newly allocated string */ -char * -tor_join_win_cmdline(const char *argv[]) -{ - smartlist_t *argv_list; - char *joined_argv; - int i; - - /* Format each argument and put the result in a smartlist */ - argv_list = smartlist_new(); - for (i=0; argv[i] != NULL; i++) { - smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i])); - } - - /* Join the arguments with whitespace */ - joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL); - - /* Free the newly allocated arguments, and the smartlist */ - SMARTLIST_FOREACH(argv_list, char *, arg, - { - tor_free(arg); - }); - smartlist_free(argv_list); - - return joined_argv; -} - -#ifndef _WIN32 -/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in - * <b>hex_errno</b>. Called between fork and _exit, so must be signal-handler - * safe. - * - * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available. - * - * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded - * with spaces. CHILD_STATE indicates where - * in the process of starting the child process did the failure occur (see - * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of - * errno when the failure occurred. - * - * On success return the number of characters added to hex_errno, not counting - * the terminating NUL; return -1 on error. - */ -STATIC int -format_helper_exit_status(unsigned char child_state, int saved_errno, - char *hex_errno) -{ - unsigned int unsigned_errno; - int written, left; - char *cur; - size_t i; - int res = -1; - - /* Fill hex_errno with spaces, and a trailing newline (memset may - not be signal handler safe, so we can't use it) */ - for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++) - hex_errno[i] = ' '; - hex_errno[HEX_ERRNO_SIZE - 1] = '\n'; - - /* Convert errno to be unsigned for hex conversion */ - if (saved_errno < 0) { - // Avoid overflow on the cast to unsigned int when result is INT_MIN - // by adding 1 to the signed int negative value, - // then, after it has been negated and cast to unsigned, - // adding the original 1 back (the double-addition is intentional). - // Otherwise, the cast to signed could cause a temporary int - // to equal INT_MAX + 1, which is undefined. - unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1; - } else { - unsigned_errno = (unsigned int) saved_errno; - } - - /* - * Count how many chars of space we have left, and keep a pointer into the - * current point in the buffer. - */ - left = HEX_ERRNO_SIZE+1; - cur = hex_errno; - - /* Emit child_state */ - written = format_hex_number_sigsafe(child_state, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - if (left <= 0) - goto err; - - /* Now the '/' */ - *cur = '/'; - - /* Adjust left and cur */ - ++cur; - --left; - if (left <= 0) - goto err; - - /* Need minus? */ - if (saved_errno < 0) { - *cur = '-'; - ++cur; - --left; - if (left <= 0) - goto err; - } - - /* Emit unsigned_errno */ - written = format_hex_number_sigsafe(unsigned_errno, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - - /* Check that we have enough space left for a newline and a NUL */ - if (left <= 1) - goto err; - - /* Emit the newline and NUL */ - *cur++ = '\n'; - *cur++ = '\0'; - - res = (int)(cur - hex_errno - 1); - - goto done; - - err: - /* - * In error exit, just write a '\0' in the first char so whatever called - * this at least won't fall off the end. - */ - *hex_errno = '\0'; - - done: - return res; -} -#endif /* !defined(_WIN32) */ - -/* Maximum number of file descriptors, if we cannot get it via sysconf() */ -#define DEFAULT_MAX_FD 256 - -/** Terminate the process of <b>process_handle</b>, if that process has not - * already exited. - * - * Return 0 if we succeeded in terminating the process (or if the process - * already exited), and -1 if we tried to kill the process but failed. - * - * Based on code originally borrowed from Python's os.kill. */ -int -tor_terminate_process(process_handle_t *process_handle) -{ -#ifdef _WIN32 - if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) { - HANDLE handle = process_handle->pid.hProcess; - - if (!TerminateProcess(handle, 0)) - return -1; - else - return 0; - } -#else /* !(defined(_WIN32)) */ - if (process_handle->waitpid_cb) { - /* We haven't got a waitpid yet, so we can just kill off the process. */ - return kill(process_handle->pid, SIGTERM); - } -#endif /* defined(_WIN32) */ - - return 0; /* We didn't need to kill the process, so report success */ -} - -/** Return the Process ID of <b>process_handle</b>. */ -int -tor_process_get_pid(process_handle_t *process_handle) -{ -#ifdef _WIN32 - return (int) process_handle->pid.dwProcessId; -#else - return (int) process_handle->pid; -#endif -} - -#ifdef _WIN32 -HANDLE -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#else /* !(defined(_WIN32)) */ -/* DOCDOC tor_process_get_stdout_pipe */ -int -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#endif /* defined(_WIN32) */ - -/* DOCDOC process_handle_new */ -static process_handle_t * -process_handle_new(void) -{ - process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t)); - -#ifdef _WIN32 - out->stdin_pipe = INVALID_HANDLE_VALUE; - out->stdout_pipe = INVALID_HANDLE_VALUE; - out->stderr_pipe = INVALID_HANDLE_VALUE; -#else - out->stdin_pipe = -1; - out->stdout_pipe = -1; - out->stderr_pipe = -1; -#endif /* defined(_WIN32) */ - - return out; -} - -#ifndef _WIN32 -/** Invoked when a process that we've launched via tor_spawn_background() has - * been found to have terminated. - */ -static void -process_handle_waitpid_cb(int status, void *arg) -{ - process_handle_t *process_handle = arg; - - process_handle->waitpid_exit_status = status; - clear_waitpid_callback(process_handle->waitpid_cb); - if (process_handle->status == PROCESS_STATUS_RUNNING) - process_handle->status = PROCESS_STATUS_NOTRUNNING; - process_handle->waitpid_cb = 0; -} -#endif /* !defined(_WIN32) */ - -/** - * @name child-process states - * - * Each of these values represents a possible state that a child process can - * be in. They're used to determine what to say when telling the parent how - * far along we were before failure. - * - * @{ - */ -#define CHILD_STATE_INIT 0 -#define CHILD_STATE_PIPE 1 -#define CHILD_STATE_MAXFD 2 -#define CHILD_STATE_FORK 3 -#define CHILD_STATE_DUPOUT 4 -#define CHILD_STATE_DUPERR 5 -#define CHILD_STATE_DUPIN 6 -#define CHILD_STATE_CLOSEFD 7 -#define CHILD_STATE_EXEC 8 -#define CHILD_STATE_FAILEXEC 9 -/** @} */ -/** - * Boolean. If true, then Tor may call execve or CreateProcess via - * tor_spawn_background. - **/ -static int may_spawn_background_process = 1; -/** - * Turn off may_spawn_background_process, so that all future calls to - * tor_spawn_background are guaranteed to fail. - **/ -void -tor_disable_spawning_background_processes(void) -{ - may_spawn_background_process = 0; -} -/** Start a program in the background. If <b>filename</b> contains a '/', then - * it will be treated as an absolute or relative path. Otherwise, on - * non-Windows systems, the system path will be searched for <b>filename</b>. - * On Windows, only the current directory will be searched. Here, to search the - * system path (as well as the application directory, current working - * directory, and system directories), set filename to NULL. - * - * The strings in <b>argv</b> will be passed as the command line arguments of - * the child program (following convention, argv[0] should normally be the - * filename of the executable, and this must be the case if <b>filename</b> is - * NULL). The last element of argv must be NULL. A handle to the child process - * will be returned in process_handle (which must be non-NULL). Read - * process_handle.status to find out if the process was successfully launched. - * For convenience, process_handle.status is returned by this function. - * - * Some parts of this code are based on the POSIX subprocess module from - * Python, and example code from - * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx. - */ -int -tor_spawn_background(const char *const filename, const char **argv, - process_environment_t *env, - process_handle_t **process_handle_out) -{ - if (BUG(may_spawn_background_process == 0)) { - /* We should never reach this point if we're forbidden to spawn - * processes. Instead we should have caught the attempt earlier. */ - return PROCESS_STATUS_ERROR; - } - -#ifdef _WIN32 - HANDLE stdout_pipe_read = NULL; - HANDLE stdout_pipe_write = NULL; - HANDLE stderr_pipe_read = NULL; - HANDLE stderr_pipe_write = NULL; - HANDLE stdin_pipe_read = NULL; - HANDLE stdin_pipe_write = NULL; - process_handle_t *process_handle; - int status; - - STARTUPINFOA siStartInfo; - BOOL retval = FALSE; - - SECURITY_ATTRIBUTES saAttr; - char *joined_argv; - - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - saAttr.bInheritHandle = TRUE; - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - saAttr.lpSecurityDescriptor = NULL; - - /* Assume failure to start process */ - status = PROCESS_STATUS_ERROR; - - /* Set up pipe for stdout */ - if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdout communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdout communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stderr */ - if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stderr communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stderr communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stdin */ - if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdin communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdin communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Create the child process */ - - /* Windows expects argv to be a whitespace delimited string, so join argv up - */ - joined_argv = tor_join_win_cmdline(argv); - - process_handle = process_handle_new(); - process_handle->status = status; - - ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION)); - ZeroMemory(&siStartInfo, sizeof(STARTUPINFO)); - siStartInfo.cb = sizeof(STARTUPINFO); - siStartInfo.hStdError = stderr_pipe_write; - siStartInfo.hStdOutput = stdout_pipe_write; - siStartInfo.hStdInput = stdin_pipe_read; - siStartInfo.dwFlags |= STARTF_USESTDHANDLES; - - /* Create the child process */ - - retval = CreateProcessA(filename, // module name - joined_argv, // command line - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - NULL, // process security attributes - NULL, // primary thread security attributes - TRUE, // handles are inherited - /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess() - * work?) */ - CREATE_NO_WINDOW, // creation flags - (env==NULL) ? NULL : env->windows_environment_block, - NULL, // use parent's current directory - &siStartInfo, // STARTUPINFO pointer - &(process_handle->pid)); // receives PROCESS_INFORMATION - - tor_free(joined_argv); - - if (!retval) { - log_warn(LD_GENERAL, - "Failed to create child process %s: %s", filename?filename:argv[0], - format_win32_error(GetLastError())); - tor_free(process_handle); - } else { - /* TODO: Close hProcess and hThread in process_handle->pid? */ - process_handle->stdout_pipe = stdout_pipe_read; - process_handle->stderr_pipe = stderr_pipe_read; - process_handle->stdin_pipe = stdin_pipe_write; - status = process_handle->status = PROCESS_STATUS_RUNNING; - } - - /* TODO: Close pipes on exit */ - *process_handle_out = process_handle; - return status; -#else /* !(defined(_WIN32)) */ - pid_t pid; - int stdout_pipe[2]; - int stderr_pipe[2]; - int stdin_pipe[2]; - int fd, retval; - process_handle_t *process_handle; - int status; - - const char *error_message = SPAWN_ERROR_MESSAGE; - size_t error_message_length; - - /* Represents where in the process of spawning the program is; - this is used for printing out the error message */ - unsigned char child_state = CHILD_STATE_INIT; - - char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */ - - static int max_fd = -1; - - status = PROCESS_STATUS_ERROR; - - /* We do the strlen here because strlen() is not signal handler safe, - and we are not allowed to use unsafe functions between fork and exec */ - error_message_length = strlen(error_message); - - // child_state = CHILD_STATE_PIPE; - - /* Set up pipe for redirecting stdout, stderr, and stdin of child */ - retval = pipe(stdout_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdout communication with child process: %s", - strerror(errno)); - return status; - } - - retval = pipe(stderr_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stderr communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - - return status; - } - - retval = pipe(stdin_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdin communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - - return status; - } - - // child_state = CHILD_STATE_MAXFD; - -#ifdef _SC_OPEN_MAX - if (-1 == max_fd) { - max_fd = (int) sysconf(_SC_OPEN_MAX); - if (max_fd == -1) { - max_fd = DEFAULT_MAX_FD; - log_warn(LD_GENERAL, - "Cannot find maximum file descriptor, assuming %d", max_fd); - } - } -#else /* !(defined(_SC_OPEN_MAX)) */ - max_fd = DEFAULT_MAX_FD; -#endif /* defined(_SC_OPEN_MAX) */ - - // child_state = CHILD_STATE_FORK; - - pid = fork(); - if (0 == pid) { - /* In child */ - -#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) - /* Attempt to have the kernel issue a SIGTERM if the parent - * goes away. Certain attributes of the binary being execve()ed - * will clear this during the execve() call, but it's better - * than nothing. - */ - prctl(PR_SET_PDEATHSIG, SIGTERM); -#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ - - child_state = CHILD_STATE_DUPOUT; - - /* Link child stdout to the write end of the pipe */ - retval = dup2(stdout_pipe[1], STDOUT_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPERR; - - /* Link child stderr to the write end of the pipe */ - retval = dup2(stderr_pipe[1], STDERR_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPIN; - - /* Link child stdin to the read end of the pipe */ - retval = dup2(stdin_pipe[0], STDIN_FILENO); - if (-1 == retval) - goto error; - - // child_state = CHILD_STATE_CLOSEFD; - - close(stderr_pipe[0]); - close(stderr_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - - /* Close all other fds, including the read end of the pipe */ - /* XXX: We should now be doing enough FD_CLOEXEC setting to make - * this needless. */ - for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) { - close(fd); - } - - // child_state = CHILD_STATE_EXEC; - - /* Call the requested program. We need the cast because - execvp doesn't define argv as const, even though it - does not modify the arguments */ - if (env) - execve(filename, (char *const *) argv, env->unixoid_environment_block); - else { - static char *new_env[] = { NULL }; - execve(filename, (char *const *) argv, new_env); - } - - /* If we got here, the exec or open(/dev/null) failed */ - - child_state = CHILD_STATE_FAILEXEC; - - error: - { - /* XXX: are we leaking fds from the pipe? */ - int n, err=0; - ssize_t nbytes; - - n = format_helper_exit_status(child_state, errno, hex_errno); - - if (n >= 0) { - /* Write the error message. GCC requires that we check the return - value, but there is nothing we can do if it fails */ - /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */ - nbytes = write(STDOUT_FILENO, error_message, error_message_length); - err = (nbytes < 0); - nbytes = write(STDOUT_FILENO, hex_errno, n); - err += (nbytes < 0); - } - - _exit(err?254:255); // exit ok: in child. - } - - /* Never reached, but avoids compiler warning */ - return status; // LCOV_EXCL_LINE - } - - /* In parent */ - - if (-1 == pid) { - log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno)); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - return status; - } - - process_handle = process_handle_new(); - process_handle->status = status; - process_handle->pid = pid; - - /* TODO: If the child process forked but failed to exec, waitpid it */ - - /* Return read end of the pipes to caller, and close write end */ - process_handle->stdout_pipe = stdout_pipe[0]; - retval = close(stdout_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stdout pipe in parent process: %s", - strerror(errno)); - } - - process_handle->waitpid_cb = set_waitpid_callback(pid, - process_handle_waitpid_cb, - process_handle); - - process_handle->stderr_pipe = stderr_pipe[0]; - retval = close(stderr_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stderr pipe in parent process: %s", - strerror(errno)); - } - - /* Return write end of the stdin pipe to caller, and close the read end */ - process_handle->stdin_pipe = stdin_pipe[1]; - retval = close(stdin_pipe[0]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close read end of stdin pipe in parent process: %s", - strerror(errno)); - } - - status = process_handle->status = PROCESS_STATUS_RUNNING; - /* Set stdin/stdout/stderr pipes to be non-blocking */ - if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) { - log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " - "nonblocking in parent process: %s", strerror(errno)); - } - - *process_handle_out = process_handle; - return status; -#endif /* defined(_WIN32) */ -} - -/** Destroy all resources allocated by the process handle in - * <b>process_handle</b>. - * If <b>also_terminate_process</b> is true, also terminate the - * process of the process handle. */ -MOCK_IMPL(void, -tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)) -{ - if (!process_handle) - return; - - if (also_terminate_process) { - if (tor_terminate_process(process_handle) < 0) { - const char *errstr = -#ifdef _WIN32 - format_win32_error(GetLastError()); -#else - strerror(errno); -#endif - log_notice(LD_GENERAL, "Failed to terminate process with " - "PID '%d' ('%s').", tor_process_get_pid(process_handle), - errstr); - } else { - log_info(LD_GENERAL, "Terminated process with PID '%d'.", - tor_process_get_pid(process_handle)); - } - } - - process_handle->status = PROCESS_STATUS_NOTRUNNING; - -#ifdef _WIN32 - if (process_handle->stdout_pipe) - CloseHandle(process_handle->stdout_pipe); - - if (process_handle->stderr_pipe) - CloseHandle(process_handle->stderr_pipe); - - if (process_handle->stdin_pipe) - CloseHandle(process_handle->stdin_pipe); -#else /* !(defined(_WIN32)) */ - close(process_handle->stdout_pipe); - close(process_handle->stderr_pipe); - close(process_handle->stdin_pipe); - - clear_waitpid_callback(process_handle->waitpid_cb); -#endif /* defined(_WIN32) */ - - memset(process_handle, 0x0f, sizeof(process_handle_t)); - tor_free(process_handle); -} - -/** Get the exit code of a process specified by <b>process_handle</b> and store - * it in <b>exit_code</b>, if set to a non-NULL value. If <b>block</b> is set - * to true, the call will block until the process has exited. Otherwise if - * the process is still running, the function will return - * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns - * PROCESS_EXIT_EXITED if the process did exit. If there is a failure, - * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if - * non-NULL) will be undefined. N.B. Under *nix operating systems, this will - * probably not work in Tor, because waitpid() is called in main.c to reap any - * terminated child processes.*/ -int -tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code) -{ -#ifdef _WIN32 - DWORD retval; - BOOL success; - - if (block) { - /* Wait for the process to exit */ - retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE); - if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } else { - retval = WaitForSingleObject(process_handle->pid.hProcess, 0); - if (WAIT_TIMEOUT == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } - - if (exit_code != NULL) { - success = GetExitCodeProcess(process_handle->pid.hProcess, - (PDWORD)exit_code); - if (!success) { - log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s", - format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } -#else /* !(defined(_WIN32)) */ - int stat_loc; - int retval; - - if (process_handle->waitpid_cb) { - /* We haven't processed a SIGCHLD yet. */ - retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG); - if (retval == process_handle->pid) { - clear_waitpid_callback(process_handle->waitpid_cb); - process_handle->waitpid_cb = NULL; - process_handle->waitpid_exit_status = stat_loc; - } - } else { - /* We already got a SIGCHLD for this process, and handled it. */ - retval = process_handle->pid; - stat_loc = process_handle->waitpid_exit_status; - } - - if (!block && 0 == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != process_handle->pid) { - log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s", - (int)process_handle->pid, strerror(errno)); - return PROCESS_EXIT_ERROR; - } - - if (!WIFEXITED(stat_loc)) { - log_warn(LD_GENERAL, "Process %d did not exit normally", - (int)process_handle->pid); - return PROCESS_EXIT_ERROR; - } - - if (exit_code != NULL) - *exit_code = WEXITSTATUS(stat_loc); -#endif /* defined(_WIN32) */ - - return PROCESS_EXIT_EXITED; -} - -#ifdef _WIN32 -/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes. If - * <b>hProcess</b> is NULL, the function will return immediately if there is - * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle - * to the process owning the <b>h</b>. In this case, the function will exit - * only once the process has exited, or <b>count</b> bytes are read. Returns - * the number of bytes read, or -1 on error. */ -ssize_t -tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process) -{ - size_t numread = 0; - BOOL retval; - DWORD byte_count; - BOOL process_exited = FALSE; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - /* Check if there is anything to read */ - retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); - if (!retval) { - log_warn(LD_GENERAL, - "Failed to peek from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* Nothing available: process exited or it is busy */ - - /* Exit if we don't know whether the process is running */ - if (NULL == process) - break; - - /* The process exited and there's nothing left to read from it */ - if (process_exited) - break; - - /* If process is not running, check for output one more time in case - it wrote something after the peek was performed. Otherwise keep on - waiting for output */ - tor_assert(process != NULL); - byte_count = WaitForSingleObject(process->pid.hProcess, 0); - if (WAIT_TIMEOUT != byte_count) - process_exited = TRUE; - - continue; - } - - /* There is data to read; read it */ - retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL); - tor_assert(byte_count + numread <= count); - if (!retval) { - log_warn(LD_GENERAL, "Failed to read from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* End of file */ - break; - } - numread += byte_count; - } - return (ssize_t)numread; -} -#else /* !(defined(_WIN32)) */ -/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes. If - * <b>process</b> is NULL, the function will return immediately if there is - * nothing more to read. Otherwise data will be read until end of file, or - * <b>count</b> bytes are read. Returns the number of bytes read, or -1 on - * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the - * file has been reached. */ -ssize_t -tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof) -{ - size_t numread = 0; - ssize_t result; - - if (eof) - *eof = 0; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - result = read(fd, buf+numread, count-numread); - - if (result == 0) { - log_debug(LD_GENERAL, "read() reached end of file"); - if (eof) - *eof = 1; - break; - } else if (result < 0 && errno == EAGAIN) { - if (process) - continue; - else - break; - } else if (result < 0) { - log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); - return -1; - } - - numread += result; - } - - log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); - return (ssize_t)numread; -} -#endif /* defined(_WIN32) */ - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stdout(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stderr(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Return a string corresponding to <b>stream_status</b>. */ -const char * -stream_status_to_string(enum stream_status stream_status) -{ - switch (stream_status) { - case IO_STREAM_OKAY: - return "okay"; - case IO_STREAM_EAGAIN: - return "temporarily unavailable"; - case IO_STREAM_TERM: - return "terminated"; - case IO_STREAM_CLOSED: - return "closed"; - default: - tor_fragile_assert(); - return "unknown"; - } -} - -/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be - * modified. The resulting smartlist will consist of pointers to buf, so there - * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated - * string. <b>len</b> should be set to the length of the buffer excluding the - * NUL. Non-printable characters (including NUL) will be replaced with "." */ -int -tor_split_lines(smartlist_t *sl, char *buf, int len) -{ - /* Index in buf of the start of the current line */ - int start = 0; - /* Index in buf of the current character being processed */ - int cur = 0; - /* Are we currently in a line */ - char in_line = 0; - - /* Loop over string */ - while (cur < len) { - /* Loop until end of line or end of string */ - for (; cur < len; cur++) { - if (in_line) { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* End of line */ - buf[cur] = '\0'; - /* Point cur to the next line */ - cur++; - /* Line starts at start and ends with a nul */ - break; - } else { - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } else { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* Skip leading vertical space */ - ; - } else { - in_line = 1; - start = cur; - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } - } - /* We are at the end of the line or end of string. If in_line is true there - * is a line which starts at buf+start and ends at a NUL. cur points to - * the character after the NUL. */ - if (in_line) - smartlist_add(sl, (void *)(buf+start)); - in_line = 0; - } - return smartlist_len(sl); -} - -#ifdef _WIN32 - -/** Return a smartlist containing lines outputted from - * <b>handle</b>. Return NULL on error, and set - * <b>stream_status_out</b> appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (HANDLE *handle, - enum stream_status *stream_status_out)) -{ - int pos; - char stdout_buf[600] = {0}; - smartlist_t *lines = NULL; - - tor_assert(stream_status_out); - - *stream_status_out = IO_STREAM_TERM; - - pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL); - if (pos < 0) { - *stream_status_out = IO_STREAM_TERM; - return NULL; - } - if (pos == 0) { - *stream_status_out = IO_STREAM_EAGAIN; - return NULL; - } - - /* End with a null even if there isn't a \r\n at the end */ - /* TODO: What if this is a partial line? */ - stdout_buf[pos] = '\0'; - - /* Split up the buffer */ - lines = smartlist_new(); - tor_split_lines(lines, stdout_buf, pos); - - /* Currently 'lines' is populated with strings residing on the - stack. Replace them with their exact copies on the heap: */ - SMARTLIST_FOREACH(lines, char *, line, - SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line))); - - *stream_status_out = IO_STREAM_OKAY; - - return lines; -} - -#else /* !(defined(_WIN32)) */ - -/** Return a smartlist containing lines outputted from - * <b>fd</b>. Return NULL on error, and set - * <b>stream_status_out</b> appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) -{ - enum stream_status stream_status; - char stdout_buf[400]; - smartlist_t *lines = NULL; - - while (1) { - memset(stdout_buf, 0, sizeof(stdout_buf)); - - stream_status = get_string_from_pipe(fd, - stdout_buf, sizeof(stdout_buf) - 1); - if (stream_status != IO_STREAM_OKAY) - goto done; - - if (!lines) lines = smartlist_new(); - smartlist_split_string(lines, stdout_buf, "\n", 0, 0); - } - - done: - *stream_status_out = stream_status; - return lines; -} - -#endif /* defined(_WIN32) */ - -/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making - * sure it's below <b>count</b> bytes. - * If the string has a trailing newline, we strip it off. - * - * This function is specifically created to handle input from managed - * proxies, according to the pluggable transports spec. Make sure it - * fits your needs before using it. - * - * Returns: - * IO_STREAM_CLOSED: If the stream is closed. - * IO_STREAM_EAGAIN: If there is nothing to read and we should check back - * later. - * IO_STREAM_TERM: If something is wrong with the stream. - * IO_STREAM_OKAY: If everything went okay and we got a string - * in <b>buf_out</b>. */ -enum stream_status -get_string_from_pipe(int fd, char *buf_out, size_t count) -{ - ssize_t ret; - - tor_assert(count <= INT_MAX); - - ret = read(fd, buf_out, count); - - if (ret == 0) - return IO_STREAM_CLOSED; - else if (ret < 0 && errno == EAGAIN) - return IO_STREAM_EAGAIN; - else if (ret < 0) - return IO_STREAM_TERM; - - if (buf_out[ret - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[ret - 1] = '\0'; - } else - buf_out[ret] = '\0'; - - return IO_STREAM_OKAY; -} diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h deleted file mode 100644 index aa3127d62d..0000000000 --- a/src/lib/process/subprocess.h +++ /dev/null @@ -1,134 +0,0 @@ -/* Copyright (c) 2003-2004, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file subprocess.h - * \brief Header for subprocess.c - **/ - -#ifndef TOR_SUBPROCESS_H -#define TOR_SUBPROCESS_H - -#include "lib/cc/torint.h" -#include "lib/testsupport/testsupport.h" -#include <stddef.h> -#ifdef _WIN32 -#include <windows.h> -#endif - -struct smartlist_t; - -void tor_disable_spawning_background_processes(void); - -typedef struct process_handle_t process_handle_t; -struct process_environment_t; -int tor_spawn_background(const char *const filename, const char **argv, - struct process_environment_t *env, - process_handle_t **process_handle_out); - -#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code " - -/** Status of an I/O stream. */ -enum stream_status { - IO_STREAM_OKAY, - IO_STREAM_EAGAIN, - IO_STREAM_TERM, - IO_STREAM_CLOSED -}; - -const char *stream_status_to_string(enum stream_status stream_status); - -enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); - -/* Values of process_handle_t.status. */ -#define PROCESS_STATUS_NOTRUNNING 0 -#define PROCESS_STATUS_RUNNING 1 -#define PROCESS_STATUS_ERROR -1 - -#ifdef SUBPROCESS_PRIVATE -struct waitpid_callback_t; - -/** Structure to represent the state of a process with which Tor is - * communicating. The contents of this structure are private to util.c */ -struct process_handle_t { - /** One of the PROCESS_STATUS_* values */ - int status; -#ifdef _WIN32 - HANDLE stdin_pipe; - HANDLE stdout_pipe; - HANDLE stderr_pipe; - PROCESS_INFORMATION pid; -#else /* !(defined(_WIN32)) */ - int stdin_pipe; - int stdout_pipe; - int stderr_pipe; - pid_t pid; - /** If the process has not given us a SIGCHLD yet, this has the - * waitpid_callback_t that gets invoked once it has. Otherwise this - * contains NULL. */ - struct waitpid_callback_t *waitpid_cb; - /** The exit status reported by waitpid. */ - int waitpid_exit_status; -#endif /* defined(_WIN32) */ -}; -#endif /* defined(SUBPROCESS_PRIVATE) */ - -/* Return values of tor_get_exit_code() */ -#define PROCESS_EXIT_RUNNING 1 -#define PROCESS_EXIT_EXITED 0 -#define PROCESS_EXIT_ERROR -1 -int tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code); -int tor_split_lines(struct smartlist_t *sl, char *buf, int len); -#ifdef _WIN32 -ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process); -#else -ssize_t tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof); -#endif /* defined(_WIN32) */ -ssize_t tor_read_all_from_process_stdout( - const process_handle_t *process_handle, char *buf, size_t count); -ssize_t tor_read_all_from_process_stderr( - const process_handle_t *process_handle, char *buf, size_t count); -char *tor_join_win_cmdline(const char *argv[]); - -int tor_process_get_pid(process_handle_t *process_handle); -#ifdef _WIN32 -HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); -#else -int tor_process_get_stdout_pipe(process_handle_t *process_handle); -#endif - -#ifdef _WIN32 -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle, - enum stream_status *stream_status)); -#else -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd, - enum stream_status *stream_status)); -#endif /* defined(_WIN32) */ - -int tor_terminate_process(process_handle_t *process_handle); - -MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)); - -#ifdef SUBPROCESS_PRIVATE -/* Prototypes for private functions only used by util.c (and unit tests) */ - -#ifndef _WIN32 -STATIC int format_helper_exit_status(unsigned char child_state, - int saved_errno, char *hex_errno); - -/* Space for hex values of child state, a slash, saved_errno (with - leading minus) and newline (no null) */ -#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \ - 1 + sizeof(int) * 2 + 1) -#endif /* !defined(_WIN32) */ - -#endif /* defined(SUBPROCESS_PRIVATE) */ - -#endif diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c index 9b626394d2..2b38481aeb 100644 --- a/src/lib/process/waitpid.c +++ b/src/lib/process/waitpid.c @@ -16,7 +16,7 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" -#include "ht.h" +#include "ext/ht.h" #ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c new file mode 100644 index 0000000000..ad65886422 --- /dev/null +++ b/src/lib/process/winprocess_sys.c @@ -0,0 +1,66 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.c + * \brief Subsystem object for windows process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/winprocess_sys.h" + +#include <stdbool.h> +#include <stddef.h> + +#ifdef _WIN32 +#include <windows.h> + +#define WINPROCESS_SYS_ENABLED true + +static int +subsys_winprocess_initialize(void) +{ +#ifndef HeapEnableTerminationOnCorruption +#define HeapEnableTerminationOnCorruption 1 +#endif + + /* On heap corruption, just give up; don't try to play along. */ + HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); + + /* SetProcessDEPPolicy is only supported on 32-bit Windows. + * (On 64-bit Windows it always fails, and some compilers don't like the + * PSETDEP cast.) + * 32-bit Windows defines _WIN32. + * 64-bit Windows defines _WIN32 and _WIN64. */ +#ifndef _WIN64 + /* Call SetProcessDEPPolicy to permanently enable DEP. + The function will not resolve on earlier versions of Windows, + and failure is not dangerous. */ + HMODULE hMod = GetModuleHandleA("Kernel32.dll"); + if (hMod) { + typedef BOOL (WINAPI *PSETDEP)(DWORD); + PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, + "SetProcessDEPPolicy"); + if (setdeppolicy) { + /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ + setdeppolicy(3); + } + } +#endif /* !defined(_WIN64) */ + + return 0; +} +#else /* !defined(_WIN32) */ +#define WINPROCESS_SYS_ENABLED false +#define subsys_winprocess_initialize NULL +#endif /* defined(_WIN32) */ + +const subsys_fns_t sys_winprocess = { + .name = "winprocess", + /* HeapEnableTerminationOnCorruption and setdeppolicy() are security + * features, we want them to run first. */ + .level = -100, + .supported = WINPROCESS_SYS_ENABLED, + .initialize = subsys_winprocess_initialize, +}; diff --git a/src/lib/process/winprocess_sys.h b/src/lib/process/winprocess_sys.h new file mode 100644 index 0000000000..7ab2aa04a6 --- /dev/null +++ b/src/lib/process/winprocess_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.h + * \brief Declare subsystem object for winprocess.c + **/ + +#ifndef TOR_WINPROCESS_SYS_H +#define TOR_WINPROCESS_SYS_H + +extern const struct subsys_fns_t sys_winprocess; + +#endif /* !defined(TOR_WINPROCESS_SYS_H) */ diff --git a/src/lib/pubsub/.may_include b/src/lib/pubsub/.may_include new file mode 100644 index 0000000000..5623492f00 --- /dev/null +++ b/src/lib/pubsub/.may_include @@ -0,0 +1,10 @@ +orconfig.h + +lib/cc/*.h +lib/container/*.h +lib/dispatch/*.h +lib/intmath/*.h +lib/log/*.h +lib/malloc/*.h +lib/pubsub/*.h +lib/string/*.h diff --git a/src/lib/pubsub/include.am b/src/lib/pubsub/include.am new file mode 100644 index 0000000000..e2abebcd40 --- /dev/null +++ b/src/lib/pubsub/include.am @@ -0,0 +1,28 @@ + +noinst_LIBRARIES += src/lib/libtor-pubsub.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-pubsub-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_pubsub_a_SOURCES = \ + src/lib/pubsub/pubsub_build.c \ + src/lib/pubsub/pubsub_check.c \ + src/lib/pubsub/pubsub_publish.c + +src_lib_libtor_pubsub_testing_a_SOURCES = \ + $(src_lib_libtor_pubsub_a_SOURCES) +src_lib_libtor_pubsub_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_pubsub_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/pubsub/pub_binding_st.h \ + src/lib/pubsub/pubsub.h \ + src/lib/pubsub/pubsub_build.h \ + src/lib/pubsub/pubsub_builder_st.h \ + src/lib/pubsub/pubsub_connect.h \ + src/lib/pubsub/pubsub_flags.h \ + src/lib/pubsub/pubsub_macros.h \ + src/lib/pubsub/pubsub_publish.h diff --git a/src/lib/pubsub/pub_binding_st.h b/src/lib/pubsub/pub_binding_st.h new file mode 100644 index 0000000000..d841bf3f54 --- /dev/null +++ b/src/lib/pubsub/pub_binding_st.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pub_binding_st.h + * @brief Declaration of pub_binding_t. + * + * This is an internal type for the pubsub implementation. + */ + +#ifndef TOR_PUB_BINDING_ST_H +#define TOR_PUB_BINDING_ST_H + +#include "lib/dispatch/msgtypes.h" +struct dispatch_t; + +/** + * A pub_binding_t is an opaque object that subsystems use to publish + * messages. The DISPATCH_ADD_PUB*() macros set it up. + **/ +typedef struct pub_binding_t { + /** + * A pointer to a configured dispatch_t object. This is filled in + * when the dispatch_t is finally constructed. + **/ + struct dispatch_t *dispatch_ptr; + /** + * A template for the msg_t fields that are filled in for this message. + * This is copied into outgoing messages, ensuring that their fields are set + * corretly. + **/ + msg_t msg_template; +} pub_binding_t; + +#endif /* !defined(TOR_PUB_BINDING_ST_H) */ diff --git a/src/lib/pubsub/pubsub.h b/src/lib/pubsub/pubsub.h new file mode 100644 index 0000000000..5346b07517 --- /dev/null +++ b/src/lib/pubsub/pubsub.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub.h + * @brief Header for OO publish-subscribe functionality. + * + * This module provides a wrapper around the "dispatch" module, + * ensuring type-safety and allowing us to do static analysis on + * publication and subscriptions. + * + * With this module, we enforce: + * <ul> + * <li>that every message has (potential) publishers and subscribers; + * <li>that every message is published and subscribed from the correct + * channels, with the correct type ID, every time it is published. + * <li>that type IDs correspond to a single C type, and that the C types are + * used correctly. + * <li>that when a message is published or subscribed, it is done with + * a correct subsystem identifier + * </ul> + * + * We do this by making "publication requests" and "subscription requests" + * into objects, and doing some computation on them before we create + * a dispatch_t with them. + * + * Rather than using the dispatch module directly, a publishing module + * receives a "binding" object that it uses to send messages with the right + * settings. + * + * Most users of this module will want to use this header, and the + * pubsub_macros.h header for convenience. + */ + +/* + * + * Overview: Messages are sent over channels. Before sending a message on a + * channel, or receiving a message on a channel, a subsystem needs to register + * that it publishes, or subscribes, to that message, on that channel. + * + * Messages, channels, and subsystems are represented internally as short + * integers, though they are associated with human-readable strings for + * initialization and debugging. + * + * When registering for a message, a subsystem must say whether it is an + * exclusive publisher/subscriber to that message type, or whether other + * subsystems may also publish/subscribe to it. + * + * All messages and their publishers/subscribers must be registered early in + * the initialization process. + * + * By default, it is an error for a message type to have publishers and no + * subscribers on a channel, or subscribers and no publishers on a channel. + * + * A subsystem may register for a message with a note that delivery or + * production is disabled -- for example, because the subsystem is + * disabled at compile-time. It is not an error for a message type to + * have all of its publishers or subscribers disabled. + * + * After a message is sent, it is delivered to every recipient. This + * delivery happens from the top level of the event loop; it may be + * interleaved with network events, timers, etc. + * + * Messages may have associated data. This data is typed, and is owned + * by the message. Strings, byte-arrays, and integers have built-in + * support. Other types may be added. If objects are to be sent, + * they should be identified by handle. If an object requires cleanup, + * it should be declared with an associated free function. + * + * Semantically, if two subsystems communicate only by this kind of + * message passing, neither is considered to depend on the other, though + * both are considered to have a dependency on the message and on any + * types it contains. + * + * (Or generational index?) + */ +#ifndef TOR_PUBSUB_PUBSUB_H +#define TOR_PUBSUB_PUBSUB_H + +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_connect.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pubsub_macros.h" +#include "lib/pubsub/pubsub_publish.h" + +#endif /* !defined(TOR_PUBSUB_PUBSUB_H) */ diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c new file mode 100644 index 0000000000..e44b7d76ec --- /dev/null +++ b/src/lib/pubsub/pubsub_build.c @@ -0,0 +1,307 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_build.c + * @brief Construct a dispatch_t in safer, more OO way. + **/ + +#define PUBSUB_PRIVATE + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_cfg.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/msgtypes.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_build.h" +#include "lib/pubsub/pubsub_builder_st.h" +#include "lib/pubsub/pubsub_connect.h" + +#include "lib/container/smartlist.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" + + #include <string.h> + +/** Construct and return a new empty pubsub_items_t. */ +static pubsub_items_t * +pubsub_items_new(void) +{ + pubsub_items_t *cfg = tor_malloc_zero(sizeof(*cfg)); + cfg->items = smartlist_new(); + cfg->type_items = smartlist_new(); + return cfg; +} + +/** Release all storage held in a pubsub_items_t. */ +void +pubsub_items_free_(pubsub_items_t *cfg) +{ + if (! cfg) + return; + SMARTLIST_FOREACH(cfg->items, pubsub_cfg_t *, item, tor_free(item)); + SMARTLIST_FOREACH(cfg->type_items, + pubsub_type_cfg_t *, item, tor_free(item)); + smartlist_free(cfg->items); + smartlist_free(cfg->type_items); + tor_free(cfg); +} + +/** Construct and return a new pubsub_builder_t. */ +pubsub_builder_t * +pubsub_builder_new(void) +{ + dispatch_naming_init(); + + pubsub_builder_t *pb = tor_malloc_zero(sizeof(*pb)); + pb->cfg = dcfg_new(); + pb->items = pubsub_items_new(); + return pb; +} + +/** + * Release all storage held by a pubsub_builder_t. + * + * You'll (mostly) only want to call this function on an error case: if you're + * constructing a dispatch_t instead, you should call + * pubsub_builder_finalize() to consume the pubsub_builder_t. + */ +void +pubsub_builder_free_(pubsub_builder_t *pb) +{ + if (pb == NULL) + return; + pubsub_items_free(pb->items); + dcfg_free(pb->cfg); + tor_free(pb); +} + +/** + * Create and return a pubsub_connector_t for the subsystem with ID + * <b>subsys</b> to use in adding publications, subscriptions, and types to + * <b>builder</b>. + **/ +pubsub_connector_t * +pubsub_connector_for_subsystem(pubsub_builder_t *builder, + subsys_id_t subsys) +{ + tor_assert(builder); + ++builder->n_connectors; + + pubsub_connector_t *con = tor_malloc_zero(sizeof(*con)); + + con->builder = builder; + con->subsys_id = subsys; + + return con; +} + +/** + * Release all storage held by a pubsub_connector_t. + **/ +void +pubsub_connector_free_(pubsub_connector_t *con) +{ + if (!con) + return; + + if (con->builder) { + --con->builder->n_connectors; + tor_assert(con->builder->n_connectors >= 0); + } + tor_free(con); +} + +/** + * Use <b>con</b> to add a request for being able to publish messages of type + * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>. + **/ +int +pubsub_add_pub_(pubsub_connector_t *con, + pub_binding_t *out, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line) +{ + pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + + memset(out, 0, sizeof(*out)); + cfg->is_publish = true; + + out->msg_template.sender = cfg->subsys = con->subsys_id; + out->msg_template.channel = cfg->channel = channel; + out->msg_template.msg = cfg->msg = msg; + out->msg_template.type = cfg->type = type; + + cfg->flags = flags; + cfg->added_by_file = file; + cfg->added_by_line = line; + + /* We're grabbing a pointer to the pub_binding_t so we can tell it about + * the dispatcher later on. + */ + cfg->pub_binding = out; + + smartlist_add(con->builder->items->items, cfg); + + if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0) + goto err; + if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0) + goto err; + + return 0; + err: + ++con->builder->n_errors; + return -1; +} + +/** + * Use <b>con</b> to add a request for being able to publish messages of type + * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>, + * passing them to the callback in <b>recv_fn</b>. + **/ +int +pubsub_add_sub_(pubsub_connector_t *con, + recv_fn_t recv_fn, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line) +{ + pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + + cfg->is_publish = false; + cfg->subsys = con->subsys_id; + cfg->channel = channel; + cfg->msg = msg; + cfg->type = type; + cfg->flags = flags; + cfg->added_by_file = file; + cfg->added_by_line = line; + + cfg->recv_fn = recv_fn; + + smartlist_add(con->builder->items->items, cfg); + + if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0) + goto err; + if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0) + goto err; + if (! (flags & DISP_FLAG_STUB)) { + if (dcfg_add_recv(con->builder->cfg, msg, cfg->subsys, recv_fn) < 0) + goto err; + } + + return 0; + err: + ++con->builder->n_errors; + return -1; +} + +/** + * Use <b>con</b> to define the functions to use for manipulating the type + * <b>type</b>. Any function pointers left as NULL will be implemented as + * no-ops. + **/ +int +pubsub_connector_register_type_(pubsub_connector_t *con, + msg_type_id_t type, + dispatch_typefns_t *fns, + const char *file, + unsigned line) +{ + pubsub_type_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + cfg->type = type; + memcpy(&cfg->fns, fns, sizeof(*fns)); + cfg->subsys = con->subsys_id; + cfg->added_by_file = file; + cfg->added_by_line = line; + + smartlist_add(con->builder->items->type_items, cfg); + + if (dcfg_type_set_fns(con->builder->cfg, type, fns) < 0) + goto err; + + return 0; + err: + ++con->builder->n_errors; + return -1; +} + +/** + * Initialize the dispatch_ptr field in every relevant publish binding + * for <b>d</b>. + */ +static void +pubsub_items_install_bindings(pubsub_items_t *items, + dispatch_t *d) +{ + SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) { + if (cfg->pub_binding) { + // XXXX we could skip this for STUB publishers, and for any publishers + // XXXX where all subscribers are STUB. + cfg->pub_binding->dispatch_ptr = d; + } + } SMARTLIST_FOREACH_END(cfg); +} + +/** + * Remove the dispatch_ptr fields for all the relevant publish bindings + * in <b>items</b>. The prevents subsequent dispatch_pub_() calls from + * sending messages to a dispatcher that has been freed. + **/ +void +pubsub_items_clear_bindings(pubsub_items_t *items) +{ + SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) { + if (cfg->pub_binding) { + cfg->pub_binding->dispatch_ptr = NULL; + } + } SMARTLIST_FOREACH_END(cfg); +} + +/** + * Create a new dispatcher as configured in a pubsub_builder_t. + * + * Consumes and frees its input. + **/ +dispatch_t * +pubsub_builder_finalize(pubsub_builder_t *builder, + pubsub_items_t **items_out) +{ + dispatch_t *dispatcher = NULL; + tor_assert_nonfatal(builder->n_connectors == 0); + + if (pubsub_builder_check(builder) < 0) + goto err; + + if (builder->n_errors) { + log_warn(LD_GENERAL, "At least one error occurred previously when " + "configuring the dispatcher."); + goto err; + } + + dispatcher = dispatch_new(builder->cfg); + + if (!dispatcher) + goto err; + + pubsub_items_install_bindings(builder->items, dispatcher); + if (items_out) { + *items_out = builder->items; + builder->items = NULL; /* Prevent free */ + } + + err: + pubsub_builder_free(builder); + return dispatcher; +} diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h new file mode 100644 index 0000000000..5a0c5f5bd3 --- /dev/null +++ b/src/lib/pubsub/pubsub_build.h @@ -0,0 +1,92 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_build.h + * @brief Header used for constructing the OO publish-subscribe facility. + * + * (See pubsub.h for more general information on this API.) + **/ + +#ifndef TOR_PUBSUB_BUILD_H +#define TOR_PUBSUB_BUILD_H + +#include "lib/dispatch/msgtypes.h" + +struct dispatch_t; +struct pubsub_connector_t; + +/** + * A "dispatch builder" is an incomplete dispatcher, used when + * registering messages. It does not have the same integrity guarantees + * as a dispatcher. It cannot actually handle messages itself: once all + * subsystems have registered, it is converted into a dispatch_t. + **/ +typedef struct pubsub_builder_t pubsub_builder_t; + +/** + * A "pubsub items" holds the configuration items used to configure a + * pubsub_builder. After the builder is finalized, this field is extracted, + * and used later to tear down pointers that enable publishing. + **/ +typedef struct pubsub_items_t pubsub_items_t; + +/** + * Create a new pubsub_builder. This should only happen in the + * main-init code. + */ +pubsub_builder_t *pubsub_builder_new(void); + +/** DOCDOC */ +int pubsub_builder_check(pubsub_builder_t *); + +/** + * Free a pubsub builder. This should only happen on error paths, where + * we have decided not to construct a dispatcher for some reason. + */ +#define pubsub_builder_free(db) \ + FREE_AND_NULL(pubsub_builder_t, pubsub_builder_free_, (db)) + +/** Internal implementation of pubsub_builder_free(). */ +void pubsub_builder_free_(pubsub_builder_t *); + +/** + * Create a pubsub connector that a single subsystem will use to + * register its messages. The main-init code does this during susbsystem + * initialization. + */ +struct pubsub_connector_t *pubsub_connector_for_subsystem(pubsub_builder_t *, + subsys_id_t); + +/** + * The main-init code does this after subsystem initialization. + */ +#define pubsub_connector_free(c) \ + FREE_AND_NULL(struct pubsub_connector_t, pubsub_connector_free_, (c)) + +void pubsub_connector_free_(struct pubsub_connector_t *); + +/** + * Constructs a dispatcher from a dispatch_builder, after checking that the + * invariances on the messages, channels, and connections have been + * respected. + * + * This should happen after every subsystem has initialized, and before + * entering the mainloop. + */ +struct dispatch_t *pubsub_builder_finalize(pubsub_builder_t *, + pubsub_items_t **items_out); + +/** + * Clear all pub_binding_t backpointers in <b>items</b>. + **/ +void pubsub_items_clear_bindings(pubsub_items_t *items); + +#define pubsub_items_free(cfg) \ + FREE_AND_NULL(pubsub_items_t, pubsub_items_free_, (cfg)) +void pubsub_items_free_(pubsub_items_t *cfg); + +#endif /* !defined(TOR_PUBSUB_BUILD_H) */ diff --git a/src/lib/pubsub/pubsub_builder_st.h b/src/lib/pubsub/pubsub_builder_st.h new file mode 100644 index 0000000000..545aa3f3ef --- /dev/null +++ b/src/lib/pubsub/pubsub_builder_st.h @@ -0,0 +1,161 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_builder_st.h + * + * @brief private structures used for configuring dispatchers and messages. + */ + +#ifndef TOR_PUBSUB_BUILDER_ST_H +#define TOR_PUBSUB_BUILDER_ST_H + +#ifdef PUBSUB_PRIVATE + +#include <stdbool.h> +#include <stddef.h> + +struct dispatch_cfg_t; +struct smartlist_t; +struct pub_binding_t; + +/** + * Configuration for a single publication or subscription request. + * + * These can be stored while the dispatcher is in use, but are only used for + * setup, teardown, and debugging. + * + * There are various fields in this request describing the message; all of + * them must match other descriptions of the message, or a bug has occurred. + **/ +typedef struct pubsub_cfg_t { + /** True if this is a publishing request; false for a subscribing request. */ + bool is_publish; + /** The system making this request. */ + subsys_id_t subsys; + /** The channel on which the message is to be sent. */ + channel_id_t channel; + /** The message ID to be sent or received. */ + message_id_t msg; + /** The C type associated with the message. */ + msg_type_id_t type; + /** One or more DISP_FLAGS_* items, combined with bitwise OR. */ + unsigned flags; + + /** + * Publishing only: a pub_binding object that will receive the binding for + * this request. We will finish filling this in when the dispatcher is + * constructed, so that the subsystem can publish then and not before. + */ + struct pub_binding_t *pub_binding; + + /** + * Subscribing only: a function to receive message objects for this request. + */ + recv_fn_t recv_fn; + + /** The file from which this message was configured */ + const char *added_by_file; + /** The line at which this message was configured */ + unsigned added_by_line; +} pubsub_cfg_t; + +/** + * Configuration request for a single C type. + * + * These are stored while the dispatcher is in use, but are only used for + * setup, teardown, and debugging. + **/ +typedef struct pubsub_type_cfg_t { + /** + * The identifier for this type. + */ + msg_type_id_t type; + /** + * Functions to use when manipulating the type. + */ + dispatch_typefns_t fns; + + /** The subsystem that configured this type. */ + subsys_id_t subsys; + /** The file from which this type was configured */ + const char *added_by_file; + /** The line at which this type was configured */ + unsigned added_by_line; +} pubsub_type_cfg_t; + +/** + * The set of configuration requests for a dispatcher, as made by various + * subsystems. + **/ +struct pubsub_items_t { + /** List of pubsub_cfg_t. */ + struct smartlist_t *items; + /** List of pubsub_type_cfg_t. */ + struct smartlist_t *type_items; +}; + +/** + * Type used to construct a dispatcher. We use this type to build up the + * configuration for a dispatcher, and then pass ownership of that + * configuration to the newly constructed dispatcher. + **/ +struct pubsub_builder_t { + /** Number of outstanding pubsub_connector_t objects pointing to this + * pubsub_builder_t. */ + int n_connectors; + /** Number of errors encountered while constructing this object so far. */ + int n_errors; + /** In-progress configuration that we're constructing, as a list of the + * requests that have been made. */ + struct pubsub_items_t *items; + /** In-progress configuration that we're constructing, in a form that can + * be converted to a dispatch_t. */ + struct dispatch_cfg_t *cfg; +}; + +/** + * Type given to a subsystem when adding connections to a pubsub_builder_t. + * We use this type to force each subsystem to get blamed for the + * publications, subscriptions, and types that it adds. + **/ +struct pubsub_connector_t { + /** The pubsub_builder that this connector refers to. */ + struct pubsub_builder_t *builder; + /** The subsystem that has been given this connector. */ + subsys_id_t subsys_id; +}; + +/** + * Helper structure used when constructing a dispatcher that sorts the + * pubsub_cfg_t objects in various ways. + **/ +typedef struct pubsub_adjmap_t { + /* XXXX The next three fields are currently constructed but not yet + * XXXX used. I believe we'll want them in the future, though. -nickm + */ + /** Number of subsystems; length of the *_by_subsys arrays. */ + size_t n_subsystems; + /** Array of lists of publisher pubsub_cfg_t objects, indexed by + * subsystem. */ + struct smartlist_t **pub_by_subsys; + /** Array of lists of subscriber pubsub_cfg_t objects, indexed by + * subsystem. */ + struct smartlist_t **sub_by_subsys; + + /** Number of message IDs; length of the *_by_msg arrays. */ + size_t n_msgs; + /** Array of lists of publisher pubsub_cfg_t objects, indexed by + * message ID. */ + struct smartlist_t **pub_by_msg; + /** Array of lists of subscriber pubsub_cfg_t objects, indexed by + * message ID. */ + struct smartlist_t **sub_by_msg; +} pubsub_adjmap_t; + +#endif /* defined(PUBSUB_PRIVATE) */ + +#endif /* !defined(TOR_PUBSUB_BUILDER_ST_H) */ diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c new file mode 100644 index 0000000000..bf1196df2c --- /dev/null +++ b/src/lib/pubsub/pubsub_check.c @@ -0,0 +1,412 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_check.c + * @brief Enforce various requirements on a pubsub_builder. + **/ + +#define PUBSUB_PRIVATE + +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/msgtypes.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pubsub_builder_st.h" +#include "lib/pubsub/pubsub_build.h" + +#include "lib/container/bitarray.h" +#include "lib/container/smartlist.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/compat_string.h" + +#include <string.h> + +static void pubsub_adjmap_add(pubsub_adjmap_t *map, + const pubsub_cfg_t *item); + +/** + * Helper: contruct and return a new pubsub_adjacency_map from <b>cfg</b>. + * Return NULL on error. + **/ +static pubsub_adjmap_t * +pubsub_build_adjacency_map(const pubsub_items_t *cfg) +{ + pubsub_adjmap_t *map = tor_malloc_zero(sizeof(*map)); + const size_t n_subsystems = get_num_subsys_ids(); + const size_t n_msgs = get_num_message_ids(); + + map->n_subsystems = n_subsystems; + map->n_msgs = n_msgs; + + map->pub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*)); + map->sub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*)); + map->pub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*)); + map->sub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*)); + + SMARTLIST_FOREACH_BEGIN(cfg->items, const pubsub_cfg_t *, item) { + pubsub_adjmap_add(map, item); + } SMARTLIST_FOREACH_END(item); + + return map; +} + +/** + * Helper: add a single pubsub_cfg_t to an adjacency map. + **/ +static void +pubsub_adjmap_add(pubsub_adjmap_t *map, + const pubsub_cfg_t *item) +{ + smartlist_t **by_subsys; + smartlist_t **by_msg; + + tor_assert(item->subsys < map->n_subsystems); + tor_assert(item->msg < map->n_msgs); + + if (item->is_publish) { + by_subsys = &map->pub_by_subsys[item->subsys]; + by_msg = &map->pub_by_msg[item->msg]; + } else { + by_subsys = &map->sub_by_subsys[item->subsys]; + by_msg = &map->sub_by_msg[item->msg]; + } + + if (! *by_subsys) + *by_subsys = smartlist_new(); + if (! *by_msg) + *by_msg = smartlist_new(); + smartlist_add(*by_subsys, (void*) item); + smartlist_add(*by_msg, (void *) item); +} + +/** + * Release all storage held by m and set m to NULL. + **/ +#define pubsub_adjmap_free(m) \ + FREE_AND_NULL(pubsub_adjmap_t, pubsub_adjmap_free_, m) + +/** + * Free every element of an <b>n</b>-element array of smartlists, then + * free the array itself. + **/ +static void +pubsub_adjmap_free_helper(smartlist_t **lsts, size_t n) +{ + if (!lsts) + return; + + for (unsigned i = 0; i < n; ++i) { + smartlist_free(lsts[i]); + } + tor_free(lsts); +} + +/** + * Release all storage held by <b>map</b>. + **/ +static void +pubsub_adjmap_free_(pubsub_adjmap_t *map) +{ + if (!map) + return; + pubsub_adjmap_free_helper(map->pub_by_subsys, map->n_subsystems); + pubsub_adjmap_free_helper(map->sub_by_subsys, map->n_subsystems); + pubsub_adjmap_free_helper(map->pub_by_msg, map->n_msgs); + pubsub_adjmap_free_helper(map->sub_by_msg, map->n_msgs); + tor_free(map); +} + +/** + * Helper: return the length of <b>sl</b>, or 0 if sl is NULL. + **/ +static int +smartlist_len_opt(const smartlist_t *sl) +{ + if (sl) + return smartlist_len(sl); + else + return 0; +} + +/** Return a pointer to a statically allocated string encoding the + * dispatcher flags in <b>flags</b>. */ +static const char * +format_flags(unsigned flags) +{ + static char buf[32]; + buf[0] = 0; + if (flags & DISP_FLAG_EXCL) { + strlcat(buf, " EXCL", sizeof(buf)); + } + if (flags & DISP_FLAG_STUB) { + strlcat(buf, " STUB", sizeof(buf)); + } + return buf[0] ? buf+1 : buf; +} + +/** + * Log a message containing a description of <b>cfg</b> at severity, prefixed + * by the string <b>prefix</b>. + */ +static void +pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix) +{ + tor_assert(prefix); + + tor_log(severity, LD_MESG, + "%s%s %s: %s{%s} on %s (%s) <%u %u %u %u %x> [%s:%d]", + prefix, + get_subsys_id_name(cfg->subsys), + cfg->is_publish ? "PUB" : "SUB", + get_message_id_name(cfg->msg), + get_msg_type_id_name(cfg->type), + get_channel_id_name(cfg->channel), + format_flags(cfg->flags), + cfg->subsys, cfg->msg, cfg->type, cfg->channel, cfg->flags, + cfg->added_by_file, cfg->added_by_line); +} + +/** + * Helper: fill a bitarray <b>out</b> with entries corresponding to the + * subsystems listed in <b>items</b>. + **/ +static void +get_message_bitarray(const pubsub_adjmap_t *map, + const smartlist_t *items, + bitarray_t **out) +{ + *out = bitarray_init_zero((unsigned)map->n_subsystems); + if (! items) + return; + + SMARTLIST_FOREACH_BEGIN(items, const pubsub_cfg_t *, cfg) { + bitarray_set(*out, cfg->subsys); + } SMARTLIST_FOREACH_END(cfg); +} + +/** + * Helper for lint_message: check that all the pubsub_cfg_t items in the two + * respective smartlists obey our local graph topology rules. + * + * (Right now this is just a matter of "each subsystem only + * publishes/subscribes once; no subsystem is a publisher and subscriber for + * the same message.") + * + * Return 0 on success, -1 on failure. + **/ +static int +lint_message_graph(const pubsub_adjmap_t *map, + message_id_t msg, + const smartlist_t *pub, + const smartlist_t *sub) +{ + bitarray_t *published_by = NULL; + bitarray_t *subscribed_by = NULL; + bool ok = true; + + get_message_bitarray(map, pub, &published_by); + get_message_bitarray(map, sub, &subscribed_by); + + /* Check whether any subsystem is publishing and subscribing the same + * message. [??] + */ + for (unsigned i = 0; i < map->n_subsystems; ++i) { + if (bitarray_is_set(published_by, i) && + bitarray_is_set(subscribed_by, i)) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" is published and subscribed by the same " + "subsystem \"%s\".", + get_message_id_name(msg), + get_subsys_id_name(i)); + ok = false; + } + } + + bitarray_free(published_by); + bitarray_free(subscribed_by); + + return ok ? 0 : -1; +} + +/** + * Helper for lint_message: check that all the pubsub_cfg_t items in the two + * respective smartlists have compatible flags, channels, and types. + **/ +static int +lint_message_consistency(message_id_t msg, + const smartlist_t *pub, + const smartlist_t *sub) +{ + if (!smartlist_len_opt(pub) && !smartlist_len_opt(sub)) + return 0; // LCOV_EXCL_LINE -- this was already checked. + + /* The 'all' list has the publishers and the subscribers. */ + smartlist_t *all = smartlist_new(); + if (pub) + smartlist_add_all(all, pub); + if (sub) + smartlist_add_all(all, sub); + + const pubsub_cfg_t *item0 = smartlist_get(all, 0); + + /* Indicates which subsystems we've found publishing/subscribing here. */ + bool pub_excl = false, sub_excl = false, chan_same = true, type_same = true; + + /* Simple message consistency properties across messages. + */ + SMARTLIST_FOREACH_BEGIN(all, const pubsub_cfg_t *, cfg) { + chan_same &= (cfg->channel == item0->channel); + type_same &= (cfg->type == item0->type); + if (cfg->is_publish) + pub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0; + else + sub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0; + } SMARTLIST_FOREACH_END(cfg); + + bool ok = true; + + if (! chan_same) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" is associated with multiple inconsistent " + "channels.", + get_message_id_name(msg)); + ok = false; + } + if (! type_same) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" is associated with multiple inconsistent " + "message types.", + get_message_id_name(msg)); + ok = false; + } + + /* Enforce exclusive-ness for publishers and subscribers that have asked for + * it. + */ + if (pub_excl && smartlist_len_opt(pub) > 1) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" has multiple publishers, but at least one is " + "marked as exclusive.", + get_message_id_name(msg)); + ok = false; + } + if (sub_excl && smartlist_len_opt(sub) > 1) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" has multiple subscribers, but at least one is " + "marked as exclusive.", + get_message_id_name(msg)); + ok = false; + } + + smartlist_free(all); + + return ok ? 0 : -1; +} + +/** + * Check whether there are any errors or inconsistencies for the message + * described by <b>msg</b> in <b>map</b>. If there are problems, log about + * them, and return -1. Otherwise return 0. + **/ +static int +lint_message(const pubsub_adjmap_t *map, message_id_t msg) +{ + /* NOTE: Some of the checks in this function are maybe over-zealous, and we + * might not want to have them forever. I've marked them with [?] below. + */ + if (BUG(msg >= map->n_msgs)) + return 0; // LCOV_EXCL_LINE + + const smartlist_t *pub = map->pub_by_msg[msg]; + const smartlist_t *sub = map->sub_by_msg[msg]; + + const size_t n_pub = smartlist_len_opt(pub); + const size_t n_sub = smartlist_len_opt(sub); + + if (n_pub == 0 && n_sub == 0) { + log_info(LD_MESG, "Nobody is publishing or subscribing to message " + "\"%s\".", + get_message_id_name(msg)); + return 0; // No publishers or subscribers: nothing to do. + } + /* We'll set this to false if there are any problems. */ + bool ok = true; + + /* First make sure that if there are publishers, there are subscribers. */ + if (n_pub == 0) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" has subscribers, but no publishers.", + get_message_id_name(msg)); + ok = false; + } else if (n_sub == 0) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" has publishers, but no subscribers.", + get_message_id_name(msg)); + ok = false; + } + + /* Check the message graph topology. */ + if (lint_message_graph(map, msg, pub, sub) < 0) + ok = false; + + /* Check whether the messages have the same fields set on them. */ + if (lint_message_consistency(msg, pub, sub) < 0) + ok = false; + + if (!ok) { + /* There was a problem -- let's log all the publishers and subscribers on + * this message */ + if (pub) { + SMARTLIST_FOREACH(pub, pubsub_cfg_t *, cfg, + pubsub_cfg_dump(cfg, LOG_WARN, " ")); + } + if (sub) { + SMARTLIST_FOREACH(sub, pubsub_cfg_t *, cfg, + pubsub_cfg_dump(cfg, LOG_WARN, " ")); + } + } + + return ok ? 0 : -1; +} + +/** + * Check all the messages in <b>map</b> for consistency. Return 0 on success, + * -1 on problems. + **/ +static int +pubsub_adjmap_check(const pubsub_adjmap_t *map) +{ + bool all_ok = true; + for (unsigned i = 0; i < map->n_msgs; ++i) { + if (lint_message(map, i) < 0) { + all_ok = false; + } + } + return all_ok ? 0 : -1; +} + +/** + * Check builder for consistency and various constraints. Return 0 on success, + * -1 on failure. + **/ +int +pubsub_builder_check(pubsub_builder_t *builder) +{ + pubsub_adjmap_t *map = pubsub_build_adjacency_map(builder->items); + int rv = -1; + + if (!map) + goto err; // should be impossible + + if (pubsub_adjmap_check(map) < 0) + goto err; + + rv = 0; + err: + pubsub_adjmap_free(map); + return rv; +} diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h new file mode 100644 index 0000000000..0ad106044e --- /dev/null +++ b/src/lib/pubsub/pubsub_connect.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_connect.h + * @brief Header for functions that add relationships to a pubsub builder. + * + * These functions are used by modules that need to add publication and + * subscription requests. Most users will want to call these functions + * indirectly, via the macros in pubsub_macros.h. + **/ + +#ifndef TOR_PUBSUB_CONNECT_H +#define TOR_PUBSUB_CONNECT_H + +#include "lib/dispatch/msgtypes.h" + +struct pub_binding_t; +/** + * A "dispatch connector" is a view of the dispatcher that a subsystem + * uses while initializing itself. It is specific to the subsystem, and + * ensures that each subsystem doesn't need to identify itself + * repeatedly while registering its messages. + **/ +typedef struct pubsub_connector_t pubsub_connector_t; + +int pubsub_add_pub_(struct pubsub_connector_t *con, + struct pub_binding_t *out, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line); + +int pubsub_add_sub_(struct pubsub_connector_t *con, + recv_fn_t recv_fn, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line); + +int pubsub_connector_register_type_(struct pubsub_connector_t *, + msg_type_id_t, + dispatch_typefns_t *, + const char *file, + unsigned line); + +#endif /* !defined(TOR_PUBSUB_CONNECT_H) */ diff --git a/src/lib/pubsub/pubsub_flags.h b/src/lib/pubsub/pubsub_flags.h new file mode 100644 index 0000000000..53c6e49565 --- /dev/null +++ b/src/lib/pubsub/pubsub_flags.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_flags.h + * @brief Flags that can be set on publish/subscribe messages. + **/ + +#ifndef TOR_PUBSUB_FLAGS_H +#define TOR_PUBSUB_FLAGS_H + +/** + * Flag for registering a message: declare that no other module is allowed to + * publish this message if we are publishing it, or subscribe to it if we are + * subscribing to it. + */ +#define DISP_FLAG_EXCL (1u<<0) + +/** + * Flag for registering a message: declare that this message is a stub, and we + * will not actually publish/subscribe it, but that the dispatcher should + * treat us as if we did when typechecking. + * + * We use this so that messages aren't treated as "dangling" if they are + * potentially used by some other build of Tor. + */ +#define DISP_FLAG_STUB (1u<<1) + +#endif /* !defined(TOR_PUBSUB_FLAGS_H) */ diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h new file mode 100644 index 0000000000..357e59fd54 --- /dev/null +++ b/src/lib/pubsub/pubsub_macros.h @@ -0,0 +1,373 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file pubsub_macros.h + * \brief Macros to help with the publish/subscribe dispatch API. + * + * The dispatch API allows different subsystems of Tor to communicate with + * another asynchronously via a shared "message" system. Some subsystems + * declare that they publish a given message, and others declare that they + * subscribe to it. Both subsystems depend on the message, but not upon one + * another. + * + * To declare a message, use DECLARE_MESSAGE() (for messages that take their + * data as a pointer) or DECLARE_MESSAGE_INT() (for messages that take their + * data as an integer. For example, you might say + * + * DECLARE_MESSAGE(new_circuit, circ, circuit_handle_t *); + * or + * DECLARE_MESSAGE_INT(shutdown_requested, boolean, bool); + * + * Every message has a unique name, a "type name" that the dispatch system + * uses to manage associated data, and a C type name. You'll want to put + * these declarations in a header, to be included by all publishers and all + * subscribers. + * + * When a subsystem wants to publish a message, it uses DECLARE_PUBLISH() at + * file scope to create necessary static functions. Then, in its subsystem + * initialization (in the "bind to dispatcher" callback) (TODO: name this + * properly!), it calls DISPATCH_ADD_PUB() to tell the dispatcher about its + * intent to publish. When it actually wants to publish, it uses the + * PUBLISH() macro. For example: + * + * // At file scope + * DECLARE_PUBLISH(shutdown_requested); + * + * static void bind_to_dispatcher(pubsub_connector_t *con) + * { + * DISPATCH_ADD_PUB(con, mainchannel, shutdown_requested); + * } + * + * // somewhere in a function + * { + * PUBLISH(shutdown_requested, true); + * } + * + * When a subsystem wants to subscribe to a message, it uses + * DECLARE_SUBSCRIBE() at file scope to declare static functions. It must + * declare a hook function that receives the message type. Then, in its "bind + * to dispatcher" function, it calls DISPATCHER_ADD_SUB() to tell the + * dispatcher about its intent to subscribe. When another module publishes + * the message, the dispatcher will call the provided hook function. + * + * // At file scope. The first argument is the message that you're + * // subscribing to; the second argument is the hook function to declare. + * DECLARE_SUBSCRIBE(shutdown_requested, on_shutdown_req_cb); + * + * // You need to declare this function. + * static void on_shutdown_req_cb(const msg_t *msg, + * bool value) + * { + * // (do something here.) + * } + * + * static void bind_to_dispatcher(pubsub_connector_t *con) + * { + * DISPATCH_ADD_SUB(con, mainchannel, shutdown_requested); + * } + * + * Where did these types come from? Somewhere in the code, you need to call + * DISPATCH_REGISTER_TYPE() to make sure that the dispatcher can manage the + * message auxiliary data. It associates a vtbl-like structure with the + * type name, so that the dispatcher knows how to manipulate the type you're + * giving it. + * + * For example, the "boolean" type we're using above could be defined as: + * + * static char *boolean_fmt(msg_aux_data_t d) + * { + * // This is used for debugging and dumping messages. + * if (d.u64) + * return tor_strdup("true"); + * else + * return tor_strdup("false"); + * } + * + * static void boolean_free(msg_aux_data_t d) + * { + * // We don't actually need to do anything to free a boolean. + * // We could use "NULL" instead of this function, but I'm including + * // it as an example. + * } + * + * static void bind_to_dispatcher(pubsub_connector_t *con) + * { + * dispatch_typefns_t boolean_fns = { + * .fmt_fn = boolean_fmt, + * .free_fn = boolean_free, + * }; + * DISPATCH_REGISTER_TYPE(con, boolean, &boolean_fns); + * } + * + * + * + * So, how does this all work? (You can stop reading here, unless you're + * debugging something.) + * + * When you declare a message in a header with DECLARE_MESSAGE() or + * DECLARE_MESSAGE_INT(), it creates five things: + * + * * two typedefs for the message argument (constant and non-constant + * variants). + * * a constant string to hold the declared message type name + * * two inline functions, to coerce the message argument type to and from + * a "msg_aux_data_t" union. + * + * All of these declarations have names based on the message name. + * + * Later, when you say DECLARE_PUBLISH() or DECLARE_SUBSCRIBE(), we use the + * elements defined by DECLARE_MESSAGE() to make sure that the publish + * function takes the correct argument type, and that the subscription hook is + * declared with the right argument type. + **/ + +#ifndef TOR_DISPATCH_MSG_H +#define TOR_DISPATCH_MSG_H + +#include "lib/cc/compat_compiler.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_connect.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pubsub_publish.h" + +/* Implemenation notes: + * + * For a messagename "foo", the DECLARE_MESSAGE*() macros must declare: + * + * msg_arg_type__foo -- a typedef for the argument type of the foo message. + * msg_arg_consttype__foo -- a typedef for the const argument type of the + * foo message. + * msg_arg_name__foo[] -- a static string constant holding the unique + * identifier for the type of the foo message. + * msg_arg_get__foo() -- an inline function taking a msg_aux_data_t and + * returning the C data type. + * msg_arg_set__foo() -- an inline function taking a msg_aux_data_t and + * the C type, setting the msg_aux_data_t to hold the C type. + * + * For a messagename "foo", the DECLARE_PUBLISH() macro must declare: + * + * pub_binding__foo -- A static pub_binding_t object used to send messages + * from this module. + * publish_fn__foo -- A function taking an argument of the appropriate + * C type, to be invoked by PUBLISH(). + * + * For a messagename "foo", the DECLARE_SUBSCRIBE() macro must declare: + * + * hookfn -- A user-provided function name, with the correct signature. + * recv_fn__foo -- A wrapper callback that takes a msg_t *, and calls + * hookfn with the appropriate arguments. + */ + +/* Macro to declare common elements shared by DECLARE_MESSAGE and + * DECLARE_MESSAGE_INT. Don't call this directly. + * + * Note that the "msg_arg_name" string constant is defined in each + * translation unit. This might be undesirable; we can tweak it in the + * future if need be. + */ +#define DECLARE_MESSAGE_COMMON__(messagename, typename, c_type) \ + typedef c_type msg_arg_type__ ##messagename; \ + typedef const c_type msg_arg_consttype__ ##messagename; \ + ATTR_UNUSED static const char msg_arg_name__ ##messagename[] = # typename; + +/** + * Use this macro in a header to declare the existence of a given message, + * taking a pointer as auxiliary data. + * + * "messagename" is a unique identifier for the message; it must be a valid + * C identifier. + * + * "typename" is a unique identifier for the type of the auxiliary data. + * It needs to be defined somewhere in Tor, using + * "DISPATCH_REGISTER_TYPE." + * + * "c_ptr_type" is a C pointer type (like "char *" or "struct foo *"). + * The "*" needs to be included. + */ +#define DECLARE_MESSAGE(messagename, typename, c_ptr_type) \ + DECLARE_MESSAGE_COMMON__(messagename, typename, c_ptr_type) \ + ATTR_UNUSED static inline c_ptr_type \ + msg_arg_get__ ##messagename(msg_aux_data_t m) \ + { \ + return m.ptr; \ + } \ + ATTR_UNUSED static inline void \ + msg_arg_set__ ##messagename(msg_aux_data_t *m, c_ptr_type v) \ + { \ + m->ptr = v; \ + } \ + EAT_SEMICOLON + +/** + * Use this macro in a header to declare the existence of a given message, + * taking an integer as auxiliary data. + * + * "messagename" is a unique identifier for the message; it must be a valid + * C identifier. + * + * "typename" is a unique identifier for the type of the auxiliary data. It + * needs to be defined somewhere in Tor, using "DISPATCH_REGISTER_TYPE." + * + * "c_type" is a C integer type, like "int" or "bool". It needs to fit inside + * a uint64_t. + */ +#define DECLARE_MESSAGE_INT(messagename, typename, c_type) \ + DECLARE_MESSAGE_COMMON__(messagename, typename, c_type) \ + ATTR_UNUSED static inline c_type \ + msg_arg_get__ ##messagename(msg_aux_data_t m) \ + { \ + return (c_type)m.u64; \ + } \ + ATTR_UNUSED static inline void \ + msg_arg_set__ ##messagename(msg_aux_data_t *m, c_type v) \ + { \ + m->u64 = (uint64_t)v; \ + } \ + EAT_SEMICOLON + +/** + * Use this macro inside a C module declare that we'll be publishing a given + * message type from within this module. + * + * It creates necessary functions and wrappers to publish a message whose + * unique identifier is "messagename". + * + * Before you use this, you need to include the header where DECLARE_MESSAGE*() + * was used for this message. + * + * You can only use this once per message in each subsystem. + */ +#define DECLARE_PUBLISH(messagename) \ + static pub_binding_t pub_binding__ ##messagename; \ + static void \ + publish_fn__ ##messagename(msg_arg_type__ ##messagename arg) \ + { \ + msg_aux_data_t data; \ + msg_arg_set__ ##messagename(&data, arg); \ + pubsub_pub_(&pub_binding__ ##messagename, data); \ + } \ + EAT_SEMICOLON + +/** + * Use this macro inside a C file to declare that we're subscribing to a + * given message and associating it with a given "hook function". It + * declares the hook function static, and helps with strong typing. + * + * Before you use this, you need to include the header where + * DECLARE_MESSAGE*() was used for the message whose unique identifier is + * "messagename". + * + * You will need to define a function with the name that you provide for + * "hookfn". The type of this function will be: + * static void hookfn(const msg_t *, const c_type) + * where c_type is the c type that you declared in the header. + * + * You can only use this once per message in each subsystem. + */ +#define DECLARE_SUBSCRIBE(messagename, hookfn) \ + static void hookfn(const msg_t *, \ + const msg_arg_consttype__ ##messagename); \ + static void recv_fn__ ## messagename(const msg_t *m) \ + { \ + msg_arg_type__ ## messagename arg; \ + arg = msg_arg_get__ ##messagename(m->aux_data__); \ + hookfn(m, arg); \ + } \ + EAT_SEMICOLON + +/** + * Add a fake use of the publish function for 'messagename', so that + * the compiler does not call it unused. + */ +#define DISPATCH__FAKE_USE_OF_PUBFN_(messagename) \ + ( 0 ? (publish_fn__ ##messagename((msg_arg_type__##messagename)0), 1) \ + : 1) + +/* + * This macro is for internal use. It backs DISPATCH_ADD_PUB*() + */ +#define DISPATCH_ADD_PUB_(connector, channel, messagename, flags) \ + ( \ + DISPATCH__FAKE_USE_OF_PUBFN_(messagename), \ + pubsub_add_pub_((connector), \ + &pub_binding__ ##messagename, \ + get_channel_id(# channel), \ + get_message_id(# messagename), \ + get_msg_type_id(msg_arg_name__ ## messagename), \ + (flags), \ + __FILE__, \ + __LINE__) \ + ) + +/** + * Use a given connector and channel name to declare that this subsystem will + * publish a given message type. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_PUB(connector, channel, messagename) \ + DISPATCH_ADD_PUB_(connector, channel, messagename, 0) + +/** + * Use a given connector and channel name to declare that this subsystem will + * publish a given message type, and that no other subsystem is allowed to. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_PUB_EXCL(connector, channel, messagename) \ + DISPATCH_ADD_PUB_(connector, channel, messagename, DISP_FLAG_EXCL) + +/* + * This macro is for internal use. It backs DISPATCH_ADD_SUB*() + */ +#define DISPATCH_ADD_SUB_(connector, channel, messagename, flags) \ + pubsub_add_sub_((connector), \ + recv_fn__ ##messagename, \ + get_channel_id(#channel), \ + get_message_id(# messagename), \ + get_msg_type_id(msg_arg_name__ ##messagename), \ + (flags), \ + __FILE__, \ + __LINE__) +/* + * Use a given connector and channel name to declare that this subsystem will + * receive a given message type. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_SUB(connector, channel, messagename) \ + DISPATCH_ADD_SUB_(connector, channel, messagename, 0) +/** + * Use a given connector and channel name to declare that this subsystem will + * receive a given message type, and that no other subsystem is allowed to do + * so. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_SUB_EXCL(connector, channel, messagename) \ + DISPATCH_ADD_SUB_(connector, channel, messagename, DISP_FLAG_EXCL) + +/** + * Publish a given message with a given argument. (Takes ownership of the + * argument if it is a pointer.) + */ +#define PUBLISH(messagename, arg) \ + publish_fn__ ##messagename(arg) + +/** + * Use a given connector to declare that the functions to be used to manipuate + * a certain C type. + **/ +#define DISPATCH_REGISTER_TYPE(con, type, fns) \ + pubsub_connector_register_type_((con), \ + get_msg_type_id(#type), \ + (fns), \ + __FILE__, \ + __LINE__) + +#endif /* !defined(TOR_DISPATCH_MSG_H) */ diff --git a/src/lib/pubsub/pubsub_publish.c b/src/lib/pubsub/pubsub_publish.c new file mode 100644 index 0000000000..454a335a78 --- /dev/null +++ b/src/lib/pubsub/pubsub_publish.c @@ -0,0 +1,72 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_publish.c + * @brief Header for functions to publish using a pub_binding_t. + **/ + +#define PUBSUB_PRIVATE +#define DISPATCH_PRIVATE +#include "orconfig.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" + +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_publish.h" + +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +#include <string.h> + +/** + * Publish a message from the publication binding <b>pub</b> using the + * auxiliary data <b>auxdata</b>. + * + * Return 0 on success, -1 on failure. + **/ +int +pubsub_pub_(const pub_binding_t *pub, msg_aux_data_t auxdata) +{ + dispatch_t *d = pub->dispatch_ptr; + if (BUG(! d)) { + /* Tried to publish a message before the dispatcher was configured. */ + /* (Without a dispatcher, we don't know how to free auxdata.) */ + return -1; + } + + if (BUG(pub->msg_template.type >= d->n_types)) { + /* The type associated with this message is not known to the dispatcher. */ + /* (Without a correct type, we don't know how to free auxdata.) */ + return -1; + } + + if (BUG(pub->msg_template.msg >= d->n_msgs) || + BUG(pub->msg_template.channel >= d->n_queues)) { + /* The message ID or channel ID was out of bounds. */ + // LCOV_EXCL_START + d->typefns[pub->msg_template.type].free_fn(auxdata); + return -1; + // LCOV_EXCL_STOP + } + + if (! d->table[pub->msg_template.msg]) { + /* Fast path: nobody wants this data. */ + + // XXXX Faster path: we could store this in the pub_binding_t. + d->typefns[pub->msg_template.type].free_fn(auxdata); + return 0; + } + + /* Construct the message object */ + msg_t *m = tor_malloc(sizeof(msg_t)); + memcpy(m, &pub->msg_template, sizeof(msg_t)); + m->aux_data__ = auxdata; + + return dispatch_send_msg_unchecked(d, m); +} diff --git a/src/lib/pubsub/pubsub_publish.h b/src/lib/pubsub/pubsub_publish.h new file mode 100644 index 0000000000..0686a465de --- /dev/null +++ b/src/lib/pubsub/pubsub_publish.h @@ -0,0 +1,15 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_PUBSUB_PUBLISH_H +#define TOR_PUBSUB_PUBLISH_H + +#include "lib/dispatch/msgtypes.h" +struct pub_binding_t; + +int pubsub_pub_(const struct pub_binding_t *pub, msg_aux_data_t auxdata); + +#endif /* !defined(TOR_PUBSUB_PUBLISH_H) */ diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include index 84906dfb3d..853dae7880 100644 --- a/src/lib/sandbox/.may_include +++ b/src/lib/sandbox/.may_include @@ -5,11 +5,10 @@ lib/container/*.h lib/err/*.h lib/log/*.h lib/malloc/*.h -lib/net/*.h lib/sandbox/*.h lib/sandbox/*.inc lib/string/*.h -ht.h -siphash.h -tor_queue.h +ext/ht.h +ext/siphash.h +ext/tor_queue.h diff --git a/src/lib/sandbox/include.am b/src/lib/sandbox/include.am index adfda6bde5..e81f14b55f 100644 --- a/src/lib/sandbox/include.am +++ b/src/lib/sandbox/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-sandbox-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_sandbox_a_SOURCES = \ src/lib/sandbox/sandbox.c @@ -13,6 +14,7 @@ src_lib_libtor_sandbox_testing_a_SOURCES = \ src_lib_libtor_sandbox_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_sandbox_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/sandbox/linux_syscalls.inc \ src/lib/sandbox/sandbox.h diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 8f577b0660..3515e8854e 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -38,13 +38,12 @@ #include "lib/err/torerr.h" #include "lib/log/log.h" #include "lib/cc/torint.h" -#include "lib/net/resolve.h" #include "lib/malloc/malloc.h" #include "lib/string/scanf.h" -#include "tor_queue.h" -#include "ht.h" -#include "siphash.h" +#include "ext/tor_queue.h" +#include "ext/ht.h" +#include "ext/siphash.h" #define DEBUGGING_CLOSE @@ -144,6 +143,7 @@ static int filter_nopar_gen[] = { SCMP_SYS(clock_gettime), SCMP_SYS(close), SCMP_SYS(clone), + SCMP_SYS(dup), SCMP_SYS(epoll_create), SCMP_SYS(epoll_wait), #ifdef __NR_epoll_pwait @@ -295,6 +295,7 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter) unsigned i; int rc; int param[] = { SIGINT, SIGTERM, SIGPIPE, SIGUSR1, SIGUSR2, SIGHUP, SIGCHLD, + SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, SIGIO, #ifdef SIGXFSZ SIGXFSZ #endif @@ -444,7 +445,7 @@ libc_uses_openat_for_everything(void) return 1; else return 0; -#else /* !(defined(CHECK_LIBC_VERSION)) */ +#else /* !defined(CHECK_LIBC_VERSION) */ return 0; #endif /* defined(CHECK_LIBC_VERSION) */ } @@ -798,6 +799,12 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) if (rc) return rc; + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), + SCMP_CMP(2, SCMP_CMP_EQ, SO_ACCEPTCONN)); + if (rc) + return rc; + #ifdef HAVE_SYSTEMD rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), @@ -1517,7 +1524,6 @@ install_syscall_filter(sandbox_cfg_t* cfg) // marking the sandbox as active sandbox_active = 1; - tor_make_getaddrinfo_cache_active(); end: seccomp_release(ctx); @@ -1764,9 +1770,4 @@ sandbox_is_active(void) return 0; } -void -sandbox_disable_getaddrinfo_cache(void) -{ -} - #endif /* !defined(USE_LIBSECCOMP) */ diff --git a/src/lib/sandbox/sandbox.h b/src/lib/sandbox/sandbox.h index 5bec09a36a..b4ae6e5c07 100644 --- a/src/lib/sandbox/sandbox.h +++ b/src/lib/sandbox/sandbox.h @@ -108,7 +108,7 @@ typedef struct { * it matches the parameter. */ const char* sandbox_intern_string(const char *param); -#else /* !(defined(USE_LIBSECCOMP)) */ +#else /* !defined(USE_LIBSECCOMP) */ #define sandbox_intern_string(s) (s) #endif /* defined(USE_LIBSECCOMP) */ diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include index a8507761a4..2f0c8d341e 100644 --- a/src/lib/smartlist_core/.may_include +++ b/src/lib/smartlist_core/.may_include @@ -4,4 +4,4 @@ lib/malloc/*.h lib/err/*.h lib/string/*.h lib/smartlist_core/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/smartlist_core/include.am b/src/lib/smartlist_core/include.am index 99d65f0b23..548179bc4f 100644 --- a/src/lib/smartlist_core/include.am +++ b/src/lib/smartlist_core/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-smartlist-core-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_smartlist_core_a_SOURCES = \ src/lib/smartlist_core/smartlist_core.c \ src/lib/smartlist_core/smartlist_split.c @@ -15,6 +16,7 @@ src_lib_libtor_smartlist_core_testing_a_CPPFLAGS = \ $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_smartlist_core_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/smartlist_core/smartlist_core.h \ src/lib/smartlist_core/smartlist_foreach.h \ diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c index ac85a6cc84..6b0a305a93 100644 --- a/src/lib/smartlist_core/smartlist_core.c +++ b/src/lib/smartlist_core/smartlist_core.c @@ -88,6 +88,30 @@ smartlist_ensure_capacity(smartlist_t *sl, size_t size) #undef MAX_CAPACITY } +/** Expand <b>sl</b> so that its length is at least <b>new_size</b>, + * filling in previously unused entries with NULL> + * + * Do nothing if <b>sl</b> already had at least <b>new_size</b> elements. + */ +void +smartlist_grow(smartlist_t *sl, size_t new_size) +{ + smartlist_ensure_capacity(sl, new_size); + + if (new_size > (size_t)sl->num_used) { + /* This memset() should be a no-op: everything else in the smartlist code + * tries to make sure that unused entries are always NULL. Still, that is + * meant as a safety mechanism, so let's clear the memory here. + */ + memset(sl->list + sl->num_used, 0, + sizeof(void *) * (new_size - sl->num_used)); + + /* This cast is safe, since we already asserted that we were below + * MAX_CAPACITY in smartlist_ensure_capacity(). */ + sl->num_used = (int)new_size; + } +} + /** Append element to the end of the list. */ void smartlist_add(smartlist_t *sl, void *element) @@ -153,6 +177,8 @@ smartlist_remove_keeporder(smartlist_t *sl, const void *element) sl->list[i++] = sl->list[j]; } } + memset(sl->list + sl->num_used, 0, + sizeof(void *) * (num_used_orig - sl->num_used)); } /** If <b>sl</b> is nonempty, remove and return the final element. Otherwise, diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h index a7fbaa099b..36f23e2009 100644 --- a/src/lib/smartlist_core/smartlist_core.h +++ b/src/lib/smartlist_core/smartlist_core.h @@ -43,6 +43,7 @@ void smartlist_clear(smartlist_t *sl); void smartlist_add(smartlist_t *sl, void *element); void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2); void smartlist_add_strdup(struct smartlist_t *sl, const char *string); +void smartlist_grow(smartlist_t *sl, size_t new_size); void smartlist_remove(smartlist_t *sl, const void *element); void smartlist_remove_keeporder(smartlist_t *sl, const void *element); @@ -76,7 +77,7 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) { raw_assert(sl->num_used > idx); sl->list[idx] = val; } -#else /* !(defined(DEBUG_SMARTLIST)) */ +#else /* !defined(DEBUG_SMARTLIST) */ #define smartlist_len(sl) ((sl)->num_used) #define smartlist_get(sl, idx) ((sl)->list[idx]) #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val)) @@ -97,4 +98,4 @@ void smartlist_del(smartlist_t *sl, int idx); void smartlist_del_keeporder(smartlist_t *sl, int idx); void smartlist_insert(smartlist_t *sl, int idx, void *val); -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_SMARTLIST_CORE_H) */ diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h index 0f6fe30074..a1fbcd444c 100644 --- a/src/lib/smartlist_core/smartlist_foreach.h +++ b/src/lib/smartlist_core/smartlist_foreach.h @@ -83,6 +83,19 @@ ++var ## _sl_idx) { \ var = (sl)->list[var ## _sl_idx]; +/** Iterates over the items in smartlist <b>sl</b> in reverse order, similar to + * SMARTLIST_FOREACH_BEGIN + * + * NOTE: This macro is incompatible with SMARTLIST_DEL_CURRENT. + */ +#define SMARTLIST_FOREACH_REVERSE_BEGIN(sl, type, var) \ + STMT_BEGIN \ + int var ## _sl_idx, var ## _sl_len=(sl)->num_used; \ + type var; \ + for (var ## _sl_idx = var ## _sl_len-1; var ## _sl_idx >= 0; \ + --var ## _sl_idx) { \ + var = (sl)->list[var ## _sl_idx]; + #define SMARTLIST_FOREACH_END(var) \ var = NULL; \ (void) var ## _sl_idx; \ diff --git a/src/lib/smartlist_core/smartlist_split.h b/src/lib/smartlist_core/smartlist_split.h index 4f72376125..e2cc7245b7 100644 --- a/src/lib/smartlist_core/smartlist_split.h +++ b/src/lib/smartlist_core/smartlist_split.h @@ -17,4 +17,4 @@ int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max); -#endif +#endif /* !defined(TOR_SMARTLIST_SPLIT_H) */ diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include index ec5c769831..1fb9127f19 100644 --- a/src/lib/string/.may_include +++ b/src/lib/string/.may_include @@ -6,5 +6,5 @@ lib/malloc/*.h lib/ctime/*.h lib/string/*.h -strlcat.c -strlcpy.c +ext/strlcat.c +ext/strlcpy.c diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c index e125c921a4..187f784be5 100644 --- a/src/lib/string/compat_string.c +++ b/src/lib/string/compat_string.c @@ -14,10 +14,10 @@ /* Inline the strl functions if the platform doesn't have them. */ #ifndef HAVE_STRLCPY -#include "strlcpy.c" +#include "ext/strlcpy.c" #endif #ifndef HAVE_STRLCAT -#include "strlcat.c" +#include "ext/strlcat.c" #endif #include <stdlib.h> diff --git a/src/lib/string/compat_string.h b/src/lib/string/compat_string.h index a0e37bb6dc..ffc892c3e5 100644 --- a/src/lib/string/compat_string.h +++ b/src/lib/string/compat_string.h @@ -25,20 +25,23 @@ static inline int strncasecmp(const char *a, const char *b, size_t n); static inline int strncasecmp(const char *a, const char *b, size_t n) { return _strnicmp(a,b,n); } -#endif +#endif /* !defined(HAVE_STRNCASECMP) */ #ifndef HAVE_STRCASECMP static inline int strcasecmp(const char *a, const char *b); static inline int strcasecmp(const char *a, const char *b) { return _stricmp(a,b); } -#endif -#endif +#endif /* !defined(HAVE_STRCASECMP) */ +#endif /* defined(_WIN32) */ #if defined __APPLE__ /* On OSX 10.9 and later, the overlap-checking code for strlcat would * appear to have a severe bug that can sometimes cause aborts in Tor. * Instead, use the non-checking variants. This is sad. * + * (If --enable-fragile-hardening is passed to configure, we use the hardened + * variants, which do not suffer from this issue.) + * * See https://trac.torproject.org/projects/tor/ticket/15205 */ #undef strlcat @@ -59,4 +62,4 @@ char *tor_strtok_r_impl(char *str, const char *sep, char **lasts); #define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts) #endif -#endif +#endif /* !defined(TOR_COMPAT_STRING_H) */ diff --git a/src/lib/string/include.am b/src/lib/string/include.am index edd74b8a3e..82d35cc5af 100644 --- a/src/lib/string/include.am +++ b/src/lib/string/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-string-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_string_a_SOURCES = \ src/lib/string/compat_ctype.c \ src/lib/string/compat_string.c \ @@ -18,6 +19,7 @@ src_lib_libtor_string_testing_a_SOURCES = \ src_lib_libtor_string_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_string_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/string/compat_ctype.h \ src/lib/string/compat_string.h \ diff --git a/src/lib/string/parse_int.h b/src/lib/string/parse_int.h index 925547942e..50d48b44c5 100644 --- a/src/lib/string/parse_int.h +++ b/src/lib/string/parse_int.h @@ -22,4 +22,4 @@ double tor_parse_double(const char *s, double min, double max, int *ok, uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, uint64_t max, int *ok, char **next); -#endif +#endif /* !defined(TOR_PARSE_INT_H) */ diff --git a/src/lib/string/printf.c b/src/lib/string/printf.c index a5cb71ce09..26203932e4 100644 --- a/src/lib/string/printf.c +++ b/src/lib/string/printf.c @@ -117,8 +117,8 @@ tor_vasprintf(char **strp, const char *fmt, va_list args) *strp = NULL; return -1; } - strp_tmp = tor_malloc(len + 1); - r = _vsnprintf(strp_tmp, len+1, fmt, args); + strp_tmp = tor_malloc((size_t)len + 1); + r = _vsnprintf(strp_tmp, (size_t)len+1, fmt, args); if (r != len) { tor_free(strp_tmp); *strp = NULL; @@ -153,9 +153,9 @@ tor_vasprintf(char **strp, const char *fmt, va_list args) *strp = tor_strdup(buf); return len; } - strp_tmp = tor_malloc(len+1); + strp_tmp = tor_malloc((size_t)len+1); /* use of tor_vsnprintf() will ensure string is null terminated */ - r = tor_vsnprintf(strp_tmp, len+1, fmt, args); + r = tor_vsnprintf(strp_tmp, (size_t)len+1, fmt, args); if (r != len) { tor_free(strp_tmp); *strp = NULL; diff --git a/src/lib/string/printf.h b/src/lib/string/printf.h index 2cc13d6bee..6e90770f81 100644 --- a/src/lib/string/printf.h +++ b/src/lib/string/printf.h @@ -27,4 +27,4 @@ int tor_asprintf(char **strp, const char *fmt, ...) int tor_vasprintf(char **strp, const char *fmt, va_list args) CHECK_PRINTF(2,0); -#endif /* !defined(TOR_UTIL_STRING_H) */ +#endif /* !defined(TOR_UTIL_PRINTF_H) */ diff --git a/src/lib/string/scanf.h b/src/lib/string/scanf.h index 6673173de5..b642e242db 100644 --- a/src/lib/string/scanf.h +++ b/src/lib/string/scanf.h @@ -21,4 +21,4 @@ int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \ int tor_sscanf(const char *buf, const char *pattern, ...) CHECK_SCANF(2, 3); -#endif /* !defined(TOR_UTIL_STRING_H) */ +#endif /* !defined(TOR_UTIL_SCANF_H) */ diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index f934f66f02..f5061a11d2 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -71,7 +71,7 @@ tor_memstr(const void *haystack, size_t hlen, const char *needle) /** Return true iff the 'len' bytes at 'mem' are all zero. */ int -tor_mem_is_zero(const char *mem, size_t len) +fast_mem_is_zero(const char *mem, size_t len) { static const char ZERO[] = { 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, @@ -95,17 +95,14 @@ tor_mem_is_zero(const char *mem, size_t len) int tor_digest_is_zero(const char *digest) { - static const uint8_t ZERO_DIGEST[] = { - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 - }; - return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN); + return safe_mem_is_zero(digest, DIGEST_LEN); } /** Return true iff the DIGEST256_LEN bytes in digest are all zero. */ int tor_digest256_is_zero(const char *digest) { - return tor_mem_is_zero(digest, DIGEST256_LEN); + return safe_mem_is_zero(digest, DIGEST256_LEN); } /** Remove from the string <b>s</b> every character which appears in @@ -212,21 +209,6 @@ strcmpstart(const char *s1, const char *s2) return strncmp(s1, s2, n); } -/** Compare the s1_len-byte string <b>s1</b> with <b>s2</b>, - * without depending on a terminating nul in s1. Sorting order is first by - * length, then lexically; return values are as for strcmp. - */ -int -strcmp_len(const char *s1, const char *s2, size_t s1_len) -{ - size_t s2_len = strlen(s2); - if (s1_len < s2_len) - return -1; - if (s1_len > s2_len) - return 1; - return fast_memcmp(s1, s2, s2_len); -} - /** Compares the first strlen(s2) characters of s1 with s2. Returns as for * strcasecmp. */ @@ -541,3 +523,16 @@ string_is_utf8(const char *str, size_t len) } return true; } + +/** As string_is_utf8(), but returns false if the string begins with a UTF-8 + * byte order mark (BOM). + */ +int +string_is_utf8_no_bom(const char *str, size_t len) +{ + if (len >= 3 && (!strcmpstart(str, "\uFEFF") || + !strcmpstart(str, "\uFFFE"))) { + return false; + } + return string_is_utf8(str, len); +} diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index d9fbf8c61e..b3c6841d41 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -20,7 +20,10 @@ const void *tor_memmem(const void *haystack, size_t hlen, const void *needle, size_t nlen); const void *tor_memstr(const void *haystack, size_t hlen, const char *needle); -int tor_mem_is_zero(const char *mem, size_t len); +int fast_mem_is_zero(const char *mem, size_t len); +#define fast_digest_is_zero(d) fast_mem_is_zero((d), DIGEST_LEN) +#define fast_digetst256_is_zero(d) fast_mem_is_zero((d), DIGEST256_LEN) + int tor_digest_is_zero(const char *digest); int tor_digest256_is_zero(const char *digest); @@ -33,7 +36,6 @@ int tor_strisnonupper(const char *s); int tor_strisspace(const char *s); int strcmp_opt(const char *s1, const char *s2); int strcmpstart(const char *s1, const char *s2); -int strcmp_len(const char *s1, const char *s2, size_t len); int strcasecmpstart(const char *s1, const char *s2); int strcmpend(const char *s1, const char *s2); int strcasecmpend(const char *s1, const char *s2); @@ -53,5 +55,6 @@ const char *find_str_at_start_of_line(const char *haystack, int string_is_C_identifier(const char *string); int string_is_utf8(const char *str, size_t len); +int string_is_utf8_no_bom(const char *str, size_t len); #endif /* !defined(TOR_UTIL_STRING_H) */ diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include new file mode 100644 index 0000000000..2b06e8519c --- /dev/null +++ b/src/lib/subsys/.may_include @@ -0,0 +1 @@ +orconfig.h diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am new file mode 100644 index 0000000000..c9ab54ca73 --- /dev/null +++ b/src/lib/subsys/include.am @@ -0,0 +1,4 @@ + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/subsys/subsys.h diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h new file mode 100644 index 0000000000..21f984f32d --- /dev/null +++ b/src/lib/subsys/subsys.h @@ -0,0 +1,95 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SUBSYS_T +#define TOR_SUBSYS_T + +#include <stdbool.h> + +struct pubsub_connector_t; + +/** + * A subsystem is a part of Tor that is initialized, shut down, configured, + * and connected to other parts of Tor. + * + * All callbacks are optional -- if a callback is set to NULL, the subsystem + * manager will treat it as a no-op. + * + * You should use c99 named-field initializers with this structure: we + * will be adding more fields, often in the middle of the structure. + **/ +typedef struct subsys_fns_t { + /** + * The name of this subsystem. It should be a programmer-readable + * identifier. + **/ + const char *name; + + /** + * Whether this subsystem is supported -- that is, whether it is compiled + * into Tor. For most subsystems, this should be true. + **/ + bool supported; + + /** + * The 'initialization level' for the subsystem. It should run from -100 + * through +100. The subsystems are initialized from lowest level to + * highest, and shut down from highest level to lowest. + **/ + int level; + + /** + * Initialize any global components of this subsystem. + * + * This function MAY rely on any lower-level subsystem being initialized. + * + * This function MUST NOT rely on any runtime configuration information; + * it is only for global state or pre-configuration state. + * + * (If you need to do any setup that depends on configuration, you'll need + * to declare a configuration callback. (Not yet designed)) + * + * This function MUST NOT have any parts that can fail. + **/ + int (*initialize)(void); + + /** + * Connect a subsystem to the message dispatch system. + **/ + int (*add_pubsub)(struct pubsub_connector_t *); + + /** + * Perform any necessary pre-fork cleanup. This function may not fail. + */ + void (*prefork)(void); + + /** + * Perform any necessary post-fork setup. This function may not fail. + */ + void (*postfork)(void); + + /** + * Free any thread-local resources held by this subsystem. Called before + * the thread exits. + */ + void (*thread_cleanup)(void); + + /** + * Free all resources held by this subsystem. + * + * This function is not allowed to fail. + **/ + void (*shutdown)(void); + +} subsys_fns_t; + +#define MIN_SUBSYS_LEVEL -100 +#define MAX_SUBSYS_LEVEL 100 + +/* All tor "libraries" (in src/libs) should have a subsystem level equal to or + * less than this value. */ +#define SUBSYS_LEVEL_LIBS -10 + +#endif /* !defined(TOR_SUBSYS_T) */ diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include index c93a06e59e..306fa57b7a 100644 --- a/src/lib/term/.may_include +++ b/src/lib/term/.may_include @@ -5,5 +5,4 @@ lib/log/*.h lib/term/*.h lib/malloc/*.h -# From src/ext -tor_readpassphrase.h +ext/tor_readpassphrase.h diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c index c6459f250f..8741344acf 100644 --- a/src/lib/term/getpass.c +++ b/src/lib/term/getpass.c @@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #elif defined(HAVE_READPASSPHRASE_H) #include <readpassphrase.h> #else -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #endif /* defined(_WIN32) || ... */ #include <stdlib.h> diff --git a/src/lib/term/getpass.h b/src/lib/term/getpass.h index a9c146ea8f..aa597ec423 100644 --- a/src/lib/term/getpass.h +++ b/src/lib/term/getpass.h @@ -15,4 +15,4 @@ ssize_t tor_getpass(const char *prompt, char *output, size_t buflen); -#endif +#endif /* !defined(TOR_GETPASS_H) */ diff --git a/src/lib/term/include.am b/src/lib/term/include.am index 55fe548ebc..a120bba0cb 100644 --- a/src/lib/term/include.am +++ b/src/lib/term/include.am @@ -11,6 +11,7 @@ else readpassphrase_source= endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_term_a_SOURCES = \ src/lib/term/getpass.c \ $(readpassphrase_source) @@ -20,5 +21,6 @@ src_lib_libtor_term_testing_a_SOURCES = \ src_lib_libtor_term_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_term_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/term/getpass.h diff --git a/src/lib/testsupport/include.am b/src/lib/testsupport/include.am index b2aa620985..a5ed46eb67 100644 --- a/src/lib/testsupport/include.am +++ b/src/lib/testsupport/include.am @@ -1,3 +1,4 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/testsupport/testsupport.h diff --git a/src/lib/testsupport/testsupport.h b/src/lib/testsupport/testsupport.h index 9363a9ba66..90b7c43b19 100644 --- a/src/lib/testsupport/testsupport.h +++ b/src/lib/testsupport/testsupport.h @@ -21,7 +21,7 @@ * tests. */ #define STATIC #define EXTERN(type, name) extern type name; -#else +#else /* !defined(TOR_UNIT_TESTS) */ #define STATIC static #define EXTERN(type, name) #endif /* defined(TOR_UNIT_TESTS) */ @@ -90,7 +90,7 @@ do { \ func = func ##__real; \ } while (0) -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ #define MOCK_DECL(rv, funcname, arglist) \ rv funcname arglist #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \ diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include index fc56f46836..02711348c5 100644 --- a/src/lib/thread/.may_include +++ b/src/lib/thread/.may_include @@ -2,6 +2,7 @@ orconfig.h lib/cc/*.h lib/lock/*.h lib/log/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h lib/wallclock/*.h diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 94ab021c52..5c8ffa55c6 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -14,9 +14,11 @@ #include "orconfig.h" #include <stdlib.h> #include "lib/thread/threads.h" +#include "lib/thread/thread_sys.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" +#include "lib/subsys/subsys.h" #include <string.h> @@ -65,7 +67,15 @@ atomic_counter_init(atomic_counter_t *counter) memset(counter, 0, sizeof(*counter)); tor_mutex_init_nonrecursive(&counter->mutex); } -/** Clean up all resources held by an atomic counter. */ +/** Clean up all resources held by an atomic counter. + * + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ void atomic_counter_destroy(atomic_counter_t *counter) { @@ -109,3 +119,19 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return oldval; } #endif /* !defined(HAVE_WORKING_STDATOMIC) */ + +static int +subsys_threads_initialize(void) +{ + tor_threads_init(); + return 0; +} + +const subsys_fns_t sys_threads = { + .name = "threads", + .supported = true, + /* Threads is used by logging, which is a diagnostic feature, we want it to + * init right after low-level error handling and approx time. */ + .level = -95, + .initialize = subsys_threads_initialize, +}; diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am index 9ec23d166e..cd8016b5df 100644 --- a/src/lib/thread/include.am +++ b/src/lib/thread/include.am @@ -12,6 +12,7 @@ if THREADS_WIN32 threads_impl_source=src/lib/thread/compat_winthreads.c endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_thread_a_SOURCES = \ src/lib/thread/compat_threads.c \ src/lib/thread/numcpus.c \ @@ -22,6 +23,8 @@ src_lib_libtor_thread_testing_a_SOURCES = \ src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ - src/lib/thread/threads.h \ - src/lib/thread/numcpus.h + src/lib/thread/numcpus.h \ + src/lib/thread/thread_sys.h \ + src/lib/thread/threads.h diff --git a/src/lib/thread/numcpus.h b/src/lib/thread/numcpus.h index 3f0a29ce7c..2f1ea16eb9 100644 --- a/src/lib/thread/numcpus.h +++ b/src/lib/thread/numcpus.h @@ -13,4 +13,4 @@ int compute_num_cpus(void); -#endif +#endif /* !defined(TOR_NUMCPUS_H) */ diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h new file mode 100644 index 0000000000..c0daf2b5e9 --- /dev/null +++ b/src/lib/thread/thread_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file threads_sys.h + * \brief Declare subsystem object for threads library + **/ + +#ifndef TOR_THREADS_SYS_H +#define TOR_THREADS_SYS_H + +extern const struct subsys_fns_t sys_threads; + +#endif /* !defined(TOR_THREADS_SYS_H) */ diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h index ecf60641b5..4b42b9abd9 100644 --- a/src/lib/thread/threads.h +++ b/src/lib/thread/threads.h @@ -107,7 +107,7 @@ typedef struct atomic_counter_t { atomic_size_t val; } atomic_counter_t; #define ATOMIC_LINKAGE static -#else /* !(defined(HAVE_WORKING_STDATOMIC)) */ +#else /* !defined(HAVE_WORKING_STDATOMIC) */ typedef struct atomic_counter_t { tor_mutex_t mutex; size_t val; @@ -131,7 +131,17 @@ atomic_counter_init(atomic_counter_t *counter) { atomic_init(&counter->val, 0); } -/** Clean up all resources held by an atomic counter. */ +/** Clean up all resources held by an atomic counter. + * + * This usage note applies to the compat_threads implementation of + * atomic_counter_destroy(): + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ static inline void atomic_counter_destroy(atomic_counter_t *counter) { @@ -162,7 +172,7 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return atomic_exchange(&counter->val, newval); } -#else /* !(defined(HAVE_WORKING_STDATOMIC)) */ +#else /* !defined(HAVE_WORKING_STDATOMIC) */ #endif /* defined(HAVE_WORKING_STDATOMIC) */ #endif /* !defined(TOR_COMPAT_THREADS_H) */ diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include index 2c7e37a836..ae01431b60 100644 --- a/src/lib/time/.may_include +++ b/src/lib/time/.may_include @@ -4,8 +4,10 @@ lib/cc/*.h lib/err/*.h lib/intmath/*.h lib/log/*.h +lib/subsys/*.h lib/time/*.h lib/wallclock/*.h +lib/defs/time.h # For load_windows_system_lib. lib/fs/winlib.h
\ No newline at end of file diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index 98854bad2c..ab45224a7f 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -164,6 +164,8 @@ static int64_t last_tick_count = 0; * to be monotonic; increments them as appropriate so that they actually * _are_ monotonic. * + * The returned time may be the same as the previous returned time. + * * Caller must hold lock. */ STATIC int64_t ratchet_performance_counter(int64_t count_raw) @@ -202,6 +204,8 @@ static struct timeval timeofday_offset = { 0, 0 }; * supposed to be monotonic; increments them as appropriate so that they * actually _are_ monotonic. * + * The returned time may be the same as the previous returned time. + * * Caller must hold lock. */ STATIC void ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out) @@ -270,7 +274,9 @@ monotime_init_internal(void) } /** - * Set "out" to the most recent monotonic time value + * Set "out" to the most recent monotonic time value. + * + * The returned time may be the same as the previous returned time. */ void monotime_get(monotime_t *out) @@ -298,10 +304,12 @@ monotime_coarse_get(monotime_coarse_t *out) #endif /* defined(TOR_UNIT_TESTS) */ out->abstime_ = mach_approximate_time(); } -#endif +#endif /* defined(HAVE_MACH_APPROXIMATE_TIME) */ /** * Return the number of nanoseconds between <b>start</b> and <b>end</b>. + * + * The returned value may be equal to zero. */ int64_t monotime_diff_nsec(const monotime_t *start, @@ -522,7 +530,9 @@ monotime_init_internal(void) GetTickCount64_fn = (GetTickCount64_fn_t) (void(*)(void)) GetProcAddress(h, "GetTickCount64"); } - // FreeLibrary(h) ? + // We can't call FreeLibrary(h) here, because freeing the handle may + // unload the library, and cause future calls to GetTickCount64_fn() + // to fail. See 29642 for details. } void @@ -757,7 +767,7 @@ monotime_coarse_zero(monotime_coarse_t *out) { memset(out, 0, sizeof(*out)); } -#endif +#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ int64_t monotime_diff_usec(const monotime_t *start, @@ -787,8 +797,8 @@ monotime_absolute_nsec(void) return monotime_diff_nsec(&initialized_at, &now); } -uint64_t -monotime_absolute_usec(void) +MOCK_IMPL(uint64_t, +monotime_absolute_usec,(void)) { return monotime_absolute_nsec() / 1000; } @@ -823,7 +833,7 @@ monotime_coarse_absolute_msec(void) { return monotime_coarse_absolute_nsec() / ONE_MILLION; } -#else +#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ #define initialized_at_coarse initialized_at #endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ @@ -855,7 +865,7 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) mach_time_info.numer; return abstime_val >> monotime_shift; } -#else +#else /* !defined(__APPLE__) */ uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units) { @@ -866,4 +876,4 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) { return (msec * STAMP_TICKS_PER_SECOND) / 1000; } -#endif +#endif /* defined(__APPLE__) */ diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 480d426ac7..4d16effd29 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -15,6 +15,120 @@ * of tens of milliseconds. */ +/* Q: When should I use monotonic time? + * + * A: If you need a time that never decreases, use monotonic time. If you need + * to send a time to a user or another process, or store a time, use the + * wall-clock time. + * + * Q: Should you use monotime or monotime_coarse as your source? + * + * A: Generally, you get better precision with monotime, but better + * performance with monotime_coarse. + * + * Q: What is a "monotonic" time, exactly? + * + * A: Monotonic times are strictly non-decreasing. The difference between any + * previous monotonic time, and the current monotonic time, is always greater + * than *or equal to* zero. + * Zero deltas happen more often: + * - on Windows (due to an OS bug), + * - when using monotime_coarse, or on systems with low-resolution timers, + * - on platforms where we emulate monotonic time using wall-clock time, and + * - when using time units that are larger than nanoseconds (due to + * truncation on division). + * + * Q: Should you use monotime_t or monotime_coarse_t directly? Should you use + * usec? msec? "stamp units?" + * + * A: Using monotime_t and monotime_coarse_t directly is most time-efficient, + * since no conversion needs to happen. But they can potentially use more + * memory than you would need for a usec/msec/"stamp unit" count. + * + * Converting to usec or msec on some platforms, and working with them in + * general, creates a risk of doing a 64-bit division. 64-bit division is + * expensive on 32-bit platforms, which still do exist. + * + * The "stamp unit" type is designed to give a type that is cheap to convert + * from monotime_coarse, has resolution of about 1-2ms, and fits nicely in a + * 32-bit integer. Its downside is that it does not correspond directly + * to a natural unit of time. + * + * There is not much point in using "coarse usec" or "coarse nsec", since the + * current coarse monotime implementations give you on the order of + * milliseconds of precision. + * + * Q: So, what backends is monotime_coarse using? + * + * A: Generally speaking, it uses "whatever monotonic-ish time implemenation + * does not require a context switch." The various implementations provide + * this by having a view of the current time in a read-only memory page that + * is updated with a frequency corresponding to the kernel's tick count. + * + * On Windows, monotime_coarse uses GetCount64() [or GetTickCount() on + * obsolete systems]. MSDN claims that the resolution is "typically in the + * range of 10-16 msec", but it has said that for years. Storing + * monotime_coarse_t uses 8 bytes. + * + * On OSX/iOS, monotime_coarse uses uses mach_approximate_time() where + * available, and falls back to regular monotime. The precision is not + * documented, but the implementation is open-source: it reads from a page + * that the kernel updates. Storing monotime_coarse_t uses 8 bytes. + * + * On unixy systems, monotime_coarse uses clock_gettime() with + * CLOCK_MONOTONIC_COARSE where available, and falls back to CLOCK_MONOTONIC. + * It typically uses vdso tricks to read from a page that the kernel updates. + * Its precision fixed, but you can get it with clock_getres(): on my Linux + * desktop, it claims to be 1 msec, but it will depend on the system HZ + * setting. Storing monotime_coarse_t uses 16 bytes. + * + * [TODO: Try CLOCK_MONOTONIC_FAST on foobsd.] + * + * Q: What backends is regular monotonic time using? + * + * A: In general, regular monotime uses something that requires a system call. + * On platforms where system calls are cheap, you win! Otherwise, you lose. + * + * On Windows, monotonic time uses QuereyPerformanceCounter. Storing + * monotime_t costs 8 bytes. + * + * On OSX/Apple, monotonic time uses mach_absolute_time. Storing + * monotime_t costs 8 bytes. + * + * On unixy systems, monotonic time uses CLOCK_MONOTONIC. Storing + * monotime_t costs 16 bytes. + * + * Q: Tell me about the costs of converting to a 64-bit nsec, usec, or msec + * count. + * + * A: Windows, coarse: Cheap, since it's all multiplication. + * + * Windows, precise: Expensive on 32-bit: it needs 64-bit division. + * + * Apple, all: Expensive on 32-bit: it needs 64-bit division. + * + * Unixy, all: Fairly cheap, since the only division required is dividing + * tv_nsec 1000, and nanoseconds-per-second fits in a 32-bit value. + * + * All, "timestamp units": Cheap everywhere: it never divides. + * + * Q: This is only somewhat related, but how much precision could I hope for + * from a libevent time? + * + * A: Actually, it's _very_ related if you're timing in order to have a + * timeout happen. + * + * On Windows, it uses select: you could in theory have a microsecond + * resolution, but it usually isn't that accurate. + * + * On OSX, iOS, and BSD, you have kqueue: You could in theory have a nanosecond + * resolution, but it usually isn't that accurate. + * + * On Linux, you have epoll: It has a millisecond resolution. Some recent + * Libevents can also use timerfd for higher resolution if + * EVENT_BASE_FLAG_PRECISE_TIMER is set: Tor doesn't set that flag. + */ + #ifndef TOR_COMPAT_TIME_H #define TOR_COMPAT_TIME_H @@ -86,26 +200,36 @@ void monotime_init(void); void monotime_get(monotime_t *out); /** * Return the number of nanoseconds between <b>start</b> and <b>end</b>. + * The returned value may be equal to zero. */ int64_t monotime_diff_nsec(const monotime_t *start, const monotime_t *end); /** * Return the number of microseconds between <b>start</b> and <b>end</b>. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ int64_t monotime_diff_usec(const monotime_t *start, const monotime_t *end); /** * Return the number of milliseconds between <b>start</b> and <b>end</b>. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ int64_t monotime_diff_msec(const monotime_t *start, const monotime_t *end); /** * Return the number of nanoseconds since the timer system was initialized. + * The returned value may be equal to zero. */ uint64_t monotime_absolute_nsec(void); /** * Return the number of microseconds since the timer system was initialized. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ -uint64_t monotime_absolute_usec(void); +MOCK_DECL(uint64_t, monotime_absolute_usec,(void)); /** * Return the number of milliseconds since the timer system was initialized. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ uint64_t monotime_absolute_msec(void); @@ -129,10 +253,13 @@ void monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec); * Set <b>out</b> to the current coarse time. */ void monotime_coarse_get(monotime_coarse_t *out); +/** + * Like monotime_absolute_*(), but faster on some platforms. + */ uint64_t monotime_coarse_absolute_nsec(void); uint64_t monotime_coarse_absolute_usec(void); uint64_t monotime_coarse_absolute_msec(void); -#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */ +#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ #define monotime_coarse_get monotime_get #define monotime_coarse_absolute_nsec monotime_absolute_nsec #define monotime_coarse_absolute_usec monotime_absolute_usec @@ -152,23 +279,32 @@ uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t); /** * Convert a difference, expressed in the units of monotime_coarse_to_stamp, * into an approximate number of milliseconds. + * + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units); uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec); uint32_t monotime_coarse_get_stamp(void); #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) +/** + * Like monotime_diff_*(), but faster on some platforms. + */ int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start, const monotime_coarse_t *end); int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start, const monotime_coarse_t *end); int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start, const monotime_coarse_t *end); +/** + * Like monotime_*(), but faster on some platforms. + */ void monotime_coarse_zero(monotime_coarse_t *out); int monotime_coarse_is_zero(const monotime_coarse_t *val); void monotime_coarse_add_msec(monotime_coarse_t *out, const monotime_coarse_t *val, uint32_t msec); -#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */ +#else /* !defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ #define monotime_coarse_diff_nsec monotime_diff_nsec #define monotime_coarse_diff_usec monotime_diff_usec #define monotime_coarse_diff_msec monotime_diff_msec @@ -182,6 +318,9 @@ void monotime_coarse_add_msec(monotime_coarse_t *out, * * Requires that the difference fit into an int32_t; not for use with * large time differences. + * + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start, const monotime_coarse_t *end); @@ -191,6 +330,9 @@ int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start, * * Requires that the difference fit into an int32_t; not for use with * large time differences. + * + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ static inline int32_t monotime_coarse_diff_msec32(const monotime_coarse_t *start, @@ -202,7 +344,7 @@ monotime_coarse_diff_msec32(const monotime_coarse_t *start, #else #define USING_32BIT_MSEC_HACK return monotime_coarse_diff_msec32_(start, end); -#endif +#endif /* SIZEOF_VOID_P == 8 */ } #ifdef TOR_UNIT_TESTS diff --git a/src/lib/time/include.am b/src/lib/time/include.am index a3f93a3744..dcb199b142 100644 --- a/src/lib/time/include.am +++ b/src/lib/time/include.am @@ -5,8 +5,10 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-time-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_time_a_SOURCES = \ src/lib/time/compat_time.c \ + src/lib/time/time_sys.c \ src/lib/time/tvdiff.c src_lib_libtor_time_testing_a_SOURCES = \ @@ -14,6 +16,8 @@ src_lib_libtor_time_testing_a_SOURCES = \ src_lib_libtor_time_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/time/compat_time.h \ + src/lib/time/time_sys.h \ src/lib/time/tvdiff.h diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c new file mode 100644 index 0000000000..8b9aa2856c --- /dev/null +++ b/src/lib/time/time_sys.c @@ -0,0 +1,28 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.c + * \brief Subsystem object for monotime setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/time/time_sys.h" +#include "lib/time/compat_time.h" + +static int +subsys_time_initialize(void) +{ + monotime_init(); + return 0; +} + +const subsys_fns_t sys_time = { + .name = "time", + /* Monotonic time depends on logging, and a lot of other modules depend on + * monotonic time. */ + .level = -80, + .supported = true, + .initialize = subsys_time_initialize, +}; diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h new file mode 100644 index 0000000000..6a860ffd08 --- /dev/null +++ b/src/lib/time/time_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.h + * \brief Declare subsystem object for the time module. + **/ + +#ifndef TOR_TIME_SYS_H +#define TOR_TIME_SYS_H + +extern const struct subsys_fns_t sys_time; + +#endif /* !defined(TOR_TIME_SYS_H) */ diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c index a87d0d96dc..d7c245f57a 100644 --- a/src/lib/time/tvdiff.c +++ b/src/lib/time/tvdiff.c @@ -11,6 +11,7 @@ #include "lib/time/tvdiff.h" #include "lib/cc/compat_compiler.h" +#include "lib/defs/time.h" #include "lib/log/log.h" #ifdef _WIN32 @@ -20,8 +21,6 @@ #include <sys/time.h> #endif -#define TOR_USEC_PER_SEC 1000000 - /** Return the difference between start->tv_sec and end->tv_sec. * Returns INT64_MAX on overflow and underflow. */ diff --git a/src/lib/time/tvdiff.h b/src/lib/time/tvdiff.h index 724af1528a..657cb99553 100644 --- a/src/lib/time/tvdiff.h +++ b/src/lib/time/tvdiff.h @@ -20,4 +20,4 @@ int64_t tv_to_msec(const struct timeval *tv); time_t time_diff(const time_t from, const time_t to); -#endif +#endif /* !defined(TOR_TVDIFF_H) */ diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 2840e590b8..c550bde024 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -1,6 +1,7 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/crypt_ops/*.h @@ -11,7 +12,7 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/subsys/*.h +lib/testsupport/*.h lib/tls/*.h - -ciphers.inc +lib/tls/*.inc diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c index b570216df0..ed0f821ce8 100644 --- a/src/lib/tls/buffers_tls.c +++ b/src/lib/tls/buffers_tls.c @@ -12,7 +12,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include <stddef.h> -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/cc/torint.h" #include "lib/log/log.h" diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am index a664b29fb2..7e05ef4f8c 100644 --- a/src/lib/tls/include.am +++ b/src/lib/tls/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-tls-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_tls_a_SOURCES = \ src/lib/tls/buffers_tls.c \ src/lib/tls/tortls.c \ @@ -29,6 +30,7 @@ src_lib_libtor_tls_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_tls_testing_a_CFLAGS = \ $(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/tls/ciphers.inc \ src/lib/tls/buffers_tls.h \ @@ -36,5 +38,6 @@ noinst_HEADERS += \ src/lib/tls/tortls.h \ src/lib/tls/tortls_internal.h \ src/lib/tls/tortls_st.h \ + src/lib/tls/tortls_sys.h \ src/lib/tls/x509.h \ src/lib/tls/x509_internal.h diff --git a/src/lib/tls/nss_countbytes.h b/src/lib/tls/nss_countbytes.h index 8b31603923..47f220c4c1 100644 --- a/src/lib/tls/nss_countbytes.h +++ b/src/lib/tls/nss_countbytes.h @@ -22,4 +22,4 @@ int tor_get_prfiledesc_byte_counts(struct PRFileDesc *fd, uint64_t *n_read_out, uint64_t *n_written_out); -#endif +#endif /* !defined(TOR_NSS_COUNTBYTES_H) */ diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c index 4ca7c7d9d3..1aff40c437 100644 --- a/src/lib/tls/tortls.c +++ b/src/lib/tls/tortls.c @@ -7,6 +7,7 @@ #define TOR_X509_PRIVATE #include "lib/tls/x509.h" #include "lib/tls/x509_internal.h" +#include "lib/tls/tortls_sys.h" #include "lib/tls/tortls.h" #include "lib/tls/tortls_st.h" #include "lib/tls/tortls_internal.h" @@ -15,6 +16,7 @@ #include "lib/crypt_ops/crypto_rsa.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/net/socket.h" +#include "lib/subsys/subsys.h" #ifdef _WIN32 #include <winsock2.h> @@ -440,3 +442,15 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity) return rv; } + +static void +subsys_tortls_shutdown(void) +{ + tor_tls_free_all(); +} + +const subsys_fns_t sys_tortls = { + .name = "tortls", + .level = -50, + .shutdown = subsys_tortls_shutdown +}; diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h index 8efc7a1c98..799bd6aaeb 100644 --- a/src/lib/tls/tortls.h +++ b/src/lib/tls/tortls.h @@ -25,12 +25,12 @@ struct ssl_ctx_st; struct ssl_session_st; typedef struct ssl_ctx_st tor_tls_context_impl_t; typedef struct ssl_st tor_tls_impl_t; -#else +#else /* !defined(ENABLE_OPENSSL) */ struct PRFileDesc; typedef struct PRFileDesc tor_tls_context_impl_t; typedef struct PRFileDesc tor_tls_impl_t; -#endif -#endif +#endif /* defined(ENABLE_OPENSSL) */ +#endif /* defined(TORTLS_PRIVATE) */ struct tor_x509_cert_t; @@ -144,9 +144,9 @@ void check_no_tls_errors_(const char *fname, int line); void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err, int severity, int domain, const char *doing); -#else +#else /* !defined(ENABLE_OPENSSL) */ #define check_no_tls_errors() STMT_NIL -#endif +#endif /* defined(ENABLE_OPENSSL) */ int tor_tls_get_my_certs(int server, const struct tor_x509_cert_t **link_cert_out, diff --git a/src/lib/tls/tortls_internal.h b/src/lib/tls/tortls_internal.h index 071c506561..866483a94c 100644 --- a/src/lib/tls/tortls_internal.h +++ b/src/lib/tls/tortls_internal.h @@ -61,8 +61,8 @@ STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret, void *arg); STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher); -#endif -#endif +#endif /* defined(TORTLS_OPENSSL_PRIVATE) */ +#endif /* defined(ENABLE_OPENSSL) */ #ifdef TOR_UNIT_TESTS extern int tor_tls_object_ex_data_index; @@ -73,4 +73,4 @@ extern uint64_t total_bytes_written_over_tls; extern uint64_t total_bytes_written_by_tls; #endif /* defined(TOR_UNIT_TESTS) */ -#endif /* defined(TORTLS_INTERNAL_H) */ +#endif /* !defined(TORTLS_INTERNAL_H) */ diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index 80b0df301f..5bafcf676d 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -25,7 +25,7 @@ * <winsock.h> and mess things up, in at least some openssl versions. */ #include <winsock2.h> #include <ws2tcpip.h> -#endif +#endif /* defined(_WIN32) */ #include "lib/crypt_ops/crypto_cipher.h" #include "lib/crypt_ops/crypto_rand.h" @@ -318,7 +318,7 @@ tor_tls_init(void) #else SSL_library_init(); SSL_load_error_strings(); -#endif +#endif /* defined(OPENSSL_1_1_API) */ #if (SIZEOF_VOID_P >= 8 && \ OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1)) @@ -383,7 +383,7 @@ static const char SERVER_CIPHER_LIST[] = * conclude that it has no valid ciphers if it's running with TLS1.3. */ TLS1_3_TXT_AES_128_GCM_SHA256 ":" -#endif +#endif /* defined(TLS1_3_TXT_AES_128_GCM_SHA256) */ TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":" TLS1_TXT_DHE_RSA_WITH_AES_128_SHA; @@ -464,7 +464,7 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /** List of ciphers that clients should advertise, omitting items that * our OpenSSL doesn't know about. */ static const char CLIENT_CIPHER_LIST[] = -#include "ciphers.inc" +#include "lib/tls/ciphers.inc" /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version * of any cipher we say. */ "!SSLv2" @@ -657,7 +657,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, if (r < 0) goto error; } -#else +#else /* !(defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SE...)) */ if (! is_client) { int nid; EC_KEY *ec_key; @@ -673,7 +673,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, SSL_CTX_set_tmp_ecdh(result->ctx, ec_key); EC_KEY_free(ec_key); } -#endif +#endif /* defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1...) */ SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER, always_accept_verify_cb); /* let us realloc bufs that we're writing from */ @@ -764,7 +764,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher) tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher); return c != NULL; } -#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */ +#else /* !defined(HAVE_SSL_CIPHER_FIND) */ # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR) if (m && m->get_cipher_by_char) { @@ -1062,7 +1062,7 @@ tor_tls_new(tor_socket_t sock, int isServer) /* We can't actually use TLS 1.3 until this bug is fixed. */ SSL_set_max_proto_version(result->ssl, TLS1_2_VERSION); } -#endif +#endif /* defined(SSL_CTRL_SET_MAX_PROTO_VERSION) */ if (!SSL_set_cipher_list(result->ssl, isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) { @@ -1728,7 +1728,7 @@ tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out, else return -1; } -#endif +#endif /* defined(TLS1_3_VERSION) */ return (r == 1) ? 0 : -1; } diff --git a/src/lib/tls/tortls_st.h b/src/lib/tls/tortls_st.h index 3f7ea8ac6a..73f6e6ecca 100644 --- a/src/lib/tls/tortls_st.h +++ b/src/lib/tls/tortls_st.h @@ -64,7 +64,7 @@ struct tor_tls_t { void (*negotiated_callback)(tor_tls_t *tls, void *arg); /** Argument to pass to negotiated_callback. */ void *callback_arg; -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS /** Last values retried from tor_get_prfiledesc_byte_counts(). */ uint64_t last_write_count; @@ -72,4 +72,4 @@ struct tor_tls_t { #endif }; -#endif +#endif /* !defined(TOR_TORTLS_ST_H) */ diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h new file mode 100644 index 0000000000..4b04f85f0c --- /dev/null +++ b/src/lib/tls/tortls_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tortls_sys.h + * \brief Declare subsystem object for the tortls module + **/ + +#ifndef TOR_TORTLS_SYS_H +#define TOR_TORTLS_SYS_H + +extern const struct subsys_fns_t sys_tortls; + +#endif /* !defined(TOR_TORTLS_SYS_H) */ diff --git a/src/lib/tls/x509.h b/src/lib/tls/x509.h index 5e6660de5c..0390a5464d 100644 --- a/src/lib/tls/x509.h +++ b/src/lib/tls/x509.h @@ -35,7 +35,7 @@ struct tor_x509_cert_t { common_digests_t cert_digests; common_digests_t pkey_digests; }; -#endif +#endif /* defined(TOR_X509_PRIVATE) */ void tor_tls_pick_certificate_lifetime(time_t now, unsigned cert_lifetime, @@ -47,7 +47,7 @@ tor_x509_cert_t *tor_x509_cert_replace_expiration( const tor_x509_cert_t *inp, time_t new_expiration_time, crypto_pk_t *signing_key); -#endif +#endif /* defined(TOR_UNIT_TESTS) */ tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert); @@ -72,4 +72,4 @@ int tor_tls_cert_is_valid(int severity, time_t now, int check_rsa_1024); -#endif +#endif /* !defined(TOR_X509_H) */ diff --git a/src/lib/tls/x509_internal.h b/src/lib/tls/x509_internal.h index bf2bec9689..f858baae98 100644 --- a/src/lib/tls/x509_internal.h +++ b/src/lib/tls/x509_internal.h @@ -50,4 +50,4 @@ int tor_x509_cert_set_cached_der_encoding(tor_x509_cert_t *cert); #define tor_x509_cert_set_cached_der_encoding(cert) (0) #endif -#endif +#endif /* !defined(TOR_X509_INTERNAL_H) */ diff --git a/src/lib/tls/x509_nss.c b/src/lib/tls/x509_nss.c index fb4af54c52..e04afaf07b 100644 --- a/src/lib/tls/x509_nss.c +++ b/src/lib/tls/x509_nss.c @@ -120,13 +120,13 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa, der.data, der.len, (SECKEYPrivateKey *)signing_key,//const &cert->signature); -#else +#else /* !(0) */ s = SEC_DerSignData(cert->arena, &signed_der, der.data, der.len, (SECKEYPrivateKey *)signing_key,//const SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION); -#endif +#endif /* 0 */ if (s != SECSuccess) goto err; @@ -145,7 +145,7 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa, &result_cert->signatureWrap, issuer_pk, NULL); tor_assert(cert_ok == SECSuccess); } -#endif +#endif /* 1 */ err: if (subject_spki) @@ -455,4 +455,4 @@ tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp, return newcert ? tor_x509_cert_new(newcert) : NULL; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c index a344279c22..7724288279 100644 --- a/src/lib/tls/x509_openssl.c +++ b/src/lib/tls/x509_openssl.c @@ -59,12 +59,12 @@ ENABLE_GCC_WARNING(redundant-decls) #define X509_get_notAfter(cert) \ X509_getm_notAfter(cert) #endif -#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */ +#else /* !defined(OPENSSL_1_1_API) */ #define X509_get_notBefore_const(cert) \ ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert)) #define X509_get_notAfter_const(cert) \ ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert)) -#endif +#endif /* defined(OPENSSL_1_1_API) */ /** Return a newly allocated X509 name with commonName <b>cname</b>. */ static X509_NAME * diff --git a/src/lib/trace/debug.h b/src/lib/trace/debug.h index e35616cf50..92bb95c883 100644 --- a/src/lib/trace/debug.h +++ b/src/lib/trace/debug.h @@ -27,4 +27,4 @@ "\"" XSTR(subsystem) "\" hit. " \ "(line "XSTR(__LINE__) ")") -#endif /* TOR_TRACE_LOG_DEBUG_H */ +#endif /* !defined(TOR_TRACE_LOG_DEBUG_H) */ diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h index 1e1e7b9d16..9de86d63f2 100644 --- a/src/lib/trace/events.h +++ b/src/lib/trace/events.h @@ -34,12 +34,12 @@ #include "lib/trace/debug.h" #endif -#else /* TOR_EVENT_TRACING_ENABLED */ +#else /* !defined(TOR_EVENT_TRACING_ENABLED) */ /* Reaching this point, we NOP every event declaration because event tracing * is not been enabled at compile time. */ #define tor_trace(subsystem, name, args...) -#endif /* TOR_EVENT_TRACING_ENABLED */ +#endif /* defined(TOR_EVENT_TRACING_ENABLED) */ -#endif /* TOR_TRACE_EVENTS_H */ +#endif /* !defined(TOR_TRACE_EVENTS_H) */ diff --git a/src/lib/trace/include.am b/src/lib/trace/include.am index 6f10c98744..98098c87f4 100644 --- a/src/lib/trace/include.am +++ b/src/lib/trace/include.am @@ -2,6 +2,7 @@ noinst_LIBRARIES += \ src/lib/libtor-trace.a +# ADD_C_FILE: INSERT HEADERS HERE. TRACEHEADERS = \ src/lib/trace/trace.h \ src/lib/trace/events.h @@ -11,7 +12,7 @@ TRACEHEADERS += \ src/lib/trace/debug.h endif -# Library source files. +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_trace_a_SOURCES = \ src/lib/trace/trace.c diff --git a/src/lib/trace/trace.h b/src/lib/trace/trace.h index 606d435568..5001b28a1d 100644 --- a/src/lib/trace/trace.h +++ b/src/lib/trace/trace.h @@ -11,4 +11,4 @@ void tor_trace_init(void); -#endif // TOR_TRACE_TRACE_H +#endif /* !defined(TOR_TRACE_TRACE_H) */ diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include new file mode 100644 index 0000000000..d159ceb41f --- /dev/null +++ b/src/lib/version/.may_include @@ -0,0 +1,3 @@ +orconfig.h +micro-revision.i +lib/version/*.h
\ No newline at end of file diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c index 7d27549cad..900a1e12a0 100644 --- a/src/lib/log/git_revision.c +++ b/src/lib/version/git_revision.c @@ -4,7 +4,7 @@ /* See LICENSE for licensing information */ #include "orconfig.h" -#include "lib/log/git_revision.h" +#include "lib/version/git_revision.h" /** String describing which Tor Git repository version the source was * built from. This string is generated by a bit of shell kludging in diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h index 79e3c6684b..79e3c6684b 100644 --- a/src/lib/log/git_revision.h +++ b/src/lib/version/git_revision.h diff --git a/src/lib/version/include.am b/src/lib/version/include.am new file mode 100644 index 0000000000..0ae31be1b2 --- /dev/null +++ b/src/lib/version/include.am @@ -0,0 +1,27 @@ + +noinst_LIBRARIES += src/lib/libtor-version.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-version-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_version_a_SOURCES = \ + src/lib/version/git_revision.c \ + src/lib/version/version.c + +src_lib_libtor_version_testing_a_SOURCES = \ + $(src_lib_libtor_version_a_SOURCES) +src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# Declare that these object files depend on micro-revision.i. Without this +# rule, we could try to build them before micro-revision.i was created. +src/lib/version/git_revision.$(OBJEXT) \ + src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \ + micro-revision.i + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/version/git_revision.h \ + src/lib/version/torversion.h diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h new file mode 100644 index 0000000000..7b0fb66ec0 --- /dev/null +++ b/src/lib/version/torversion.h @@ -0,0 +1,12 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_VERSION_H +#define TOR_VERSION_H + +const char *get_version(void); +const char *get_short_version(void); + +#endif /* !defined(TOR_VERSION_H) */ diff --git a/src/lib/version/version.c b/src/lib/version/version.c new file mode 100644 index 0000000000..434e6fb424 --- /dev/null +++ b/src/lib/version/version.c @@ -0,0 +1,50 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "lib/version/torversion.h" +#include "lib/version/git_revision.h" + +#include <stdio.h> +#include <string.h> + +/** A shorter version of this Tor process's version, for export in our router + * descriptor. (Does not include the git version, if any.) */ +static const char the_short_tor_version[] = + VERSION +#ifdef TOR_BUILD_TAG + " ("TOR_BUILD_TAG")" +#endif + ""; + +#define MAX_VERSION_LEN 128 + +/** The version of this Tor process, possibly including git version */ +static char the_tor_version[MAX_VERSION_LEN] = ""; + +/** Return the current Tor version. */ +const char * +get_version(void) +{ + if (the_tor_version[0] == 0) { + if (strlen(tor_git_revision)) { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s (git-%s)", the_short_tor_version, tor_git_revision); + } else { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s", the_short_tor_version); + } + the_tor_version[sizeof(the_tor_version)-1] = 0; + } + + return the_tor_version; +} + +/** Return the current Tor version, without any git tag. */ +const char * +get_short_version(void) +{ + return the_short_tor_version; +} diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include index dc010da063..ce7a26472b 100644 --- a/src/lib/wallclock/.may_include +++ b/src/lib/wallclock/.may_include @@ -3,4 +3,5 @@ lib/cc/*.h lib/err/*.h lib/wallclock/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index ee498702d5..77eeddaf56 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -9,7 +9,9 @@ **/ #include "orconfig.h" +#include "lib/subsys/subsys.h" #include "lib/wallclock/approx_time.h" +#include "lib/wallclock/wallclock_sys.h" /* ===== * Cached time @@ -41,3 +43,19 @@ update_approx_time(time_t now) cached_approx_time = now; } #endif /* !defined(TIME_IS_FAST) */ + +static int +subsys_wallclock_initialize(void) +{ + update_approx_time(time(NULL)); + return 0; +} + +const subsys_fns_t sys_wallclock = { + .name = "wallclock", + .supported = true, + /* Approximate time is a diagnostic feature, we want it to init right after + * low-level error handling. */ + .level = -98, + .initialize = subsys_wallclock_initialize, +}; diff --git a/src/lib/wallclock/approx_time.h b/src/lib/wallclock/approx_time.h index e6b53f2c27..e7da160122 100644 --- a/src/lib/wallclock/approx_time.h +++ b/src/lib/wallclock/approx_time.h @@ -22,4 +22,4 @@ time_t approx_time(void); void update_approx_time(time_t now); #endif /* defined(TIME_IS_FAST) */ -#endif +#endif /* !defined(TOR_APPROX_TIME_H) */ diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am index 1961639bd7..2b50d6ccbb 100644 --- a/src/lib/wallclock/include.am +++ b/src/lib/wallclock/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-wallclock-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_wallclock_a_SOURCES = \ src/lib/wallclock/approx_time.c \ src/lib/wallclock/time_to_tm.c \ @@ -15,8 +16,10 @@ src_lib_libtor_wallclock_testing_a_SOURCES = \ src_lib_libtor_wallclock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_wallclock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/wallclock/approx_time.h \ src/lib/wallclock/timeval.h \ src/lib/wallclock/time_to_tm.h \ - src/lib/wallclock/tor_gettimeofday.h + src/lib/wallclock/tor_gettimeofday.h \ + src/lib/wallclock/wallclock_sys.h diff --git a/src/lib/wallclock/time_to_tm.h b/src/lib/wallclock/time_to_tm.h index abe78c0efe..da27fcaba1 100644 --- a/src/lib/wallclock/time_to_tm.h +++ b/src/lib/wallclock/time_to_tm.h @@ -19,4 +19,4 @@ struct tm *tor_localtime_r_msg(const time_t *timep, struct tm *result, struct tm *tor_gmtime_r_msg(const time_t *timep, struct tm *result, char **err_out); -#endif +#endif /* !defined(TOR_WALLCLOCK_TIME_TO_TM_H) */ diff --git a/src/lib/wallclock/timeval.h b/src/lib/wallclock/timeval.h index 4967e939bf..e632d04a04 100644 --- a/src/lib/wallclock/timeval.h +++ b/src/lib/wallclock/timeval.h @@ -20,6 +20,27 @@ #include <sys/time.h> #endif +#ifdef TOR_COVERAGE +/* For coverage builds, we use a slower definition of these macros without + * branches, to make coverage consistent. */ +#undef timeradd +#undef timersub +#define timeradd(tv1,tv2,tvout) \ + do { \ + (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec; \ + (tvout)->tv_usec = (tv1)->tv_usec + (tv2)->tv_usec; \ + (tvout)->tv_sec += (tvout)->tv_usec / 1000000; \ + (tvout)->tv_usec %= 1000000; \ + } while (0) +#define timersub(tv1,tv2,tvout) \ + do { \ + (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec - 1; \ + (tvout)->tv_usec = (tv1)->tv_usec - (tv2)->tv_usec + 1000000; \ + (tvout)->tv_sec += (tvout)->tv_usec / 1000000; \ + (tvout)->tv_usec %= 1000000; \ + } while (0) +#endif /* defined(TOR_COVERAGE) */ + #ifndef timeradd /** Replacement for timeradd on platforms that do not have it: sets tvout to * the sum of tv1 and tv2. */ @@ -62,4 +83,4 @@ ((tv1)->tv_sec op (tv2)->tv_sec)) #endif /* !defined(timercmp) */ -#endif +#endif /* !defined(TOR_TIMEVAL_H) */ diff --git a/src/lib/wallclock/tor_gettimeofday.h b/src/lib/wallclock/tor_gettimeofday.h index c7fff9747a..6fec2fc893 100644 --- a/src/lib/wallclock/tor_gettimeofday.h +++ b/src/lib/wallclock/tor_gettimeofday.h @@ -17,4 +17,4 @@ struct timeval; MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval)); -#endif +#endif /* !defined(TOR_GETTIMEOFDAY_H) */ diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h new file mode 100644 index 0000000000..a30912b8fb --- /dev/null +++ b/src/lib/wallclock/wallclock_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file wallclock_sys.h + * \brief Declare subsystem object for the wallclock module. + **/ + +#ifndef TOR_WALLCLOCK_SYS_H +#define TOR_WALLCLOCK_SYS_H + +extern const struct subsys_fns_t sys_wallclock; + +#endif /* !defined(TOR_WALLCLOCK_SYS_H) */ |