diff options
Diffstat (limited to 'src/lib')
457 files changed, 20560 insertions, 3503 deletions
diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h index fa82241b28..c72ac3eb8e 100644 --- a/src/lib/arch/bytes.h +++ b/src/lib/arch/bytes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BYTES_H @@ -16,12 +16,17 @@ #include <string.h> #include "lib/cc/torint.h" -/* The uint8 variants are defined to make the code more uniform. */ +/** + * Read an 8-bit from <b>cp</b>. + */ static inline uint8_t get_uint8(const void *cp) { return *(const uint8_t*)(cp); } +/** + * Store an 8-bit value from <b>v</b> to <b>cp</b>. + */ static inline void set_uint8(void *cp, uint8_t v) { @@ -93,7 +98,7 @@ set_uint64(void *cp, uint64_t v) memcpy(cp,&v,8); } -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) static inline uint16_t tor_htons(uint32_t a) { @@ -129,7 +134,10 @@ tor_ntohll(uint64_t a) { return a; } -#else +#else /* !defined(WORDS_BIGENDIAN) */ +/** + * Convert a 16-bit value from host order to network order (big-endian). + **/ static inline uint16_t tor_htons(uint16_t a) { @@ -139,12 +147,18 @@ tor_htons(uint16_t a) ((a & 0xff00) >> 8); } +/** + * Convert a 16-bit value from network order (big-endian) to host order. + **/ static inline uint16_t tor_ntohs(uint16_t a) { return tor_htons(a); } +/** + * Convert a 32-bit value from host order to network order (big-endian). + **/ static inline uint32_t tor_htonl(uint32_t a) { @@ -156,6 +170,9 @@ tor_htonl(uint32_t a) ((a & 0xff000000) >>24); } +/** + * Convert a 32-bit value from network order (big-endian) to host order. + **/ static inline uint32_t tor_ntohl(uint32_t a) { @@ -177,6 +194,6 @@ tor_ntohll(uint64_t a) { return tor_htonll(a); } -#endif +#endif /* defined(WORDS_BIGENDIAN) */ -#endif +#endif /* !defined(TOR_BYTES_H) */ diff --git a/src/lib/arch/include.am b/src/lib/arch/include.am index f92ee9222f..c5926c6330 100644 --- a/src/lib/arch/include.am +++ b/src/lib/arch/include.am @@ -1,3 +1,4 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/arch/bytes.h diff --git a/src/lib/arch/lib_arch.md b/src/lib/arch/lib_arch.md new file mode 100644 index 0000000000..9b8bccdf16 --- /dev/null +++ b/src/lib/arch/lib_arch.md @@ -0,0 +1,2 @@ +@dir /lib/arch +@brief lib/arch: Compatibility code for handling different CPU architectures. diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include new file mode 100644 index 0000000000..c4be73bce2 --- /dev/null +++ b/src/lib/buf/.may_include @@ -0,0 +1,10 @@ +orconfig.h + +lib/buf/*.h +lib/cc/*.h +lib/ctime/*.h +lib/malloc/*.h +lib/testsupport/*.h +lib/log/*.h +lib/string/*.h +lib/time/*.h diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c index fe4cf7c385..a5031a47a6 100644 --- a/src/lib/container/buffers.c +++ b/src/lib/buf/buffers.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -25,7 +25,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include <stddef.h> -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/cc/torint.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" @@ -99,6 +99,7 @@ #define DBG_S(s) (void)0 #endif +#ifndef COCCI #ifdef DISABLE_MEMORY_SENTINELS #define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL #else @@ -109,6 +110,7 @@ memset(a,0,SENTINEL_LEN); \ } while (0) #endif /* defined(DISABLE_MEMORY_SENTINELS) */ +#endif /* !defined(COCCI) */ /** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem, * to free up space at the end. */ @@ -158,7 +160,7 @@ chunk_new_with_alloc_size(size_t alloc) static inline chunk_t * chunk_grow(chunk_t *chunk, size_t sz) { - off_t offset; + ptrdiff_t offset; const size_t memlen_orig = chunk->memlen; const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig); const size_t new_alloc = CHUNK_ALLOC_SIZE(sz); @@ -283,7 +285,7 @@ buf_t * buf_new_with_data(const char *cp, size_t sz) { /* Validate arguments */ - if (!cp || sz <= 0 || sz >= INT_MAX) { + if (!cp || sz <= 0 || sz > BUF_MAX_LEN) { return NULL; } @@ -440,7 +442,7 @@ chunk_copy(const chunk_t *in_chunk) #endif newch->next = NULL; if (in_chunk->data) { - off_t offset = in_chunk->data - in_chunk->mem; + ptrdiff_t offset = in_chunk->data - in_chunk->mem; newch->data = newch->mem + offset; } return newch; @@ -528,9 +530,9 @@ buf_add(buf_t *buf, const char *string, size_t string_len) return (int)buf->datalen; check(); - if (BUG(buf->datalen >= INT_MAX)) + if (BUG(buf->datalen > BUF_MAX_LEN)) return -1; - if (BUG(buf->datalen >= INT_MAX - string_len)) + if (BUG(buf->datalen > BUF_MAX_LEN - string_len)) return -1; while (string_len) { @@ -549,7 +551,7 @@ buf_add(buf_t *buf, const char *string, size_t string_len) } check(); - tor_assert(buf->datalen < INT_MAX); + tor_assert(buf->datalen <= BUF_MAX_LEN); return (int)buf->datalen; } @@ -578,6 +580,7 @@ buf_add_vprintf(buf_t *buf, const char *format, va_list args) /* XXXX Faster implementations are easy enough, but let's optimize later */ char *tmp; tor_vasprintf(&tmp, format, args); + tor_assert(tmp != NULL); buf_add(buf, tmp, strlen(tmp)); tor_free(tmp); } @@ -642,7 +645,7 @@ buf_get_bytes(buf_t *buf, char *string, size_t string_len) buf_peek(buf, string, string_len); buf_drain(buf, string_len); check(); - tor_assert(buf->datalen < INT_MAX); + tor_assert(buf->datalen <= BUF_MAX_LEN); return (int)buf->datalen; } @@ -657,9 +660,9 @@ buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen) char b[4096]; size_t cp, len; - if (BUG(buf_out->datalen >= INT_MAX || *buf_flushlen >= INT_MAX)) + if (BUG(buf_out->datalen > BUF_MAX_LEN || *buf_flushlen > BUF_MAX_LEN)) return -1; - if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen)) + if (BUG(buf_out->datalen > BUF_MAX_LEN - *buf_flushlen)) return -1; len = *buf_flushlen; @@ -667,7 +670,7 @@ buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen) len = buf_in->datalen; cp = len; /* Remember the number of bytes we intend to copy. */ - tor_assert(cp < INT_MAX); + tor_assert(cp <= BUF_MAX_LEN); while (len) { /* This isn't the most efficient implementation one could imagine, since * it does two copies instead of 1, but I kinda doubt that this will be @@ -691,9 +694,9 @@ buf_move_all(buf_t *buf_out, buf_t *buf_in) return; if (buf_datalen(buf_in) == 0) return; - if (BUG(buf_out->datalen >= INT_MAX || buf_in->datalen >= INT_MAX)) + if (BUG(buf_out->datalen > BUF_MAX_LEN || buf_in->datalen > BUF_MAX_LEN)) return; - if (BUG(buf_out->datalen >= INT_MAX - buf_in->datalen)) + if (BUG(buf_out->datalen > BUF_MAX_LEN - buf_in->datalen)) return; if (buf_out->head == NULL) { @@ -712,7 +715,8 @@ buf_move_all(buf_t *buf_out, buf_t *buf_in) /** Internal structure: represents a position in a buffer. */ typedef struct buf_pos_t { const chunk_t *chunk; /**< Which chunk are we pointing to? */ - int pos;/**< Which character inside the chunk's data are we pointing to? */ + ptrdiff_t pos;/**< Which character inside the chunk's data are we pointing + * to? */ size_t chunk_pos; /**< Total length of all previous chunks. */ } buf_pos_t; @@ -728,15 +732,15 @@ buf_pos_init(const buf_t *buf, buf_pos_t *out) /** Advance <b>out</b> to the first appearance of <b>ch</b> at the current * position of <b>out</b>, or later. Return -1 if no instances are found; * otherwise returns the absolute position of the character. */ -static off_t +static ptrdiff_t buf_find_pos_of_char(char ch, buf_pos_t *out) { const chunk_t *chunk; - int pos; + ptrdiff_t pos; tor_assert(out); if (out->chunk) { if (out->chunk->datalen) { - tor_assert(out->pos < (off_t)out->chunk->datalen); + tor_assert(out->pos < (ptrdiff_t)out->chunk->datalen); } else { tor_assert(out->pos == 0); } @@ -746,7 +750,7 @@ buf_find_pos_of_char(char ch, buf_pos_t *out) char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos); if (cp) { out->chunk = chunk; - tor_assert(cp - chunk->data < INT_MAX); + tor_assert(cp - chunk->data <= BUF_MAX_LEN); out->pos = (int)(cp - chunk->data); return out->chunk_pos + out->pos; } else { @@ -762,9 +766,9 @@ buf_find_pos_of_char(char ch, buf_pos_t *out) static inline int buf_pos_inc(buf_pos_t *pos) { - tor_assert(pos->pos < INT_MAX - 1); + tor_assert(pos->pos < BUF_MAX_LEN); ++pos->pos; - if (pos->pos == (off_t)pos->chunk->datalen) { + if (pos->pos == (ptrdiff_t)pos->chunk->datalen) { if (!pos->chunk->next) return -1; pos->chunk_pos += pos->chunk->datalen; @@ -809,7 +813,7 @@ buf_find_string_offset(const buf_t *buf, const char *s, size_t n) buf_pos_init(buf, &pos); while (buf_find_pos_of_char(*s, &pos) >= 0) { if (buf_matches_at_pos(&pos, s, n)) { - tor_assert(pos.chunk_pos + pos.pos < INT_MAX); + tor_assert(pos.chunk_pos + pos.pos <= BUF_MAX_LEN); return (int)(pos.chunk_pos + pos.pos); } else { if (buf_pos_inc(&pos)<0) @@ -838,12 +842,12 @@ buf_peek_startswith(const buf_t *buf, const char *cmd) /** Return the index within <b>buf</b> at which <b>ch</b> first appears, * or -1 if <b>ch</b> does not appear on buf. */ -static off_t +static ptrdiff_t buf_find_offset_of_char(buf_t *buf, char ch) { chunk_t *chunk; - off_t offset = 0; - tor_assert(buf->datalen < INT_MAX); + ptrdiff_t offset = 0; + tor_assert(buf->datalen <= BUF_MAX_LEN); for (chunk = buf->head; chunk; chunk = chunk->next) { char *cp = memchr(chunk->data, ch, chunk->datalen); if (cp) @@ -865,7 +869,7 @@ int buf_get_line(buf_t *buf, char *data_out, size_t *data_len) { size_t sz; - off_t offset; + ptrdiff_t offset; if (!buf->head) return 0; @@ -913,7 +917,7 @@ buf_assert_ok(buf_t *buf) for (ch = buf->head; ch; ch = ch->next) { total += ch->datalen; tor_assert(ch->datalen <= ch->memlen); - tor_assert(ch->datalen < INT_MAX); + tor_assert(ch->datalen <= BUF_MAX_LEN); tor_assert(ch->data >= &ch->mem[0]); tor_assert(ch->data <= &ch->mem[0]+ch->memlen); if (ch->data == &ch->mem[0]+ch->memlen) { diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h index c103b93a82..d8a77feb72 100644 --- a/src/lib/container/buffers.h +++ b/src/lib/buf/buffers.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,6 +29,9 @@ void buf_free_(buf_t *buf); void buf_clear(buf_t *buf); buf_t *buf_copy(const buf_t *buf); +/** Maximum bytes in a buffer, inclusive. */ +#define BUF_MAX_LEN (INT_MAX - 1) + MOCK_DECL(size_t, buf_datalen, (const buf_t *buf)); size_t buf_allocation(const buf_t *buf); size_t buf_slack(const buf_t *buf); diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am new file mode 100644 index 0000000000..27430d1d38 --- /dev/null +++ b/src/lib/buf/include.am @@ -0,0 +1,19 @@ + +noinst_LIBRARIES += src/lib/libtor-buf.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-buf-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_buf_a_SOURCES = \ + src/lib/buf/buffers.c + +src_lib_libtor_buf_testing_a_SOURCES = \ + $(src_lib_libtor_buf_a_SOURCES) +src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/buf/buffers.h diff --git a/src/lib/buf/lib_buf.md b/src/lib/buf/lib_buf.md new file mode 100644 index 0000000000..519ab50a2d --- /dev/null +++ b/src/lib/buf/lib_buf.md @@ -0,0 +1,13 @@ +@dir /lib/buf +@brief lib/buf: An efficient byte queue. + +This module defines the buf_t type, which is used throughout our networking +code. The implementation is a singly-linked queue of buffer chunks, similar +to the BSD kernel's +["mbuf"](https://www.freebsd.org/cgi/man.cgi?query=mbuf&sektion=9) structure. + +The buf_t type is also reasonable for use in constructing long strings. + +See \refdir{lib/net} for networking code that uses buf_t, and +\refdir{lib/tls} for cryptographic code that uses buf_t. + diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include index 2b06e8519c..fa1478ce46 100644 --- a/src/lib/cc/.may_include +++ b/src/lib/cc/.may_include @@ -1 +1,2 @@ orconfig.h +lib/cc/*.h
\ No newline at end of file diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h index fbe6a38f1f..96aa912652 100644 --- a/src/lib/cc/compat_compiler.h +++ b/src/lib/cc/compat_compiler.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -25,11 +25,11 @@ #endif /* defined(__has_feature) */ #ifndef NULL_REP_IS_ZERO_BYTES -#error "It seems your platform does not represent NULL as zero. We can't cope." +#error "Your platform does not represent NULL as zero. We can't cope." #endif #ifndef DOUBLE_0_REP_IS_ZERO_BYTES -#error "It seems your platform does not represent 0.0 as zeros. We can't cope." +#error "Your platform does not represent 0.0 as zeros. We can't cope." #endif #if 'a'!=97 || 'z'!=122 || 'A'!=65 || ' '!=32 @@ -65,8 +65,6 @@ /* Temporarily enable and disable warnings. */ #ifdef __GNUC__ -# define PRAGMA_STRINGIFY_(s) #s -# define PRAGMA_JOIN_STRINGIFY_(a,b) PRAGMA_STRINGIFY_(a ## b) /* Support for macro-generated pragmas (c99) */ # define PRAGMA_(x) _Pragma (#x) # ifdef __clang__ @@ -78,17 +76,17 @@ /* we have push/pop support */ # define DISABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(push) \ - PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) + PRAGMA_DIAGNOSTIC_(ignored warningopt) # define ENABLE_GCC_WARNING(warningopt) \ PRAGMA_DIAGNOSTIC_(pop) #else /* !(defined(__clang__) || GCC_VERSION >= 406) */ /* older version of gcc: no push/pop support. */ # define DISABLE_GCC_WARNING(warningopt) \ - PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) + PRAGMA_DIAGNOSTIC_(ignored warningopt) # define ENABLE_GCC_WARNING(warningopt) \ - PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt)) + PRAGMA_DIAGNOSTIC_(warning warningopt) #endif /* defined(__clang__) || GCC_VERSION >= 406 */ -#else /* !(defined(__GNUC__)) */ +#else /* !defined(__GNUC__) */ /* not gcc at all */ # define DISABLE_GCC_WARNING(warning) # define ENABLE_GCC_WARNING(warning) @@ -200,8 +198,8 @@ /** Macro: yield a pointer to the field at position <b>off</b> within the * structure <b>st</b>. Example: * <pre> - * struct a { int foo; int bar; } x; - * off_t bar_offset = offsetof(struct a, bar); + * struct a_t { int foo; int bar; } x; + * ptrdiff_t bar_offset = offsetof(struct a_t, bar); * int *bar_p = STRUCT_VAR_P(&x, bar_offset); * *bar_p = 3; * </pre> @@ -211,10 +209,10 @@ /** Macro: yield a pointer to an enclosing structure given a pointer to * a substructure at offset <b>off</b>. Example: * <pre> - * struct base { ... }; - * struct subtype { int x; struct base b; } x; - * struct base *bp = &x.base; - * struct *sp = SUBTYPE_P(bp, struct subtype, b); + * struct base_t { ... }; + * struct subtype_t { int x; struct base_t b; } x; + * struct base_t *bp = &x.base; + * struct *sp = SUBTYPE_P(bp, struct subtype_t, b); * </pre> */ #define SUBTYPE_P(p, subtype, basemember) \ @@ -223,4 +221,29 @@ /** Macro: Yields the number of elements in array x. */ #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0])) -#endif /* !defined(TOR_COMPAT_H) */ +/** + * "Eat" a semicolon that somebody puts at the end of a top-level macro. + * + * Frequently, we want to declare a macro that people will use at file scope, + * and we want to allow people to put a semicolon after the macro. + * + * This declaration of a struct can be repeated any number of times, and takes + * a trailing semicolon afterwards. + **/ +#define EAT_SEMICOLON \ + struct dummy_semicolon_eater__ + +/** + * Tell our static analysis tool to believe that (clang's scan-build or + * coverity scan) that an expression might be true. We use this to suppress + * dead-code warnings. + **/ +#if defined(__COVERITY__) || defined(__clang_analyzer__) +/* By calling getenv, we force the analyzer not to conclude that 'expr' is + * false. */ +#define POSSIBLE(expr) ((expr) || getenv("STATIC_ANALYZER_DEADCODE_DUMMY_")) +#else +#define POSSIBLE(expr) (expr) +#endif /* defined(__COVERITY__) || defined(__clang_analyzer__) */ + +#endif /* !defined(TOR_COMPAT_COMPILER_H) */ diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h new file mode 100644 index 0000000000..d9d3aa40b0 --- /dev/null +++ b/src/lib/cc/ctassert.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2018 The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file ctassert.h + * + * \brief Compile-time assertions: CTASSERT(expression). + */ + +#ifndef TOR_CTASSERT_H +#define TOR_CTASSERT_H + +#include "lib/cc/compat_compiler.h" + +/** + * CTASSERT(expression) + * + * Trigger a compiler error if expression is false. + */ +#if __STDC_VERSION__ >= 201112L + +/* If C11 is available, just use _Static_assert. */ +#define CTASSERT(x) _Static_assert((x), #x) + +#else /* !(__STDC_VERSION__ >= 201112L) */ + +/* + * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__ + * and __LINE__, or just __LINE__, with an intermediate preprocessor + * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the + * expansions together into a unique name. + * + * We use this name as a typedef of an array type with a positive + * length if the assertion is true, and a negative length of the + * assertion is false, which is invalid and hence triggers a compiler + * error. + */ +#if defined(__COUNTER__) +#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__) +#elif defined(__INCLUDE_LEVEL__) +#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__) +#else +/* hope it's unique enough */ +#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__) +#endif /* defined(__COUNTER__) || ... */ + +#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b) +#define CTASSERT_DECL(x, a, b) \ + typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED; EAT_SEMICOLON + +#endif /* __STDC_VERSION__ >= 201112L */ + +#endif /* !defined(TOR_CTASSERT_H) */ diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am index 2ae90f97dd..d2a415e956 100644 --- a/src/lib/cc/include.am +++ b/src/lib/cc/include.am @@ -1,4 +1,7 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/cc/compat_compiler.h \ + src/lib/cc/ctassert.h \ + src/lib/cc/tokpaste.h \ src/lib/cc/torint.h diff --git a/src/lib/cc/lib_cc.md b/src/lib/cc/lib_cc.md new file mode 100644 index 0000000000..bd49005ba2 --- /dev/null +++ b/src/lib/cc/lib_cc.md @@ -0,0 +1,2 @@ +@dir /lib/cc +@brief lib/cc: Macros for managing the C compiler and language. diff --git a/src/lib/cc/tokpaste.h b/src/lib/cc/tokpaste.h new file mode 100644 index 0000000000..068621b5bd --- /dev/null +++ b/src/lib/cc/tokpaste.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file tokpaste.h + * @brief Token-pasting macros. + **/ + +#ifndef TOR_LIB_CC_TOKPASTE_H +#define TOR_LIB_CC_TOKPASTE_H + +/** + * Concatenate `a` and `b` in a way that allows their result itself to be + * expanded by the preprocessor. + * + * Ordinarily you could just say `a ## b` in a macro definition. But doing so + * results in a symbol which the preprocessor will not then expand. If you + * wanted to use `a ## b` to create the name of a macro and have the + * preprocessor expand _that_ macro, you need to have another level of + * indirection, as this macro provides. + **/ +#define PASTE(a,b) PASTE__(a,b) + +/** Helper for PASTE(). */ +#define PASTE__(a,b) a ## b + +#endif /* !defined(TOR_LIB_CC_TOKPASTE_H) */ diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h index c9b2d329f2..af7a90431c 100644 --- a/src/lib/cc/torint.h +++ b/src/lib/cc/torint.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -49,7 +49,7 @@ typedef int32_t ssize_t; * aren't 2's complement, and you don't define LONG_MAX, then you're so * bizarre that I want nothing to do with you. */ #ifndef USING_TWOS_COMPLEMENT -#error "Seems that your platform doesn't use 2's complement arithmetic. Argh." +#error "Your platform doesn't use 2's complement arithmetic." #endif #ifndef TIME_MAX @@ -96,9 +96,9 @@ typedef int32_t ssize_t; # else # define TOR_PRIuSZ PRIu32 # endif -#else +#else /* !defined(_WIN32) */ # define TOR_PRIuSZ "zu" -#endif +#endif /* defined(_WIN32) */ #ifdef _WIN32 # ifdef _WIN64 @@ -106,9 +106,9 @@ typedef int32_t ssize_t; # else # define TOR_PRIdSZ PRId32 # endif -#else +#else /* !defined(_WIN32) */ # define TOR_PRIdSZ "zd" -#endif +#endif /* defined(_WIN32) */ #ifndef SSIZE_MAX #if (SIZEOF_SIZE_T == 4) @@ -125,4 +125,12 @@ typedef int32_t ssize_t; /** Any size_t larger than this amount is likely to be an underflow. */ #define SIZE_T_CEILING ((size_t)(SSIZE_MAX-16)) +#if SIZEOF_INT > SIZEOF_VOID_P +#error "sizeof(int) > sizeof(void *) - Can't build Tor here." +#endif + +#if SIZEOF_UNSIGNED_INT > SIZEOF_VOID_P +#error "sizeof(unsigned int) > sizeof(void *) - Can't build Tor here." +#endif + #endif /* !defined(TOR_TORINT_H) */ diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include index 68fe9f1c54..6cd80086e6 100644 --- a/src/lib/compress/.may_include +++ b/src/lib/compress/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/compress/*.h lib/container/*.h @@ -8,5 +9,6 @@ lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c index 95fd73bb32..7ce3910d84 100644 --- a/src/lib/compress/compress.c +++ b/src/lib/compress/compress.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,10 +29,12 @@ #include "lib/compress/compress.h" #include "lib/compress/compress_lzma.h" #include "lib/compress/compress_none.h" +#include "lib/compress/compress_sys.h" #include "lib/compress/compress_zlib.h" #include "lib/compress/compress_zstd.h" #include "lib/intmath/cmp.h" #include "lib/malloc/malloc.h" +#include "lib/subsys/subsys.h" #include "lib/thread/threads.h" /** Total number of bytes allocated for compression state overhead. */ @@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state) } /** Initialize all compression modules. */ -void +int tor_compress_init(void) { atomic_counter_init(&total_compress_allocation); @@ -668,6 +670,8 @@ tor_compress_init(void) tor_zlib_init(); tor_lzma_init(); tor_zstd_init(); + + return 0; } /** Warn if we had any problems while setting up our compression libraries. @@ -677,5 +681,21 @@ tor_compress_init(void) void tor_compress_log_init_warnings(void) { + // XXXX can we move this into tor_compress_init() after all? log.c queues + // XXXX log messages at startup. tor_zstd_warn_if_version_mismatched(); } + +static int +subsys_compress_initialize(void) +{ + return tor_compress_init(); +} + +const subsys_fns_t sys_compress = { + .name = "compress", + SUBSYS_DECLARE_LOCATION(), + .supported = true, + .level = -55, + .initialize = subsys_compress_initialize, +}; diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h index 5f16a2ab27..f36cdb82aa 100644 --- a/src/lib/compress/compress.h +++ b/src/lib/compress/compress.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); -void tor_compress_init(void); +int tor_compress_init(void); void tor_compress_log_init_warnings(void); struct buf_t; diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c index 198128b261..d1941c9da6 100644 --- a/src/lib/compress/compress_buf.c +++ b/src/lib/compress/compress_buf.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/cc/compat_compiler.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/compress/compress.h" #include "lib/log/util_bug.h" diff --git a/src/lib/compress/compress_lzma.c b/src/lib/compress/compress_lzma.c index 2dab37e433..8884b020e8 100644 --- a/src/lib/compress/compress_lzma.c +++ b/src/lib/compress/compress_lzma.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -221,7 +221,7 @@ tor_lzma_compress_new(int compress, tor_free(result); return NULL; /* LCOV_EXCL_STOP */ -#else /* !(defined(HAVE_LZMA)) */ +#else /* !defined(HAVE_LZMA) */ (void)compress; (void)method; (void)level; @@ -312,7 +312,7 @@ tor_lzma_compress_process(tor_lzma_compress_state_t *state, lzma_error_str(retval)); return TOR_COMPRESS_ERROR; } -#else /* !(defined(HAVE_LZMA)) */ +#else /* !defined(HAVE_LZMA) */ (void)state; (void)out; (void)out_len; diff --git a/src/lib/compress/compress_lzma.h b/src/lib/compress/compress_lzma.h index 556ab437dc..de03cda91c 100644 --- a/src/lib/compress/compress_lzma.h +++ b/src/lib/compress/compress_lzma.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_none.c b/src/lib/compress/compress_none.c index 0b5760773a..43c021c788 100644 --- a/src/lib/compress/compress_none.c +++ b/src/lib/compress/compress_none.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_none.h b/src/lib/compress/compress_none.h index 2bb9c3d66c..df696a11aa 100644 --- a/src/lib/compress/compress_none.h +++ b/src/lib/compress/compress_none.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h new file mode 100644 index 0000000000..dce0549924 --- /dev/null +++ b/src/lib/compress/compress_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_sys.h + * \brief Declare subsystem object for the compress module + **/ + +#ifndef TOR_COMPRESS_SYS_H +#define TOR_COMPRESS_SYS_H + +extern const struct subsys_fns_t sys_compress; + +#endif /* !defined(TOR_COMPRESS_SYS_H) */ diff --git a/src/lib/compress/compress_zlib.c b/src/lib/compress/compress_zlib.c index df0d1bff5f..3711e46222 100644 --- a/src/lib/compress/compress_zlib.c +++ b/src/lib/compress/compress_zlib.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_zlib.h b/src/lib/compress/compress_zlib.h index e4f248cd9b..7328ce899b 100644 --- a/src/lib/compress/compress_zlib.h +++ b/src/lib/compress/compress_zlib.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c index 45d0d4d602..5913d823e1 100644 --- a/src/lib/compress/compress_zstd.c +++ b/src/lib/compress/compress_zstd.c @@ -1,6 +1,6 @@ /* Copyright (c) 2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -25,17 +25,17 @@ * all invocations of zstd's static-only functions in a check to make sure * that the compile-time version matches the run-time version. */ #define ZSTD_STATIC_LINKING_ONLY -#endif +#endif /* defined(ENABLE_ZSTD_ADVANCED_APIS) */ #ifdef HAVE_ZSTD #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE -DISABLE_GCC_WARNING(unused-const-variable) +DISABLE_GCC_WARNING("-Wunused-const-variable") #endif #include <zstd.h> #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE -ENABLE_GCC_WARNING(unused-const-variable) -#endif +ENABLE_GCC_WARNING("-Wunused-const-variable") #endif +#endif /* defined(HAVE_ZSTD) */ /** Total number of bytes allocated for Zstandard state. */ static atomic_counter_t total_zstd_allocation; @@ -77,7 +77,7 @@ tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number) version_number / 100 % 100, version_number % 100); } -#endif +#endif /* defined(HAVE_ZSTD) */ #define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */ @@ -93,7 +93,7 @@ tor_zstd_get_version_str(void) ZSTD_versionNumber()); return version_str; -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ return NULL; #endif /* defined(HAVE_ZSTD) */ } @@ -125,9 +125,9 @@ tor_zstd_can_use_static_apis(void) } #endif return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber()); -#else +#else /* !(defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)) */ return 0; -#endif +#endif /* defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD) */ } /** Internal Zstandard state for incremental compression/decompression. @@ -237,7 +237,7 @@ tor_zstd_state_size_precalc(int compress, int preset) #endif } } -#endif +#endif /* defined(ZSTD_STATIC_LINKING_ONLY) */ return tor_zstd_state_size_precalc_fake(compress, preset); } #endif /* defined(HAVE_ZSTD) */ @@ -317,7 +317,7 @@ tor_zstd_compress_new(int compress, tor_free(result); return NULL; // LCOV_EXCL_STOP -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ (void)compress; (void)method; (void)level; @@ -454,7 +454,7 @@ tor_zstd_compress_process(tor_zstd_compress_state_t *state, return TOR_COMPRESS_OK; } -#else /* !(defined(HAVE_ZSTD)) */ +#else /* !defined(HAVE_ZSTD) */ (void)state; (void)out; (void)out_len; @@ -527,7 +527,7 @@ tor_zstd_warn_if_version_mismatched(void) "For safety, we'll avoid using advanced zstd functionality.", header_version, runtime_version); } -#endif +#endif /* defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS) */ } #ifdef TOR_UNIT_TESTS @@ -538,4 +538,4 @@ tor_zstd_set_static_apis_disabled_for_testing(int disabled) { static_apis_disable_for_testing = disabled; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/lib/compress/compress_zstd.h b/src/lib/compress/compress_zstd.h index 47f950b9e0..0fc71db749 100644 --- a/src/lib/compress/compress_zstd.h +++ b/src/lib/compress/compress_zstd.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am index 75c9032bd2..60dd447d4e 100644 --- a/src/lib/compress/include.am +++ b/src/lib/compress/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-compress-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_compress_a_SOURCES = \ src/lib/compress/compress.c \ src/lib/compress/compress_buf.c \ @@ -18,9 +19,11 @@ src_lib_libtor_compress_testing_a_SOURCES = \ src_lib_libtor_compress_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_compress_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/compress/compress.h \ src/lib/compress/compress_lzma.h \ src/lib/compress/compress_none.h \ + src/lib/compress/compress_sys.h \ src/lib/compress/compress_zlib.h \ src/lib/compress/compress_zstd.h diff --git a/src/lib/compress/lib_compress.md b/src/lib/compress/lib_compress.md new file mode 100644 index 0000000000..c43f223fe7 --- /dev/null +++ b/src/lib/compress/lib_compress.md @@ -0,0 +1,6 @@ +@dir /lib/compress +@brief lib/compress: Wraps several compression libraries + +Currently supported are zlib (mandatory), zstd (optional), and lzma +(optional). + diff --git a/src/lib/conf/.may_include b/src/lib/conf/.may_include new file mode 100644 index 0000000000..629e2f897d --- /dev/null +++ b/src/lib/conf/.may_include @@ -0,0 +1,3 @@ +orconfig.h +lib/cc/*.h +lib/conf/*.h diff --git a/src/lib/conf/confdecl.h b/src/lib/conf/confdecl.h new file mode 100644 index 0000000000..c2d3fb335d --- /dev/null +++ b/src/lib/conf/confdecl.h @@ -0,0 +1,221 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file confdecl.h + * @brief Macros for generating a configuration struct from a list + * of its individual fields. + * + * This header defines three important macros: BEGIN_CONF_STRUCT(), + * END_CONF_STRUCT(), and CONF_VAR(). They're meant to be used together to + * define a configuration structure and the means for encoding and decoding + * it. + * + * To use them, make a new header with a name like `MOD_options.inc`. Start + * it with a BEGIN_CONF_STRUCT(), then define your variables with CONF_VAR(), + * then end the header with END_CONF_STRUCT(), as in: + * + * BEGIN_CONF_STRUCT(module_options_t) + * CONF_VAR(ModuleIsActive, BOOLEAN, 0, "1") + * END_CONF_STRUCT(module_options_t) + * + * Once you've done that, you can use that header to define a configuration + * structure by saying: + * + * typedef struct module_options_t module_options_t; + * #define CONF_CONTEXT STRUCT + * #include "MOD_options.inc" + * #undef CONF_CONTEXT + * + * And you can define your field definition table by saying: + * + * #define CONF_CONTEXT TABLE + * #include "MOD_options.inc" + * #undef CONF_CONTEXT + * + * The two above snippets will define a structure called `module_options_t` + * with appropriate members, and a table of config_var_t objects called + * `module_options_t_vars[]`. + * + * For lower-level modules, you can say <tt>\#define CONF_TABLE LL_TABLE</tt>, + * and get a table definition suitable for use in modules that are at a lower + * level than lib/confmgt. Note that the types for these tables cannot + * include any extended types. + **/ + +#ifndef TOR_LIB_CONF_CONFDECL_H +#define TOR_LIB_CONF_CONFDECL_H + +#undef CONF_CONTEXT +#include "lib/cc/tokpaste.h" +#include "lib/cc/torint.h" + +/** + * Begin the definition of a configuration object called `name`. + **/ +#define BEGIN_CONF_STRUCT(name) \ + PASTE(BEGIN_CONF_STRUCT__, CONF_CONTEXT)(name) +/** + * End the definition of a configuration object called `name`. + **/ +#define END_CONF_STRUCT(name) \ + PASTE(END_CONF_STRUCT__, CONF_CONTEXT)(name) +/** + * Declare a single configuration field with name `varname`, type `vartype`, + * flags `varflags`, and initial value `initval`. + **/ +#define CONF_VAR(varname, vartype, varflags, initval) \ + PASTE(CONF_VAR__, CONF_CONTEXT)(varname, vartype, varflags, initval) + +#ifndef COCCI +/** + * @defgroup STRUCT_MACROS Internal macros: struct definitions. + * Implementation helpers: the regular confdecl macros expand to these + * when CONF_CONTEXT is defined to STRUCT. Don't use them directly. + * @{*/ +#define BEGIN_CONF_STRUCT__STRUCT(name) \ + struct name { \ + uint32_t magic; +#define END_CONF_STRUCT__STRUCT(name) \ + }; +#define CONF_VAR__STRUCT(varname, vartype, varflags, initval) \ + config_decl_ ## vartype varname; +/** @} */ + +/** + * @defgroup TABLE_MACROS Internal macros: table definitions. + * Implementation helpers: the regular confdecl macros expand to these + * when CONF_CONTEXT is defined to TABLE. Don't use them directly. + * @{*/ +#define BEGIN_CONF_STRUCT__TABLE(structname) \ + /* We use this typedef so we can refer to the config type */ \ + /* without having its name as a macro argument to CONF_VAR. */ \ + typedef struct structname config_var_reference__obj; \ + static const config_var_t structname##_vars[] = { +#define END_CONF_STRUCT__TABLE(structname) \ + { .member = { .name = NULL } } \ + }; +#define CONF_VAR__TABLE(varname, vartype, varflags, initval) \ + { \ + .member = \ + { .name = #varname, \ + .type = CONFIG_TYPE_EXTENDED, \ + .type_def = &vartype ## _type_defn, \ + .offset=offsetof(config_var_reference__obj, varname), \ + }, \ + .flags = varflags, \ + .initvalue = initval \ + }, +/**@}*/ + +/** + * @defgroup LL_TABLE_MACROS Internal macros: low-level table definitions. + * Implementation helpers: the regular confdecl macros expand to these + * when CONF_CONTEXT is defined to LL_TABLE. Don't use them directly. + * @{*/ +#define BEGIN_CONF_STRUCT__LL_TABLE(structname) \ + /* We use this typedef so we can refer to the config type */ \ + /* without having its name as a macro argument to CONF_VAR. */ \ + typedef struct structname config_var_reference__obj; \ + static const config_var_t structname##_vars[] = { +#define END_CONF_STRUCT__LL_TABLE(structname) \ + { .member = { .name = NULL } } \ + }; +#define CONF_VAR__LL_TABLE(varname, vartype, varflags, initval) \ + { \ + .member = \ + { .name = #varname, \ + .type = CONFIG_TYPE_ ## vartype, \ + .offset=offsetof(config_var_reference__obj, varname), \ + }, \ + .flags = varflags, \ + .initvalue = initval \ + }, +/**@}*/ + +/** @defgroup STUB_TABLE_MACROS Internal macros: stub table declarations, + * for use when a module is disabled. + * Implementation helpers: the regular confdecl macros expand to these + * when CONF_CONTEXT is defined to LL_TABLE. Don't use them directly. + * @{ */ +#define BEGIN_CONF_STRUCT__STUB_TABLE(structname) \ + static const config_var_t structname##_vars[] = { +#define END_CONF_STRUCT__STUB_TABLE(structname) \ + { .member = { .name = NULL } } \ + }; +#define CONF_VAR__STUB_TABLE(varname, vartype, varflags, initval) \ + { \ + .member = \ + { .name = #varname, \ + .type = CONFIG_TYPE_IGNORE, \ + .offset = -1, \ + }, \ + .flags = CFLG_GROUP_DISABLED, \ + }, +/**@}*/ + +#endif /* !defined(COCCI) */ + +/** Type aliases for the "commonly used" configuration types. + * + * Defining them in this way allows our CONF_VAR__STRUCT() macro to declare + * structure members corresponding to the configuration types. For example, + * when the macro sees us declare a configuration option "foo" of type STRING, + * it can emit `config_decl_STRING foo;`, which is an alias for `char *foo`. + */ +/**@{*/ +typedef char *config_decl_STRING; +typedef char *config_decl_FILENAME; +/* Yes, "POSINT" is really an int, and not an unsigned int. For + * historical reasons, many configuration values are restricted + * to the range [0,INT_MAX], and stored in signed ints. + */ +typedef int config_decl_POSINT; +typedef uint64_t config_decl_UINT64; +typedef int config_decl_INT; +typedef int config_decl_INTERVAL; +typedef int config_decl_MSEC_INTERVAL; +typedef uint64_t config_decl_MEMUNIT; +typedef double config_decl_DOUBLE; +typedef int config_decl_BOOL; +typedef int config_decl_AUTOBOOL; +typedef time_t config_decl_ISOTIME; +typedef struct smartlist_t config_decl_CSV; +typedef int config_decl_CSV_INTERVAL; +typedef struct config_line_t *config_decl_LINELIST; +typedef struct config_line_t *config_decl_LINELIST_V; +typedef struct nonexistent_struct *config_decl_LINELIST_S; +/**@}*/ + +struct var_type_def_t; + +/* Forward declarations for configuration type definitions. These are used by + * the CONF_VAR__TABLE macro to set the definition of each variable type + * correctly. + */ +/**@{*/ +extern const struct var_type_def_t STRING_type_defn; +extern const struct var_type_def_t FILENAME_type_defn; +extern const struct var_type_def_t POSINT_type_defn; +extern const struct var_type_def_t UINT64_type_defn; +extern const struct var_type_def_t INT_type_defn; +extern const struct var_type_def_t INTERVAL_type_defn; +extern const struct var_type_def_t MSEC_INTERVAL_type_defn; +extern const struct var_type_def_t MEMUNIT_type_defn; +extern const struct var_type_def_t DOUBLE_type_defn; +extern const struct var_type_def_t BOOL_type_defn; +extern const struct var_type_def_t AUTOBOOL_type_defn; +extern const struct var_type_def_t ISOTIME_type_defn; +extern const struct var_type_def_t CSV_type_defn; +extern const struct var_type_def_t CSV_INTERVAL_type_defn; +extern const struct var_type_def_t LINELIST_type_defn; +extern const struct var_type_def_t LINELIST_V_type_defn; +extern const struct var_type_def_t LINELIST_S_type_defn; +extern const struct var_type_def_t IGNORE_type_defn; +extern const struct var_type_def_t OBSOLETE_type_defn; +/**@}*/ + +#endif /* !defined(TOR_LIB_CONF_CONFDECL_H) */ diff --git a/src/lib/conf/config.md b/src/lib/conf/config.md new file mode 100644 index 0000000000..7741e21f42 --- /dev/null +++ b/src/lib/conf/config.md @@ -0,0 +1,126 @@ + +@page configuration Configuration options and persistent state + +@tableofcontents + +## Introduction + +Tor uses a shared, table-driven mechanism to handle its +configuration (torrc) files and its state files. Each module can +declare a set of named fields for these files, and get notified +whenever the configuration changes, or when the state is about to be +flushed to disk. + +## Declaring options + +Most modules will only need to use the macros in confdecl.h to +declare a configuration or state structure. + +You'll write something like this: + + // my_module_config.inc + BEGIN_CONF_STRUCT(module_options_t) + CONF_VAR(FieldOne, INT, 0, "7") + CONF_VAR(FieldTwo, STRING, 0, NULL) + END_CONF_STRUCT(module_options_t) + +The above example will result in a structure called module_config_t +with two fields: one an integer called FieldOne and one a string +called FieldTwo. The integer gets a default value of 7; the +string's default value is NULL. + +After making a definition file like that, you include it twice: once +in a header, after saying \#define CONF_CONTEXT STRUCT, and once in +a C file, after saying \#define CONF_CONTEXT TABLE. The first time +defines a module_options_t structure, and the second time defines a +table that tells the configuration manager how to use it. + +Using the table, you declare a `const` config_format_t, which +associates the fields with a set of functions for validating and +normalizing them, a list of abbreviations and deprecations, and +other features. + +See confdecl.h and conftypes.h for more information. For example +usage, see crypto_options.inc or mainloop_state.inc. + +## Getting notifications + +After using those macros, you must tell the subsystem management +code about your module's configuration/state. + +If you're writing configuration code, you'll need a function that receives +the configuration object, and acts upon it. This function needs to be safe +to call multiple times, since Tor will reconfigure its subsystems whenever it +re-reads the torrc, gets a configuration change from a controller, or +restarts in process. This function goes in your subsystem's +subsys_fns_t.set_options field. + +If you're writing state code, you'll need a function that receives +state (subsys_fns_t.set_state), and a function that flushes the +application state into a state object (subsys_fns_t.flush_state). +The `set_state` function will be called once (@ref config_once_per +"1") when Tor is starting, whereas the `flush_state` function will +be called whenever Tor is about to save the state to disk. + +See subsys_fns_t for more information here, and \ref initialization +for more information about initialization and subsystems in general. + +> @anchor config_once_per 1. Technically, state is set once _per startup_. +> Remember that Tor can be stopped and started multiple times in +> the same process. If this happens, then your set_state() function +> is called once every time Tor starts. + +## How it works + +The common logic used to handle configuration and state files lives +in @refdir{lib/confmgt}. At the highest level, a configuration +manager object (config_mgr_t) maintains a list of each module's +configuration objects, and a list of all their fields. When the +user specifies a configuration value, the manager finds out how to +parse it, where to store it, and which configuration object is +affected. + +The top-level configuration module (config.c) and state module +(statefile.c) use config_mgr_t to create, initialize, set, compare, +and free a "top level configuration object". This object contains a +list of sub-objects: one for each module that participates in the +configuration/state system. This top-level code then invokes the +subsystem manager code (subsysmgr.c) to pass the corresponding +configuration or state objects to each module that has one. + +Note that the top level code does not have easy access to the +configuration objects used by the sub-modules. This is by design. A +module _may_ expose some or all of its configuration or state object via +accessor functions, if it likes, but if it does not, that object should +be considered module-local. + +## Adding new types + +Configuration and state fields each have a "type". These types +specify how the fields' values are represented in C; how they are +stored in files; and how they are encoded back and forth. + +There is a set of built-in types listed in conftypes.h, but +higher-level code can define its own types. To do so, you make an +instance of var_type_fns_t that describes how to manage your type, +and an instance of var_type_def_t that wraps your var_type_fns_t +with a name and optional parameters and flags. + +For an example of how a higher-level type is defined, see +ROUTERSET_type_defn in routerset.c. Also see the typedef +`config_decl_ROUTERSET`. Together, these let the routerset type be +used with the macros in confdecl.h. + +## Legacy configuration and state + +As of this writing (November 2019), most of the configuration and state is +still handled directly in config.c and statefile.c, and stored in the +monolithic structures or_options_t and or_state_t respectively. + +These top-level structures are accessed with get_options() and +get_state(), and used throughout much of the code, at the level of +@refdir{core} and higher. + +With time we hope to refactor this configuration into more +reasonable pieces, so that they are no longer (effectively) global +variables used throughout the code. diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h new file mode 100644 index 0000000000..9f85d21740 --- /dev/null +++ b/src/lib/conf/confmacros.h @@ -0,0 +1,72 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file confmacros.h + * @brief Macro definitions for declaring configuration variables + **/ + +#ifndef TOR_LIB_CONF_CONFMACROS_H +#define TOR_LIB_CONF_CONFMACROS_H + +#include "orconfig.h" +#include "lib/conf/conftesting.h" + +#ifndef COCCI +/** + * Used to indicate the end of an array of configuration variables. + **/ +#define END_OF_CONFIG_VARS \ + { .member = { .name = NULL } DUMMY_CONF_TEST_MEMBERS } +#endif /* !defined(COCCI) */ + +/** + * Declare a config_var_t as a member named <b>membername</b> of the structure + * <b>structtype</b>, whose user-visible name is <b>varname</b>, whose + * type corresponds to the config_type_t member CONFIG_TYPE_<b>vartype</b>, + * and whose initial value is <b>intval</b>. + * + * Most modules that use this macro should wrap it in a local macro that + * sets structtype to the local configuration type. + **/ +#define CONFIG_VAR_ETYPE(structtype, varname, vartype, membername, \ + varflags, initval) \ + { .member = \ + { .name = varname, \ + .type = CONFIG_TYPE_ ## vartype, \ + .offset = offsetof(structtype, membername), \ + }, \ + .flags = varflags, \ + .initvalue = initval \ + CONF_TEST_MEMBERS(structtype, vartype, membername) \ + } + +/** + * As CONFIG_VAR_ETYPE, but declares a value using an extension type whose + * type definition is <b>vartype</b>_type_defn. + **/ +#define CONFIG_VAR_DEFN(structtype, varname, vartype, membername, \ + varflags, initval) \ + { .member = \ + { .name = varname, \ + .type = CONFIG_TYPE_EXTENDED, \ + .type_def = &vartype ## _type_defn, \ + .offset = offsetof(structtype, membername), \ + }, \ + .flags = varflags, \ + .initvalue = initval \ + CONF_TEST_MEMBERS(structtype, vartype, membername) \ + } + +/** + * Declare an obsolete configuration variable with a given name. + **/ +#define CONFIG_VAR_OBSOLETE(varname) \ + { .member = { .name = varname, .type = CONFIG_TYPE_OBSOLETE }, \ + .flags = CFLG_GROUP_OBSOLETE \ + } + +#endif /* !defined(TOR_LIB_CONF_CONFMACROS_H) */ diff --git a/src/lib/conf/conftesting.h b/src/lib/conf/conftesting.h new file mode 100644 index 0000000000..4707c919d3 --- /dev/null +++ b/src/lib/conf/conftesting.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file conftesting.h + * @brief Macro and type declarations for testing + **/ + +#ifndef TOR_LIB_CONF_CONFTESTING_H +#define TOR_LIB_CONF_CONFTESTING_H + +#include "lib/cc/torint.h" + +#ifndef COCCI +#ifdef TOR_UNIT_TESTS +#define USE_CONF_TESTING +/** + * Union used when building in test mode typechecking the members of a type + * used with confmgt.c. See CONF_CHECK_VAR_TYPE for a description of how + * it is used. */ +typedef union { + char **STRING; + char **FILENAME; + int *POSINT; /* yes, this is really an int, and not an unsigned int. For + * historical reasons, many configuration values are restricted + * to the range [0,INT_MAX], and stored in signed ints. + */ + uint64_t *UINT64; + int *INT; + int *INTERVAL; + int *MSEC_INTERVAL; + uint64_t *MEMUNIT; + double *DOUBLE; + int *BOOL; + int *AUTOBOOL; + time_t *ISOTIME; + struct smartlist_t **CSV; + int *CSV_INTERVAL; + struct config_line_t **LINELIST; + struct config_line_t **LINELIST_S; + struct config_line_t **LINELIST_V; + // XXXX this doesn't belong at this level of abstraction. + struct routerset_t **ROUTERSET; +} confparse_dummy_values_t; + +/* Macros to define extra members inside config_var_t fields, and at the + * end of a list of them. + */ +/* This is a somewhat magic type-checking macro for users of confmgt.c. + * It initializes a union member "confparse_dummy_values_t.conftype" with + * the address of a static member "tp_dummy.member". This + * will give a compiler warning unless the member field is of the correct + * type. + * + * (This warning is mandatory, because a type mismatch here violates the type + * compatibility constraint for simple assignment, and requires a diagnostic, + * according to the C spec.) + * + * For example, suppose you say: + * "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)". + * Then this macro will evaluate to: + * { .STRING = &or_options_t_dummy.Address } + * And since confparse_dummy_values_t.STRING has type "char **", that + * expression will create a warning unless or_options_t.Address also + * has type "char *". + */ +#define CONF_CHECK_VAR_TYPE(tp, conftype, member) \ + { . conftype = &tp ## _dummy . member } +#define CONF_TEST_MEMBERS(tp, conftype, member) \ + , .var_ptr_dummy=CONF_CHECK_VAR_TYPE(tp, conftype, member) +#define DUMMY_CONF_TEST_MEMBERS , .var_ptr_dummy={ .INT=NULL } +#define DUMMY_TYPECHECK_INSTANCE(tp) \ + static tp tp ## _dummy +#endif /* defined(TOR_UNIT_TESTS) */ +#endif /* !defined(COCCI) */ + +#ifndef USE_CONF_TESTING +#define CONF_TEST_MEMBERS(tp, conftype, member) +/* Repeatedly declarable incomplete struct to absorb redundant semicolons */ +#define DUMMY_TYPECHECK_INSTANCE(tp) \ + struct tor_semicolon_eater +#define DUMMY_CONF_TEST_MEMBERS + +#endif /* !defined(USE_CONF_TESTING) */ + +#endif /* !defined(TOR_LIB_CONF_CONFTESTING_H) */ diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h new file mode 100644 index 0000000000..081ebf397f --- /dev/null +++ b/src/lib/conf/conftypes.h @@ -0,0 +1,383 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file conftypes.h + * @brief Types used to specify configurable options. + * + * This header defines the types that different modules will use in order to + * declare their configuration and state variables, and tell the configuration + * management code about those variables. From the individual module's point + * of view, its configuration and state are simply data structures. + * + * For defining new variable types, see var_type_def_st.h. + * + * For the code that manipulates variables defined via this module, see + * lib/confmgt/, especially typedvar.h and (later) structvar.h. The + * configuration manager is responsible for encoding, decoding, and + * maintaining the configuration structures used by the various modules. + * + * STATUS NOTE: This is a work in process refactoring. It is not yet possible + * for modules to define their own variables, and much of the configuration + * management code is still in src/app/config/. + **/ + +#ifndef TOR_SRC_LIB_CONF_CONFTYPES_H +#define TOR_SRC_LIB_CONF_CONFTYPES_H + +#include "lib/cc/torint.h" +#ifdef TOR_UNIT_TESTS +#include "lib/conf/conftesting.h" +#endif + +#include <stddef.h> + +/** Enumeration of types which option values can take */ +typedef enum config_type_t { + CONFIG_TYPE_STRING = 0, /**< An arbitrary string. */ + CONFIG_TYPE_FILENAME, /**< A filename: some prefixes get expanded. */ + CONFIG_TYPE_POSINT, /**< A non-negative integer less than MAX_INT */ + CONFIG_TYPE_INT, /**< Any integer. */ + CONFIG_TYPE_UINT64, /**< A value in range 0..UINT64_MAX */ + CONFIG_TYPE_INTERVAL, /**< A number of seconds, with optional units*/ + CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional + * units */ + CONFIG_TYPE_MEMUNIT, /**< A number of bytes, with optional units*/ + CONFIG_TYPE_DOUBLE, /**< A floating-point value */ + CONFIG_TYPE_BOOL, /**< A boolean value, expressed as 0 or 1. */ + CONFIG_TYPE_AUTOBOOL, /**< A boolean+auto value, expressed 0 for false, + * 1 for true, and -1 for auto */ + CONFIG_TYPE_ISOTIME, /**< An ISO-formatted time relative to UTC. */ + CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and + * optional whitespace. */ + CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and + * optional whitespace, representing intervals in + * seconds, with optional units. We allow + * multiple values here for legacy reasons, but + * ignore every value after the first. */ + CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ + CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, + * mixed with other keywords. */ + CONFIG_TYPE_LINELIST_V, /**< Catch-all "virtual" option to summarize + * context-sensitive config lines when fetching. + */ + /** Ignored (obsolete) option. Uses no storage. + * + * Reported as "obsolete" when its type is queried. + */ + CONFIG_TYPE_OBSOLETE, + /** Ignored option. Uses no storage. + * + * Reported as "ignored" when its type is queried. For use with options used + * by disabled modules. + **/ + CONFIG_TYPE_IGNORE, + + /** + * Extended type: definition appears in the <b>type_def</b> pointer + * of the corresponding struct_member_t. + * + * For some types, we cannot define them as particular values of this + * enumeration, since those types are abstractions defined at a higher level + * than this module. (For example, parsing a routerset_t is higher-level + * than this module.) To handle this, we use CONFIG_TYPE_EXTENDED for those + * types, and give a definition for them in the struct_member_t.type_def. + **/ + CONFIG_TYPE_EXTENDED, +} config_type_t; + +/* Forward delcaration for var_type_def_t, for extended types. */ +struct var_type_def_t; + +/** Structure to specify a named, typed member within a structure. */ +typedef struct struct_member_t { + /** Name of the field. */ + const char *name; + /** + * Type of the field, according to the config_type_t enumeration. + * + * For any type not otherwise listed in config_type_t, this field's value + * should be CONFIG_TYPE_EXTENDED. When it is, the <b>type_def</b> pointer + * must be set. + **/ + /* + * NOTE: In future refactoring, we might remove this field entirely, along + * with its corresponding enumeration. In that case, we will require that + * type_def be set in all cases. If we do, we will also need a new mechanism + * to enforce consistency between configuration variable types and their + * corresponding structures, since our current design in + * lib/conf/conftesting.h won't work any more. + */ + config_type_t type; + /** + * Pointer to a type definition for the type of this field. Overrides + * <b>type</b> if it is not NULL. Must be set when <b>type</b> is + * CONFIG_TYPE_EXTENDED. + **/ + const struct var_type_def_t *type_def; + /** + * Offset of this field within the structure. Compute this with + * offsetof(structure, fieldname). + **/ + ptrdiff_t offset; +} struct_member_t; + +/** + * Structure to describe the location and preferred value of a "magic number" + * field within a structure. + * + * These 'magic numbers' are 32-bit values used to tag objects to make sure + * that they have the correct type. + * + * If all fields in this structure are zero or 0, the magic-number check is + * not performed. + */ +typedef struct struct_magic_decl_t { + /** The name of the structure */ + const char *typename; + /** A value used to recognize instances of this structure. */ + uint32_t magic_val; + /** The location within the structure at which we expect to find + * <b>magic_val</b>. */ + ptrdiff_t magic_offset; +} struct_magic_decl_t; + +/** + * Flag to indicate that an option or type is "undumpable". An + * undumpable option is never saved to disk. + * + * For historical reasons its name is usually is prefixed with __. + **/ +#define CFLG_NODUMP (1u<<0) +/** + * Flag to indicate that an option or type is "unlisted". + * + * We don't tell the controller about unlisted options when it asks for a + * list of them. + **/ +#define CFLG_NOLIST (1u<<1) +/** + * Flag to indicate that an option or type is "unsettable". + * + * An unsettable option can never be set directly by name. + **/ +#define CFLG_NOSET (1u<<2) +/** + * Flag to indicate that an option or type does not need to be copied when + * copying the structure that contains it. + * + * (Usually, if an option does not need to be copied, then either it contains + * no data, or the data that it does contain is completely contained within + * another option.) + **/ +#define CFLG_NOCOPY (1u<<3) +/** + * Flag to indicate that an option or type does not need to be compared + * when telling the controller about the differences between two + * configurations. + * + * (Usually, if an option does not need to be compared, then either it + * contains no data, or the data that it does contain is completely contained + * within another option.) + **/ +#define CFLG_NOCMP (1u<<4) +/** + * Flag to indicate that an option or type should not be replaced when setting + * it. + * + * For most options, setting them replaces their old value. For some options, + * however, setting them appends to their old value. + */ +#define CFLG_NOREPLACE (1u<<5) +/** + * Flag to indicate that an option or type cannot be changed while Tor is + * running. + **/ +#define CFLG_IMMUTABLE (1u<<6) +/** + * Flag to indicate that we should warn that an option or type is obsolete + * whenever the user tries to use it. + **/ +#define CFLG_WARN_OBSOLETE (1u<<7) +/** + * Flag to indicate that we should warn that an option applies only to + * a disabled module, whenever the user tries to use it. + **/ +#define CFLG_WARN_DISABLED (1u<<8) + +/** + * A group of flags that should be set on all obsolete options and types. + **/ +#define CFLG_GROUP_OBSOLETE \ + (CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET|CFLG_NOLIST|\ + CFLG_WARN_OBSOLETE) + +/** + * A group of fflags that should be set on all disabled options. + **/ +#define CFLG_GROUP_DISABLED \ + (CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET|CFLG_NOLIST|\ + CFLG_WARN_DISABLED) + +/** A variable allowed in the configuration file or on the command line. */ +typedef struct config_var_t { + struct_member_t member; /** A struct member corresponding to this + * variable. */ + const char *initvalue; /**< String (or null) describing initial value. */ + uint32_t flags; /**< One or more flags describing special handling for this + * variable */ +#ifdef TOR_UNIT_TESTS + /** Used for compiler-magic to typecheck the corresponding field in the + * corresponding struct. Only used in unit test mode, at compile-time. */ + confparse_dummy_values_t var_ptr_dummy; +#endif +} config_var_t; + +/** + * An abbreviation or alias for a configuration option. + **/ +typedef struct config_abbrev_t { + /** The option name as abbreviated. Not case-sensitive. */ + const char *abbreviated; + /** The full name of the option. Not case-sensitive. */ + const char *full; + /** True if this abbreviation should only be allowed on the command line. */ + int commandline_only; + /** True if we should warn whenever this abbreviation is used. */ + int warn; +} config_abbrev_t; + +/** + * A note that a configuration option is deprecated, with an explanation why. + */ +typedef struct config_deprecation_t { + /** The option that is deprecated. */ + const char *name; + /** A user-facing string explaining why the option is deprecated. */ + const char *why_deprecated; +} config_deprecation_t; + +#ifndef COCCI +/** + * Handy macro for declaring "In the config file or on the command line, you + * can abbreviate <b>tok</b>s as <b>tok</b>". Used inside an array of + * config_abbrev_t. + * + * For example, to declare "NumCpu" as an abbreviation for "NumCPUs", + * you can say PLURAL(NumCpu). + **/ +#define PLURAL(tok) { (#tok), (#tok "s"), 0, 0 } +#endif /* !defined(COCCI) */ + +/** + * Validation function: verify whether a configuation object is well-formed + * and consistent. + * + * On success, return 0. On failure, set <b>msg_out</b> to a newly allocated + * string containing an error message, and return -1. */ +typedef int (*validate_fn_t)(const void *value, char **msg_out); +/** + * Validation function: verify whether a configuration object (`value`) is an + * allowable value given the previous configuration value (`old_value`). + * + * On success, return 0. On failure, set <b>msg_out</b> to a newly allocated + * string containing an error message, and return -1. */ +typedef int (*check_transition_fn_t)(const void *old_value, const void *value, + char **msg_out); +/** + * Validation function: normalize members of `value`, and compute derived + * members. + * + * This function is called before any other validation of `value`, and must + * not assume that validate_fn or check_transition_fn has passed. + * + * On success, return 0. On failure, set <b>msg_out</b> to a newly allocated + * string containing an error message, and return -1. */ +typedef int (*pre_normalize_fn_t)(void *value, char **msg_out); +/** + * Validation function: normalize members of `value`, and compute derived + * members. + * + * This function is called after validation of `value`, and may + * assume that validate_fn or check_transition_fn has passed. + * + * On success, return 0. On failure, set <b>msg_out</b> to a newly allocated + * string containing an error message, and return -1. */ +typedef int (*post_normalize_fn_t)(void *value, char **msg_out); + +/** + * Legacy function to validate whether a given configuration is + * well-formed and consistent. + * + * The configuration to validate is passed as <b>newval</b>. The previous + * configuration, if any, is provided in <b>oldval</b>. + * + * This API is deprecated, since it mixes the responsibilities of + * pre_normalize_fn_t, post_normalize_fn_t, validate_fn_t, and + * check_transition_fn_t. No new instances of this function type should + * be written. + * + * On success, return 0. On failure, set *<b>msg_out</b> to a newly allocated + * error message, and return -1. + */ +typedef int (*legacy_validate_fn_t)(const void *oldval, + void *newval, + char **msg_out); + +struct config_mgr_t; + +/** + * Callback to clear all non-managed fields of a configuration object. + * + * <b>obj</b> is the configuration object whose non-managed fields should be + * cleared. + * + * (Regular fields get cleared by config_reset(), but you might have fields + * in the object that do not correspond to configuration variables. If those + * fields need to be cleared or freed, this is where to do it.) + */ +typedef void (*clear_cfg_fn_t)(const struct config_mgr_t *mgr, void *obj); + +/** Information on the keys, value types, key-to-struct-member mappings, + * variable descriptions, validation functions, and abbreviations for a + * configuration or storage format. */ +typedef struct config_format_t { + size_t size; /**< Size of the struct that everything gets parsed into. */ + struct_magic_decl_t magic; /**< Magic number info for this struct. */ + const config_abbrev_t *abbrevs; /**< List of abbreviations that we expand + * when parsing this format. */ + const config_deprecation_t *deprecations; /** List of deprecated options */ + const config_var_t *vars; /**< List of variables we recognize, their default + * values, and where we stick them in the + * structure. */ + + /** Early-stage normalization callback. Invoked by config_validate(). */ + pre_normalize_fn_t pre_normalize_fn; + /** Configuration validation function. Invoked by config_validate(). */ + validate_fn_t validate_fn; + /** Legacy validation function. Invoked by config_validate(). */ + legacy_validate_fn_t legacy_validate_fn; + /** Transition checking function. Invoked by config_validate(). */ + check_transition_fn_t check_transition_fn; + /** Late-stage normalization callback. Invoked by config_validate(). */ + post_normalize_fn_t post_normalize_fn; + + clear_cfg_fn_t clear_fn; /**< Function to clear the configuration. */ + /** If present, extra denotes a LINELIST variable for unrecognized + * lines. Otherwise, unrecognized lines are an error. */ + const struct_member_t *extra; + /** + * If true, this format describes a top-level configuration, with + * a suite containing multiple sub-configuration objects. + */ + bool has_config_suite; + /** The position of a config_suite_t pointer within the toplevel object. + * Ignored unless have_config_suite is true. + */ + ptrdiff_t config_suite_offset; +} config_format_t; + +#endif /* !defined(TOR_SRC_LIB_CONF_CONFTYPES_H) */ diff --git a/src/lib/conf/include.am b/src/lib/conf/include.am new file mode 100644 index 0000000000..cb0b83fa64 --- /dev/null +++ b/src/lib/conf/include.am @@ -0,0 +1,7 @@ + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/conf/confdecl.h \ + src/lib/conf/conftesting.h \ + src/lib/conf/conftypes.h \ + src/lib/conf/confmacros.h diff --git a/src/lib/conf/lib_conf.md b/src/lib/conf/lib_conf.md new file mode 100644 index 0000000000..60dd04e99e --- /dev/null +++ b/src/lib/conf/lib_conf.md @@ -0,0 +1,3 @@ +@dir /lib/conf +@brief lib/conf: Types and macros for declaring configuration options. + diff --git a/src/lib/confmgt/.may_include b/src/lib/confmgt/.may_include new file mode 100644 index 0000000000..5ff949f103 --- /dev/null +++ b/src/lib/confmgt/.may_include @@ -0,0 +1,12 @@ +orconfig.h +lib/cc/*.h +lib/conf/*.h +lib/confmgt/*.h +lib/container/*.h +lib/encoding/*.h +lib/intmath/*.h +lib/log/*.h +lib/malloc/*.h +lib/string/*.h +lib/testsupport/*.h +ext/*.h diff --git a/src/lib/confmgt/confmgt.c b/src/lib/confmgt/confmgt.c new file mode 100644 index 0000000000..bf2764160e --- /dev/null +++ b/src/lib/confmgt/confmgt.c @@ -0,0 +1,1406 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file confmgt.c + * + * \brief Back-end for parsing and generating key-value files, used to + * implement the torrc file format and the state file. + * + * This module is used by config.c to parse and encode torrc + * configuration files, and by statefile.c to parse and encode the + * $DATADIR/state file. + * + * To use this module, its callers provide an instance of + * config_format_t to describe the mappings from a set of configuration + * options to a number of fields in a C structure. With this mapping, + * the functions here can convert back and forth between the C structure + * specified, and a linked list of key-value pairs. + */ + +#define CONFMGT_PRIVATE +#include "orconfig.h" +#include "lib/confmgt/confmgt.h" + +#include "lib/confmgt/structvar.h" +#include "lib/confmgt/unitparse.h" +#include "lib/container/bitarray.h" +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" + +#include "ext/siphash.h" + +/** + * A managed_var_t is an internal wrapper around a config_var_t in + * a config_format_t structure. It is used by config_mgr_t to + * keep track of which option goes with which structure. */ +typedef struct managed_var_t { + /** + * A pointer to the config_var_t for this option. + */ + const config_var_t *cvar; + /** + * The index of the object in which this option is stored. It is + * IDX_TOPLEVEL to indicate that the object is the top-level object. + **/ + int object_idx; +} managed_var_t; + +static void config_reset(const config_mgr_t *fmt, void *options, + const managed_var_t *var, int use_defaults); +static void config_mgr_register_fmt(config_mgr_t *mgr, + const config_format_t *fmt, + int object_idx); + +/** Release all storage held in a managed_var_t. */ +static void +managed_var_free_(managed_var_t *mv) +{ + if (!mv) + return; + tor_free(mv); +} +#define managed_var_free(mv) \ + FREE_AND_NULL(managed_var_t, managed_var_free_, (mv)) + +struct config_suite_t { + /** A list of configuration objects managed by a given configuration + * manager. They are stored in the same order as the config_format_t + * objects in the manager's list of subformats. */ + smartlist_t *configs; +}; + +/** + * Allocate a new empty config_suite_t. + **/ +static config_suite_t * +config_suite_new(void) +{ + config_suite_t *suite = tor_malloc_zero(sizeof(config_suite_t)); + suite->configs = smartlist_new(); + return suite; +} + +/** Release all storage held by a config_suite_t. (Does not free + * any configuration objects it holds; the caller must do that first.) */ +static void +config_suite_free_(config_suite_t *suite) +{ + if (!suite) + return; + smartlist_free(suite->configs); + tor_free(suite); +} + +#define config_suite_free(suite) \ + FREE_AND_NULL(config_suite_t, config_suite_free_, (suite)) + +struct config_mgr_t { + /** The 'top-level' configuration format. This one is used for legacy + * options that have not yet been assigned to different sub-modules. + * + * (NOTE: for now, this is the only config_format_t that a config_mgr_t + * contains. A subsequent commit will add more. XXXX) + */ + const config_format_t *toplevel; + /** + * List of second-level configuration format objects that this manager + * also knows about. + */ + smartlist_t *subconfigs; + /** A smartlist of managed_var_t objects for all configuration formats. */ + smartlist_t *all_vars; + /** A smartlist of config_abbrev_t objects for all configuration + * formats. These objects are used to track synonyms and abbreviations for + * different configuration options. */ + smartlist_t *all_abbrevs; + /** A smartlist of config_deprecation_t for all configuration formats. */ + smartlist_t *all_deprecations; + /** True if this manager has been frozen and cannot have any more formats + * added to it. A manager must be frozen before it can be used to construct + * or manipulate objects. */ + bool frozen; + /** A replacement for the magic number of the toplevel object. We override + * that number to make it unique for this particular config_mgr_t, so that + * an object constructed with one mgr can't be used with another, even if + * those managers' contents are equal. + */ + struct_magic_decl_t toplevel_magic; +}; + +#define IDX_TOPLEVEL (-1) + +/** Create a new config_mgr_t to manage a set of configuration objects to be + * wrapped under <b>toplevel_fmt</b>. */ +config_mgr_t * +config_mgr_new(const config_format_t *toplevel_fmt) +{ + config_mgr_t *mgr = tor_malloc_zero(sizeof(config_mgr_t)); + mgr->subconfigs = smartlist_new(); + mgr->all_vars = smartlist_new(); + mgr->all_abbrevs = smartlist_new(); + mgr->all_deprecations = smartlist_new(); + + config_mgr_register_fmt(mgr, toplevel_fmt, IDX_TOPLEVEL); + mgr->toplevel = toplevel_fmt; + + return mgr; +} + +/** Add a config_format_t to a manager, with a specified (unique) index. */ +static void +config_mgr_register_fmt(config_mgr_t *mgr, + const config_format_t *fmt, + int object_idx) +{ + int i; + + tor_assertf(!mgr->frozen, + "Tried to add a format to a configuration manager after " + "it had been frozen."); + + if (object_idx != IDX_TOPLEVEL) { + tor_assertf(! fmt->has_config_suite, + "Tried to register a toplevel format in a non-toplevel position"); + } + if (fmt->config_suite_offset) { + tor_assertf(fmt->has_config_suite, + "config_suite_offset was set, but has_config_suite was not."); + } + + tor_assertf(fmt != mgr->toplevel && + ! smartlist_contains(mgr->subconfigs, fmt), + "Tried to register an already-registered format."); + + /* register variables */ + for (i = 0; fmt->vars[i].member.name; ++i) { + managed_var_t *mv = tor_malloc_zero(sizeof(managed_var_t)); + mv->cvar = &fmt->vars[i]; + mv->object_idx = object_idx; + smartlist_add(mgr->all_vars, mv); + } + + /* register abbrevs */ + if (fmt->abbrevs) { + for (i = 0; fmt->abbrevs[i].abbreviated; ++i) { + smartlist_add(mgr->all_abbrevs, (void*)&fmt->abbrevs[i]); + } + } + + /* register deprecations. */ + if (fmt->deprecations) { + const config_deprecation_t *d; + for (d = fmt->deprecations; d->name; ++d) { + smartlist_add(mgr->all_deprecations, (void*)d); + } + } +} + +/** + * Add a new format to this configuration object. Asserts on failure. + * + * Returns an internal "index" value used to identify this format within + * all of those formats contained in <b>mgr</b>. This index value + * should not generally be used outside of this module. + **/ +int +config_mgr_add_format(config_mgr_t *mgr, + const config_format_t *fmt) +{ + tor_assert(mgr); + int idx = smartlist_len(mgr->subconfigs); + config_mgr_register_fmt(mgr, fmt, idx); + smartlist_add(mgr->subconfigs, (void *)fmt); + return idx; +} + +/** Return a pointer to the config_suite_t * pointer inside a + * configuration object; returns NULL if there is no such member. */ +static inline config_suite_t ** +config_mgr_get_suite_ptr(const config_mgr_t *mgr, void *toplevel) +{ + if (! mgr->toplevel->has_config_suite) + return NULL; + return STRUCT_VAR_P(toplevel, mgr->toplevel->config_suite_offset); +} + +/** + * Return a pointer to the configuration object within <b>toplevel</b> whose + * index is <b>idx</b>. + * + * NOTE: XXXX Eventually, there will be multiple objects supported within the + * toplevel object. For example, the or_options_t will contain pointers + * to configuration objects for other modules. This function gets + * the sub-object for a particular module. + */ +void * +config_mgr_get_obj_mutable(const config_mgr_t *mgr, void *toplevel, int idx) +{ + tor_assert(mgr); + tor_assert(toplevel); + if (idx == IDX_TOPLEVEL) + return toplevel; + + tor_assertf(idx >= 0 && idx < smartlist_len(mgr->subconfigs), + "Index %d is out of range.", idx); + config_suite_t **suite = config_mgr_get_suite_ptr(mgr, toplevel); + tor_assert(suite); + tor_assert(smartlist_len(mgr->subconfigs) == + smartlist_len((*suite)->configs)); + + return smartlist_get((*suite)->configs, idx); +} + +/** As config_mgr_get_obj_mutable(), but return a const pointer. */ +const void * +config_mgr_get_obj(const config_mgr_t *mgr, const void *toplevel, int idx) +{ + return config_mgr_get_obj_mutable(mgr, (void*)toplevel, idx); +} + +/** Sorting helper for smartlist of managed_var_t */ +static int +managed_var_cmp(const void **a, const void **b) +{ + const managed_var_t *mv1 = *(const managed_var_t**)a; + const managed_var_t *mv2 = *(const managed_var_t**)b; + + return strcasecmp(mv1->cvar->member.name, mv2->cvar->member.name); +} + +/** + * Mark a configuration manager as "frozen", so that no more formats can be + * added, and so that it can be used for manipulating configuration objects. + **/ +void +config_mgr_freeze(config_mgr_t *mgr) +{ + static uint64_t mgr_count = 0; + + smartlist_sort(mgr->all_vars, managed_var_cmp); + memcpy(&mgr->toplevel_magic, &mgr->toplevel->magic, + sizeof(struct_magic_decl_t)); + uint64_t magic_input[3] = { mgr->toplevel_magic.magic_val, + (uint64_t) (uintptr_t) mgr, + ++mgr_count }; + mgr->toplevel_magic.magic_val = + (uint32_t)siphash24g(magic_input, sizeof(magic_input)); + mgr->frozen = true; +} + +/** Release all storage held in <b>mgr</b> */ +void +config_mgr_free_(config_mgr_t *mgr) +{ + if (!mgr) + return; + SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv, managed_var_free(mv)); + smartlist_free(mgr->all_vars); + smartlist_free(mgr->all_abbrevs); + smartlist_free(mgr->all_deprecations); + smartlist_free(mgr->subconfigs); + memset(mgr, 0, sizeof(*mgr)); + tor_free(mgr); +} + +/** Return a new smartlist_t containing a config_var_t for every variable that + * <b>mgr</b> knows about. The elements of this smartlist do not need + * to be freed; they have the same lifespan as <b>mgr</b>. */ +smartlist_t * +config_mgr_list_vars(const config_mgr_t *mgr) +{ + smartlist_t *result = smartlist_new(); + tor_assert(mgr); + SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv, + smartlist_add(result, (void*) mv->cvar)); + return result; +} + +/** Return a new smartlist_t containing the names of all deprecated variables. + * The elements of this smartlist do not need to be freed; they have the same + * lifespan as <b>mgr</b>. + */ +smartlist_t * +config_mgr_list_deprecated_vars(const config_mgr_t *mgr) +{ + smartlist_t *result = smartlist_new(); + tor_assert(mgr); + SMARTLIST_FOREACH(mgr->all_deprecations, config_deprecation_t *, d, + smartlist_add(result, (char*)d->name)); + return result; +} + +/** + * Check the magic number on <b>object</b> to make sure it's a valid toplevel + * object, created with <b>mgr</b>. Exit with an assertion if it isn't. + **/ +void +config_check_toplevel_magic(const config_mgr_t *mgr, + const void *object) +{ + struct_check_magic(object, &mgr->toplevel_magic); +} + +/** Assert that the magic fields in <b>options</b> and its subsidiary + * objects are all okay. */ +static void +config_mgr_assert_magic_ok(const config_mgr_t *mgr, + const void *options) +{ + tor_assert(mgr); + tor_assert(options); + tor_assert(mgr->frozen); + struct_check_magic(options, &mgr->toplevel_magic); + + config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, (void*)options); + if (suitep == NULL) { + tor_assert(smartlist_len(mgr->subconfigs) == 0); + return; + } + + tor_assert(smartlist_len((*suitep)->configs) == + smartlist_len(mgr->subconfigs)); + SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) { + void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx); + tor_assert(obj); + struct_check_magic(obj, &fmt->magic); + } SMARTLIST_FOREACH_END(fmt); +} + +/** Macro: assert that <b>cfg</b> has the right magic field for + * <b>mgr</b>. */ +#define CONFIG_CHECK(mgr, cfg) STMT_BEGIN \ + config_mgr_assert_magic_ok((mgr), (cfg)); \ + STMT_END + +/** Allocate an empty configuration object of a given format type. */ +void * +config_new(const config_mgr_t *mgr) +{ + tor_assert(mgr->frozen); + void *opts = tor_malloc_zero(mgr->toplevel->size); + struct_set_magic(opts, &mgr->toplevel_magic); + config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, opts); + if (suitep) { + *suitep = config_suite_new(); + SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) { + void *obj = tor_malloc_zero(fmt->size); + struct_set_magic(obj, &fmt->magic); + smartlist_add((*suitep)->configs, obj); + } SMARTLIST_FOREACH_END(fmt); + } + CONFIG_CHECK(mgr, opts); + return opts; +} + +/* + * Functions to parse config options + */ + +/** If <b>option</b> is an official abbreviation for a longer option, + * return the longer option. Otherwise return <b>option</b>. + * If <b>command_line</b> is set, apply all abbreviations. Otherwise, only + * apply abbreviations that work for the config file and the command line. + * If <b>warn_obsolete</b> is set, warn about deprecated names. */ +const char * +config_expand_abbrev(const config_mgr_t *mgr, const char *option, + int command_line, int warn_obsolete) +{ + SMARTLIST_FOREACH_BEGIN(mgr->all_abbrevs, const config_abbrev_t *, abbrev) { + /* Abbreviations are case insensitive. */ + if (!strcasecmp(option, abbrev->abbreviated) && + (command_line || !abbrev->commandline_only)) { + if (warn_obsolete && abbrev->warn) { + log_warn(LD_CONFIG, + "The configuration option '%s' is deprecated; " + "use '%s' instead.", + abbrev->abbreviated, + abbrev->full); + } + /* Keep going through the list in case we want to rewrite it more. + * (We could imagine recursing here, but I don't want to get the + * user into an infinite loop if we craft our list wrong.) */ + option = abbrev->full; + } + } SMARTLIST_FOREACH_END(abbrev); + return option; +} + +/** If <b>key</b> is a deprecated configuration option, return the message + * explaining why it is deprecated (which may be an empty string). Return NULL + * if it is not deprecated. The <b>key</b> field must be fully expanded. */ +const char * +config_find_deprecation(const config_mgr_t *mgr, const char *key) +{ + if (BUG(mgr == NULL) || BUG(key == NULL)) + return NULL; // LCOV_EXCL_LINE + + SMARTLIST_FOREACH_BEGIN(mgr->all_deprecations, const config_deprecation_t *, + d) { + if (!strcasecmp(d->name, key)) { + return d->why_deprecated ? d->why_deprecated : ""; + } + } SMARTLIST_FOREACH_END(d); + return NULL; +} + +/** + * Find the managed_var_t object for a variable whose name is <b>name</b> + * according to <b>mgr</b>. Return that object, or NULL if none exists. + * + * If <b>allow_truncated</b> is true, then accept any variable whose + * name begins with <b>name</b>. + * + * If <b>idx_out</b> is not NULL, set *<b>idx_out</b> to the position of + * that variable within mgr->all_vars, or to -1 if the variable is + * not found. + */ +static const managed_var_t * +config_mgr_find_var(const config_mgr_t *mgr, + const char *key, + bool allow_truncated, int *idx_out) +{ + const size_t keylen = strlen(key); + if (idx_out) + *idx_out = -1; + + if (!keylen) + return NULL; /* if they say "--" on the command line, it's not an option */ + + /* First, check for an exact (case-insensitive) match */ + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!strcasecmp(mv->cvar->member.name, key)) { + if (idx_out) + *idx_out = mv_sl_idx; + return mv; + } + } SMARTLIST_FOREACH_END(mv); + + if (!allow_truncated) + return NULL; + + /* If none, check for an abbreviated match */ + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!strncasecmp(key, mv->cvar->member.name, keylen)) { + log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. " + "Please use '%s' instead", + key, mv->cvar->member.name); + if (idx_out) + *idx_out = mv_sl_idx; + return mv; + } + } SMARTLIST_FOREACH_END(mv); + + /* Okay, unrecognized option */ + return NULL; +} + +/** + * If <b>key</b> is a name or an abbreviation configuration option, return + * the corresponding canonical name for it. Warn if the abbreviation is + * non-standard. Return NULL if the option does not exist. + */ +const char * +config_find_option_name(const config_mgr_t *mgr, const char *key) +{ + key = config_expand_abbrev(mgr, key, 0, 0); + const managed_var_t *mv = config_mgr_find_var(mgr, key, true, NULL); + if (mv) + return mv->cvar->member.name; + else + return NULL; +} + +/** Return the number of option entries in <b>fmt</b>. */ +static int +config_count_options(const config_mgr_t *mgr) +{ + return smartlist_len(mgr->all_vars); +} + +/** + * Return true iff at least one bit from <b>flag</b> is set on <b>var</b>, + * either in <b>var</b>'s flags, or on the flags of its type. + **/ +static bool +config_var_has_flag(const config_var_t *var, uint32_t flag) +{ + uint32_t have_flags = var->flags | struct_var_get_flags(&var->member); + + return (have_flags & flag) != 0; +} + +/** + * Return true if assigning a value to <b>var</b> replaces the previous + * value. Return false if assigning a value to <b>var</b> appends + * to the previous value. + **/ +static bool +config_var_is_replaced_on_set(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOREPLACE); +} + +/** + * Return true iff <b>var</b> may be assigned by name (e.g., via the + * CLI, the configuration files, or the controller API). + **/ +bool +config_var_is_settable(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOSET); +} + +/** + * Return true iff the controller is allowed to fetch the value of + * <b>var</b>. + **/ +static bool +config_var_is_gettable(const config_var_t *var) +{ + /* Arguably, invisible or obsolete options should not be gettable. However, + * they have been gettable for a long time, and making them ungettable could + * have compatibility effects. For now, let's leave them alone. + */ + + // return ! config_var_has_flag(var, CVFLAG_OBSOLETE|CFGLAGS_INVISIBLE); + (void)var; + return true; +} + +/** + * Return true iff we need to check <b>var</b> for changes when we are + * comparing config options for changes. + * + * A false result might mean that the variable is a derived variable, and that + * comparing the variable it derives from compares this one too-- or it might + * mean that there is no data to compare. + **/ +static bool +config_var_should_list_changes(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOCMP); +} + +/** + * Return true iff we need to copy the data for <b>var</b> when we are + * copying a config option. + * + * A false option might mean that the variable is a derived variable, and that + * copying the variable it derives from copies it-- or it might mean that + * there is no data to copy. + **/ +static bool +config_var_needs_copy(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOCOPY); +} + +/** + * Return true iff variable <b>var</b> should appear on list of variable + * names given to the controller or the CLI. + * + * (Note that this option is imperfectly obeyed. The + * --list-torrc-options command looks at the "settable" flag, whereas + * "GETINFO config/defaults" and "list_deprecated_*()" do not filter + * their results. It would be good for consistency to try to converge + * these behaviors in the future.) + **/ +bool +config_var_is_listable(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NOLIST); +} + +/** + * Return true iff variable <b>var</b> should be written out when we + * are writing our configuration to disk, to a controller, or via the + * --dump-config command. + * + * This option may be set because a variable is hidden, or because it is + * derived from another variable which will already be written out. + **/ +static bool +config_var_is_dumpable(const config_var_t *var) +{ + return ! config_var_has_flag(var, CFLG_NODUMP); +} + +/* + * Functions to assign config options. + */ + +/** <b>c</b>-\>key is known to be a real key. Update <b>options</b> + * with <b>c</b>-\>value and return 0, or return -1 if bad value. + * + * Called from config_assign_line() and option_reset(). + */ +static int +config_assign_value(const config_mgr_t *mgr, void *options, + config_line_t *c, char **msg) +{ + const managed_var_t *var; + + CONFIG_CHECK(mgr, options); + + var = config_mgr_find_var(mgr, c->key, true, NULL); + tor_assert(var); + tor_assert(!strcmp(c->key, var->cvar->member.name)); + void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx); + + if (config_var_has_flag(var->cvar, CFLG_WARN_OBSOLETE)) { + log_warn(LD_GENERAL, "Skipping obsolete configuration option \"%s\".", + var->cvar->member.name); + } else if (config_var_has_flag(var->cvar, CFLG_WARN_DISABLED)) { + log_warn(LD_GENERAL, "This copy of Tor was built without support for " + "the option \"%s\". Skipping.", var->cvar->member.name); + } + + return struct_var_kvassign(object, c, msg, &var->cvar->member); +} + +/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments + * to it will replace old ones. */ +static void +config_mark_lists_fragile(const config_mgr_t *mgr, void *options) +{ + tor_assert(mgr); + tor_assert(options); + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + void *object = config_mgr_get_obj_mutable(mgr, options, mv->object_idx); + struct_var_mark_fragile(object, &mv->cvar->member); + } SMARTLIST_FOREACH_END(mv); +} + +/** + * Log a warning that declaring that the option called <b>what</b> + * is deprecated because of the reason in <b>why</b>. + * + * (Both arguments must be non-NULL.) + **/ +void +warn_deprecated_option(const char *what, const char *why) +{ + const char *space = (why && strlen(why)) ? " " : ""; + log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely " + "be removed in a future version of Tor.%s%s (If you think this is " + "a mistake, please let us know!)", + what, space, why); +} + +/** If <b>c</b> is a syntactically valid configuration line, update + * <b>options</b> with its value and return 0. Otherwise return -1 for bad + * key, -2 for bad value. + * + * If <b>clear_first</b> is set, clear the value first. Then if + * <b>use_defaults</b> is set, set the value to the default. + * + * Called from config_assign(). + */ +static int +config_assign_line(const config_mgr_t *mgr, void *options, + config_line_t *c, unsigned flags, + bitarray_t *options_seen, char **msg) +{ + const unsigned use_defaults = flags & CAL_USE_DEFAULTS; + const unsigned clear_first = flags & CAL_CLEAR_FIRST; + const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS; + const managed_var_t *mvar; + + CONFIG_CHECK(mgr, options); + + int var_index = -1; + mvar = config_mgr_find_var(mgr, c->key, true, &var_index); + if (!mvar) { + const config_format_t *fmt = mgr->toplevel; + if (fmt->extra) { + void *lvalue = STRUCT_VAR_P(options, fmt->extra->offset); + log_info(LD_CONFIG, + "Found unrecognized option '%s'; saving it.", c->key); + config_line_append((config_line_t**)lvalue, c->key, c->value); + return 0; + } else { + tor_asprintf(msg, + "Unknown option '%s'. Failing.", c->key); + return -1; + } + } + + const config_var_t *cvar = mvar->cvar; + tor_assert(cvar); + + /* Put keyword into canonical case. */ + if (strcmp(cvar->member.name, c->key)) { + tor_free(c->key); + c->key = tor_strdup(cvar->member.name); + } + + const char *deprecation_msg; + if (warn_deprecations && + (deprecation_msg = config_find_deprecation(mgr, cvar->member.name))) { + warn_deprecated_option(cvar->member.name, deprecation_msg); + } + + if (!strlen(c->value)) { + /* reset or clear it, then return */ + if (!clear_first) { + if (! config_var_is_replaced_on_set(cvar) && + c->command != CONFIG_LINE_CLEAR) { + /* We got an empty linelist from the torrc or command line. + As a special case, call this an error. Warn and ignore. */ + log_warn(LD_CONFIG, + "Linelist option '%s' has no value. Skipping.", c->key); + } else { /* not already cleared */ + config_reset(mgr, options, mvar, use_defaults); + } + } + return 0; + } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) { + // This block is unreachable, since a CLEAR line always has an + // empty value, and so will trigger be handled by the previous + // "if (!strlen(c->value))" block. + + // LCOV_EXCL_START + tor_assert_nonfatal_unreached(); + config_reset(mgr, options, mvar, use_defaults); + // LCOV_EXCL_STOP + } + + if (options_seen && config_var_is_replaced_on_set(cvar)) { + /* We're tracking which options we've seen, and this option is not + * supposed to occur more than once. */ + tor_assert(var_index >= 0); + if (bitarray_is_set(options_seen, var_index)) { + log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last " + "value will be ignored.", cvar->member.name); + } + bitarray_set(options_seen, var_index); + } + + if (config_assign_value(mgr, options, c, msg) < 0) + return -2; + return 0; +} + +/** Restore the option named <b>key</b> in options to its default value. + * Called from config_assign(). */ +STATIC void +config_reset_line(const config_mgr_t *mgr, void *options, + const char *key, int use_defaults) +{ + const managed_var_t *var; + + CONFIG_CHECK(mgr, options); + + var = config_mgr_find_var(mgr, key, true, NULL); + if (!var) + return; /* give error on next pass. */ + + config_reset(mgr, options, var, use_defaults); +} + +/** Return true iff value needs to be quoted and escaped to be used in + * a configuration file. */ +static int +config_value_needs_escape(const char *value) +{ + if (*value == '\"') + return 1; + while (*value) { + switch (*value) + { + case '\r': + case '\n': + case '#': + /* Note: quotes and backspaces need special handling when we are using + * quotes, not otherwise, so they don't trigger escaping on their + * own. */ + return 1; + default: + if (!TOR_ISPRINT(*value)) + return 1; + } + ++value; + } + return 0; +} + +/** Return newly allocated line or lines corresponding to <b>key</b> in the + * configuration <b>options</b>. If <b>escape_val</b> is true and a + * value needs to be quoted before it's put in a config file, quote and + * escape that value. Return NULL if no such key exists. */ +config_line_t * +config_get_assigned_option(const config_mgr_t *mgr, const void *options, + const char *key, int escape_val) +{ + const managed_var_t *var; + config_line_t *result; + + tor_assert(options && key); + + CONFIG_CHECK(mgr, options); + + var = config_mgr_find_var(mgr, key, true, NULL); + if (!var) { + log_warn(LD_CONFIG, "Unknown option '%s'. Failing.", key); + return NULL; + } + if (! config_var_is_gettable(var->cvar)) { + log_warn(LD_CONFIG, "Option '%s' is obsolete or unfetchable. Failing.", + key); + return NULL; + } + const void *object = config_mgr_get_obj(mgr, options, var->object_idx); + + result = struct_var_kvencode(object, &var->cvar->member); + + if (escape_val) { + config_line_t *line; + for (line = result; line; line = line->next) { + if (line->value && config_value_needs_escape(line->value)) { + char *newval = esc_for_log(line->value); + tor_free(line->value); + line->value = newval; + } + } + } + + return result; +} +/** Iterate through the linked list of requested options <b>list</b>. + * For each item, convert as appropriate and assign to <b>options</b>. + * If an item is unrecognized, set *msg and return -1 immediately, + * else return 0 for success. + * + * If <b>clear_first</b>, interpret config options as replacing (not + * extending) their previous values. If <b>clear_first</b> is set, + * then <b>use_defaults</b> to decide if you set to defaults after + * clearing, or make the value 0 or NULL. + * + * Here are the use cases: + * 1. A non-empty AllowInvalid line in your torrc. Appends to current + * if linelist, replaces current if csv. + * 2. An empty AllowInvalid line in your torrc. Should clear it. + * 3. "RESETCONF AllowInvalid" sets it to default. + * 4. "SETCONF AllowInvalid" makes it NULL. + * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo". + * + * Use_defaults Clear_first + * 0 0 "append" + * 1 0 undefined, don't use + * 0 1 "set to null first" + * 1 1 "set to defaults first" + * Return 0 on success, -1 on bad key, -2 on bad value. + * + * As an additional special case, if a LINELIST config option has + * no value and clear_first is 0, then warn and ignore it. + */ + +/* +There are three call cases for config_assign() currently. + +Case one: Torrc entry +options_init_from_torrc() calls config_assign(0, 0) + calls config_assign_line(0, 0). + if value is empty, calls config_reset(0) and returns. + calls config_assign_value(), appends. + +Case two: setconf +options_trial_assign() calls config_assign(0, 1) + calls config_reset_line(0) + calls config_reset(0) + calls option_clear(). + calls config_assign_line(0, 1). + if value is empty, returns. + calls config_assign_value(), appends. + +Case three: resetconf +options_trial_assign() calls config_assign(1, 1) + calls config_reset_line(1) + calls config_reset(1) + calls option_clear(). + calls config_assign_value(default) + calls config_assign_line(1, 1). + returns. +*/ +int +config_assign(const config_mgr_t *mgr, void *options, config_line_t *list, + unsigned config_assign_flags, char **msg) +{ + config_line_t *p; + bitarray_t *options_seen; + const int n_options = config_count_options(mgr); + const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST; + const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS; + + CONFIG_CHECK(mgr, options); + + /* pass 1: normalize keys */ + for (p = list; p; p = p->next) { + const char *full = config_expand_abbrev(mgr, p->key, 0, 1); + if (strcmp(full,p->key)) { + tor_free(p->key); + p->key = tor_strdup(full); + } + } + + /* pass 2: if we're reading from a resetting source, clear all + * mentioned config options, and maybe set to their defaults. */ + if (clear_first) { + for (p = list; p; p = p->next) + config_reset_line(mgr, options, p->key, use_defaults); + } + + options_seen = bitarray_init_zero(n_options); + /* pass 3: assign. */ + while (list) { + int r; + if ((r=config_assign_line(mgr, options, list, config_assign_flags, + options_seen, msg))) { + bitarray_free(options_seen); + return r; + } + list = list->next; + } + bitarray_free(options_seen); + + /** Now we're done assigning a group of options to the configuration. + * Subsequent group assignments should _replace_ linelists, not extend + * them. */ + config_mark_lists_fragile(mgr, options); + + return 0; +} + +/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent. + * Called from config_reset() and config_free(). */ +static void +config_clear(const config_mgr_t *mgr, void *options, const managed_var_t *var) +{ + void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx); + struct_var_free(object, &var->cvar->member); +} + +/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if + * <b>use_defaults</b>, set it to its default value. + * Called by config_init() and option_reset_line() and option_assign_line(). */ +static void +config_reset(const config_mgr_t *mgr, void *options, + const managed_var_t *var, int use_defaults) +{ + config_line_t *c; + char *msg = NULL; + CONFIG_CHECK(mgr, options); + config_clear(mgr, options, var); /* clear it first */ + + if (!use_defaults) + return; /* all done */ + + if (var->cvar->initvalue) { + c = tor_malloc_zero(sizeof(config_line_t)); + c->key = tor_strdup(var->cvar->member.name); + c->value = tor_strdup(var->cvar->initvalue); + if (config_assign_value(mgr, options, c, &msg) < 0) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Failed to assign default: %s", msg); + tor_free(msg); /* if this happens it's a bug */ + // LCOV_EXCL_STOP + } + config_free_lines(c); + } +} + +/** Release storage held by <b>options</b>. */ +void +config_free_(const config_mgr_t *mgr, void *options) +{ + if (!options) + return; + + tor_assert(mgr); + + if (mgr->toplevel->clear_fn) { + mgr->toplevel->clear_fn(mgr, options); + } + config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, options); + if (suitep) { + tor_assert(smartlist_len((*suitep)->configs) == + smartlist_len(mgr->subconfigs)); + SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) { + void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx); + if (fmt->clear_fn) { + fmt->clear_fn(mgr, obj); + } + } SMARTLIST_FOREACH_END(fmt); + } + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + config_clear(mgr, options, mv); + } SMARTLIST_FOREACH_END(mv); + + if (mgr->toplevel->extra) { + config_line_t **linep = STRUCT_VAR_P(options, + mgr->toplevel->extra->offset); + config_free_lines(*linep); + *linep = NULL; + } + + if (suitep) { + SMARTLIST_FOREACH((*suitep)->configs, void *, obj, tor_free(obj)); + config_suite_free(*suitep); + } + + tor_free(options); +} + +/** Return true iff the option <b>name</b> has the same value in <b>o1</b> + * and <b>o2</b>. Must not be called for LINELIST_S or OBSOLETE options. + */ +int +config_is_same(const config_mgr_t *mgr, + const void *o1, const void *o2, + const char *name) +{ + CONFIG_CHECK(mgr, o1); + CONFIG_CHECK(mgr, o2); + + const managed_var_t *var = config_mgr_find_var(mgr, name, true, NULL); + if (!var) { + return true; + } + const void *obj1 = config_mgr_get_obj(mgr, o1, var->object_idx); + const void *obj2 = config_mgr_get_obj(mgr, o2, var->object_idx); + + return struct_var_eq(obj1, obj2, &var->cvar->member); +} + +/** + * Return a list of the options which have changed between <b>options1</b> and + * <b>options2</b>. If an option has reverted to its default value, it has a + * value entry of NULL. + * + * <b>options1</b> and <b>options2</b> must be top-level configuration objects + * of the type managed by <b>mgr</b>. + **/ +config_line_t * +config_get_changes(const config_mgr_t *mgr, + const void *options1, const void *options2) +{ + config_line_t *result = NULL; + config_line_t **next = &result; + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) { + if (! config_var_should_list_changes(mv->cvar)) { + /* something else will check this var, or it doesn't need checking */ + continue; + } + const void *obj1 = config_mgr_get_obj(mgr, options1, mv->object_idx); + const void *obj2 = config_mgr_get_obj(mgr, options2, mv->object_idx); + + if (struct_var_eq(obj1, obj2, &mv->cvar->member)) { + continue; + } + + const char *varname = mv->cvar->member.name; + config_line_t *line = + config_get_assigned_option(mgr, options2, varname, 1); + + if (line) { + *next = line; + } else { + *next = tor_malloc_zero(sizeof(config_line_t)); + (*next)->key = tor_strdup(varname); + } + while (*next) + next = &(*next)->next; + } SMARTLIST_FOREACH_END(mv); + + return result; +} + +/** Copy storage held by <b>old</b> into a new or_options_t and return it. */ +void * +config_dup(const config_mgr_t *mgr, const void *old) +{ + void *newopts; + + newopts = config_new(mgr); + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) { + if (! config_var_needs_copy(mv->cvar)) { + // Something else will copy this option, or it doesn't need copying. + continue; + } + const void *oldobj = config_mgr_get_obj(mgr, old, mv->object_idx); + void *newobj = config_mgr_get_obj_mutable(mgr, newopts, mv->object_idx); + if (struct_var_copy(newobj, oldobj, &mv->cvar->member) < 0) { + // LCOV_EXCL_START + log_err(LD_BUG, "Unable to copy value for %s.", + mv->cvar->member.name); + tor_assert_unreached(); + // LCOV_EXCL_STOP + } + } SMARTLIST_FOREACH_END(mv); + + return newopts; +} +/** Set all vars in the configuration object <b>options</b> to their default + * values. */ +void +config_init(const config_mgr_t *mgr, void *options) +{ + CONFIG_CHECK(mgr, options); + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!mv->cvar->initvalue) + continue; /* defaults to NULL or 0 */ + config_reset(mgr, options, mv, 1); + } SMARTLIST_FOREACH_END(mv); +} + +/** + * Helper for config_validate_single: see whether any immutable option + * has changed between old_options and new_options. + * + * On success return 0; on failure set *msg_out to a newly allocated + * string explaining what is wrong, and return -1. + */ +static int +config_check_immutable_flags(const config_format_t *fmt, + const void *old_options, + const void *new_options, + char **msg_out) +{ + tor_assert(fmt); + tor_assert(new_options); + if (BUG(! old_options)) + return 0; + + unsigned i; + for (i = 0; fmt->vars[i].member.name; ++i) { + const config_var_t *v = &fmt->vars[i]; + if (! config_var_has_flag(v, CFLG_IMMUTABLE)) + continue; + + if (! struct_var_eq(old_options, new_options, &v->member)) { + tor_asprintf(msg_out, + "While Tor is running, changing %s is not allowed", + v->member.name); + return -1; + } + } + + return 0; +} + +/** + * Normalize and validate a single object `options` within a configuration + * suite, according to its format. `options` may be modified as appropriate + * in order to set ancillary data. If `old_options` is provided, make sure + * that the transition from `old_options` to `options` is permitted. + * + * On success return VSTAT_OK; on failure set *msg_out to a newly allocated + * string explaining what is wrong, and return a different validation_status_t + * to describe which step failed. + **/ +static validation_status_t +config_validate_single(const config_format_t *fmt, + const void *old_options, void *options, + char **msg_out) +{ + tor_assert(fmt); + tor_assert(options); + + if (fmt->pre_normalize_fn) { + if (fmt->pre_normalize_fn(options, msg_out) < 0) { + return VSTAT_PRE_NORMALIZE_ERR; + } + } + + if (fmt->legacy_validate_fn) { + if (fmt->legacy_validate_fn(old_options, options, msg_out) < 0) { + return VSTAT_LEGACY_ERR; + } + } + + if (fmt->validate_fn) { + if (fmt->validate_fn(options, msg_out) < 0) { + return VSTAT_VALIDATE_ERR; + } + } + + if (old_options) { + if (config_check_immutable_flags(fmt, old_options, options, msg_out) < 0) { + return VSTAT_TRANSITION_ERR; + } + + if (fmt->check_transition_fn) { + if (fmt->check_transition_fn(old_options, options, msg_out) < 0) { + return VSTAT_TRANSITION_ERR; + } + } + } + + if (fmt->post_normalize_fn) { + if (fmt->post_normalize_fn(options, msg_out) < 0) { + return VSTAT_POST_NORMALIZE_ERR; + } + } + + return VSTAT_OK; +} + +/** + * Normalize and validate all the options in configuration object `options` + * and its sub-objects. `options` may be modified as appropriate in order to + * set ancillary data. If `old_options` is provided, make sure that the + * transition from `old_options` to `options` is permitted. + * + * On success return VSTAT_OK; on failure set *msg_out to a newly allocated + * string explaining what is wrong, and return a different validation_status_t + * to describe which step failed. + **/ +validation_status_t +config_validate(const config_mgr_t *mgr, + const void *old_options, void *options, + char **msg_out) +{ + validation_status_t rv; + CONFIG_CHECK(mgr, options); + if (old_options) { + CONFIG_CHECK(mgr, old_options); + } + + config_suite_t **suitep_new = config_mgr_get_suite_ptr(mgr, options); + config_suite_t **suitep_old = NULL; + if (old_options) + suitep_old = config_mgr_get_suite_ptr(mgr, (void*) old_options); + + /* Validate the sub-objects */ + if (suitep_new) { + SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) { + void *obj = smartlist_get((*suitep_new)->configs, fmt_sl_idx); + const void *obj_old=NULL; + if (suitep_old) + obj_old = smartlist_get((*suitep_old)->configs, fmt_sl_idx); + + rv = config_validate_single(fmt, obj_old, obj, msg_out); + if (rv < 0) + return rv; + } SMARTLIST_FOREACH_END(fmt); + } + + /* Validate the top-level object. */ + rv = config_validate_single(mgr->toplevel, old_options, options, msg_out); + if (rv < 0) + return rv; + + return VSTAT_OK; +} + +/** Allocate and return a new string holding the written-out values of the vars + * in 'options'. If 'minimal', do not write out any default-valued vars. + * Else, if comment_defaults, write default values as comments. + */ +char * +config_dump(const config_mgr_t *mgr, const void *default_options, + const void *options, int minimal, + int comment_defaults) +{ + const config_format_t *fmt = mgr->toplevel; + smartlist_t *elements; + const void *defaults = default_options; + void *defaults_tmp = NULL; + config_line_t *line, *assigned; + char *result; + char *msg = NULL; + + if (defaults == NULL) { + defaults = defaults_tmp = config_new(mgr); + config_init(mgr, defaults_tmp); + } + + /* XXX use a 1 here so we don't add a new log line while dumping */ + if (default_options == NULL) { + if (config_validate(mgr, NULL, defaults_tmp, &msg) < 0) { + // LCOV_EXCL_START + log_err(LD_BUG, "Failed to validate default config: %s", msg); + tor_free(msg); + tor_assert(0); + // LCOV_EXCL_STOP + } + } + + elements = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) { + int comment_option = 0; + /* Don't save 'hidden' control variables. */ + if (! config_var_is_dumpable(mv->cvar)) + continue; + const char *name = mv->cvar->member.name; + if (minimal && config_is_same(mgr, options, defaults, name)) + continue; + else if (comment_defaults && + config_is_same(mgr, options, defaults, name)) + comment_option = 1; + + line = assigned = + config_get_assigned_option(mgr, options, name, 1); + + for (; line; line = line->next) { + if (!strcmpstart(line->key, "__")) { + /* This check detects "hidden" variables inside LINELIST_V structures. + */ + continue; + } + int value_exists = line->value && *(line->value); + smartlist_add_asprintf(elements, "%s%s%s%s\n", + comment_option ? "# " : "", + line->key, value_exists ? " " : "", line->value); + } + config_free_lines(assigned); + } SMARTLIST_FOREACH_END(mv); + + if (fmt->extra) { + line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->offset); + for (; line; line = line->next) { + int value_exists = line->value && *(line->value); + smartlist_add_asprintf(elements, "%s%s%s\n", + line->key, value_exists ? " " : "", line->value); + } + } + + result = smartlist_join_strings(elements, "", 0, NULL); + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + config_free(mgr, defaults_tmp); + return result; +} + +/** + * Return true if every member of <b>options</b> is in-range and well-formed. + * Return false otherwise. Log errors at level <b>severity</b>. + */ +bool +config_check_ok(const config_mgr_t *mgr, const void *options, int severity) +{ + bool all_ok = true; + + SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) { + if (!struct_var_ok(options, &mv->cvar->member)) { + log_fn(severity, LD_BUG, "Invalid value for %s", + mv->cvar->member.name); + all_ok = false; + } + } SMARTLIST_FOREACH_END(mv); + + return all_ok; +} diff --git a/src/lib/confmgt/confmgt.h b/src/lib/confmgt/confmgt.h new file mode 100644 index 0000000000..5065c13b60 --- /dev/null +++ b/src/lib/confmgt/confmgt.h @@ -0,0 +1,136 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file confmgt.h + * + * \brief Header for confmgt.c. + */ + +#ifndef TOR_CONFMGT_H +#define TOR_CONFMGT_H + +#include "lib/conf/conftypes.h" +#include "lib/conf/confmacros.h" +#include "lib/testsupport/testsupport.h" + +/** + * A collection of config_format_t objects to describe several objects + * that are all configured with the same configuration file. + * + * (NOTE: for now, this only handles a single config_format_t.) + **/ +typedef struct config_mgr_t config_mgr_t; + +config_mgr_t *config_mgr_new(const config_format_t *toplevel_fmt); +void config_mgr_free_(config_mgr_t *mgr); +int config_mgr_add_format(config_mgr_t *mgr, + const config_format_t *fmt); +void config_mgr_freeze(config_mgr_t *mgr); +#define config_mgr_free(mgr) \ + FREE_AND_NULL(config_mgr_t, config_mgr_free_, (mgr)) +struct smartlist_t *config_mgr_list_vars(const config_mgr_t *mgr); +struct smartlist_t *config_mgr_list_deprecated_vars(const config_mgr_t *mgr); + +/** A collection of managed configuration objects. */ +typedef struct config_suite_t config_suite_t; + +/** + * Flag for config_assign: if set, then "resetting" an option changes it to + * its default value, as specified in the config_var_t. Otherwise, + * "resetting" an option changes it to a type-dependent null value -- + * typically 0 or NULL. + * + * (An option is "reset" when it is set to an empty value, or as described in + * CAL_CLEAR_FIRST). + **/ +#define CAL_USE_DEFAULTS (1u<<0) +/** + * Flag for config_assign: if set, then we reset every provided config + * option before we set it. + * + * For example, if this flag is not set, then passing a multi-line option to + * config_assign will cause any previous value to be extended. But if this + * flag is set, then a multi-line option will replace any previous value. + **/ +#define CAL_CLEAR_FIRST (1u<<1) +/** + * Flag for config_assign: if set, we warn about deprecated options. + **/ +#define CAL_WARN_DEPRECATIONS (1u<<2) + +void *config_new(const config_mgr_t *fmt); +void config_free_(const config_mgr_t *fmt, void *options); +#define config_free(mgr, options) do { \ + config_free_((mgr), (options)); \ + (options) = NULL; \ + } while (0) + +struct config_line_t *config_get_assigned_option(const config_mgr_t *mgr, + const void *options, const char *key, + int escape_val); +int config_is_same(const config_mgr_t *fmt, + const void *o1, const void *o2, + const char *name); +struct config_line_t *config_get_changes(const config_mgr_t *mgr, + const void *options1, const void *options2); +void config_init(const config_mgr_t *mgr, void *options); + +/** An enumeration to report which validation step failed. */ +typedef enum { + VSTAT_PRE_NORMALIZE_ERR = -5, + VSTAT_VALIDATE_ERR = -4, + VSTAT_LEGACY_ERR = -3, + VSTAT_TRANSITION_ERR = -2, + VSTAT_POST_NORMALIZE_ERR = -1, + VSTAT_OK = 0, +} validation_status_t; + +validation_status_t config_validate(const config_mgr_t *mgr, + const void *old_options, void *options, + char **msg_out); +void *config_dup(const config_mgr_t *mgr, const void *old); +char *config_dump(const config_mgr_t *mgr, const void *default_options, + const void *options, int minimal, + int comment_defaults); +void config_check_toplevel_magic(const config_mgr_t *mgr, + const void *object); +bool config_check_ok(const config_mgr_t *mgr, const void *options, + int severity); +int config_assign(const config_mgr_t *mgr, void *options, + struct config_line_t *list, + unsigned flags, char **msg); +const char *config_find_deprecation(const config_mgr_t *mgr, + const char *key); +const char *config_find_option_name(const config_mgr_t *mgr, + const char *key); +const char *config_expand_abbrev(const config_mgr_t *mgr, + const char *option, + int command_line, int warn_obsolete); +void warn_deprecated_option(const char *what, const char *why); + +bool config_var_is_settable(const config_var_t *var); +bool config_var_is_listable(const config_var_t *var); + +/* Helper macros to compare an option across two configuration objects */ +#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt) +#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt) +#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt)) +#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt) +#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt) +#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt) + +void *config_mgr_get_obj_mutable(const config_mgr_t *mgr, + void *toplevel, int idx); +const void *config_mgr_get_obj(const config_mgr_t *mgr, + const void *toplevel, int idx); + +#ifdef CONFMGT_PRIVATE +STATIC void config_reset_line(const config_mgr_t *mgr, void *options, + const char *key, int use_defaults); +#endif /* defined(CONFMGT_PRIVATE) */ + +#endif /* !defined(TOR_CONFMGT_H) */ diff --git a/src/lib/confmgt/include.am b/src/lib/confmgt/include.am new file mode 100644 index 0000000000..d3a7a7cd69 --- /dev/null +++ b/src/lib/confmgt/include.am @@ -0,0 +1,27 @@ +noinst_LIBRARIES += src/lib/libtor-confmgt.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-confmgt-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_confmgt_a_SOURCES = \ + src/lib/confmgt/confmgt.c \ + src/lib/confmgt/structvar.c \ + src/lib/confmgt/type_defs.c \ + src/lib/confmgt/typedvar.c \ + src/lib/confmgt/unitparse.c + +src_lib_libtor_confmgt_testing_a_SOURCES = \ + $(src_lib_libtor_confmgt_a_SOURCES) +src_lib_libtor_confmgt_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_confmgt_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/confmgt/confmgt.h \ + src/lib/confmgt/structvar.h \ + src/lib/confmgt/type_defs.h \ + src/lib/confmgt/typedvar.h \ + src/lib/confmgt/unitparse.h \ + src/lib/confmgt/var_type_def_st.h diff --git a/src/lib/confmgt/lib_confmgt.md b/src/lib/confmgt/lib_confmgt.md new file mode 100644 index 0000000000..861e720f64 --- /dev/null +++ b/src/lib/confmgt/lib_confmgt.md @@ -0,0 +1,7 @@ +@dir /lib/confmgt +@brief lib/confmgt: Parse, encode, manipulate configuration files. + +This logic is used in common by our state files (statefile.c) and +configuration files (config.c) to manage a set of named, typed fields, +reading and writing them to disk and to the controller. + diff --git a/src/lib/confmgt/structvar.c b/src/lib/confmgt/structvar.c new file mode 100644 index 0000000000..55deb4759c --- /dev/null +++ b/src/lib/confmgt/structvar.c @@ -0,0 +1,239 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file structvar.c + * @brief Functions to manipulate named and typed elements of + * a structure. + * + * These functions represent a low-level API for accessing a member of a + * structure. They use typedvar.c to work, and they are used in turn by the + * configuration system to examine and set fields in configuration objects + * used by individual modules. + * + * Almost no code should call these directly. + **/ + +#include "orconfig.h" +#include "lib/confmgt/structvar.h" +#include "lib/cc/compat_compiler.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/typedvar.h" +#include "lib/log/util_bug.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include <stddef.h> + +/** + * Return true iff all fields on <b>decl</b> are NULL or 0, indicating that + * there is no object or no magic number to check. + **/ +static inline bool +magic_is_null(const struct_magic_decl_t *decl) +{ + return decl->typename == NULL && + decl->magic_offset == 0 && + decl->magic_val == 0; +} + +/** + * Set the 'magic number' on <b>object</b> to correspond to decl. + **/ +void +struct_set_magic(void *object, const struct_magic_decl_t *decl) +{ + tor_assert(decl); + if (magic_is_null(decl)) + return; + + tor_assert(object); + uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset); + *ptr = decl->magic_val; +} + +/** + * Assert that the 'magic number' on <b>object</b> to corresponds to decl. + **/ +void +struct_check_magic(const void *object, const struct_magic_decl_t *decl) +{ + tor_assert(decl); + if (magic_is_null(decl)) + return; + + tor_assert(object); + + const uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset); + tor_assertf(*ptr == decl->magic_val, + "Bad magic number on purported %s object. " + "Expected %"PRIu32"x but got %"PRIu32"x.", + decl->typename, decl->magic_val, *ptr); +} + +/** + * Return a mutable pointer to the member of <b>object</b> described + * by <b>member</b>. + **/ +void * +struct_get_mptr(void *object, const struct_member_t *member) +{ + tor_assert(object); + return STRUCT_VAR_P(object, member->offset); +} + +/** + * Return a const pointer to the member of <b>object</b> described + * by <b>member</b>. + **/ +const void * +struct_get_ptr(const void *object, const struct_member_t *member) +{ + tor_assert(object); + return STRUCT_VAR_P(object, member->offset); +} + +/** + * Helper: given a struct_member_t, look up the type definition for its + * variable. + */ +static const var_type_def_t * +get_type_def(const struct_member_t *member) +{ + if (member->type_def) + return member->type_def; + + return lookup_type_def(member->type); +} + +/** + * (As typed_var_free, but free and clear the member of <b>object</b> defined + * by <b>member</b>.) + **/ +void +struct_var_free(void *object, const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + + typed_var_free(p, def); +} + +/** + * (As typed_var_copy, but copy from <b>src</b> to <b>dest</b> the member + * defined by <b>member</b>.) + **/ +int +struct_var_copy(void *dest, const void *src, const struct_member_t *member) +{ + void *p_dest = struct_get_mptr(dest, member); + const void *p_src = struct_get_ptr(src, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_copy(p_dest, p_src, def); +} + +/** + * (As typed_var_eq, but compare the members of <b>a</b> and <b>b</b> + * defined by <b>member</b>.) + **/ +bool +struct_var_eq(const void *a, const void *b, const struct_member_t *member) +{ + const void *p_a = struct_get_ptr(a, member); + const void *p_b = struct_get_ptr(b, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_eq(p_a, p_b, def); +} + +/** + * (As typed_var_ok, but validate the member of <b>object</b> defined by + * <b>member</b>.) + **/ +bool +struct_var_ok(const void *object, const struct_member_t *member) +{ + const void *p = struct_get_ptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_ok(p, def); +} + +/** + * (As typed_var_kvassign, but assign a value to the member of <b>object</b> + * defined by <b>member</b>.) + **/ +int +struct_var_kvassign(void *object, const struct config_line_t *line, + char **errmsg, + const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_kvassign(p, line, errmsg, def); +} + +/** + * (As typed_var_kvencode, but encode the value of the member of <b>object</b> + * defined by <b>member</b>.) + **/ +struct config_line_t * +struct_var_kvencode(const void *object, const struct_member_t *member) +{ + const void *p = struct_get_ptr(object, member); + const var_type_def_t *def = get_type_def(member); + + return typed_var_kvencode(member->name, p, def); +} + +/** + * Mark the field in <b>object</b> determined by <b>member</b> -- a variable + * that ordinarily would be extended by assignment -- as "fragile", so that it + * will get replaced by the next assignment instead. + */ +void +struct_var_mark_fragile(void *object, const struct_member_t *member) +{ + void *p = struct_get_mptr(object, member); + const var_type_def_t *def = get_type_def(member); + return typed_var_mark_fragile(p, def); +} + +/** + * Return the official name of this struct member. + **/ +const char * +struct_var_get_name(const struct_member_t *member) +{ + return member->name; +} + +/** + * Return the type name for this struct member. + * + * Do not use the output of this function to inspect a type within Tor. It is + * suitable for debugging, informing the controller or user of a variable's + * type, etc. + **/ +const char * +struct_var_get_typename(const struct_member_t *member) +{ + const var_type_def_t *def = get_type_def(member); + + return def ? def->name : NULL; +} + +/** Return all of the flags set for this struct member. */ +uint32_t +struct_var_get_flags(const struct_member_t *member) +{ + const var_type_def_t *def = get_type_def(member); + + return def ? def->flags : 0; +} diff --git a/src/lib/confmgt/structvar.h b/src/lib/confmgt/structvar.h new file mode 100644 index 0000000000..91334fa8c5 --- /dev/null +++ b/src/lib/confmgt/structvar.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file structvar.h + * @brief Header for lib/confmgt/structvar.c + **/ + +#ifndef TOR_LIB_CONFMGT_STRUCTVAR_H +#define TOR_LIB_CONFMGT_STRUCTVAR_H + +struct struct_magic_decl_t; +struct struct_member_t; +struct config_line_t; + +#include <stdbool.h> +#include "lib/cc/torint.h" + +void struct_set_magic(void *object, + const struct struct_magic_decl_t *decl); +void struct_check_magic(const void *object, + const struct struct_magic_decl_t *decl); + +void *struct_get_mptr(void *object, + const struct struct_member_t *member); +const void *struct_get_ptr(const void *object, + const struct struct_member_t *member); + +void struct_var_free(void *object, + const struct struct_member_t *member); +int struct_var_copy(void *dest, const void *src, + const struct struct_member_t *member); +bool struct_var_eq(const void *a, const void *b, + const struct struct_member_t *member); +bool struct_var_ok(const void *object, + const struct struct_member_t *member); +void struct_var_mark_fragile(void *object, + const struct struct_member_t *member); + +const char *struct_var_get_name(const struct struct_member_t *member); +const char *struct_var_get_typename(const struct struct_member_t *member); +uint32_t struct_var_get_flags(const struct struct_member_t *member); + +int struct_var_kvassign(void *object, const struct config_line_t *line, + char **errmsg, + const struct struct_member_t *member); +struct config_line_t *struct_var_kvencode( + const void *object, + const struct struct_member_t *member); + +#endif /* !defined(TOR_LIB_CONFMGT_STRUCTVAR_H) */ diff --git a/src/lib/confmgt/type_defs.c b/src/lib/confmgt/type_defs.c new file mode 100644 index 0000000000..d9e5e1e4c2 --- /dev/null +++ b/src/lib/confmgt/type_defs.c @@ -0,0 +1,849 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file type_defs.c + * @brief Definitions for various low-level configuration types. + * + * This module creates a number of var_type_def_t objects, to be used by + * typedvar.c in manipulating variables. + * + * The types here are common types that can be implemented with Tor's + * low-level functionality. To define new types, see var_type_def_st.h. + **/ + +#include "orconfig.h" +#include "lib/conf/conftypes.h" +#include "lib/conf/confdecl.h" +#include "lib/confmgt/typedvar.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/unitparse.h" + +#include "lib/cc/compat_compiler.h" +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/time_fmt.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/parse_int.h" +#include "lib/string/printf.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include <stddef.h> +#include <string.h> +#include <errno.h> + +////// +// CONFIG_TYPE_STRING +// CONFIG_TYPE_FILENAME +// +// These two types are the same for now, but they have different names. +// +// Warning: For this type, the default value (NULL) and "" are considered +// different values. That is generally risky, and best avoided for other +// types in the future. +////// + +static int +string_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)params; + (void)errmsg; + char **p = (char**)target; + *p = tor_strdup(value); + return 0; +} + +static char * +string_encode(const void *value, const void *params) +{ + (void)params; + const char **p = (const char**)value; + return *p ? tor_strdup(*p) : NULL; +} + +static void +string_clear(void *value, const void *params) +{ + (void)params; + char **p = (char**)value; + tor_free(*p); // sets *p to NULL. +} + +static const var_type_fns_t string_fns = { + .parse = string_parse, + .encode = string_encode, + .clear = string_clear, +}; + +///// +// CONFIG_TYPE_INT +// CONFIG_TYPE_POSINT +// +// These types are implemented as int, possibly with a restricted range. +///// + +/** + * Parameters for parsing an integer type. + **/ +typedef struct int_type_params_t { + int minval; /**< Lowest allowed value */ + int maxval; /**< Highest allowed value */ +} int_parse_params_t; + +static const int_parse_params_t INT_PARSE_UNRESTRICTED = { + .minval = INT_MIN, + .maxval = INT_MAX, +}; + +static const int_parse_params_t INT_PARSE_POSINT = { + .minval = 0, + .maxval = INT_MAX, +}; + +static int +int_parse(void *target, const char *value, char **errmsg, const void *params) +{ + const int_parse_params_t *pp; + if (params) { + pp = params; + } else { + pp = &INT_PARSE_UNRESTRICTED; + } + int *p = target; + int ok=0; + *p = (int)tor_parse_long(value, 10, pp->minval, pp->maxval, &ok, NULL); + if (!ok) { + tor_asprintf(errmsg, "Integer %s is malformed or out of bounds. " + "Allowed values are between %d and %d.", + value, pp->minval, pp->maxval); + return -1; + } + return 0; +} + +static char * +int_encode(const void *value, const void *params) +{ + (void)params; + int v = *(int*)value; + char *result; + tor_asprintf(&result, "%d", v); + return result; +} + +static void +int_clear(void *value, const void *params) +{ + (void)params; + *(int*)value = 0; +} + +static bool +int_ok(const void *value, const void *params) +{ + const int_parse_params_t *pp = params; + if (pp) { + int v = *(int*)value; + return pp->minval <= v && v <= pp->maxval; + } else { + return true; + } +} + +static const var_type_fns_t int_fns = { + .parse = int_parse, + .encode = int_encode, + .clear = int_clear, + .ok = int_ok, +}; + +///// +// CONFIG_TYPE_UINT64 +// +// This type is an unrestricted u64. +///// + +static int +uint64_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)params; + (void)errmsg; + uint64_t *p = target; + int ok=0; + *p = tor_parse_uint64(value, 10, 0, UINT64_MAX, &ok, NULL); + if (!ok) { + tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.", + value); + return -1; + } + return 0; +} + +static char * +uint64_encode(const void *value, const void *params) +{ + (void)params; + uint64_t v = *(uint64_t*)value; + char *result; + tor_asprintf(&result, "%"PRIu64, v); + return result; +} + +static void +uint64_clear(void *value, const void *params) +{ + (void)params; + *(uint64_t*)value = 0; +} + +static const var_type_fns_t uint64_fns = { + .parse = uint64_parse, + .encode = uint64_encode, + .clear = uint64_clear, +}; + +///// +// CONFIG_TYPE_INTERVAL +// CONFIG_TYPE_MSEC_INTERVAL +// CONFIG_TYPE_MEMUNIT +// +// These types are implemented using the config_parse_units() function. +// The intervals are stored as ints, whereas memory units are stored as +// uint64_ts. +///// + +static int +units_parse_u64(void *target, const char *value, char **errmsg, + const void *params) +{ + const unit_table_t *table = params; + tor_assert(table); + uint64_t *v = (uint64_t*)target; + int ok=1; + char *msg = NULL; + *v = config_parse_units(value, table, &ok, &msg); + if (!ok) { + tor_asprintf(errmsg, "Provided value is malformed or out of bounds: %s", + msg); + tor_free(msg); + return -1; + } + if (BUG(msg)) { + tor_free(msg); + } + return 0; +} + +static int +units_parse_int(void *target, const char *value, char **errmsg, + const void *params) +{ + const unit_table_t *table = params; + tor_assert(table); + int *v = (int*)target; + int ok=1; + char *msg = NULL; + uint64_t u64 = config_parse_units(value, table, &ok, &msg); + if (!ok) { + tor_asprintf(errmsg, "Provided value is malformed or out of bounds: %s", + msg); + tor_free(msg); + return -1; + } + if (BUG(msg)) { + tor_free(msg); + } + if (u64 > INT_MAX) { + tor_asprintf(errmsg, "Provided value %s is too large", value); + return -1; + } + *v = (int) u64; + return 0; +} + +static bool +units_ok_int(const void *value, const void *params) +{ + (void)params; + int v = *(int*)value; + return v >= 0; +} + +static const var_type_fns_t memunit_fns = { + .parse = units_parse_u64, + .encode = uint64_encode, // doesn't use params + .clear = uint64_clear, // doesn't use params +}; + +static const var_type_fns_t interval_fns = { + .parse = units_parse_int, + .encode = int_encode, // doesn't use params + .clear = int_clear, // doesn't use params, + .ok = units_ok_int // can't use int_ok, since that expects int params. +}; + +///// +// CONFIG_TYPE_DOUBLE +// +// This is a nice simple double. +///// + +static int +double_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)params; + (void)errmsg; + double *v = (double*)target; + char *endptr=NULL; + errno = 0; + *v = strtod(value, &endptr); + if (endptr == value || *endptr != '\0') { + // Either there are no converted characters, or there were some characters + // that didn't get converted. + tor_asprintf(errmsg, "Could not convert %s to a number.", escaped(value)); + return -1; + } + if (errno == ERANGE) { + // strtod will set errno to ERANGE on underflow or overflow. + bool underflow = -.00001 < *v && *v < .00001; + tor_asprintf(errmsg, + "%s is too %s to express as a floating-point number.", + escaped(value), underflow ? "small" : "large"); + return -1; + } + return 0; +} + +static char * +double_encode(const void *value, const void *params) +{ + (void)params; + double v = *(double*)value; + char *result; + tor_asprintf(&result, "%f", v); + return result; +} + +static void +double_clear(void *value, const void *params) +{ + (void)params; + double *v = (double *)value; + *v = 0.0; +} + +static const var_type_fns_t double_fns = { + .parse = double_parse, + .encode = double_encode, + .clear = double_clear, +}; + +///// +// CONFIG_TYPE_BOOL +// CONFIG_TYPE_AUTOBOOL +// +// These types are implemented as a case-insensitive string-to-integer +// mapping. +///// + +typedef struct enumeration_table_t { + const char *name; + int value; +} enumeration_table_t; + +typedef struct enumeration_params_t { + const char *allowed_val_string; + const enumeration_table_t *table; +} enumeration_params_t; + +static int +enum_parse(void *target, const char *value, char **errmsg, + const void *params_) +{ + const enumeration_params_t *params = params_; + const enumeration_table_t *table = params->table; + int *p = (int *)target; + for (; table->name; ++table) { + if (!strcasecmp(value, table->name)) { + *p = table->value; + return 0; + } + } + tor_asprintf(errmsg, "Unrecognized value %s. %s", + value, params->allowed_val_string); + return -1; +} + +static char * +enum_encode(const void *value, const void *params_) +{ + int v = *(const int*)value; + const enumeration_params_t *params = params_; + const enumeration_table_t *table = params->table; + for (; table->name; ++table) { + if (v == table->value) + return tor_strdup(table->name); + } + return NULL; // error. +} + +static void +enum_clear(void *value, const void *params_) +{ + int *p = (int*)value; + const enumeration_params_t *params = params_; + const enumeration_table_t *table = params->table; + tor_assert(table->name); + *p = table->value; +} + +static bool +enum_ok(const void *value, const void *params_) +{ + int v = *(const int*)value; + const enumeration_params_t *params = params_; + const enumeration_table_t *table = params->table; + for (; table->name; ++table) { + if (v == table->value) + return true; + } + return false; +} + +static const enumeration_table_t enum_table_bool[] = { + { "0", 0 }, + { "1", 1 }, + { NULL, 0 }, +}; + +static const enumeration_params_t enum_params_bool = { + "Allowed values are 0 and 1.", + enum_table_bool +}; + +static const enumeration_table_t enum_table_autobool[] = { + { "0", 0 }, + { "1", 1 }, + { "auto", -1 }, + { NULL, 0 }, +}; + +static const enumeration_params_t enum_params_autobool = { + "Allowed values are 0, 1, and auto.", + enum_table_autobool +}; + +static const var_type_fns_t enum_fns = { + .parse = enum_parse, + .encode = enum_encode, + .clear = enum_clear, + .ok = enum_ok, +}; + +///// +// CONFIG_TYPE_ISOTIME +// +// This is a time_t, encoded in ISO8601 format. +///// + +static int +time_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void) params; + time_t *p = target; + if (parse_iso_time(value, p) < 0) { + tor_asprintf(errmsg, "Invalid time %s", escaped(value)); + return -1; + } + return 0; +} + +static char * +time_encode(const void *value, const void *params) +{ + (void)params; + time_t v = *(const time_t *)value; + char *result = tor_malloc(ISO_TIME_LEN+1); + format_iso_time(result, v); + return result; +} + +static void +time_clear(void *value, const void *params) +{ + (void)params; + time_t *t = value; + *t = 0; +} + +static const var_type_fns_t time_fns = { + .parse = time_parse, + .encode = time_encode, + .clear = time_clear, +}; + +///// +// CONFIG_TYPE_CSV +// +// This type is a comma-separated list of strings, stored in a smartlist_t. +// An empty list may be encoded either as an empty smartlist, or as NULL. +///// + +static int +csv_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)params; + (void)errmsg; + smartlist_t **sl = (smartlist_t**)target; + *sl = smartlist_new(); + smartlist_split_string(*sl, value, ",", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + return 0; +} + +static char * +csv_encode(const void *value, const void *params) +{ + (void)params; + const smartlist_t *sl = *(const smartlist_t **)value; + if (! sl) + return tor_strdup(""); + + return smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL); +} + +static void +csv_clear(void *value, const void *params) +{ + (void)params; + smartlist_t **sl = (smartlist_t**)value; + if (!*sl) + return; + SMARTLIST_FOREACH(*sl, char *, cp, tor_free(cp)); + smartlist_free(*sl); // clears pointer. +} + +static const var_type_fns_t csv_fns = { + .parse = csv_parse, + .encode = csv_encode, + .clear = csv_clear, +}; + +///// +// CONFIG_TYPE_CSV_INTERVAL +// +// This type used to be a list of time intervals, used to determine a download +// schedule. Now, only the first interval counts: everything after the first +// comma is discarded. +///// + +static int +legacy_csv_interval_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)params; + /* We used to have entire smartlists here. But now that all of our + * download schedules use exponential backoff, only the first part + * matters. */ + const char *comma = strchr(value, ','); + const char *val = value; + char *tmp = NULL; + if (comma) { + tmp = tor_strndup(val, comma - val); + val = tmp; + } + + int rv = units_parse_int(target, val, errmsg, &time_units); + tor_free(tmp); + return rv; +} + +static const var_type_fns_t legacy_csv_interval_fns = { + .parse = legacy_csv_interval_parse, + .encode = int_encode, + .clear = int_clear, +}; + +///// +// CONFIG_TYPE_LINELIST +// CONFIG_TYPE_LINELIST_S +// CONFIG_TYPE_LINELIST_V +// +// A linelist is a raw config_line_t list. Order is preserved. +// +// The LINELIST type is used for homogeneous lists, where all the lines +// have the same key. +// +// The LINELIST_S and LINELIST_V types are used for the case where multiple +// lines of different keys are kept in a single list, to preserve their +// relative order. The unified list is stored as a "virtual" variable whose +// type is LINELIST_V; the individual sublists are treated as variables of +// type LINELIST_S. +// +// A linelist may be fragile or non-fragile. Assigning a line to a fragile +// linelist replaces the list with the line. If the line has the "APPEND" +// command set on it, or if the list is non-fragile, the line is appended. +// Either way, the new list is non-fragile. +///// + +static int +linelist_kv_parse(void *target, const struct config_line_t *line, + char **errmsg, const void *params) +{ + (void)params; + (void)errmsg; + config_line_t **lines = target; + + if (*lines && (*lines)->fragile) { + if (line->command == CONFIG_LINE_APPEND) { + (*lines)->fragile = 0; + } else { + config_free_lines(*lines); // sets it to NULL + } + } + + config_line_append(lines, line->key, line->value); + return 0; +} + +static int +linelist_kv_virt_noparse(void *target, const struct config_line_t *line, + char **errmsg, const void *params) +{ + (void)target; + (void)line; + (void)params; + *errmsg = tor_strdup("Cannot assign directly to virtual option."); + return -1; +} + +static struct config_line_t * +linelist_kv_encode(const char *key, const void *value, + const void *params) +{ + (void)key; + (void)params; + config_line_t *lines = *(config_line_t **)value; + return config_lines_dup(lines); +} + +static struct config_line_t * +linelist_s_kv_encode(const char *key, const void *value, + const void *params) +{ + (void)params; + config_line_t *lines = *(config_line_t **)value; + return config_lines_dup_and_filter(lines, key); +} + +static void +linelist_clear(void *target, const void *params) +{ + (void)params; + config_line_t **lines = target; + config_free_lines(*lines); // sets it to NULL +} + +static bool +linelist_eq(const void *a, const void *b, const void *params) +{ + (void)params; + const config_line_t *lines_a = *(const config_line_t **)a; + const config_line_t *lines_b = *(const config_line_t **)b; + return config_lines_eq(lines_a, lines_b); +} + +static int +linelist_copy(void *target, const void *value, const void *params) +{ + (void)params; + config_line_t **ptr = (config_line_t **)target; + const config_line_t *val = *(const config_line_t **)value; + config_free_lines(*ptr); + *ptr = config_lines_dup(val); + return 0; +} + +static void +linelist_mark_fragile(void *target, const void *params) +{ + (void)params; + config_line_t **ptr = (config_line_t **)target; + if (*ptr) + (*ptr)->fragile = 1; +} + +static const var_type_fns_t linelist_fns = { + .kv_parse = linelist_kv_parse, + .kv_encode = linelist_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, + .mark_fragile = linelist_mark_fragile, +}; + +static const var_type_fns_t linelist_v_fns = { + .kv_parse = linelist_kv_virt_noparse, + .kv_encode = linelist_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, + .mark_fragile = linelist_mark_fragile, +}; + +static const var_type_fns_t linelist_s_fns = { + .kv_parse = linelist_kv_parse, + .kv_encode = linelist_s_kv_encode, + .clear = linelist_clear, + .eq = linelist_eq, + .copy = linelist_copy, +}; + +///// +// CONFIG_TYPE_ROUTERSET +// +// XXXX to this module. +///// + +///// +// CONFIG_TYPE_IGNORE +// +// Used to indicate an option that cannot be stored or encoded. +///// + +static int +ignore_parse(void *target, const char *value, char **errmsg, + const void *params) +{ + (void)target; + (void)value; + (void)errmsg; + (void)params; + return 0; +} + +static char * +ignore_encode(const void *value, const void *params) +{ + (void)value; + (void)params; + return NULL; +} + +static const var_type_fns_t ignore_fns = { + .parse = ignore_parse, + .encode = ignore_encode, +}; + +const var_type_def_t STRING_type_defn = { + .name="String", .fns=&string_fns }; +const var_type_def_t FILENAME_type_defn = { + .name="Filename", .fns=&string_fns }; +const var_type_def_t INT_type_defn = { + .name="SignedInteger", .fns=&int_fns, + .params=&INT_PARSE_UNRESTRICTED }; +const var_type_def_t POSINT_type_defn = { + .name="Integer", .fns=&int_fns, + .params=&INT_PARSE_POSINT }; +const var_type_def_t UINT64_type_defn = { + .name="Integer", .fns=&uint64_fns, }; +const var_type_def_t MEMUNIT_type_defn = { + .name="DataSize", .fns=&memunit_fns, + .params=&memory_units }; +const var_type_def_t INTERVAL_type_defn = { + .name="TimeInterval", .fns=&interval_fns, + .params=&time_units }; +const var_type_def_t MSEC_INTERVAL_type_defn = { + .name="TimeMsecInterval", + .fns=&interval_fns, + .params=&time_msec_units }; +const var_type_def_t DOUBLE_type_defn = { + .name="Float", .fns=&double_fns, }; +const var_type_def_t BOOL_type_defn = { + .name="Boolean", .fns=&enum_fns, + .params=&enum_params_bool }; +const var_type_def_t AUTOBOOL_type_defn = { + .name="Boolean+Auto", .fns=&enum_fns, + .params=&enum_params_autobool }; +const var_type_def_t ISOTIME_type_defn = { + .name="Time", .fns=&time_fns, }; +const var_type_def_t CSV_type_defn = { + .name="CommaList", .fns=&csv_fns, }; +const var_type_def_t CSV_INTERVAL_type_defn = { + .name="TimeInterval", + .fns=&legacy_csv_interval_fns, }; +const var_type_def_t LINELIST_type_defn = { + .name="LineList", .fns=&linelist_fns, + .flags=CFLG_NOREPLACE }; +/* + * A "linelist_s" is a derived view of a linelist_v: inspecting + * it gets part of a linelist_v, and setting it adds to the linelist_v. + */ +const var_type_def_t LINELIST_S_type_defn = { + .name="Dependent", .fns=&linelist_s_fns, + .flags=CFLG_NOREPLACE| + /* The operations we disable here are + * handled by the linelist_v. */ + CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP }; +const var_type_def_t LINELIST_V_type_defn = { + .name="Virtual", .fns=&linelist_v_fns, + .flags=CFLG_NOREPLACE|CFLG_NOSET }; +const var_type_def_t IGNORE_type_defn = { + .name="Ignored", .fns=&ignore_fns, + .flags=CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET, +}; +const var_type_def_t OBSOLETE_type_defn = { + .name="Obsolete", .fns=&ignore_fns, + .flags=CFLG_GROUP_OBSOLETE, +}; + +/** + * Table mapping conf_type_t values to var_type_def_t objects. + **/ +static const var_type_def_t *type_definitions_table[] = { + [CONFIG_TYPE_STRING] = &STRING_type_defn, + [CONFIG_TYPE_FILENAME] = &FILENAME_type_defn, + [CONFIG_TYPE_INT] = &INT_type_defn, + [CONFIG_TYPE_POSINT] = &POSINT_type_defn, + [CONFIG_TYPE_UINT64] = &UINT64_type_defn, + [CONFIG_TYPE_MEMUNIT] = &MEMUNIT_type_defn, + [CONFIG_TYPE_INTERVAL] = &INTERVAL_type_defn, + [CONFIG_TYPE_MSEC_INTERVAL] = &MSEC_INTERVAL_type_defn, + [CONFIG_TYPE_DOUBLE] = &DOUBLE_type_defn, + [CONFIG_TYPE_BOOL] = &BOOL_type_defn, + [CONFIG_TYPE_AUTOBOOL] = &AUTOBOOL_type_defn, + [CONFIG_TYPE_ISOTIME] = &ISOTIME_type_defn, + [CONFIG_TYPE_CSV] = &CSV_type_defn, + [CONFIG_TYPE_CSV_INTERVAL] = &CSV_INTERVAL_type_defn, + [CONFIG_TYPE_LINELIST] = &LINELIST_type_defn, + [CONFIG_TYPE_LINELIST_S] = &LINELIST_S_type_defn, + [CONFIG_TYPE_LINELIST_V] = &LINELIST_V_type_defn, + [CONFIG_TYPE_IGNORE] = &IGNORE_type_defn, + [CONFIG_TYPE_OBSOLETE] = &OBSOLETE_type_defn, +}; + +/** + * Return a pointer to the var_type_def_t object for the given + * config_type_t value, or NULL if no such type definition exists. + **/ +const var_type_def_t * +lookup_type_def(config_type_t type) +{ + int t = type; + tor_assert(t >= 0); + if (t >= (int)ARRAY_LENGTH(type_definitions_table)) + return NULL; + return type_definitions_table[t]; +} diff --git a/src/lib/confmgt/type_defs.h b/src/lib/confmgt/type_defs.h new file mode 100644 index 0000000000..fec002b1d3 --- /dev/null +++ b/src/lib/confmgt/type_defs.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file type_defs.h + * @brief Header for lib/confmgt/type_defs.c + **/ + +#ifndef TOR_LIB_CONFMGT_TYPE_DEFS_H +#define TOR_LIB_CONFMGT_TYPE_DEFS_H + +const struct var_type_def_t *lookup_type_def(config_type_t type); + +#endif /* !defined(TOR_LIB_CONFMGT_TYPE_DEFS_H) */ diff --git a/src/lib/confmgt/typedvar.c b/src/lib/confmgt/typedvar.c new file mode 100644 index 0000000000..1955302cdc --- /dev/null +++ b/src/lib/confmgt/typedvar.c @@ -0,0 +1,236 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file typedvar.c + * @brief Functions for accessing a pointer as an object of a given type. + * + * These functions represent a low-level API for accessing a typed variable. + * They are used in the configuration system to examine and set fields in + * configuration objects used by individual modules. + * + * Almost no code should call these directly. + **/ + +#include "orconfig.h" +#include "lib/conf/conftypes.h" +#include "lib/confmgt/type_defs.h" +#include "lib/confmgt/typedvar.h" +#include "lib/encoding/confline.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" + +#include "lib/confmgt/var_type_def_st.h" + +#include <stddef.h> +#include <string.h> + +/** + * Try to parse a string in <b>value</b> that encodes an object of the type + * defined by <b>def</b>. + * + * On success, adjust the lvalue pointed to by <b>target</b> to hold that + * value, and return 0. On failure, set *<b>errmsg</b> to a newly allocated + * string holding an error message, and return -1. + **/ +int +typed_var_assign(void *target, const char *value, char **errmsg, + const var_type_def_t *def) +{ + if (BUG(!def)) + return -1; // LCOV_EXCL_LINE + // clear old value if needed. + typed_var_free(target, def); + + tor_assert(def->fns->parse); + return def->fns->parse(target, value, errmsg, def->params); +} + +/** + * Try to parse a single line from the head of<b>line</b> that encodes an + * object of the type defined in <b>def</b>. On success and failure, behave as + * typed_var_assign(). + * + * All types for which keys are significant should use this function. + * + * Note that although multiple lines may be provided in <b>line</b>, + * only the first one is handled by this function. + **/ +int +typed_var_kvassign(void *target, const config_line_t *line, + char **errmsg, const var_type_def_t *def) +{ + if (BUG(!def)) + return -1; // LCOV_EXCL_LINE + + if (def->fns->kv_parse) { + // We do _not_ free the old value here, since linelist options + // sometimes have append semantics. + return def->fns->kv_parse(target, line, errmsg, def->params); + } + + int rv = typed_var_assign(target, line->value, errmsg, def); + if (rv < 0 && *errmsg != NULL) { + /* typed_var_assign() didn't know the line's keyword, but we do. + * Let's add it to the error message. */ + char *oldmsg = *errmsg; + tor_asprintf(errmsg, "Could not parse %s: %s", line->key, oldmsg); + tor_free(oldmsg); + } + return rv; +} + +/** + * Release storage held by a variable in <b>target</b> of type defined by + * <b>def</b>, and set <b>target</b> to a reasonable default. + **/ +void +typed_var_free(void *target, const var_type_def_t *def) +{ + if (BUG(!def)) + return; // LCOV_EXCL_LINE + if (def->fns->clear) { + def->fns->clear(target, def->params); + } +} + +/** + * Encode a value of type <b>def</b> pointed to by <b>value</b>, and return + * its result in a newly allocated string. The string may need to be escaped. + * + * Returns NULL if this option has a NULL value, or on internal error. + **/ +char * +typed_var_encode(const void *value, const var_type_def_t *def) +{ + if (BUG(!def)) + return NULL; // LCOV_EXCL_LINE + tor_assert(def->fns->encode); + return def->fns->encode(value, def->params); +} + +/** + * As typed_var_encode(), but returns a newly allocated config_line_t + * object. The provided <b>key</b> is used as the key of the lines, unless + * the type is one (line a linelist) that encodes its own keys. + * + * This function may return a list of multiple lines. + * + * Returns NULL if there are no lines to encode, or on internal error. + */ +config_line_t * +typed_var_kvencode(const char *key, const void *value, + const var_type_def_t *def) +{ + if (BUG(!def)) + return NULL; // LCOV_EXCL_LINE + if (def->fns->kv_encode) { + return def->fns->kv_encode(key, value, def->params); + } + char *encoded_value = typed_var_encode(value, def); + if (!encoded_value) + return NULL; + + config_line_t *result = tor_malloc_zero(sizeof(config_line_t)); + result->key = tor_strdup(key); + result->value = encoded_value; + return result; +} + +/** + * Set <b>dest</b> to contain the same value as <b>src</b>. Both types + * must be as defined by <b>def</b>. + * + * Return 0 on success, and -1 on failure. + **/ +int +typed_var_copy(void *dest, const void *src, const var_type_def_t *def) +{ + if (BUG(!def)) + return -1; // LCOV_EXCL_LINE + if (def->fns->copy) { + // If we have been provided a copy fuction, use it. + return def->fns->copy(dest, src, def); + } + + // Otherwise, encode 'src' and parse the result into 'def'. + char *enc = typed_var_encode(src, def); + if (!enc) { + typed_var_free(dest, def); + return 0; + } + char *err = NULL; + int rv = typed_var_assign(dest, enc, &err, def); + if (BUG(rv < 0)) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Encoded value %s was not parseable as a %s: %s", + escaped(enc), def->name, err?err:""); + // LCOV_EXCL_STOP + } + tor_free(err); + tor_free(enc); + return rv; +} + +/** + * Return true if <b>a</b> and <b>b</b> are semantically equivalent. + * Both types must be as defined by <b>def</b>. + **/ +bool +typed_var_eq(const void *a, const void *b, const var_type_def_t *def) +{ + if (BUG(!def)) + return false; // LCOV_EXCL_LINE + + if (def->fns->eq) { + // Use a provided eq function if we got one. + return def->fns->eq(a, b, def->params); + } + + // Otherwise, encode the values and compare them. + char *enc_a = typed_var_encode(a, def); + char *enc_b = typed_var_encode(b, def); + bool eq = !strcmp_opt(enc_a,enc_b); + tor_free(enc_a); + tor_free(enc_b); + return eq; +} + +/** + * Check whether <b>value</b> encodes a valid value according to the + * type definition in <b>def</b>. + */ +bool +typed_var_ok(const void *value, const var_type_def_t *def) +{ + if (BUG(!def)) + return false; // LCOV_EXCL_LINE + + if (def->fns->ok) + return def->fns->ok(value, def->params); + + return true; +} + +/** + * Mark <b>value</b> -- a variable that ordinarily would be extended by + * assignment -- as "fragile", so that it will get replaced by the next + * assignment instead. + **/ +void +typed_var_mark_fragile(void *value, const var_type_def_t *def) +{ + if (BUG(!def)) { + return; // LCOV_EXCL_LINE + } + + if (def->fns->mark_fragile) + def->fns->mark_fragile(value, def->params); +} diff --git a/src/lib/confmgt/typedvar.h b/src/lib/confmgt/typedvar.h new file mode 100644 index 0000000000..cc90ed10a3 --- /dev/null +++ b/src/lib/confmgt/typedvar.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file typedvar.h + * @brief Header for lib/confmgt/typedvar.c + **/ + +#ifndef TOR_LIB_CONFMGT_TYPEDVAR_H +#define TOR_LIB_CONFMGT_TYPEDVAR_H + +#include <stdbool.h> + +enum config_type_t; +struct config_line_t; + +typedef struct var_type_fns_t var_type_fns_t; +typedef struct var_type_def_t var_type_def_t; + +int typed_var_assign(void *target, const char *value, char **errmsg, + const var_type_def_t *def); +void typed_var_free(void *target, const var_type_def_t *def); +char *typed_var_encode(const void *value, const var_type_def_t *def); +int typed_var_copy(void *dest, const void *src, const var_type_def_t *def); +bool typed_var_eq(const void *a, const void *b, const var_type_def_t *def); +bool typed_var_ok(const void *value, const var_type_def_t *def); + +int typed_var_kvassign(void *target, const struct config_line_t *line, + char **errmsg, const var_type_def_t *def); +struct config_line_t *typed_var_kvencode(const char *key, const void *value, + const var_type_def_t *def); + +void typed_var_mark_fragile(void *value, const var_type_def_t *def); + +#endif /* !defined(TOR_LIB_CONFMGT_TYPEDVAR_H) */ diff --git a/src/lib/confmgt/unitparse.c b/src/lib/confmgt/unitparse.c new file mode 100644 index 0000000000..99716e8d9d --- /dev/null +++ b/src/lib/confmgt/unitparse.c @@ -0,0 +1,260 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file unitparse.c + * @brief Functions for parsing values with units from a configuration file. + **/ + +#include "orconfig.h" +#include "lib/confmgt/unitparse.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/parse_int.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" +#include "lib/intmath/muldiv.h" + +#include <string.h> + +/** Table to map the names of memory units to the number of bytes they + * contain. */ +// clang-format off +const struct unit_table_t memory_units[] = { + { "", 1 }, + { "b", 1<< 0 }, + { "byte", 1<< 0 }, + { "bytes", 1<< 0 }, + { "kb", 1<<10 }, + { "kbyte", 1<<10 }, + { "kbytes", 1<<10 }, + { "kilobyte", 1<<10 }, + { "kilobytes", 1<<10 }, + { "kilobits", 1<<7 }, + { "kilobit", 1<<7 }, + { "kbits", 1<<7 }, + { "kbit", 1<<7 }, + { "m", 1<<20 }, + { "mb", 1<<20 }, + { "mbyte", 1<<20 }, + { "mbytes", 1<<20 }, + { "megabyte", 1<<20 }, + { "megabytes", 1<<20 }, + { "megabits", 1<<17 }, + { "megabit", 1<<17 }, + { "mbits", 1<<17 }, + { "mbit", 1<<17 }, + { "gb", 1<<30 }, + { "gbyte", 1<<30 }, + { "gbytes", 1<<30 }, + { "gigabyte", 1<<30 }, + { "gigabytes", 1<<30 }, + { "gigabits", 1<<27 }, + { "gigabit", 1<<27 }, + { "gbits", 1<<27 }, + { "gbit", 1<<27 }, + { "tb", UINT64_C(1)<<40 }, + { "tbyte", UINT64_C(1)<<40 }, + { "tbytes", UINT64_C(1)<<40 }, + { "terabyte", UINT64_C(1)<<40 }, + { "terabytes", UINT64_C(1)<<40 }, + { "terabits", UINT64_C(1)<<37 }, + { "terabit", UINT64_C(1)<<37 }, + { "tbits", UINT64_C(1)<<37 }, + { "tbit", UINT64_C(1)<<37 }, + { NULL, 0 }, +}; +// clang-format on + +/** Table to map the names of time units to the number of seconds they + * contain. */ +// clang-format off +const struct unit_table_t time_units[] = { + { "", 1 }, + { "second", 1 }, + { "seconds", 1 }, + { "minute", 60 }, + { "minutes", 60 }, + { "hour", 60*60 }, + { "hours", 60*60 }, + { "day", 24*60*60 }, + { "days", 24*60*60 }, + { "week", 7*24*60*60 }, + { "weeks", 7*24*60*60 }, + { "month", 2629728, }, /* about 30.437 days */ + { "months", 2629728, }, + { NULL, 0 }, +}; +// clang-format on + +/** Table to map the names of time units to the number of milliseconds + * they contain. */ +// clang-format off +const struct unit_table_t time_msec_units[] = { + { "", 1 }, + { "msec", 1 }, + { "millisecond", 1 }, + { "milliseconds", 1 }, + { "second", 1000 }, + { "seconds", 1000 }, + { "minute", 60*1000 }, + { "minutes", 60*1000 }, + { "hour", 60*60*1000 }, + { "hours", 60*60*1000 }, + { "day", 24*60*60*1000 }, + { "days", 24*60*60*1000 }, + { "week", 7*24*60*60*1000 }, + { "weeks", 7*24*60*60*1000 }, + { NULL, 0 }, +}; +// clang-format on + +/** Parse a string <b>val</b> containing a number, zero or more + * spaces, and an optional unit string. If the unit appears in the + * table <b>u</b>, then multiply the number by the unit multiplier. + * On success, set *<b>ok</b> to 1 and return this product. + * Otherwise, set *<b>ok</b> to 0. + * + * If an error (like overflow or a negative value is detected), put an error + * message in *<b>errmsg_out</b> if that pointer is non-NULL, and otherwise + * log a warning. + */ +uint64_t +config_parse_units(const char *val, const unit_table_t *u, int *ok, + char **errmsg_out) +{ + uint64_t v = 0; + double d = 0; + int use_float = 0; + char *cp; + char *errmsg = NULL; + + tor_assert(ok); + + v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); + if (!*ok || (cp && *cp == '.')) { + d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp); + if (!*ok) { + tor_asprintf(&errmsg, "Unable to parse %s as a number", val); + goto done; + } + use_float = 1; + } + + if (BUG(!cp)) { + // cp should always be non-NULL if the parse operation succeeds. + + // LCOV_EXCL_START + *ok = 1; + v = use_float ? ((uint64_t)d) : v; + goto done; + // LCOV_EXCL_STOP + } + + cp = (char*) eat_whitespace(cp); + + for ( ;u->unit;++u) { + if (!strcasecmp(u->unit, cp)) { + if (use_float) { + d = u->multiplier * d; + + if (d < 0) { + tor_asprintf(&errmsg, "Got a negative value while parsing %s %s", + val, u->unit); + *ok = 0; + goto done; + } + + // Some compilers may warn about casting a double to an unsigned type + // because they don't know if d is >= 0 + if (d >= 0 && (d > (double)INT64_MAX || (uint64_t)d > INT64_MAX)) { + tor_asprintf(&errmsg, "Overflow while parsing %s %s", + val, u->unit); + *ok = 0; + goto done; + } + + v = (uint64_t) d; + } else { + v = tor_mul_u64_nowrap(v, u->multiplier); + + if (v > INT64_MAX) { + tor_asprintf(&errmsg, "Overflow while parsing %s %s", + val, u->unit); + *ok = 0; + goto done; + } + } + + *ok = 1; + goto done; + } + } + tor_asprintf(&errmsg, "Unknown unit in %s", val); + *ok = 0; + done: + + if (errmsg) { + tor_assert_nonfatal(!*ok); + if (errmsg_out) { + *errmsg_out = errmsg; + } else { + log_warn(LD_CONFIG, "%s", errmsg); + tor_free(errmsg); + } + } + + if (*ok) + return v; + else + return 0; +} + +/** Parse a string in the format "number unit", where unit is a unit of + * information (byte, KB, M, etc). On success, set *<b>ok</b> to true + * and return the number of bytes specified. Otherwise, set + * *<b>ok</b> to false and return 0. */ +uint64_t +config_parse_memunit(const char *s, int *ok) +{ + uint64_t u = config_parse_units(s, memory_units, ok, NULL); + return u; +} + +/** Parse a string in the format "number unit", where unit is a unit of + * time in milliseconds. On success, set *<b>ok</b> to true and return + * the number of milliseconds in the provided interval. Otherwise, set + * *<b>ok</b> to 0 and return -1. */ +int +config_parse_msec_interval(const char *s, int *ok) +{ + uint64_t r; + r = config_parse_units(s, time_msec_units, ok, NULL); + if (r > INT_MAX) { + log_warn(LD_CONFIG, "Msec interval '%s' is too long", s); + *ok = 0; + return -1; + } + return (int)r; +} + +/** Parse a string in the format "number unit", where unit is a unit of time. + * On success, set *<b>ok</b> to true and return the number of seconds in + * the provided interval. Otherwise, set *<b>ok</b> to 0 and return -1. + */ +int +config_parse_interval(const char *s, int *ok) +{ + uint64_t r; + r = config_parse_units(s, time_units, ok, NULL); + if (r > INT_MAX) { + log_warn(LD_CONFIG, "Interval '%s' is too long", s); + *ok = 0; + return -1; + } + return (int)r; +} diff --git a/src/lib/confmgt/unitparse.h b/src/lib/confmgt/unitparse.h new file mode 100644 index 0000000000..047e11b424 --- /dev/null +++ b/src/lib/confmgt/unitparse.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file unitparse.h + * @brief Header for lib/confmgt/unitparse.c + **/ + +#ifndef TOR_LIB_CONFMGT_UNITPARSE_H +#define TOR_LIB_CONFMGT_UNITPARSE_H + +#include <lib/cc/torint.h> + +/** Mapping from a unit name to a multiplier for converting that unit into a + * base unit. Used by config_parse_unit. */ +typedef struct unit_table_t { + const char *unit; /**< The name of the unit */ + uint64_t multiplier; /**< How many of the base unit appear in this unit */ +} unit_table_t; + +extern const unit_table_t memory_units[]; +extern const unit_table_t time_units[]; +extern const struct unit_table_t time_msec_units[]; + +uint64_t config_parse_units(const char *val, const unit_table_t *u, int *ok, + char **errmsg_out); + +uint64_t config_parse_memunit(const char *s, int *ok); +int config_parse_msec_interval(const char *s, int *ok); +int config_parse_interval(const char *s, int *ok); + +#endif /* !defined(TOR_LIB_CONFMGT_UNITPARSE_H) */ diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h new file mode 100644 index 0000000000..2519b86aa0 --- /dev/null +++ b/src/lib/confmgt/var_type_def_st.h @@ -0,0 +1,167 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file var_type_def_st.h + * @brief Structure declarations for typedvar type definitions. + * + * This structure is used for defining new variable types. If you are not + * defining a new variable type for use by the configuration management + * system, you don't need this structure. + * + * For defining new variables, see the types in conftypes.h. + * + * For data-driven access to configuration variables, see the other members of + * lib/confmgt/. + * + * STATUS NOTE: It is not yet possible to actually define new variables + * outside of config.c, and many of the types that will eventually be used + * to do so are not yet moved. This will change as more of #29211 is + * completed. + **/ + +#ifndef TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H +#define TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H + +#include <stdbool.h> + +struct config_line_t; + +/** + * A structure full of functions pointers to implement a variable type. + * + * Every type MUST implement parse or kv_parse and encode or kv_encode; + * the other functions pointers MAY be NULL. + * + * All functions here take a <b>params</b> argument, whose value + * is determined by the type definition. Two types may have the + * same functions, but differ only in parameters. + * + * Implementation considerations: If "" encodes a valid value for a type, try + * to make sure that it encodes the same thing as the default value for the + * type (that is, the value that is set by config_clear() or memset(0)). If + * this is not the case, you need to make extra certain that your parse/encode + * implementations preserve the NULL/"" distinction. + **/ +struct var_type_fns_t { + /** + * Try to parse a string in <b>value</b> that encodes an object of this + * type. On success, adjust the lvalue pointed to by <b>target</b> to hold + * that value, and return 0. On failure, set *<b>errmsg</b> to a newly + * allocated string holding an error message, and return -1. + **/ + int (*parse)(void *target, const char *value, char **errmsg, + const void *params); + /** + * Try to parse a single line from the head of<b>line</b> that encodes + * an object of this type. On success and failure, behave as in the parse() + * function. + * + * If this function is absent, it is implemented in terms of parse(). + * + * All types for which keys are significant should use this method. For + * example, a "linelist" type records the actual keys that are given + * for each line, and so should use this method. + * + * Note that although multiple lines may be provided in <b>line</b>, + * only the first one should be handled by this function. + **/ + int (*kv_parse)(void *target, const struct config_line_t *line, + char **errmsg, const void *params); + /** + * Encode a value pointed to by <b>value</b> and return its result + * in a newly allocated string. The string may need to be escaped. + * + * If this function is absent, it is implemented in terms of kv_encode(). + * + * Returns NULL if this option has a NULL value, or on internal error. + * + * Requirement: all strings generated by encode() should produce a + * semantically equivalent value when given to parse(). + **/ + char *(*encode)(const void *value, const void *params); + /** + * As encode(), but returns a newly allocated config_line_t object. The + * provided <b>key</b> is used as the key of the lines, unless the type is + * one that encodes its own keys. + * + * Unlike kv_parse(), this function will return a list of multiple lines, + * if <b>value</b> is such that it must be encoded by multiple lines. + * + * Returns NULL if there are no lines to encode, or on internal error. + * + * If this function is absent, it is implemented in terms of encode(). + **/ + struct config_line_t *(*kv_encode)(const char *key, const void *value, + const void *params); + /** + * Free all storage held in <b>arg</b>, and set <b>arg</b> to a default + * value -- usually zero or NULL. + * + * If this function is absent, the default implementation does nothing. + **/ + void (*clear)(void *arg, const void *params); + /** + * Return true if <b>a</b> and <b>b</b> hold the same value, and false + * otherwise. + * + * If this function is absent, it is implemented by encoding both a and + * b and comparing their encoded strings for equality. + **/ + bool (*eq)(const void *a, const void *b, const void *params); + /** + * Try to copy the value from <b>value</b> into <b>target</b>. + * On success return 0; on failure return -1. + * + * If this function is absent, it is implemented by encoding the value + * into a string, and then parsing it into the target. + **/ + int (*copy)(void *target, const void *value, const void *params); + /** + * Check whether <b>value</b> holds a valid value according to the + * rules of this type; return true if it does and false if it doesn't. + * + * The default implementation for this function assumes that all + * values are valid. + **/ + bool (*ok)(const void *value, const void *params); + /** + * Mark a value of this variable as "fragile", so that future attempts to + * assign to this variable will replace rather than extending it. + * + * The default implementation for this function does nothing. + * + * Only meaningful for types with is_cumulative set. + **/ + void (*mark_fragile)(void *value, const void *params); +}; + +/** + * A structure describing a type that can be manipulated with the typedvar_* + * functions. + **/ +struct var_type_def_t { + /** + * The name of this type. Should not include spaces. Used for + * debugging, log messages, and the controller API. */ + const char *name; + /** + * A function table for this type. + */ + const struct var_type_fns_t *fns; + /** + * A pointer to a value that should be passed as the 'params' argument when + * calling the functions in this type's function table. + */ + const void *params; + /** + * A bitwise OR of one or more VTFLAG_* values, describing properties + * for all values of this type. + **/ + uint32_t flags; +}; + +#endif /* !defined(TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H) */ diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include index 90de5eda40..81507527d3 100644 --- a/src/lib/container/.may_include +++ b/src/lib/container/.may_include @@ -7,12 +7,9 @@ lib/malloc/*.h lib/err/*.h lib/smartlist_core/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h lib/intmath/*.h lib/log/*.h -# XXXX I am unsure about this one. It's only here for buffers.c -lib/time/*.h - -ht.h -siphash.h +ext/ht.h +ext/siphash.h diff --git a/src/lib/container/bitarray.h b/src/lib/container/bitarray.h index 910d5fea65..41409e350a 100644 --- a/src/lib/container/bitarray.h +++ b/src/lib/container/bitarray.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BITARRAY_H @@ -83,4 +83,4 @@ bitarray_is_set(bitarray_t *b, int bit) return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK)); } -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_BITARRAY_H) */ diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c index 9aa9b1ee56..34b1265d81 100644 --- a/src/lib/container/bloomfilt.c +++ b/src/lib/container/bloomfilt.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,7 +14,7 @@ #include "lib/container/bloomfilt.h" #include "lib/intmath/bits.h" #include "lib/log/util_bug.h" -#include "siphash.h" +#include "ext/siphash.h" /** How many bloom-filter bits we set per address. This is twice the * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit diff --git a/src/lib/container/bloomfilt.h b/src/lib/container/bloomfilt.h index 0ce18bd3ec..6d36056b5a 100644 --- a/src/lib/container/bloomfilt.h +++ b/src/lib/container/bloomfilt.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BLOOMFILT_H diff --git a/src/lib/container/handles.h b/src/lib/container/handles.h index ca7c94559e..6b1bbd5167 100644 --- a/src/lib/container/handles.h +++ b/src/lib/container/handles.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2019, The Tor Project, Inc. */ +/* Copyright (c) 2016-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,33 +16,33 @@ * To enable a type to have handles, add a HANDLE_ENTRY() field in its * definition, as in: * - * struct walrus { - * HANDLE_ENTRY(wlr, walrus); + * struct walrus_t { + * HANDLE_ENTRY(wlr, walrus_t); * // ... * }; * - * And invoke HANDLE_DECL(wlr, walrus, [static]) to declare the handle + * And invoke HANDLE_DECL(wlr, walrus_t, [static]) to declare the handle * manipulation functions (typically in a header): * * // opaque handle to walrus. * typedef struct wlr_handle_t wlr_handle_t; * * // make a new handle - * struct wlr_handle_t *wlr_handle_new(struct walrus *); + * struct wlr_handle_t *wlr_handle_new(struct walrus_t *); * * // release a handle * void wlr_handle_free(wlr_handle_t *); * * // return the pointed-to walrus, or NULL. - * struct walrus *wlr_handle_get(wlr_handle_t *). + * struct walrus_t *wlr_handle_get(wlr_handle_t *). * * // call this function when you're about to free the walrus; * // it invalidates all handles. (IF YOU DON'T, YOU WILL HAVE * // DANGLING REFERENCES) - * void wlr_handles_clear(struct walrus *); + * void wlr_handles_clear(struct walrus_t *); * * Finally, use HANDLE_IMPL() to define the above functions in some - * appropriate C file: HANDLE_IMPL(wlr, walrus, [static]) + * appropriate C file: HANDLE_IMPL(wlr, walrus_t, [static]) * **/ @@ -57,12 +57,13 @@ #define HANDLE_ENTRY(name, structname) \ struct name ## _handle_head_t *handle_head -#define HANDLE_DECL(name, structname, linkage) \ +#define HANDLE_DECL(name, structname_t, linkage) \ typedef struct name ## _handle_t name ## _handle_t; \ - linkage name ## _handle_t *name ## _handle_new(struct structname *object); \ + linkage name ## _handle_t *name ## _handle_new( \ + struct structname_t *object); \ linkage void name ## _handle_free_(name ## _handle_t *); \ - linkage struct structname *name ## _handle_get(name ## _handle_t *); \ - linkage void name ## _handles_clear(struct structname *object); + linkage struct structname_t *name ## _handle_get(name ## _handle_t *); \ + linkage void name ## _handles_clear(struct structname_t *object); /* * Implementation notes: there are lots of possible implementations here. We diff --git a/src/lib/container/include.am b/src/lib/container/include.am index e6492098b5..00d7b8e587 100644 --- a/src/lib/container/include.am +++ b/src/lib/container/include.am @@ -5,10 +5,11 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-container-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_container_a_SOURCES = \ src/lib/container/bloomfilt.c \ - src/lib/container/buffers.c \ src/lib/container/map.c \ + src/lib/container/namemap.c \ src/lib/container/order.c \ src/lib/container/smartlist.c @@ -17,11 +18,13 @@ src_lib_libtor_container_testing_a_SOURCES = \ src_lib_libtor_container_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/container/bitarray.h \ src/lib/container/bloomfilt.h \ - src/lib/container/buffers.h \ src/lib/container/handles.h \ src/lib/container/map.h \ + src/lib/container/namemap.h \ + src/lib/container/namemap_st.h \ src/lib/container/order.h \ src/lib/container/smartlist.h diff --git a/src/lib/container/lib_container.md b/src/lib/container/lib_container.md new file mode 100644 index 0000000000..f4902ca44a --- /dev/null +++ b/src/lib/container/lib_container.md @@ -0,0 +1,49 @@ +@dir /lib/container +@brief lib/container: Hash tables, dynamic arrays, bit arrays, etc. + +### Smartlists: Neither lists, nor especially smart. + +For historical reasons, we call our dynamic-allocated array type +`smartlist_t`. It can grow or shrink as elements are added and removed. + +All smartlists hold an array of `void *`. Whenever you expose a smartlist +in an API you *must* document which types its pointers actually hold. + +<!-- It would be neat to fix that, wouldn't it? -NM --> + +Smartlists are created empty with `smartlist_new()` and freed with +`smartlist_free()`. See the `containers.h` header documentation for more +information; there are many convenience functions for commonly needed +operations. + +For low-level operations on smartlists, see also +\refdir{lib/smartlist_core}. + +<!-- TODO: WRITE more about what you can do with smartlists. --> + +### Digest maps, string maps, and more. + +Tor makes frequent use of maps from 160-bit digests, 256-bit digests, +or nul-terminated strings to `void *`. These types are `digestmap_t`, +`digest256map_t`, and `strmap_t` respectively. See the containers.h +module documentation for more information. + +### Intrusive lists and hashtables + +For performance-sensitive cases, we sometimes want to use "intrusive" +collections: ones where the bookkeeping pointers are stuck inside the +structures that belong to the collection. If you've used the +BSD-style sys/queue.h macros, you'll be familiar with these. + +Unfortunately, the `sys/queue.h` macros vary significantly between the +platforms that have them, so we provide our own variants in +`ext/tor_queue.h`. + +We also provide an intrusive hashtable implementation in `ext/ht.h`. +When you're using it, you'll need to define your own hash +functions. If attacker-induced collisions are a worry here, use the +cryptographic siphash24g function to extract hashes. + +<!-- TODO: WRITE about bloom filters, namemaps, bit-arrays, order functions. +--> + diff --git a/src/lib/container/map.c b/src/lib/container/map.c index d213ad50bf..7db84313ea 100644 --- a/src/lib/container/map.c +++ b/src/lib/container/map.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -21,7 +21,7 @@ #include <stdlib.h> #include <string.h> -#include "ht.h" +#include "ext/ht.h" /** Helper: Declare an entry type and a map type to implement a mapping using * ht.h. The map type will be called <b>maptype</b>. The key part of each @@ -85,21 +85,21 @@ digest256map_entry_hash(const digest256map_entry_t *a) } HT_PROTOTYPE(strmap_impl, strmap_entry_t, node, strmap_entry_hash, - strmap_entries_eq) + strmap_entries_eq); HT_GENERATE2(strmap_impl, strmap_entry_t, node, strmap_entry_hash, - strmap_entries_eq, 0.6, tor_reallocarray_, tor_free_) + strmap_entries_eq, 0.6, tor_reallocarray_, tor_free_); HT_PROTOTYPE(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash, - digestmap_entries_eq) + digestmap_entries_eq); HT_GENERATE2(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash, - digestmap_entries_eq, 0.6, tor_reallocarray_, tor_free_) + digestmap_entries_eq, 0.6, tor_reallocarray_, tor_free_); HT_PROTOTYPE(digest256map_impl, digest256map_entry_t, node, digest256map_entry_hash, - digest256map_entries_eq) + digest256map_entries_eq); HT_GENERATE2(digest256map_impl, digest256map_entry_t, node, digest256map_entry_hash, - digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_) + digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_); #define strmap_entry_free(ent) \ FREE_AND_NULL(strmap_entry_t, strmap_entry_free_, (ent)) diff --git a/src/lib/container/map.h b/src/lib/container/map.h index a2d1b01d12..dbc1967247 100644 --- a/src/lib/container/map.h +++ b/src/lib/container/map.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_MAP_H @@ -15,36 +15,37 @@ #include "lib/testsupport/testsupport.h" #include "lib/cc/torint.h" -#include "siphash.h" +#include "ext/siphash.h" -#define DECLARE_MAP_FNS(maptype, keytype, prefix) \ - typedef struct maptype maptype; \ - typedef struct prefix##entry_t *prefix##iter_t; \ - MOCK_DECL(maptype*, prefix##new, (void)); \ - void* prefix##set(maptype *map, keytype key, void *val); \ - void* prefix##get(const maptype *map, keytype key); \ - void* prefix##remove(maptype *map, keytype key); \ - MOCK_DECL(void, prefix##free_, (maptype *map, void (*free_val)(void*))); \ - int prefix##isempty(const maptype *map); \ - int prefix##size(const maptype *map); \ - prefix##iter_t *prefix##iter_init(maptype *map); \ - prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter); \ - prefix##iter_t *prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter); \ - void prefix##iter_get(prefix##iter_t *iter, keytype *keyp, void **valp); \ - int prefix##iter_done(prefix##iter_t *iter); \ - void prefix##assert_ok(const maptype *map) +#define DECLARE_MAP_FNS(mapname_t, keytype, prefix) \ + typedef struct mapname_t mapname_t; \ + typedef struct prefix##_entry_t *prefix##_iter_t; \ + MOCK_DECL(mapname_t*, prefix##_new, (void)); \ + void* prefix##_set(mapname_t *map, keytype key, void *val); \ + void* prefix##_get(const mapname_t *map, keytype key); \ + void* prefix##_remove(mapname_t *map, keytype key); \ + MOCK_DECL(void, prefix##_free_, (mapname_t *map, void (*free_val)(void*))); \ + int prefix##_isempty(const mapname_t *map); \ + int prefix##_size(const mapname_t *map); \ + prefix##_iter_t *prefix##_iter_init(mapname_t *map); \ + prefix##_iter_t *prefix##_iter_next(mapname_t *map, prefix##_iter_t *iter); \ + prefix##_iter_t *prefix##_iter_next_rmv(mapname_t *map, \ + prefix##_iter_t *iter); \ + void prefix##_iter_get(prefix##_iter_t *iter, keytype *keyp, void **valp); \ + int prefix##_iter_done(prefix##_iter_t *iter); \ + void prefix##_assert_ok(const mapname_t *map) /* Map from const char * to void *. Implemented with a hash table. */ -DECLARE_MAP_FNS(strmap_t, const char *, strmap_); +DECLARE_MAP_FNS(strmap_t, const char *, strmap); /* Map from const char[DIGEST_LEN] to void *. Implemented with a hash table. */ -DECLARE_MAP_FNS(digestmap_t, const char *, digestmap_); +DECLARE_MAP_FNS(digestmap_t, const char *, digestmap); /* Map from const uint8_t[DIGEST256_LEN] to void *. Implemented with a hash * table. */ -DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_); +DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map); -#define MAP_FREE_AND_NULL(maptype, map, fn) \ +#define MAP_FREE_AND_NULL(mapname_t, map, fn) \ do { \ - maptype ## _free_((map), (fn)); \ + mapname_t ## _free_((map), (fn)); \ (map) = NULL; \ } while (0) @@ -55,12 +56,12 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_); #undef DECLARE_MAP_FNS /** Iterates over the key-value pairs in a map <b>map</b> in order. - * <b>prefix</b> is as for DECLARE_MAP_FNS (i.e., strmap_ or digestmap_). + * <b>prefix</b> is as for DECLARE_MAP_FNS (i.e., strmap or digestmap). * The map's keys and values are of type keytype and valtype respectively; * each iteration assigns them to keyvar and valvar. * * Example use: - * MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) { + * MAP_FOREACH(digestmap, m, const char *, k, routerinfo_t *, r) { * // use k and r * } MAP_FOREACH_END. */ @@ -80,21 +81,21 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_); */ #define MAP_FOREACH(prefix, map, keytype, keyvar, valtype, valvar) \ STMT_BEGIN \ - prefix##iter_t *keyvar##_iter; \ - for (keyvar##_iter = prefix##iter_init(map); \ - !prefix##iter_done(keyvar##_iter); \ - keyvar##_iter = prefix##iter_next(map, keyvar##_iter)) { \ + prefix##_iter_t *keyvar##_iter; \ + for (keyvar##_iter = prefix##_iter_init(map); \ + !prefix##_iter_done(keyvar##_iter); \ + keyvar##_iter = prefix##_iter_next(map, keyvar##_iter)) { \ keytype keyvar; \ void *valvar##_voidp; \ valtype valvar; \ - prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp); \ + prefix##_iter_get(keyvar##_iter, &keyvar, &valvar##_voidp); \ valvar = valvar##_voidp; /** As MAP_FOREACH, except allows members to be removed from the map * during the iteration via MAP_DEL_CURRENT. Example use: * * Example use: - * MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) { + * MAP_FOREACH(digestmap, m, const char *, k, routerinfo_t *, r) { * if (is_very_old(r)) * MAP_DEL_CURRENT(k); * } MAP_FOREACH_END. @@ -120,18 +121,18 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_); */ #define MAP_FOREACH_MODIFY(prefix, map, keytype, keyvar, valtype, valvar) \ STMT_BEGIN \ - prefix##iter_t *keyvar##_iter; \ + prefix##_iter_t *keyvar##_iter; \ int keyvar##_del=0; \ - for (keyvar##_iter = prefix##iter_init(map); \ - !prefix##iter_done(keyvar##_iter); \ + for (keyvar##_iter = prefix##_iter_init(map); \ + !prefix##_iter_done(keyvar##_iter); \ keyvar##_iter = keyvar##_del ? \ - prefix##iter_next_rmv(map, keyvar##_iter) : \ - prefix##iter_next(map, keyvar##_iter)) { \ + prefix##_iter_next_rmv(map, keyvar##_iter) : \ + prefix##_iter_next(map, keyvar##_iter)) { \ keytype keyvar; \ void *valvar##_voidp; \ valtype valvar; \ keyvar##_del=0; \ - prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp); \ + prefix##_iter_get(keyvar##_iter, &keyvar, &valvar##_voidp); \ valvar = valvar##_voidp; /** Used with MAP_FOREACH_MODIFY to remove the currently-iterated-upon @@ -151,7 +152,7 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_); * } DIGESTMAP_FOREACH_END. */ #define DIGESTMAP_FOREACH(map, keyvar, valtype, valvar) \ - MAP_FOREACH(digestmap_, map, const char *, keyvar, valtype, valvar) + MAP_FOREACH(digestmap, map, const char *, keyvar, valtype, valvar) /** As MAP_FOREACH_MODIFY, but does not require declaration of prefix or * keytype. @@ -162,89 +163,89 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_); * } DIGESTMAP_FOREACH_END. */ #define DIGESTMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar) \ - MAP_FOREACH_MODIFY(digestmap_, map, const char *, keyvar, valtype, valvar) + MAP_FOREACH_MODIFY(digestmap, map, const char *, keyvar, valtype, valvar) /** Used to end a DIGESTMAP_FOREACH() block. */ #define DIGESTMAP_FOREACH_END MAP_FOREACH_END #define DIGEST256MAP_FOREACH(map, keyvar, valtype, valvar) \ - MAP_FOREACH(digest256map_, map, const uint8_t *, keyvar, valtype, valvar) + MAP_FOREACH(digest256map, map, const uint8_t *, keyvar, valtype, valvar) #define DIGEST256MAP_FOREACH_MODIFY(map, keyvar, valtype, valvar) \ - MAP_FOREACH_MODIFY(digest256map_, map, const uint8_t *, \ + MAP_FOREACH_MODIFY(digest256map, map, const uint8_t *, \ keyvar, valtype, valvar) #define DIGEST256MAP_FOREACH_END MAP_FOREACH_END #define STRMAP_FOREACH(map, keyvar, valtype, valvar) \ - MAP_FOREACH(strmap_, map, const char *, keyvar, valtype, valvar) + MAP_FOREACH(strmap, map, const char *, keyvar, valtype, valvar) #define STRMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar) \ - MAP_FOREACH_MODIFY(strmap_, map, const char *, keyvar, valtype, valvar) + MAP_FOREACH_MODIFY(strmap, map, const char *, keyvar, valtype, valvar) #define STRMAP_FOREACH_END MAP_FOREACH_END void* strmap_set_lc(strmap_t *map, const char *key, void *val); void* strmap_get_lc(const strmap_t *map, const char *key); void* strmap_remove_lc(strmap_t *map, const char *key); -#define DECLARE_TYPED_DIGESTMAP_FNS(prefix, maptype, valtype) \ - typedef struct maptype maptype; \ - typedef struct prefix##iter_t *prefix##iter_t; \ - ATTR_UNUSED static inline maptype* \ - prefix##new(void) \ +#define DECLARE_TYPED_DIGESTMAP_FNS(prefix, mapname_t, valtype) \ + typedef struct mapname_t mapname_t; \ + typedef struct prefix##_iter_t *prefix##_iter_t; \ + ATTR_UNUSED static inline mapname_t* \ + prefix##_new(void) \ { \ - return (maptype*)digestmap_new(); \ + return (mapname_t*)digestmap_new(); \ } \ ATTR_UNUSED static inline digestmap_t* \ - prefix##to_digestmap(maptype *map) \ + prefix##_to_digestmap(mapname_t *map) \ { \ return (digestmap_t*)map; \ } \ ATTR_UNUSED static inline valtype* \ - prefix##get(maptype *map, const char *key) \ + prefix##_get(mapname_t *map, const char *key) \ { \ return (valtype*)digestmap_get((digestmap_t*)map, key); \ } \ ATTR_UNUSED static inline valtype* \ - prefix##set(maptype *map, const char *key, valtype *val) \ + prefix##_set(mapname_t *map, const char *key, valtype *val) \ { \ return (valtype*)digestmap_set((digestmap_t*)map, key, val); \ } \ ATTR_UNUSED static inline valtype* \ - prefix##remove(maptype *map, const char *key) \ + prefix##_remove(mapname_t *map, const char *key) \ { \ return (valtype*)digestmap_remove((digestmap_t*)map, key); \ } \ ATTR_UNUSED static inline void \ - prefix##f##ree_(maptype *map, void (*free_val)(void*)) \ + prefix##_f##ree_(mapname_t *map, void (*free_val)(void*)) \ { \ digestmap_free_((digestmap_t*)map, free_val); \ } \ ATTR_UNUSED static inline int \ - prefix##isempty(maptype *map) \ + prefix##_isempty(mapname_t *map) \ { \ return digestmap_isempty((digestmap_t*)map); \ } \ ATTR_UNUSED static inline int \ - prefix##size(maptype *map) \ + prefix##_size(mapname_t *map) \ { \ return digestmap_size((digestmap_t*)map); \ } \ ATTR_UNUSED static inline \ - prefix##iter_t *prefix##iter_init(maptype *map) \ + prefix##_iter_t *prefix##_iter_init(mapname_t *map) \ { \ - return (prefix##iter_t*) digestmap_iter_init((digestmap_t*)map); \ + return (prefix##_iter_t*) digestmap_iter_init((digestmap_t*)map); \ } \ ATTR_UNUSED static inline \ - prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter) \ + prefix##_iter_t *prefix##_iter_next(mapname_t *map, prefix##_iter_t *iter) \ { \ - return (prefix##iter_t*) digestmap_iter_next( \ + return (prefix##_iter_t*) digestmap_iter_next( \ (digestmap_t*)map, (digestmap_iter_t*)iter); \ } \ - ATTR_UNUSED static inline prefix##iter_t* \ - prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter) \ + ATTR_UNUSED static inline prefix##_iter_t* \ + prefix##_iter_next_rmv(mapname_t *map, prefix##_iter_t *iter) \ { \ - return (prefix##iter_t*) digestmap_iter_next_rmv( \ + return (prefix##_iter_t*) digestmap_iter_next_rmv( \ (digestmap_t*)map, (digestmap_iter_t*)iter); \ } \ ATTR_UNUSED static inline void \ - prefix##iter_get(prefix##iter_t *iter, \ + prefix##_iter_get(prefix##_iter_t *iter, \ const char **keyp, \ valtype **valp) \ { \ @@ -253,9 +254,9 @@ void* strmap_remove_lc(strmap_t *map, const char *key); *valp = v; \ } \ ATTR_UNUSED static inline int \ - prefix##iter_done(prefix##iter_t *iter) \ + prefix##_iter_done(prefix##_iter_t *iter) \ { \ return digestmap_iter_done((digestmap_iter_t*)iter); \ } -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_MAP_H) */ diff --git a/src/lib/container/namemap.c b/src/lib/container/namemap.c new file mode 100644 index 0000000000..e286cad947 --- /dev/null +++ b/src/lib/container/namemap.c @@ -0,0 +1,189 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file namemap.c + * @brief Mappings between identifiers and 16-bit ints. + **/ + +#include "orconfig.h" +#include "lib/container/smartlist.h" +#include "lib/container/namemap.h" +#include "lib/container/namemap_st.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +#include "ext/siphash.h" + +#include <string.h> + +/** Helper for namemap hashtable implementation: compare two entries. */ +static inline int +mapped_name_eq(const mapped_name_t *a, const mapped_name_t *b) +{ + return !strcmp(a->name, b->name); +} + +/** Helper for namemap hashtable implementation: hash an entry. */ +static inline unsigned +mapped_name_hash(const mapped_name_t *a) +{ + return (unsigned) siphash24g(a->name, strlen(a->name)); +} + +HT_PROTOTYPE(namemap_ht, mapped_name_t, node, mapped_name_hash, + mapped_name_eq); +HT_GENERATE2(namemap_ht, mapped_name_t, node, mapped_name_hash, + mapped_name_eq, 0.6, tor_reallocarray_, tor_free_); + +/** Set up an uninitialized <b>map</b>. */ +void +namemap_init(namemap_t *map) +{ + memset(map, 0, sizeof(*map)); + HT_INIT(namemap_ht, &map->ht); + map->names = smartlist_new(); +} + +/** Return the name that <b>map</b> associates with a given <b>id</b>, or + * NULL if there is no such name. */ +const char * +namemap_get_name(const namemap_t *map, unsigned id) +{ + if (map->names && id < (unsigned)smartlist_len(map->names)) { + mapped_name_t *name = smartlist_get(map->names, (int)id); + return name->name; + } else { + return NULL; + } +} + +/** + * Return the name that <b>map</b> associates with a given <b>id</b>, or a + * pointer to a statically allocated string describing the value of <b>id</b> + * if no such name exists. + **/ +const char * +namemap_fmt_name(const namemap_t *map, unsigned id) +{ + static char buf[32]; + + const char *name = namemap_get_name(map, id); + if (name) + return name; + + tor_snprintf(buf, sizeof(buf), "{%u}", id); + + return buf; +} + +/** + * Helper: As namemap_get_id(), but requires that <b>name</b> is + * <b>namelen</b> charaters long, and that <b>namelen</b> is no more than + * MAX_NAMEMAP_NAME_LEN. + */ +static unsigned +namemap_get_id_unchecked(const namemap_t *map, + const char *name, + size_t namelen) +{ + union { + mapped_name_t n; + char storage[MAX_NAMEMAP_NAME_LEN + sizeof(mapped_name_t) + 1]; + } u; + memcpy(u.n.name, name, namelen); + u.n.name[namelen] = 0; + const mapped_name_t *found = HT_FIND(namemap_ht, &map->ht, &u.n); + if (found) { + tor_assert(map->names); + tor_assert(smartlist_get(map->names, found->intval) == found); + return found->intval; + } + + return NAMEMAP_ERR; +} + +/** + * Return the identifier currently associated by <b>map</b> with the name + * <b>name</b>, or NAMEMAP_ERR if no such identifier exists. + **/ +unsigned +namemap_get_id(const namemap_t *map, + const char *name) +{ + size_t namelen = strlen(name); + if (namelen > MAX_NAMEMAP_NAME_LEN) { + return NAMEMAP_ERR; + } + + return namemap_get_id_unchecked(map, name, namelen); +} + +/** + * Return the identifier associated by <b>map</b> with the name + * <b>name</b>, allocating a new identifier in <b>map</b> if none exists. + * + * Return NAMEMAP_ERR if <b>name</b> is too long, or if there are no more + * identifiers we can allocate. + **/ +unsigned +namemap_get_or_create_id(namemap_t *map, + const char *name) +{ + size_t namelen = strlen(name); + if (namelen > MAX_NAMEMAP_NAME_LEN) { + return NAMEMAP_ERR; + } + + if (PREDICT_UNLIKELY(map->names == NULL)) + map->names = smartlist_new(); + + unsigned found = namemap_get_id_unchecked(map, name, namelen); + if (found != NAMEMAP_ERR) + return found; + + unsigned new_id = (unsigned)smartlist_len(map->names); + if (new_id == NAMEMAP_ERR) + return NAMEMAP_ERR; /* Can't allocate any more. */ + + mapped_name_t *insert = tor_malloc_zero( + offsetof(mapped_name_t, name) + namelen + 1); + memcpy(insert->name, name, namelen+1); + insert->intval = new_id; + + HT_INSERT(namemap_ht, &map->ht, insert); + smartlist_add(map->names, insert); + + return new_id; +} + +/** Return the number of entries in 'names' */ +size_t +namemap_get_size(const namemap_t *map) +{ + if (PREDICT_UNLIKELY(map->names == NULL)) + return 0; + + return smartlist_len(map->names); +} + +/** + * Release all storage held in <b>map</b>. + */ +void +namemap_clear(namemap_t *map) +{ + if (!map) + return; + + HT_CLEAR(namemap_ht, &map->ht); + if (map->names) { + SMARTLIST_FOREACH(map->names, mapped_name_t *, n, + tor_free(n)); + smartlist_free(map->names); + } + memset(map, 0, sizeof(*map)); +} diff --git a/src/lib/container/namemap.h b/src/lib/container/namemap.h new file mode 100644 index 0000000000..b451c18c68 --- /dev/null +++ b/src/lib/container/namemap.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_NAMEMAP_H +#define TOR_NAMEMAP_H + +/** + * \file namemap.h + * + * \brief Header for namemap.c + **/ + +#include "lib/cc/compat_compiler.h" +#include "ext/ht.h" + +#include <stddef.h> + +typedef struct namemap_t namemap_t; + +/** Returned in place of an identifier when an error occurs. */ +#define NAMEMAP_ERR UINT_MAX + +void namemap_init(namemap_t *map); +const char *namemap_get_name(const namemap_t *map, unsigned id); +const char *namemap_fmt_name(const namemap_t *map, unsigned id); +unsigned namemap_get_id(const namemap_t *map, + const char *name); +unsigned namemap_get_or_create_id(namemap_t *map, + const char *name); +size_t namemap_get_size(const namemap_t *map); +void namemap_clear(namemap_t *map); + +#endif /* !defined(TOR_NAMEMAP_H) */ diff --git a/src/lib/container/namemap_st.h b/src/lib/container/namemap_st.h new file mode 100644 index 0000000000..39aa85cc09 --- /dev/null +++ b/src/lib/container/namemap_st.h @@ -0,0 +1,41 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef NAMEMAP_ST_H +#define NAMEMAP_ST_H + +/** + * @file namemap_st.h + * @brief Internal declarations for namemap structure. + **/ + +#include "lib/cc/compat_compiler.h" +#include "ext/ht.h" + +struct smartlist_t; + +/** Longest allowed name that's allowed in a namemap_t. */ +#define MAX_NAMEMAP_NAME_LEN 128 + +/** An entry inside a namemap_t. Maps a string to a numeric identifier. */ +typedef struct mapped_name_t { + HT_ENTRY(mapped_name_t) node; + unsigned intval; + char name[FLEXIBLE_ARRAY_MEMBER]; +} mapped_name_t; + +/** A structure that allocates small numeric identifiers for names and maps + * back and forth between them. */ +struct namemap_t { + HT_HEAD(namemap_ht, mapped_name_t) ht; + struct smartlist_t *names; +}; + +#ifndef COCCI +/** Macro to initialize a namemap. */ +#define NAMEMAP_INIT() { HT_INITIALIZER(), NULL } +#endif + +#endif /* !defined(NAMEMAP_ST_H) */ diff --git a/src/lib/container/order.c b/src/lib/container/order.c index f6503a124e..cac241f027 100644 --- a/src/lib/container/order.c +++ b/src/lib/container/order.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/container/order.h b/src/lib/container/order.h index a176d6d8a6..5bca095f35 100644 --- a/src/lib/container/order.h +++ b/src/lib/container/order.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_ORDER_H @@ -57,4 +57,4 @@ third_quartile_uint32(uint32_t *array, int n_elements) return find_nth_uint32(array, n_elements, (n_elements*3)/4); } -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_ORDER_H) */ diff --git a/src/lib/container/smartlist.c b/src/lib/container/smartlist.c index 3ab2797d68..7784f83957 100644 --- a/src/lib/container/smartlist.c +++ b/src/lib/container/smartlist.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -652,7 +652,7 @@ smartlist_sort_pointers(smartlist_t *sl) #define LEFT_CHILD(i) ( 2*(i) + 1 ) #define RIGHT_CHILD(i) ( 2*(i) + 2 ) #define PARENT(i) ( ((i)-1) / 2 ) -/** }@ */ +/** @} */ /** @{ */ /** Helper macros for heaps: Given a local variable <b>idx_field_offset</b> @@ -678,7 +678,7 @@ smartlist_sort_pointers(smartlist_t *sl) static inline void smartlist_heapify(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, int idx) { while (1) { @@ -725,7 +725,7 @@ smartlist_heapify(smartlist_t *sl, void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item) { int idx; @@ -754,7 +754,7 @@ smartlist_pqueue_add(smartlist_t *sl, void * smartlist_pqueue_pop(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset) + ptrdiff_t idx_field_offset) { void *top; tor_assert(sl->num_used); @@ -778,7 +778,7 @@ smartlist_pqueue_pop(smartlist_t *sl, void smartlist_pqueue_remove(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item) { int idx = IDX_OF_ITEM(item); @@ -802,7 +802,7 @@ smartlist_pqueue_remove(smartlist_t *sl, void smartlist_pqueue_assert_ok(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset) + ptrdiff_t idx_field_offset) { int i; for (i = sl->num_used - 1; i >= 0; --i) { diff --git a/src/lib/container/smartlist.h b/src/lib/container/smartlist.h index 77682db03e..458d564cd5 100644 --- a/src/lib/container/smartlist.h +++ b/src/lib/container/smartlist.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SMARTLIST_H @@ -13,6 +13,7 @@ **/ #include <stdarg.h> +#include <stddef.h> #include "lib/smartlist_core/smartlist_core.h" #include "lib/smartlist_core/smartlist_foreach.h" @@ -72,18 +73,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key, void smartlist_pqueue_add(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item); void *smartlist_pqueue_pop(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset); + ptrdiff_t idx_field_offset); void smartlist_pqueue_remove(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset, + ptrdiff_t idx_field_offset, void *item); void smartlist_pqueue_assert_ok(smartlist_t *sl, int (*compare)(const void *a, const void *b), - int idx_field_offset); + ptrdiff_t idx_field_offset); char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out) ATTR_MALLOC; @@ -91,6 +92,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, size_t join_len, int terminate, size_t *len_out) ATTR_MALLOC; +#ifndef COCCI /* Helper: Given two lists of items, possibly of different types, such that * both lists are sorted on some common field (as determined by a comparison * expression <b>cmpexpr</b>), and such that one list (<b>sl1</b>) has no @@ -164,5 +166,6 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join, #define SMARTLIST_FOREACH_JOIN_END(var1, var2) \ } \ STMT_END +#endif /* !defined(COCCI) */ -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_SMARTLIST_H) */ diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include index a0fa4ec05c..810e777271 100644 --- a/src/lib/crypt_ops/.may_include +++ b/src/lib/crypt_ops/.may_include @@ -1,6 +1,7 @@ orconfig.h lib/arch/*.h lib/cc/*.h +lib/conf/*.h lib/container/*.h lib/crypt_ops/*.h lib/ctime/*.h @@ -12,13 +13,16 @@ lib/malloc/*.h lib/intmath/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/subsys/*.h +lib/testsupport/*.h lib/thread/*.h lib/log/*.h +lib/crypt_ops/*.inc + trunnel/pwbox.h keccak-tiny/*.h ed25519/*.h -siphash.h +ext/siphash.h diff --git a/src/lib/crypt_ops/aes.h b/src/lib/crypt_ops/aes.h index 7c774062d9..c25417b4e6 100644 --- a/src/lib/crypt_ops/aes.h +++ b/src/lib/crypt_ops/aes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /* Implements a minimal interface to counter-mode AES. */ @@ -16,7 +16,7 @@ #include "lib/cc/torint.h" #include "lib/malloc/malloc.h" -typedef struct aes_cnt_cipher aes_cnt_cipher_t; +typedef struct aes_cnt_cipher_t aes_cnt_cipher_t; aes_cnt_cipher_t* aes_new_cipher(const uint8_t *key, const uint8_t *iv, int key_bits); diff --git a/src/lib/crypt_ops/aes_nss.c b/src/lib/crypt_ops/aes_nss.c index 4eda5e5902..71d2f01449 100644 --- a/src/lib/crypt_ops/aes_nss.c +++ b/src/lib/crypt_ops/aes_nss.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,10 +15,10 @@ #include "lib/crypt_ops/crypto_util.h" #include "lib/log/util_bug.h" -DISABLE_GCC_WARNING(strict-prototypes) +DISABLE_GCC_WARNING("-Wstrict-prototypes") #include <pk11pub.h> #include <secerr.h> -ENABLE_GCC_WARNING(strict-prototypes) +ENABLE_GCC_WARNING("-Wstrict-prototypes") aes_cnt_cipher_t * aes_new_cipher(const uint8_t *key, const uint8_t *iv, diff --git a/src/lib/crypt_ops/aes_openssl.c b/src/lib/crypt_ops/aes_openssl.c index 2f985d4512..502f7703bd 100644 --- a/src/lib/crypt_ops/aes_openssl.c +++ b/src/lib/crypt_ops/aes_openssl.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -28,7 +28,7 @@ #error "We require OpenSSL >= 1.0.0" #endif -DISABLE_GCC_WARNING(redundant-decls) +DISABLE_GCC_WARNING("-Wredundant-decls") #include <stdlib.h> #include <string.h> @@ -37,9 +37,8 @@ DISABLE_GCC_WARNING(redundant-decls) #include <openssl/engine.h> #include <openssl/modes.h> -ENABLE_GCC_WARNING(redundant-decls) +ENABLE_GCC_WARNING("-Wredundant-decls") -#include "lib/crypt_ops/aes.h" #include "lib/log/log.h" #include "lib/ctime/di_ops.h" @@ -148,13 +147,13 @@ evaluate_ctr_for_aes(void) { return 0; } -#else /* !(defined(USE_EVP_AES_CTR)) */ +#else /* !defined(USE_EVP_AES_CTR) */ /*======================================================================*/ /* Interface to AES code, and counter implementation */ /** Implements an AES counter-mode cipher. */ -struct aes_cnt_cipher { +struct aes_cnt_cipher_t { /** This next element (however it's defined) is the AES key. */ union { EVP_CIPHER_CTX evp; diff --git a/src/lib/crypt_ops/certs.md b/src/lib/crypt_ops/certs.md new file mode 100644 index 0000000000..f3bd8c2c96 --- /dev/null +++ b/src/lib/crypt_ops/certs.md @@ -0,0 +1,29 @@ + +@page certificates Certificates in Tor + +We have, alas, several certificate types in Tor. + +The tor_x509_cert_t type represents an X.509 certificate. This document +won't explain X.509 to you -- possibly, no document can. (OTOH, Peter +Gutmann's "x.509 style guide", though severely dated, does a good job of +explaining how awful x.509 can be.) Do not introduce any new usages of +X.509. Right now we only use it in places where TLS forces us to do so. +See x509.c for more information about using this type. + + +The authority_cert_t type is used only for directory authority keys. It +has a medium-term signing key (which the authorities actually keep +online) signed by a long-term identity key (which the authority operator +had really better be keeping offline). Don't use it for any new kind of +certificate. + +For new places where you need a certificate, consider tor_cert_t: it +represents a typed and dated _something_ signed by an Ed25519 key. The +format is described in tor-spec. Unlike x.509, you can write it on a +napkin. The torcert.c file is used for manipulating these certificates and +their associated keys. + +(Additionally, the Tor directory design uses a fairly wide variety of +documents that include keys and which are signed by keys. You can +consider these documents to be an additional kind of certificate if you +want.) diff --git a/src/lib/crypt_ops/compat_openssl.h b/src/lib/crypt_ops/compat_openssl.h index 6605d01045..c2e1459078 100644 --- a/src/lib/crypt_ops/compat_openssl.h +++ b/src/lib/crypt_ops/compat_openssl.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_COMPAT_OPENSSL_H @@ -41,7 +41,7 @@ ((st) == SSL3_ST_SW_SRVR_HELLO_B)) #define OSSL_HANDSHAKE_STATE int #define CONST_IF_OPENSSL_1_1_API -#else /* !(!defined(OPENSSL_1_1_API)) */ +#else /* defined(OPENSSL_1_1_API) */ #define STATE_IS_SW_SERVER_HELLO(st) \ ((st) == TLS_ST_SW_SRVR_HELLO) #define CONST_IF_OPENSSL_1_1_API const diff --git a/src/lib/crypt_ops/crypto_cipher.c b/src/lib/crypt_ops/crypto_cipher.c index 7bc2edad54..0a24a580ae 100644 --- a/src/lib/crypt_ops/crypto_cipher.c +++ b/src/lib/crypt_ops/crypto_cipher.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_cipher.h b/src/lib/crypt_ops/crypto_cipher.h index cc4fbf7a41..1e22a7c138 100644 --- a/src/lib/crypt_ops/crypto_cipher.h +++ b/src/lib/crypt_ops/crypto_cipher.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -25,7 +25,7 @@ /** Length of our symmetric cipher's keys of 256-bit. */ #define CIPHER256_KEY_LEN 32 -typedef struct aes_cnt_cipher crypto_cipher_t; +typedef struct aes_cnt_cipher_t crypto_cipher_t; /* environment setup */ crypto_cipher_t *crypto_cipher_new(const char *key); @@ -54,4 +54,4 @@ int crypto_cipher_decrypt_with_iv(const char *key, char *to, size_t tolen, const char *from, size_t fromlen); -#endif /* !defined(TOR_CRYPTO_H) */ +#endif /* !defined(TOR_CRYPTO_CIPHER_H) */ diff --git a/src/lib/crypt_ops/crypto_curve25519.c b/src/lib/crypt_ops/crypto_curve25519.c index de4e17a296..2a2589f07d 100644 --- a/src/lib/crypt_ops/crypto_curve25519.c +++ b/src/lib/crypt_ops/crypto_curve25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2019, The Tor Project, Inc. */ +/* Copyright (c) 2012-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_curve25519.h b/src/lib/crypt_ops/crypto_curve25519.h index 061a7a3505..154a0b94bc 100644 --- a/src/lib/crypt_ops/crypto_curve25519.h +++ b/src/lib/crypt_ops/crypto_curve25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2019, The Tor Project, Inc. */ +/* Copyright (c) 2012-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -76,8 +76,8 @@ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret); int curve25519_public_from_base64(curve25519_public_key_t *pkey, const char *input); -int curve25519_public_to_base64(char *output, - const curve25519_public_key_t *pkey); +void curve25519_public_to_base64(char *output, + const curve25519_public_key_t *pkey); void curve25519_set_impl_params(int use_ed); void curve25519_init(void); diff --git a/src/lib/crypt_ops/crypto_dh.c b/src/lib/crypt_ops/crypto_dh.c index 4be7948761..086aceed6f 100644 --- a/src/lib/crypt_ops/crypto_dh.c +++ b/src/lib/crypt_ops/crypto_dh.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_dh.h b/src/lib/crypt_ops/crypto_dh.h index 850d50c7ae..2a0e1f6912 100644 --- a/src/lib/crypt_ops/crypto_dh.h +++ b/src/lib/crypt_ops/crypto_dh.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_dh_nss.c b/src/lib/crypt_ops/crypto_dh_nss.c index 379eb84a4f..018db8bf43 100644 --- a/src/lib/crypt_ops/crypto_dh_nss.c +++ b/src/lib/crypt_ops/crypto_dh_nss.c @@ -1,11 +1,11 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** - * \file crypto_dh_nss.h + * \file crypto_dh_nss.c * * \brief NSS implementation of Diffie-Hellman over Z_p. **/ diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c index 8c6388fd5d..c5f7271596 100644 --- a/src/lib/crypt_ops/crypto_dh_openssl.c +++ b/src/lib/crypt_ops/crypto_dh_openssl.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,11 +17,11 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" -DISABLE_GCC_WARNING(redundant-decls) +DISABLE_GCC_WARNING("-Wredundant-decls") #include <openssl/dh.h> -ENABLE_GCC_WARNING(redundant-decls) +ENABLE_GCC_WARNING("-Wredundant-decls") #include <openssl/bn.h> #include <string.h> @@ -34,7 +34,7 @@ static int tor_check_dh_key(int severity, const BIGNUM *bn); struct crypto_dh_t { DH *dh; /**< The openssl DH object */ }; -#endif +#endif /* !defined(ENABLE_NSS) */ static DH *new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g); @@ -68,7 +68,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g) goto out; if (!DH_set0_pqg(dh, dh_p, NULL, dh_g)) goto out; -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ if (!(dh->p = BN_dup(p))) goto out; if (!(dh->g = BN_dup(g))) @@ -100,10 +100,10 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g) DH_free(dh); return ret; } -#endif +#endif /* 0 */ /** - * Helper: convert <b>hex<b> to a bignum, and return it. Assert that the + * Helper: convert <b>hex</b> to a bignum, and return it. Assert that the * operation was successful. */ static BIGNUM * @@ -202,7 +202,7 @@ crypto_dh_new(int dh_type) tor_free(res); // sets res to NULL. return res; } -#endif +#endif /* !defined(ENABLE_NSS) */ /** Create and return a new openssl DH from a given prime and generator. */ static DH * @@ -231,7 +231,7 @@ new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g) if (!DH_set_length(res_dh, DH_PRIVATE_KEY_BITS)) goto err; -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ res_dh->p = dh_p; res_dh->g = dh_g; res_dh->length = DH_PRIVATE_KEY_BITS; @@ -298,7 +298,7 @@ crypto_dh_generate_public(crypto_dh_t *dh) "the-universe chances really do happen. Treating as a failure."); return -1; } -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { /* LCOV_EXCL_START * If this happens, then openssl's DH implementation is busted. */ @@ -461,7 +461,7 @@ crypto_dh_free_(crypto_dh_t *dh) DH_free(dh->dh); tor_free(dh); } -#endif +#endif /* !defined(ENABLE_NSS) */ void crypto_dh_free_all_openssl(void) diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c index de81b87b7e..7775e69410 100644 --- a/src/lib/crypt_ops/crypto_digest.c +++ b/src/lib/crypt_ops/crypto_digest.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -23,171 +23,6 @@ #include "lib/arch/bytes.h" -#ifdef ENABLE_NSS -DISABLE_GCC_WARNING(strict-prototypes) -#include <pk11pub.h> -ENABLE_GCC_WARNING(strict-prototypes) -#else - -#include "lib/crypt_ops/crypto_openssl_mgt.h" - -DISABLE_GCC_WARNING(redundant-decls) - -#include <openssl/hmac.h> -#include <openssl/sha.h> - -ENABLE_GCC_WARNING(redundant-decls) -#endif - -#ifdef ENABLE_NSS -/** - * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS). - * On failure, return SEC_OID_UNKNOWN. */ -static SECOidTag -digest_alg_to_nss_oid(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: return SEC_OID_SHA1; - case DIGEST_SHA256: return SEC_OID_SHA256; - case DIGEST_SHA512: return SEC_OID_SHA512; - case DIGEST_SHA3_256: FALLTHROUGH; - case DIGEST_SHA3_512: FALLTHROUGH; - default: - return SEC_OID_UNKNOWN; - } -} - -/* Helper: get an unkeyed digest via pk11wrap */ -static int -digest_nss_internal(SECOidTag alg, - char *digest, unsigned len_out, - const char *msg, size_t msg_len) -{ - if (alg == SEC_OID_UNKNOWN) - return -1; - tor_assert(msg_len <= UINT_MAX); - - int rv = -1; - SECStatus s; - PK11Context *ctx = PK11_CreateDigestContext(alg); - if (!ctx) - return -1; - - s = PK11_DigestBegin(ctx); - if (s != SECSuccess) - goto done; - - s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len); - if (s != SECSuccess) - goto done; - - unsigned int len = 0; - s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out); - if (s != SECSuccess) - goto done; - - rv = 0; - done: - PK11_DestroyContext(ctx, PR_TRUE); - return rv; -} - -/** True iff alg is implemented in our crypto library, and we want to use that - * implementation */ -static bool -library_supports_digest(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: FALLTHROUGH; - case DIGEST_SHA256: FALLTHROUGH; - case DIGEST_SHA512: - return true; - case DIGEST_SHA3_256: FALLTHROUGH; - case DIGEST_SHA3_512: FALLTHROUGH; - default: - return false; - } -} -#endif - -/* Crypto digest functions */ - -/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in - * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. - * Return 0 on success, -1 on failure. - */ -MOCK_IMPL(int, -crypto_digest,(char *digest, const char *m, size_t len)) -{ - tor_assert(m); - tor_assert(digest); -#ifdef ENABLE_NSS - return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len); -#else - if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) { - return -1; - } -#endif - return 0; -} - -/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, - * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result - * into <b>digest</b>. Return 0 on success, -1 on failure. */ -int -crypto_digest256(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - - int ret = 0; - if (algorithm == DIGEST_SHA256) { -#ifdef ENABLE_NSS - return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len); -#else - ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); -#endif - } else { - ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) - > -1); - } - - if (!ret) - return -1; - return 0; -} - -/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, - * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result - * into <b>digest</b>. Return 0 on success, -1 on failure. */ -int -crypto_digest512(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - - int ret = 0; - if (algorithm == DIGEST_SHA512) { -#ifdef ENABLE_NSS - return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len); -#else - ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) - != NULL); -#endif - } else { - ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) - > -1); - } - - if (!ret) - return -1; - return 0; -} - /** Set the common_digests_t in <b>ds_out</b> to contain every digest on the * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on * success, -1 on failure. */ @@ -267,485 +102,6 @@ crypto_digest_algorithm_get_length(digest_algorithm_t alg) } } -/** Intermediate information about the digest of a stream of data. */ -struct crypto_digest_t { - digest_algorithm_t algorithm; /**< Which algorithm is in use? */ - /** State for the digest we're using. Only one member of the - * union is usable, depending on the value of <b>algorithm</b>. Note also - * that space for other members might not even be allocated! - */ - union { -#ifdef ENABLE_NSS - PK11Context *ctx; -#else - SHA_CTX sha1; /**< state for SHA1 */ - SHA256_CTX sha2; /**< state for SHA256 */ - SHA512_CTX sha512; /**< state for SHA512 */ -#endif - keccak_state sha3; /**< state for SHA3-[256,512] */ - } d; -}; - -#ifdef TOR_UNIT_TESTS - -digest_algorithm_t -crypto_digest_get_algorithm(crypto_digest_t *digest) -{ - tor_assert(digest); - - return digest->algorithm; -} - -#endif /* defined(TOR_UNIT_TESTS) */ - -/** - * Return the number of bytes we need to malloc in order to get a - * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe - * when we free one. - */ -static size_t -crypto_digest_alloc_bytes(digest_algorithm_t alg) -{ - /* Helper: returns the number of bytes in the 'f' field of 'st' */ -#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) - /* Gives the length of crypto_digest_t through the end of the field 'd' */ -#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ - STRUCT_FIELD_SIZE(crypto_digest_t, f)) - switch (alg) { -#ifdef ENABLE_NSS - case DIGEST_SHA1: FALLTHROUGH; - case DIGEST_SHA256: FALLTHROUGH; - case DIGEST_SHA512: - return END_OF_FIELD(d.ctx); -#else - case DIGEST_SHA1: - return END_OF_FIELD(d.sha1); - case DIGEST_SHA256: - return END_OF_FIELD(d.sha2); - case DIGEST_SHA512: - return END_OF_FIELD(d.sha512); -#endif - case DIGEST_SHA3_256: - case DIGEST_SHA3_512: - return END_OF_FIELD(d.sha3); - default: - tor_assert(0); // LCOV_EXCL_LINE - return 0; // LCOV_EXCL_LINE - } -#undef END_OF_FIELD -#undef STRUCT_FIELD_SIZE -} - -/** - * Internal function: create and return a new digest object for 'algorithm'. - * Does not typecheck the algorithm. - */ -static crypto_digest_t * -crypto_digest_new_internal(digest_algorithm_t algorithm) -{ - crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); - r->algorithm = algorithm; - - switch (algorithm) - { -#ifdef ENABLE_NSS - case DIGEST_SHA1: FALLTHROUGH; - case DIGEST_SHA256: FALLTHROUGH; - case DIGEST_SHA512: - r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm)); - if (BUG(!r->d.ctx)) { - tor_free(r); - return NULL; - } - if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) { - crypto_digest_free(r); - return NULL; - } - break; -#else - case DIGEST_SHA1: - SHA1_Init(&r->d.sha1); - break; - case DIGEST_SHA256: - SHA256_Init(&r->d.sha2); - break; - case DIGEST_SHA512: - SHA512_Init(&r->d.sha512); - break; -#endif - case DIGEST_SHA3_256: - keccak_digest_init(&r->d.sha3, 256); - break; - case DIGEST_SHA3_512: - keccak_digest_init(&r->d.sha3, 512); - break; - default: - tor_assert_unreached(); - } - - return r; -} - -/** Allocate and return a new digest object to compute SHA1 digests. - */ -crypto_digest_t * -crypto_digest_new(void) -{ - return crypto_digest_new_internal(DIGEST_SHA1); -} - -/** Allocate and return a new digest object to compute 256-bit digests - * using <b>algorithm</b>. - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` - * C_RUST_COUPLED: `crypto::digest::Sha256::default` - */ -crypto_digest_t * -crypto_digest256_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - return crypto_digest_new_internal(algorithm); -} - -/** Allocate and return a new digest object to compute 512-bit digests - * using <b>algorithm</b>. */ -crypto_digest_t * -crypto_digest512_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - return crypto_digest_new_internal(algorithm); -} - -/** Deallocate a digest object. - */ -void -crypto_digest_free_(crypto_digest_t *digest) -{ - if (!digest) - return; -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - PK11_DestroyContext(digest->d.ctx, PR_TRUE); - } -#endif - size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); - memwipe(digest, 0, bytes); - tor_free(digest); -} - -/** Add <b>len</b> bytes from <b>data</b> to the digest object. - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` - * C_RUST_COUPLED: `crypto::digest::Sha256::process` - */ -void -crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, - size_t len) -{ - tor_assert(digest); - tor_assert(data); - /* Using the SHA*_*() calls directly means we don't support doing - * SHA in hardware. But so far the delay of getting the question - * to the hardware, and hearing the answer, is likely higher than - * just doing it ourselves. Hashes are fast. - */ - switch (digest->algorithm) { -#ifdef ENABLE_NSS - case DIGEST_SHA1: FALLTHROUGH; - case DIGEST_SHA256: FALLTHROUGH; - case DIGEST_SHA512: - tor_assert(len <= UINT_MAX); - SECStatus s = PK11_DigestOp(digest->d.ctx, - (const unsigned char *)data, - (unsigned int)len); - tor_assert(s == SECSuccess); - break; -#else - case DIGEST_SHA1: - SHA1_Update(&digest->d.sha1, (void*)data, len); - break; - case DIGEST_SHA256: - SHA256_Update(&digest->d.sha2, (void*)data, len); - break; - case DIGEST_SHA512: - SHA512_Update(&digest->d.sha512, (void*)data, len); - break; -#endif - case DIGEST_SHA3_256: FALLTHROUGH; - case DIGEST_SHA3_512: - keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); - break; - default: - /* LCOV_EXCL_START */ - tor_fragile_assert(); - break; - /* LCOV_EXCL_STOP */ - } -} - -/** Compute the hash of the data that has been passed to the digest - * object; write the first out_len bytes of the result to <b>out</b>. - * <b>out_len</b> must be \<= DIGEST512_LEN. - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` - * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` - */ -void -crypto_digest_get_digest(crypto_digest_t *digest, - char *out, size_t out_len) -{ - unsigned char r[DIGEST512_LEN]; - tor_assert(digest); - tor_assert(out); - tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); - - /* The SHA-3 code handles copying into a temporary ctx, and also can handle - * short output buffers by truncating appropriately. */ - if (digest->algorithm == DIGEST_SHA3_256 || - digest->algorithm == DIGEST_SHA3_512) { - keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); - return; - } - -#ifdef ENABLE_NSS - /* Copy into a temporary buffer since DigestFinal (alters) the context */ - unsigned char buf[1024]; - unsigned int saved_len = 0; - unsigned rlen; - unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx, - buf, sizeof(buf), - &saved_len); - tor_assert(saved); - SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r)); - tor_assert(s == SECSuccess); - tor_assert(rlen >= out_len); - s = PK11_RestoreContext(digest->d.ctx, saved, saved_len); - tor_assert(s == SECSuccess); - if (saved != buf) { - PORT_ZFree(saved, saved_len); - } -#else - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - crypto_digest_t tmpenv; - /* memcpy into a temporary ctx, since SHA*_Final clears the context */ - memcpy(&tmpenv, digest, alloc_bytes); - switch (digest->algorithm) { - case DIGEST_SHA1: - SHA1_Final(r, &tmpenv.d.sha1); - break; - case DIGEST_SHA256: - SHA256_Final(r, &tmpenv.d.sha2); - break; - case DIGEST_SHA512: - SHA512_Final(r, &tmpenv.d.sha512); - break; -//LCOV_EXCL_START - case DIGEST_SHA3_256: FALLTHROUGH; - case DIGEST_SHA3_512: - default: - log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); - /* This is fatal, because it should never happen. */ - tor_assert_unreached(); - break; -//LCOV_EXCL_STOP - } -#endif - memcpy(out, r, out_len); - memwipe(r, 0, sizeof(r)); -} - -/** Allocate and return a new digest object with the same state as - * <b>digest</b> - * - * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` - * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` - */ -crypto_digest_t * -crypto_digest_dup(const crypto_digest_t *digest) -{ - tor_assert(digest); - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - crypto_digest_t *result = tor_memdup(digest, alloc_bytes); -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - result->d.ctx = PK11_CloneContext(digest->d.ctx); - } -#endif - return result; -} - -/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>. - * Asserts that <b>digest</b> is a SHA1 digest object. - */ -void -crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, - const crypto_digest_t *digest) -{ - const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); - tor_assert(bytes <= sizeof(checkpoint->mem)); -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - unsigned char *allocated; - allocated = PK11_SaveContextAlloc(digest->d.ctx, - (unsigned char *)checkpoint->mem, - sizeof(checkpoint->mem), - &checkpoint->bytes_used); - /* No allocation is allowed here. */ - tor_assert(allocated == checkpoint->mem); - return; - } -#endif - memcpy(checkpoint->mem, digest, bytes); -} - -/** Restore the state of <b>digest</b> from <b>checkpoint</b>. - * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the - * state was previously stored with crypto_digest_checkpoint() */ -void -crypto_digest_restore(crypto_digest_t *digest, - const crypto_digest_checkpoint_t *checkpoint) -{ - const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); -#ifdef ENABLE_NSS - if (library_supports_digest(digest->algorithm)) { - SECStatus s = PK11_RestoreContext(digest->d.ctx, - (unsigned char *)checkpoint->mem, - checkpoint->bytes_used); - tor_assert(s == SECSuccess); - return; - } -#endif - memcpy(digest, checkpoint->mem, bytes); -} - -/** Replace the state of the digest object <b>into</b> with the state - * of the digest object <b>from</b>. Requires that 'into' and 'from' - * have the same digest type. - */ -void -crypto_digest_assign(crypto_digest_t *into, - const crypto_digest_t *from) -{ - tor_assert(into); - tor_assert(from); - tor_assert(into->algorithm == from->algorithm); - const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); -#ifdef ENABLE_NSS - if (library_supports_digest(from->algorithm)) { - PK11_DestroyContext(into->d.ctx, PR_TRUE); - into->d.ctx = PK11_CloneContext(from->d.ctx); - return; - } -#endif - memcpy(into,from,alloc_bytes); -} - -/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest - * at <b>digest_out</b> to the hash of the concatenation of those strings, - * plus the optional string <b>append</b>, computed with the algorithm - * <b>alg</b>. - * <b>out_len</b> must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist(char *digest_out, size_t len_out, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); -} - -/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest - * at <b>digest_out</b> to the hash of the concatenation of: the - * optional string <b>prepend</b>, those strings, - * and the optional string <b>append</b>, computed with the algorithm - * <b>alg</b>. - * <b>len_out</b> must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, - const char *prepend, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_t *d = crypto_digest_new_internal(alg); - if (prepend) - crypto_digest_add_bytes(d, prepend, strlen(prepend)); - SMARTLIST_FOREACH(lst, const char *, cp, - crypto_digest_add_bytes(d, cp, strlen(cp))); - if (append) - crypto_digest_add_bytes(d, append, strlen(append)); - crypto_digest_get_digest(d, digest_out, len_out); - crypto_digest_free(d); -} - -/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using - * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte - * result in <b>hmac_out</b>. Asserts on failure. - */ -void -crypto_hmac_sha256(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len) -{ - /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ - tor_assert(key_len < INT_MAX); - tor_assert(msg_len < INT_MAX); - tor_assert(hmac_out); -#ifdef ENABLE_NSS - PK11SlotInfo *slot = NULL; - PK11SymKey *symKey = NULL; - PK11Context *hmac = NULL; - - int ok = 0; - SECStatus s; - SECItem keyItem, paramItem; - keyItem.data = (unsigned char *)key; - keyItem.len = (unsigned)key_len; - paramItem.type = siBuffer; - paramItem.data = NULL; - paramItem.len = 0; - - slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL); - if (!slot) - goto done; - symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC, - PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL); - if (!symKey) - goto done; - - hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey, - ¶mItem); - if (!hmac) - goto done; - s = PK11_DigestBegin(hmac); - if (s != SECSuccess) - goto done; - s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len); - if (s != SECSuccess) - goto done; - unsigned int len=0; - s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN); - if (s != SECSuccess || len != DIGEST256_LEN) - goto done; - ok = 1; - - done: - if (hmac) - PK11_DestroyContext(hmac, PR_TRUE); - if (symKey) - PK11_FreeSymKey(symKey); - if (slot) - PK11_FreeSlot(slot); - - tor_assert(ok); -#else - unsigned char *rv = NULL; - rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, - (unsigned char*)hmac_out, NULL); - tor_assert(rv); -#endif -} - /** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in @@ -779,7 +135,26 @@ crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, /** Internal state for a eXtendable-Output Function (XOF). */ struct crypto_xof_t { +#ifdef OPENSSL_HAS_SHAKE3_EVP + /* XXXX We can't enable this yet, because OpenSSL's + * DigestFinalXOF function can't be called repeatedly on the same + * XOF. + * + * We could in theory use the undocumented SHA3_absorb and SHA3_squeeze + * functions, but let's not mess with undocumented OpenSSL internals any + * more than we have to. + * + * We could also revise our XOF code so that it only allows a single + * squeeze operation; we don't require streaming squeeze operations + * outside the tests yet. + */ + EVP_MD_CTX *ctx; +#else /* !defined(OPENSSL_HAS_SHAKE3_EVP) */ + /** + * State of the Keccak sponge for the SHAKE-256 computation. + **/ keccak_state s; +#endif /* defined(OPENSSL_HAS_SHAKE3_EVP) */ }; /** Allocate a new XOF object backed by SHAKE-256. The security level @@ -792,7 +167,14 @@ crypto_xof_new(void) { crypto_xof_t *xof; xof = tor_malloc(sizeof(crypto_xof_t)); +#ifdef OPENSSL_HAS_SHAKE256 + xof->ctx = EVP_MD_CTX_new(); + tor_assert(xof->ctx); + int r = EVP_DigestInit(xof->ctx, EVP_shake256()); + tor_assert(r == 1); +#else /* !defined(OPENSSL_HAS_SHAKE256) */ keccak_xof_init(&xof->s, 256); +#endif /* defined(OPENSSL_HAS_SHAKE256) */ return xof; } @@ -803,8 +185,13 @@ crypto_xof_new(void) void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) { +#ifdef OPENSSL_HAS_SHAKE256 + int r = EVP_DigestUpdate(xof->ctx, data, len); + tor_assert(r == 1); +#else int i = keccak_xof_absorb(&xof->s, data, len); tor_assert(i == 0); +#endif /* defined(OPENSSL_HAS_SHAKE256) */ } /** Squeeze bytes out of a XOF object. Calling this routine will render @@ -813,8 +200,13 @@ crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len) { +#ifdef OPENSSL_HAS_SHAKE256 + int r = EVP_DigestFinalXOF(xof->ctx, out, len); + tor_assert(r == 1); +#else int i = keccak_xof_squeeze(&xof->s, out, len); tor_assert(i == 0); +#endif /* defined(OPENSSL_HAS_SHAKE256) */ } /** Cleanse and deallocate a XOF object. */ @@ -823,6 +215,34 @@ crypto_xof_free_(crypto_xof_t *xof) { if (!xof) return; +#ifdef OPENSSL_HAS_SHAKE256 + if (xof->ctx) + EVP_MD_CTX_free(xof->ctx); +#endif memwipe(xof, 0, sizeof(crypto_xof_t)); tor_free(xof); } + +/** Compute the XOF (SHAKE256) of a <b>input_len</b> bytes at <b>input</b>, + * putting <b>output_len</b> bytes at <b>output</b>. */ +void +crypto_xof(uint8_t *output, size_t output_len, + const uint8_t *input, size_t input_len) +{ +#ifdef OPENSSL_HAS_SHA3 + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + tor_assert(ctx); + int r = EVP_DigestInit(ctx, EVP_shake256()); + tor_assert(r == 1); + r = EVP_DigestUpdate(ctx, input, input_len); + tor_assert(r == 1); + r = EVP_DigestFinalXOF(ctx, output, output_len); + tor_assert(r == 1); + EVP_MD_CTX_free(ctx); +#else /* !defined(OPENSSL_HAS_SHA3) */ + crypto_xof_t *xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, input, input_len); + crypto_xof_squeeze_bytes(xof, output, output_len); + crypto_xof_free(xof); +#endif /* defined(OPENSSL_HAS_SHA3) */ +} diff --git a/src/lib/crypt_ops/crypto_digest.h b/src/lib/crypt_ops/crypto_digest.h index 47e60ce617..eefd2e3f0a 100644 --- a/src/lib/crypt_ops/crypto_digest.h +++ b/src/lib/crypt_ops/crypto_digest.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -38,6 +38,9 @@ /** Length of hex encoding of SHA512 digest, not including final NUL. */ #define HEX_DIGEST512_LEN 128 +/** + * An identifier for a cryptographic digest algorithm. + **/ typedef enum { DIGEST_SHA1 = 0, DIGEST_SHA256 = 1, @@ -45,16 +48,31 @@ typedef enum { DIGEST_SHA3_256 = 3, DIGEST_SHA3_512 = 4, } digest_algorithm_t; +/** Number of digest algorithms that we know */ #define N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1) +/** Number of digest algorithms to compute when computing "all the + * commonly used digests." + * + * (This is used in common_digests_t and related functions.) + */ #define N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1) +/** + * Bytes of storage needed to record the state of an in-progress SHA-1 digest. + * + * This is a deliberate overestimate. + **/ #define DIGEST_CHECKPOINT_BYTES (SIZEOF_VOID_P + 512) + /** Structure used to temporarily save the a digest object. Only implemented * for SHA1 digest for now. */ typedef struct crypto_digest_checkpoint_t { #ifdef ENABLE_NSS + /** The number of bytes used in <b>mem</b>. */ unsigned int bytes_used; #endif + /** A buffer to store the SHA1 state. Its contents are unspecified, and + * are managed by the underlying crypto library.*/ uint8_t mem[DIGEST_CHECKPOINT_BYTES]; } crypto_digest_checkpoint_t; @@ -67,10 +85,19 @@ typedef struct crypto_digest_checkpoint_t { * once. **/ typedef struct { + /** An array of digest outputs, one for each "common" digest algorithm. */ char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN]; } common_digests_t; +/** + * State for computing a digest over a stream of data. + **/ typedef struct crypto_digest_t crypto_digest_t; + +/** + * State for computing an "extendable-output function" (like SHAKE) over a + * stream of data, and/or streaming the output. + **/ typedef struct crypto_xof_t crypto_xof_t; struct smartlist_t; @@ -97,6 +124,9 @@ crypto_digest_t *crypto_digest_new(void); crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm); crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm); void crypto_digest_free_(crypto_digest_t *digest); +/** + * Release all storage held in <b>d</b>, and set it to NULL. + **/ #define crypto_digest_free(d) \ FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d)) void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, @@ -122,8 +152,13 @@ crypto_xof_t *crypto_xof_new(void); void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len); void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); void crypto_xof_free_(crypto_xof_t *xof); +/** + * Release all storage held in <b>xof</b>, and set it to NULL. + **/ #define crypto_xof_free(xof) \ FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof)) +void crypto_xof(uint8_t *output, size_t output_len, + const uint8_t *input, size_t input_len); #ifdef TOR_UNIT_TESTS digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest); diff --git a/src/lib/crypt_ops/crypto_digest_nss.c b/src/lib/crypt_ops/crypto_digest_nss.c new file mode 100644 index 0000000000..92c20fe9e8 --- /dev/null +++ b/src/lib/crypt_ops/crypto_digest_nss.c @@ -0,0 +1,563 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest_nss.c + * \brief Block of functions related with digest and xof utilities and + * operations (NSS specific implementations). + **/ + +#include "lib/container/smartlist.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" + +#include "keccak-tiny/keccak-tiny.h" + +#include <stdlib.h> +#include <string.h> + +#include "lib/arch/bytes.h" + +DISABLE_GCC_WARNING("-Wstrict-prototypes") +#include <pk11pub.h> +ENABLE_GCC_WARNING("-Wstrict-prototypes") + +/** + * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS). + * On failure, return SEC_OID_UNKNOWN. */ +static SECOidTag +digest_alg_to_nss_oid(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: return SEC_OID_SHA1; + case DIGEST_SHA256: return SEC_OID_SHA256; + case DIGEST_SHA512: return SEC_OID_SHA512; + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: FALLTHROUGH; + default: + return SEC_OID_UNKNOWN; + } +} + +/** Helper: Compute an unkeyed digest of the <b>msg_len</b> bytes at + * <b>msg</b>, using the digest algorithm specified by <b>alg</b>. + * Store the result in the <b>len_out</b>-byte buffer at <b>digest</b>. + * Return the number of bytes written on success, and -1 on failure. + **/ +static int +digest_nss_internal(SECOidTag alg, + char *digest, unsigned len_out, + const char *msg, size_t msg_len) +{ + if (alg == SEC_OID_UNKNOWN) + return -1; + tor_assert(msg_len <= UINT_MAX); + + int rv = -1; + SECStatus s; + PK11Context *ctx = PK11_CreateDigestContext(alg); + if (!ctx) + return -1; + + s = PK11_DigestBegin(ctx); + if (s != SECSuccess) + goto done; + + s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len); + if (s != SECSuccess) + goto done; + + unsigned int len = 0; + s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out); + if (s != SECSuccess) + goto done; + + rv = 0; + done: + PK11_DestroyContext(ctx, PR_TRUE); + return rv; +} + +/** True iff alg is implemented in our crypto library, and we want to use that + * implementation */ +static bool +library_supports_digest(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: FALLTHROUGH; + case DIGEST_SHA256: FALLTHROUGH; + case DIGEST_SHA512: + return true; + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: FALLTHROUGH; + default: + return false; + } +} + +/* Crypto digest functions */ + +/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in + * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. + * Return 0 on success, -1 on failure. + */ +MOCK_IMPL(int, +crypto_digest,(char *digest, const char *m, size_t len)) +{ + tor_assert(m); + tor_assert(digest); + return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len); +} + +/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; + if (algorithm == DIGEST_SHA256) { + return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len); + } else { + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); + } + + if (!ret) + return -1; + return 0; +} + +/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; + if (algorithm == DIGEST_SHA512) { + return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len); + } else { + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); + } + + if (!ret) + return -1; + return 0; +} + +/** Intermediate information about the digest of a stream of data. */ +struct crypto_digest_t { + digest_algorithm_t algorithm; /**< Which algorithm is in use? */ + /** State for the digest we're using. Only one member of the + * union is usable, depending on the value of <b>algorithm</b>. Note also + * that space for other members might not even be allocated! + */ + union { + PK11Context *ctx; + keccak_state sha3; /**< state for SHA3-[256,512] */ + } d; +}; + +#ifdef TOR_UNIT_TESTS + +digest_algorithm_t +crypto_digest_get_algorithm(crypto_digest_t *digest) +{ + tor_assert(digest); + + return digest->algorithm; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Return the number of bytes we need to malloc in order to get a + * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe + * when we free one. + */ +static size_t +crypto_digest_alloc_bytes(digest_algorithm_t alg) +{ + /* Helper: returns the number of bytes in the 'f' field of 'st' */ +#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) + /* Gives the length of crypto_digest_t through the end of the field 'd' */ +#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ + STRUCT_FIELD_SIZE(crypto_digest_t, f)) + switch (alg) { + case DIGEST_SHA1: FALLTHROUGH; + case DIGEST_SHA256: FALLTHROUGH; + case DIGEST_SHA512: + return END_OF_FIELD(d.ctx); + case DIGEST_SHA3_256: + case DIGEST_SHA3_512: + return END_OF_FIELD(d.sha3); + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +#undef END_OF_FIELD +#undef STRUCT_FIELD_SIZE +} + +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: FALLTHROUGH; + case DIGEST_SHA256: FALLTHROUGH; + case DIGEST_SHA512: + r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm)); + if (BUG(!r->d.ctx)) { + tor_free(r); + return NULL; + } + if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) { + crypto_digest_free(r); + return NULL; + } + break; + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; + default: + tor_assert_unreached(); + } + + return r; +} + +/** Allocate and return a new digest object to compute SHA1 digests. + */ +crypto_digest_t * +crypto_digest_new(void) +{ + return crypto_digest_new_internal(DIGEST_SHA1); +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using <b>algorithm</b>. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` + * C_RUST_COUPLED: `crypto::digest::Sha256::default` + */ +crypto_digest_t * +crypto_digest256_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + return crypto_digest_new_internal(algorithm); +} + +/** Allocate and return a new digest object to compute 512-bit digests + * using <b>algorithm</b>. */ +crypto_digest_t * +crypto_digest512_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + return crypto_digest_new_internal(algorithm); +} + +/** Deallocate a digest object. + */ +void +crypto_digest_free_(crypto_digest_t *digest) +{ + if (!digest) + return; + if (library_supports_digest(digest->algorithm)) { + PK11_DestroyContext(digest->d.ctx, PR_TRUE); + } + size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memwipe(digest, 0, bytes); + tor_free(digest); +} + +/** Add <b>len</b> bytes from <b>data</b> to the digest object. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` + * C_RUST_COUPLED: `crypto::digest::Sha256::process` + */ +void +crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len) +{ + tor_assert(digest); + tor_assert(data); + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question + * to the hardware, and hearing the answer, is likely higher than + * just doing it ourselves. Hashes are fast. + */ + switch (digest->algorithm) { + case DIGEST_SHA1: FALLTHROUGH; + case DIGEST_SHA256: FALLTHROUGH; + case DIGEST_SHA512: + tor_assert(len <= UINT_MAX); + SECStatus s = PK11_DigestOp(digest->d.ctx, + (const unsigned char *)data, + (unsigned int)len); + tor_assert(s == SECSuccess); + break; + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: + keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); + break; + default: + /* LCOV_EXCL_START */ + tor_fragile_assert(); + break; + /* LCOV_EXCL_STOP */ + } +} + +/** Compute the hash of the data that has been passed to the digest + * object; write the first out_len bytes of the result to <b>out</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` + * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` + */ +void +crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len) +{ + unsigned char r[DIGEST512_LEN]; + tor_assert(digest); + tor_assert(out); + tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); + + /* The SHA-3 code handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); + return; + } + + /* Copy into a temporary buffer since DigestFinal (alters) the context */ + unsigned char buf[1024]; + unsigned int saved_len = 0; + unsigned rlen; + unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx, + buf, sizeof(buf), + &saved_len); + tor_assert(saved); + SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r)); + tor_assert(s == SECSuccess); + tor_assert(rlen >= out_len); + s = PK11_RestoreContext(digest->d.ctx, saved, saved_len); + tor_assert(s == SECSuccess); + + if (saved != buf) { + PORT_ZFree(saved, saved_len); + } + memcpy(out, r, out_len); + memwipe(r, 0, sizeof(r)); +} + +/** Allocate and return a new digest object with the same state as + * <b>digest</b> + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` + * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` + */ +crypto_digest_t * +crypto_digest_dup(const crypto_digest_t *digest) +{ + tor_assert(digest); + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + crypto_digest_t *result = tor_memdup(digest, alloc_bytes); + + if (library_supports_digest(digest->algorithm)) { + result->d.ctx = PK11_CloneContext(digest->d.ctx); + } + + return result; +} + +/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. + */ +void +crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + tor_assert(bytes <= sizeof(checkpoint->mem)); + if (library_supports_digest(digest->algorithm)) { + unsigned char *allocated; + allocated = PK11_SaveContextAlloc(digest->d.ctx, + (unsigned char *)checkpoint->mem, + sizeof(checkpoint->mem), + &checkpoint->bytes_used); + /* No allocation is allowed here. */ + tor_assert(allocated == checkpoint->mem); + return; + } + memcpy(checkpoint->mem, digest, bytes); +} + +/** Restore the state of <b>digest</b> from <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the + * state was previously stored with crypto_digest_checkpoint() */ +void +crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + if (library_supports_digest(digest->algorithm)) { + SECStatus s = PK11_RestoreContext(digest->d.ctx, + (unsigned char *)checkpoint->mem, + checkpoint->bytes_used); + tor_assert(s == SECSuccess); + return; + } + memcpy(digest, checkpoint->mem, bytes); +} + +/** Replace the state of the digest object <b>into</b> with the state + * of the digest object <b>from</b>. Requires that 'into' and 'from' + * have the same digest type. + */ +void +crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from) +{ + tor_assert(into); + tor_assert(from); + tor_assert(into->algorithm == from->algorithm); + const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); + if (library_supports_digest(from->algorithm)) { + PK11_DestroyContext(into->d.ctx, PR_TRUE); + into->d.ctx = PK11_CloneContext(from->d.ctx); + return; + } + memcpy(into,from,alloc_bytes); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * plus the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist(char *digest_out, size_t len_out, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of: the + * optional string <b>prepend</b>, those strings, + * and the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>len_out</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_t *d = crypto_digest_new_internal(alg); + if (prepend) + crypto_digest_add_bytes(d, prepend, strlen(prepend)); + SMARTLIST_FOREACH(lst, const char *, cp, + crypto_digest_add_bytes(d, cp, strlen(cp))); + if (append) + crypto_digest_add_bytes(d, append, strlen(append)); + crypto_digest_get_digest(d, digest_out, len_out); + crypto_digest_free(d); +} + +/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using + * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte + * result in <b>hmac_out</b>. Asserts on failure. + */ +void +crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len) +{ + /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ + tor_assert(key_len < INT_MAX); + tor_assert(msg_len < INT_MAX); + tor_assert(hmac_out); + + PK11SlotInfo *slot = NULL; + PK11SymKey *symKey = NULL; + PK11Context *hmac = NULL; + + int ok = 0; + SECStatus s; + SECItem keyItem, paramItem; + keyItem.data = (unsigned char *)key; + keyItem.len = (unsigned)key_len; + paramItem.type = siBuffer; + paramItem.data = NULL; + paramItem.len = 0; + + slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL); + if (!slot) + goto done; + symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC, + PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL); + if (!symKey) + goto done; + + hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey, + ¶mItem); + if (!hmac) + goto done; + s = PK11_DigestBegin(hmac); + if (s != SECSuccess) + goto done; + s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len); + if (s != SECSuccess) + goto done; + unsigned int len=0; + s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN); + if (s != SECSuccess || len != DIGEST256_LEN) + goto done; + ok = 1; + + done: + if (hmac) + PK11_DestroyContext(hmac, PR_TRUE); + if (symKey) + PK11_FreeSymKey(symKey); + if (slot) + PK11_FreeSlot(slot); + + tor_assert(ok); +} diff --git a/src/lib/crypt_ops/crypto_digest_openssl.c b/src/lib/crypt_ops/crypto_digest_openssl.c new file mode 100644 index 0000000000..11189c7fb2 --- /dev/null +++ b/src/lib/crypt_ops/crypto_digest_openssl.c @@ -0,0 +1,521 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest_openssl.c + * \brief Block of functions related with digest and xof utilities and + * operations (OpenSSL specific implementations). + **/ + +#include "lib/container/smartlist.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" + +#include "keccak-tiny/keccak-tiny.h" + +#include <stdlib.h> +#include <string.h> + +#include "lib/arch/bytes.h" + +#include "lib/crypt_ops/crypto_openssl_mgt.h" + +DISABLE_GCC_WARNING("-Wredundant-decls") + +#include <openssl/hmac.h> +#include <openssl/sha.h> + +ENABLE_GCC_WARNING("-Wredundant-decls") + +/* Crypto digest functions */ + +/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in + * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. + * Return 0 on success, -1 on failure. + */ +MOCK_IMPL(int, +crypto_digest,(char *digest, const char *m, size_t len)) +{ + tor_assert(m); + tor_assert(digest); + if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) { + return -1; + } + return 0; +} + +/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; + if (algorithm == DIGEST_SHA256) { + ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); + } else { +#ifdef OPENSSL_HAS_SHA3 + unsigned int dlen = DIGEST256_LEN; + ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_256(), NULL); +#else + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); +#endif /* defined(OPENSSL_HAS_SHA3) */ + } + + if (!ret) + return -1; + return 0; +} + +/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; + if (algorithm == DIGEST_SHA512) { + ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) + != NULL); + } else { +#ifdef OPENSSL_HAS_SHA3 + unsigned int dlen = DIGEST512_LEN; + ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_512(), NULL); +#else + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); +#endif /* defined(OPENSSL_HAS_SHA3) */ + } + + if (!ret) + return -1; + return 0; +} + +/** Intermediate information about the digest of a stream of data. */ +struct crypto_digest_t { + digest_algorithm_t algorithm; /**< Which algorithm is in use? */ + /** State for the digest we're using. Only one member of the + * union is usable, depending on the value of <b>algorithm</b>. Note also + * that space for other members might not even be allocated! + */ + union { + SHA_CTX sha1; /**< state for SHA1 */ + SHA256_CTX sha2; /**< state for SHA256 */ + SHA512_CTX sha512; /**< state for SHA512 */ +#ifdef OPENSSL_HAS_SHA3 + EVP_MD_CTX *md; +#else + keccak_state sha3; /**< state for SHA3-[256,512] */ +#endif + } d; +}; + +#ifdef TOR_UNIT_TESTS + +digest_algorithm_t +crypto_digest_get_algorithm(crypto_digest_t *digest) +{ + tor_assert(digest); + + return digest->algorithm; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Return the number of bytes we need to malloc in order to get a + * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe + * when we free one. + */ +static size_t +crypto_digest_alloc_bytes(digest_algorithm_t alg) +{ + /** Helper: returns the number of bytes in the 'f' field of 'st' */ +#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) + /** Gives the length of crypto_digest_t through the end of the field 'd' */ +#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ + STRUCT_FIELD_SIZE(crypto_digest_t, f)) + switch (alg) { + case DIGEST_SHA1: + return END_OF_FIELD(d.sha1); + case DIGEST_SHA256: + return END_OF_FIELD(d.sha2); + case DIGEST_SHA512: + return END_OF_FIELD(d.sha512); +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: + return END_OF_FIELD(d.md); +#else + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: + return END_OF_FIELD(d.sha3); +#endif /* defined(OPENSSL_HAS_SHA3) */ + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +#undef END_OF_FIELD +#undef STRUCT_FIELD_SIZE +} + +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: + SHA1_Init(&r->d.sha1); + break; + case DIGEST_SHA256: + SHA256_Init(&r->d.sha2); + break; + case DIGEST_SHA512: + SHA512_Init(&r->d.sha512); + break; +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: + r->d.md = EVP_MD_CTX_new(); + if (!EVP_DigestInit(r->d.md, EVP_sha3_256())) { + crypto_digest_free(r); + return NULL; + } + break; + case DIGEST_SHA3_512: + r->d.md = EVP_MD_CTX_new(); + if (!EVP_DigestInit(r->d.md, EVP_sha3_512())) { + crypto_digest_free(r); + return NULL; + } + break; +#else /* !defined(OPENSSL_HAS_SHA3) */ + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; +#endif /* defined(OPENSSL_HAS_SHA3) */ + default: + tor_assert_unreached(); + } + + return r; +} + +/** Allocate and return a new digest object to compute SHA1 digests. + */ +crypto_digest_t * +crypto_digest_new(void) +{ + return crypto_digest_new_internal(DIGEST_SHA1); +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using <b>algorithm</b>. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` + * C_RUST_COUPLED: `crypto::digest::Sha256::default` + */ +crypto_digest_t * +crypto_digest256_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + return crypto_digest_new_internal(algorithm); +} + +/** Allocate and return a new digest object to compute 512-bit digests + * using <b>algorithm</b>. */ +crypto_digest_t * +crypto_digest512_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + return crypto_digest_new_internal(algorithm); +} + +/** Deallocate a digest object. + */ +void +crypto_digest_free_(crypto_digest_t *digest) +{ + if (!digest) + return; +#ifdef OPENSSL_HAS_SHA3 + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + if (digest->d.md) { + EVP_MD_CTX_free(digest->d.md); + } + } +#endif /* defined(OPENSSL_HAS_SHA3) */ + size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memwipe(digest, 0, bytes); + tor_free(digest); +} + +/** Add <b>len</b> bytes from <b>data</b> to the digest object. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` + * C_RUST_COUPLED: `crypto::digest::Sha256::process` + */ +void +crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len) +{ + tor_assert(digest); + tor_assert(data); + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question + * to the hardware, and hearing the answer, is likely higher than + * just doing it ourselves. Hashes are fast. + */ + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + case DIGEST_SHA512: + SHA512_Update(&digest->d.sha512, (void*)data, len); + break; +#ifdef OPENSSL_HAS_SHA3 + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: { + int r = EVP_DigestUpdate(digest->d.md, data, len); + tor_assert(r); + } + break; +#else /* !defined(OPENSSL_HAS_SHA3) */ + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: + keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); + break; +#endif /* defined(OPENSSL_HAS_SHA3) */ + default: + /* LCOV_EXCL_START */ + tor_fragile_assert(); + break; + /* LCOV_EXCL_STOP */ + } +} + +/** Compute the hash of the data that has been passed to the digest + * object; write the first out_len bytes of the result to <b>out</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` + * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` + */ +void +crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len) +{ + unsigned char r[DIGEST512_LEN]; + tor_assert(digest); + tor_assert(out); + tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); + + /* The SHA-3 code handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { +#ifdef OPENSSL_HAS_SHA3 + unsigned dlen = (unsigned) + crypto_digest_algorithm_get_length(digest->algorithm); + EVP_MD_CTX *tmp = EVP_MD_CTX_new(); + EVP_MD_CTX_copy(tmp, digest->d.md); + memset(r, 0xff, sizeof(r)); + int res = EVP_DigestFinal(tmp, r, &dlen); + EVP_MD_CTX_free(tmp); + tor_assert(res == 1); + goto done; +#else /* !defined(OPENSSL_HAS_SHA3) */ + /* Tiny-Keccak handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); + return; +#endif /* defined(OPENSSL_HAS_SHA3) */ + } + + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + crypto_digest_t tmpenv; + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, alloc_bytes); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Final(r, &tmpenv.d.sha1); + break; + case DIGEST_SHA256: + SHA256_Final(r, &tmpenv.d.sha2); + break; + case DIGEST_SHA512: + SHA512_Final(r, &tmpenv.d.sha512); + break; +//LCOV_EXCL_START + case DIGEST_SHA3_256: FALLTHROUGH; + case DIGEST_SHA3_512: + default: + log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); + /* This is fatal, because it should never happen. */ + tor_assert_unreached(); + break; +//LCOV_EXCL_STOP + } +#ifdef OPENSSL_HAS_SHA3 + done: +#endif + memcpy(out, r, out_len); + memwipe(r, 0, sizeof(r)); +} + +/** Allocate and return a new digest object with the same state as + * <b>digest</b> + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` + * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` + */ +crypto_digest_t * +crypto_digest_dup(const crypto_digest_t *digest) +{ + tor_assert(digest); + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + crypto_digest_t *result = tor_memdup(digest, alloc_bytes); + +#ifdef OPENSSL_HAS_SHA3 + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + result->d.md = EVP_MD_CTX_new(); + EVP_MD_CTX_copy(result->d.md, digest->d.md); + } +#endif /* defined(OPENSSL_HAS_SHA3) */ + return result; +} + +/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. + */ +void +crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + tor_assert(bytes <= sizeof(checkpoint->mem)); + memcpy(checkpoint->mem, digest, bytes); +} + +/** Restore the state of <b>digest</b> from <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the + * state was previously stored with crypto_digest_checkpoint() */ +void +crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memcpy(digest, checkpoint->mem, bytes); +} + +/** Replace the state of the digest object <b>into</b> with the state + * of the digest object <b>from</b>. Requires that 'into' and 'from' + * have the same digest type. + */ +void +crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from) +{ + tor_assert(into); + tor_assert(from); + tor_assert(into->algorithm == from->algorithm); + const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); + +#ifdef OPENSSL_HAS_SHA3 + if (from->algorithm == DIGEST_SHA3_256 || + from->algorithm == DIGEST_SHA3_512) { + EVP_MD_CTX_copy(into->d.md, from->d.md); + return; + } +#endif /* defined(OPENSSL_HAS_SHA3) */ + + memcpy(into,from,alloc_bytes); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * plus the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist(char *digest_out, size_t len_out, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of: the + * optional string <b>prepend</b>, those strings, + * and the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>len_out</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_t *d = crypto_digest_new_internal(alg); + if (prepend) + crypto_digest_add_bytes(d, prepend, strlen(prepend)); + SMARTLIST_FOREACH(lst, const char *, cp, + crypto_digest_add_bytes(d, cp, strlen(cp))); + if (append) + crypto_digest_add_bytes(d, append, strlen(append)); + crypto_digest_get_digest(d, digest_out, len_out); + crypto_digest_free(d); +} + +/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using + * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte + * result in <b>hmac_out</b>. Asserts on failure. + */ +void +crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len) +{ + /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ + tor_assert(key_len < INT_MAX); + tor_assert(msg_len < INT_MAX); + tor_assert(hmac_out); + unsigned char *rv = NULL; + rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, + (unsigned char*)hmac_out, NULL); + tor_assert(rv); +} diff --git a/src/lib/crypt_ops/crypto_ed25519.c b/src/lib/crypt_ops/crypto_ed25519.c index 0a442bb739..f242c7011e 100644 --- a/src/lib/crypt_ops/crypto_ed25519.c +++ b/src/lib/crypt_ops/crypto_ed25519.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Tor Project, Inc. */ +/* Copyright (c) 2013-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -226,7 +226,7 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong) int ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey) { - return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN); + return safe_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN); } /* Return a heap-allocated array that contains <b>msg</b> prefixed by the diff --git a/src/lib/crypt_ops/crypto_ed25519.h b/src/lib/crypt_ops/crypto_ed25519.h index 325b28244d..346de464e3 100644 --- a/src/lib/crypt_ops/crypto_ed25519.h +++ b/src/lib/crypt_ops/crypto_ed25519.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2012-2019, The Tor Project, Inc. */ +/* Copyright (c) 2012-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c index 84f73e5272..92b8b9372e 100644 --- a/src/lib/crypt_ops/crypto_format.c +++ b/src/lib/crypt_ops/crypto_format.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -104,7 +104,7 @@ crypto_read_tagged_contents_from_file(const char *fname, prefix[32] = 0; /* Check type, extract tag. */ if (strcmpstart(prefix, "== ") || strcmpend(prefix, " ==") || - ! tor_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) { + ! fast_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) { saved_errno = EINVAL; goto end; } @@ -131,20 +131,27 @@ crypto_read_tagged_contents_from_file(const char *fname, return r; } -/** Encode <b>pkey</b> as a base64-encoded string, without trailing "=" +/** Encode <b>pkey</b> as a base64-encoded string, including trailing "=" * characters, in the buffer <b>output</b>, which must have at least - * CURVE25519_BASE64_PADDED_LEN+1 bytes available. Return 0 on success, -1 on - * failure. */ -int + * CURVE25519_BASE64_PADDED_LEN+1 bytes available. + * Can not fail. + * + * Careful! CURVE25519_BASE64_PADDED_LEN is one byte longer than + * ED25519_BASE64_LEN. + */ +void curve25519_public_to_base64(char *output, const curve25519_public_key_t *pkey) { char buf[128]; - base64_encode(buf, sizeof(buf), - (const char*)pkey->public_key, CURVE25519_PUBKEY_LEN, 0); - buf[CURVE25519_BASE64_PADDED_LEN] = '\0'; + int n = base64_encode(buf, sizeof(buf), + (const char*)pkey->public_key, + CURVE25519_PUBKEY_LEN, 0); + /* These asserts should always succeed, unless there is a bug in + * base64_encode(). */ + tor_assert(n == CURVE25519_BASE64_PADDED_LEN); + tor_assert(buf[CURVE25519_BASE64_PADDED_LEN] == '\0'); memcpy(output, buf, CURVE25519_BASE64_PADDED_LEN+1); - return 0; } /** Try to decode a base64-encoded curve25519 public key from <b>input</b> @@ -181,8 +188,7 @@ ed25519_fmt(const ed25519_public_key_t *pkey) if (ed25519_public_key_is_zero(pkey)) { strlcpy(formatted, "<unset>", sizeof(formatted)); } else { - int r = ed25519_public_to_base64(formatted, pkey); - tor_assert(!r); + ed25519_public_to_base64(formatted, pkey); } } else { strlcpy(formatted, "<null>", sizeof(formatted)); @@ -202,28 +208,35 @@ ed25519_public_from_base64(ed25519_public_key_t *pkey, /** Encode the public key <b>pkey</b> into the buffer at <b>output</b>, * which must have space for ED25519_BASE64_LEN bytes of encoded key, - * plus one byte for a terminating NUL. Return 0 on success, -1 on failure. + * plus one byte for a terminating NUL. + * Can not fail. + * + * Careful! ED25519_BASE64_LEN is one byte shorter than + * CURVE25519_BASE64_PADDED_LEN. */ -int +void ed25519_public_to_base64(char *output, const ed25519_public_key_t *pkey) { - return digest256_to_base64(output, (const char *)pkey->pubkey); + digest256_to_base64(output, (const char *)pkey->pubkey); } /** Encode the signature <b>sig</b> into the buffer at <b>output</b>, * which must have space for ED25519_SIG_BASE64_LEN bytes of encoded signature, - * plus one byte for a terminating NUL. Return 0 on success, -1 on failure. + * plus one byte for a terminating NUL. + * Can not fail. */ -int +void ed25519_signature_to_base64(char *output, const ed25519_signature_t *sig) { char buf[256]; int n = base64_encode_nopad(buf, sizeof(buf), sig->sig, ED25519_SIG_LEN); + /* These asserts should always succeed, unless there is a bug in + * base64_encode_nopad(). */ tor_assert(n == ED25519_SIG_BASE64_LEN); + tor_assert(buf[ED25519_SIG_BASE64_LEN] == '\0'); memcpy(output, buf, ED25519_SIG_BASE64_LEN+1); - return 0; } /** Try to decode the string <b>input</b> into an ed25519 signature. On @@ -233,16 +246,11 @@ int ed25519_signature_from_base64(ed25519_signature_t *sig, const char *input) { - if (strlen(input) != ED25519_SIG_BASE64_LEN) return -1; - char buf[ED25519_SIG_BASE64_LEN+3]; - memcpy(buf, input, ED25519_SIG_BASE64_LEN); - buf[ED25519_SIG_BASE64_LEN+0] = '='; - buf[ED25519_SIG_BASE64_LEN+1] = '='; - buf[ED25519_SIG_BASE64_LEN+2] = 0; char decoded[128]; - int n = base64_decode(decoded, sizeof(decoded), buf, strlen(buf)); + int n = base64_decode(decoded, sizeof(decoded), input, + ED25519_SIG_BASE64_LEN); if (n < 0 || n != ED25519_SIG_LEN) return -1; memcpy(sig->sig, decoded, ED25519_SIG_LEN); @@ -250,24 +258,26 @@ ed25519_signature_from_base64(ed25519_signature_t *sig, return 0; } -/** Base64 encode DIGEST_LINE bytes from <b>digest</b>, remove the trailing = +/** Base64 encode DIGEST_LEN bytes from <b>digest</b>, remove the trailing = * characters, and store the nul-terminated result in the first - * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>. */ -/* XXXX unify with crypto_format.c code */ -int + * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>. + * Can not fail. */ +void digest_to_base64(char *d64, const char *digest) { char buf[256]; - base64_encode(buf, sizeof(buf), digest, DIGEST_LEN, 0); - buf[BASE64_DIGEST_LEN] = '\0'; + int n = base64_encode_nopad(buf, sizeof(buf), + (const uint8_t *)digest, DIGEST_LEN); + /* These asserts should always succeed, unless there is a bug in + * base64_encode_nopad(). */ + tor_assert(n == BASE64_DIGEST_LEN); + tor_assert(buf[BASE64_DIGEST_LEN] == '\0'); memcpy(d64, buf, BASE64_DIGEST_LEN+1); - return 0; } /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without * trailing newline or = characters), decode it and store the result in the * first DIGEST_LEN bytes at <b>digest</b>. */ -/* XXXX unify with crypto_format.c code */ int digest_from_base64(char *digest, const char *d64) { @@ -279,22 +289,24 @@ digest_from_base64(char *digest, const char *d64) /** Base64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the * trailing = characters, and store the nul-terminated result in the first - * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. */ - /* XXXX unify with crypto_format.c code */ -int + * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. + * Can not fail. */ +void digest256_to_base64(char *d64, const char *digest) { char buf[256]; - base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN, 0); - buf[BASE64_DIGEST256_LEN] = '\0'; + int n = base64_encode_nopad(buf, sizeof(buf), + (const uint8_t *)digest, DIGEST256_LEN); + /* These asserts should always succeed, unless there is a bug in + * base64_encode_nopad(). */ + tor_assert(n == BASE64_DIGEST256_LEN); + tor_assert(buf[BASE64_DIGEST256_LEN] == '\0'); memcpy(d64, buf, BASE64_DIGEST256_LEN+1); - return 0; } /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without * trailing newline or = characters), decode it and store the result in the * first DIGEST256_LEN bytes at <b>digest</b>. */ -/* XXXX unify with crypto_format.c code */ int digest256_from_base64(char *digest, const char *d64) { diff --git a/src/lib/crypt_ops/crypto_format.h b/src/lib/crypt_ops/crypto_format.h index fe852e6a61..91da137e1c 100644 --- a/src/lib/crypt_ops/crypto_format.h +++ b/src/lib/crypt_ops/crypto_format.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -33,18 +33,18 @@ ssize_t crypto_read_tagged_contents_from_file(const char *fname, int ed25519_public_from_base64(struct ed25519_public_key_t *pkey, const char *input); -int ed25519_public_to_base64(char *output, - const struct ed25519_public_key_t *pkey); +void ed25519_public_to_base64(char *output, + const struct ed25519_public_key_t *pkey); const char *ed25519_fmt(const struct ed25519_public_key_t *pkey); int ed25519_signature_from_base64(struct ed25519_signature_t *sig, const char *input); -int ed25519_signature_to_base64(char *output, - const struct ed25519_signature_t *sig); +void ed25519_signature_to_base64(char *output, + const struct ed25519_signature_t *sig); -int digest_to_base64(char *d64, const char *digest); +void digest_to_base64(char *d64, const char *digest); int digest_from_base64(char *digest, const char *d64); -int digest256_to_base64(char *d64, const char *digest); +void digest256_to_base64(char *d64, const char *digest); int digest256_from_base64(char *digest, const char *d64); #endif /* !defined(TOR_CRYPTO_FORMAT_H) */ diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c index fd2e701651..7b02820087 100644 --- a/src/lib/crypt_ops/crypto_hkdf.c +++ b/src/lib/crypt_ops/crypto_hkdf.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -25,7 +25,7 @@ #include <openssl/kdf.h> #define HAVE_OPENSSL_HKDF 1 #endif -#endif +#endif /* defined(ENABLE_OPENSSL) */ #include <string.h> @@ -109,7 +109,7 @@ crypto_expand_key_material_rfc5869_sha256_openssl( return 0; } -#else +#else /* !defined(HAVE_OPENSSL_HKDF) */ /** * Perform RFC5869 HKDF computation using our own legacy implementation. @@ -166,7 +166,7 @@ crypto_expand_key_material_rfc5869_sha256_legacy( memwipe(mac, 0, sizeof(mac)); return 0; } -#endif +#endif /* defined(HAVE_OPENSSL_HKDF) */ /** Expand some secret key material according to RFC5869, using SHA256 as the * underlying hash. The <b>key_in_len</b> bytes at <b>key_in</b> are the @@ -191,11 +191,11 @@ crypto_expand_key_material_rfc5869_sha256( salt_in_len, info_in, info_in_len, key_out, key_out_len); -#else +#else /* !defined(HAVE_OPENSSL_HKDF) */ return crypto_expand_key_material_rfc5869_sha256_legacy(key_in, key_in_len, salt_in, salt_in_len, info_in, info_in_len, key_out, key_out_len); -#endif +#endif /* defined(HAVE_OPENSSL_HKDF) */ } diff --git a/src/lib/crypt_ops/crypto_hkdf.h b/src/lib/crypt_ops/crypto_hkdf.h index 2994d18e3d..404f548774 100644 --- a/src/lib/crypt_ops/crypto_hkdf.h +++ b/src/lib/crypt_ops/crypto_hkdf.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c index 329c264af6..a836bd8645 100644 --- a/src/lib/crypt_ops/crypto_init.c +++ b/src/lib/crypt_ops/crypto_init.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,6 +12,8 @@ #include "orconfig.h" +#define CRYPTO_PRIVATE + #include "lib/crypt_ops/crypto_init.h" #include "lib/crypt_ops/crypto_curve25519.h" @@ -20,8 +22,14 @@ #include "lib/crypt_ops/crypto_openssl_mgt.h" #include "lib/crypt_ops/crypto_nss_mgt.h" #include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_sys.h" +#include "lib/crypt_ops/crypto_options_st.h" +#include "lib/conf/conftypes.h" +#include "lib/log/util_bug.h" + +#include "lib/subsys/subsys.h" -#include "siphash.h" +#include "ext/siphash.h" /** Boolean: has our crypto library been initialized? (early phase) */ static int crypto_early_initialized_ = 0; @@ -66,6 +74,8 @@ crypto_early_init(void) if (crypto_init_siphash_key() < 0) return -1; + crypto_rand_fast_init(); + curve25519_init(); ed25519_init(); } @@ -92,7 +102,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir) (void)useAccel; (void)accelName; (void)accelDir; -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS if (crypto_nss_late_init() < 0) return -1; @@ -108,6 +118,7 @@ crypto_thread_cleanup(void) #ifdef ENABLE_OPENSSL crypto_openssl_thread_cleanup(); #endif + destroy_thread_fast_rng(); } /** @@ -126,6 +137,8 @@ crypto_global_cleanup(void) crypto_nss_global_cleanup(); #endif + crypto_rand_fast_shutdown(); + crypto_early_initialized_ = 0; crypto_global_initialized_ = 0; have_seeded_siphash = 0; @@ -142,6 +155,12 @@ crypto_prefork(void) #ifdef ENABLE_NSS crypto_nss_prefork(); #endif + /* It is not safe to share a fast_rng object across a fork boundary unless + * we actually have zero-on-fork support in map_anon.c. If we have + * drop-on-fork support, we will crash; if we have neither, we will yield + * a copy of the parent process's rng, which is scary and insecure. + */ + destroy_thread_fast_rng(); } /** Run operations that the crypto library requires to be happy again @@ -202,3 +221,111 @@ tor_is_using_nss(void) return 0; #endif } + +static int +subsys_crypto_initialize(void) +{ + if (crypto_early_init() < 0) + return -1; + crypto_dh_init(); + return 0; +} + +static void +subsys_crypto_shutdown(void) +{ + crypto_global_cleanup(); +} + +static void +subsys_crypto_prefork(void) +{ + crypto_prefork(); +} + +static void +subsys_crypto_postfork(void) +{ + crypto_postfork(); +} + +static void +subsys_crypto_thread_cleanup(void) +{ + crypto_thread_cleanup(); +} + +/** Magic number for crypto_options_t. */ +#define CRYPTO_OPTIONS_MAGIC 0x68757368 + +/** + * Return 0 if <b>arg</b> is a valid crypto_options_t. Otherwise return -1 + * and set *<b>msg_out</b> to a freshly allocated error string. + **/ +static int +crypto_options_validate(const void *arg, char **msg_out) +{ + const crypto_options_t *opt = arg; + tor_assert(opt->magic == CRYPTO_OPTIONS_MAGIC); + tor_assert(msg_out); + + if (opt->AccelDir && !opt->AccelName) { + *msg_out = tor_strdup("Can't use hardware crypto accelerator dir " + "without engine name."); + return -1; + } + + return 0; +} + +/* Declare the options field table for crypto_options */ +#define CONF_CONTEXT LL_TABLE +#include "lib/crypt_ops/crypto_options.inc" +#undef CONF_CONTEXT + +/** + * Declares the configuration options for this module. + **/ +static const config_format_t crypto_options_fmt = { + .size = sizeof(crypto_options_t), + .magic = { "crypto_options_t", + CRYPTO_OPTIONS_MAGIC, + offsetof(crypto_options_t, magic) }, + .vars = crypto_options_t_vars, + .validate_fn = crypto_options_validate, +}; + +/** + * Invoked from subsysmgr.c when a new set of options arrives. + **/ +static int +crypto_set_options(void *arg) +{ + const crypto_options_t *options = arg; + const bool hardware_accel = options->HardwareAccel || options->AccelName; + + // This call already checks for crypto_global_initialized_, so it + // will only initialize the subsystem the first time it's called. + if (crypto_global_init(hardware_accel, + options->AccelName, + options->AccelDir)) { + log_err(LD_BUG, "Unable to initialize the crypto subsystem. Exiting."); + return -1; + } + return 0; +} + +const struct subsys_fns_t sys_crypto = { + .name = "crypto", + SUBSYS_DECLARE_LOCATION(), + .supported = true, + .level = -60, + .initialize = subsys_crypto_initialize, + .shutdown = subsys_crypto_shutdown, + .prefork = subsys_crypto_prefork, + .postfork = subsys_crypto_postfork, + .thread_cleanup = subsys_crypto_thread_cleanup, + + .options_format = &crypto_options_fmt, + .set_options = crypto_set_options, +}; diff --git a/src/lib/crypt_ops/crypto_init.h b/src/lib/crypt_ops/crypto_init.h index 540d08eb56..b11e2e34bf 100644 --- a/src/lib/crypt_ops/crypto_init.h +++ b/src/lib/crypt_ops/crypto_init.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -33,4 +33,4 @@ const char *crypto_get_header_version_string(void); int tor_is_using_nss(void); -#endif /* !defined(TOR_CRYPTO_H) */ +#endif /* !defined(TOR_CRYPTO_INIT_H) */ diff --git a/src/lib/crypt_ops/crypto_nss_mgt.c b/src/lib/crypt_ops/crypto_nss_mgt.c index 0179126e38..d82e51249c 100644 --- a/src/lib/crypt_ops/crypto_nss_mgt.c +++ b/src/lib/crypt_ops/crypto_nss_mgt.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,7 +16,7 @@ #include "lib/log/util_bug.h" #include "lib/string/printf.h" -DISABLE_GCC_WARNING(strict-prototypes) +DISABLE_GCC_WARNING("-Wstrict-prototypes") #include <nss.h> #include <pk11func.h> #include <ssl.h> @@ -24,7 +24,7 @@ DISABLE_GCC_WARNING(strict-prototypes) #include <prerror.h> #include <prtypes.h> #include <prinit.h> -ENABLE_GCC_WARNING(strict-prototypes) +ENABLE_GCC_WARNING("-Wstrict-prototypes") const char * crypto_nss_get_version_str(void) diff --git a/src/lib/crypt_ops/crypto_nss_mgt.h b/src/lib/crypt_ops/crypto_nss_mgt.h index 72fd2a1229..8686b1b8aa 100644 --- a/src/lib/crypt_ops/crypto_nss_mgt.h +++ b/src/lib/crypt_ops/crypto_nss_mgt.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,6 +29,6 @@ void crypto_nss_global_cleanup(void); void crypto_nss_prefork(void); void crypto_nss_postfork(void); -#endif +#endif /* defined(ENABLE_NSS) */ -#endif /* !defined(TOR_CRYPTO_NSS_H) */ +#endif /* !defined(TOR_CRYPTO_NSS_MGT_H) */ diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c index 2186d2a939..4cacb3dd98 100644 --- a/src/lib/crypt_ops/crypto_ope.c +++ b/src/lib/crypt_ops/crypto_ope.c @@ -1,8 +1,9 @@ -/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** - * A rudimentary order-preserving encryption scheme. + * @file crypto_ope.c + * @brief A rudimentary order-preserving encryption scheme. * * To compute the encryption of N, this scheme uses an AES-CTR stream to * generate M-byte values, and adds the first N of them together. (+1 each to @@ -57,9 +58,9 @@ ope_val_from_le(ope_val_t x) ((x) >> 8) | (((x)&0xff) << 8); } -#else +#else /* !defined(WORDS_BIGENDIAN) */ #define ope_val_from_le(x) (x) -#endif +#endif /* defined(WORDS_BIGENDIAN) */ /** * Return a new AES256-CTR stream cipher object for <b>ope</b>, ready to yield @@ -143,7 +144,7 @@ crypto_ope_new(const uint8_t *key) return ope; } -/** Free all storage held in <>ope</b>. */ +/** Free all storage held in <b>ope</b>. */ void crypto_ope_free_(crypto_ope_t *ope) { diff --git a/src/lib/crypt_ops/crypto_ope.h b/src/lib/crypt_ops/crypto_ope.h index 610d956335..7498ea6a2e 100644 --- a/src/lib/crypt_ops/crypto_ope.h +++ b/src/lib/crypt_ops/crypto_ope.h @@ -1,6 +1,11 @@ -/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * @file crypto_ope.h + * @brief header for crypto_ope.c + **/ + #ifndef CRYPTO_OPE_H #define CRYPTO_OPE_H @@ -37,10 +42,10 @@ void crypto_ope_free_(crypto_ope_t *ope); uint64_t crypto_ope_encrypt(const crypto_ope_t *ope, int plaintext); #ifdef CRYPTO_OPE_PRIVATE -struct aes_cnt_cipher; -STATIC struct aes_cnt_cipher *ope_get_cipher(const crypto_ope_t *ope, +struct aes_cnt_cipher_t; +STATIC struct aes_cnt_cipher_t *ope_get_cipher(const crypto_ope_t *ope, uint32_t initial_idx); -STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher *c, size_t n); -#endif +STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher_t *c, size_t n); +#endif /* defined(CRYPTO_OPE_PRIVATE) */ -#endif +#endif /* !defined(CRYPTO_OPE_H) */ diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c index e7d6084f6c..065cbca1cc 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.c +++ b/src/lib/crypt_ops/crypto_openssl_mgt.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -21,7 +21,7 @@ #include "lib/testsupport/testsupport.h" #include "lib/thread/threads.h" -DISABLE_GCC_WARNING(redundant-decls) +DISABLE_GCC_WARNING("-Wredundant-decls") #include <openssl/err.h> #include <openssl/rsa.h> @@ -36,7 +36,7 @@ DISABLE_GCC_WARNING(redundant-decls) #include <openssl/crypto.h> #include <openssl/ssl.h> -ENABLE_GCC_WARNING(redundant-decls) +ENABLE_GCC_WARNING("-Wredundant-decls") #include <string.h> @@ -130,10 +130,12 @@ crypto_openssl_get_header_version_str(void) return crypto_openssl_header_version_str; } +#ifndef COCCI #ifndef OPENSSL_THREADS -#error OpenSSL has been built without thread support. Tor requires an \ - OpenSSL library with thread support enabled. +#error "OpenSSL has been built without thread support. Tor requires an \ + OpenSSL library with thread support enabled." #endif +#endif /* !defined(COCCI) */ #ifndef NEW_THREAD_API /** Helper: OpenSSL uses this callback to manipulate mutexes. */ @@ -185,6 +187,10 @@ crypto_openssl_free_all(void) tor_free(crypto_openssl_version_str); tor_free(crypto_openssl_header_version_str); + /* Destroying a locked mutex is undefined behaviour. This mutex may be + * locked, because multiple threads can access it. But we need to destroy + * it, otherwise re-initialisation will trigger undefined behaviour. + * See #31735 for details. */ #ifndef NEW_THREAD_API if (n_openssl_mutexes_) { int n = n_openssl_mutexes_; @@ -209,10 +215,10 @@ crypto_openssl_early_init(void) OPENSSL_INIT_LOAD_CRYPTO_STRINGS | OPENSSL_INIT_ADD_ALL_CIPHERS | OPENSSL_INIT_ADD_ALL_DIGESTS, NULL); -#else +#else /* !defined(OPENSSL_1_1_API) */ ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); -#endif +#endif /* defined(OPENSSL_1_1_API) */ setup_openssl_threading(); @@ -278,8 +284,14 @@ log_engine(const char *fn, ENGINE *e) } #endif /* !defined(DISABLE_ENGINES) */ -/** Initialize engines for openssl (if enabled). */ -static void +/** Initialize engines for openssl (if enabled). Load all the built-in + * engines, along with the one called <b>accelName</b> (which may be NULL). + * If <b>accelName</b> is prefixed with "!", then it is required: return -1 + * if it can't be loaded. Otherwise return 0. + * + * If <b>accelDir</b> is not NULL, it is the path from which the engine should + * be loaded. */ +static int crypto_openssl_init_engines(const char *accelName, const char *accelDir) { @@ -287,7 +299,13 @@ crypto_openssl_init_engines(const char *accelName, (void)accelName; (void)accelDir; log_warn(LD_CRYPTO, "No OpenSSL hardware acceleration support enabled."); -#else + if (accelName && accelName[0] == '!') { + log_warn(LD_CRYPTO, "Unable to load required dynamic OpenSSL engine " + "\"%s\".", accelName+1); + return -1; + } + return 0; +#else /* !defined(DISABLE_ENGINES) */ ENGINE *e = NULL; log_info(LD_CRYPTO, "Initializing OpenSSL engine support."); @@ -295,6 +313,9 @@ crypto_openssl_init_engines(const char *accelName, ENGINE_register_all_complete(); if (accelName) { + const bool required = accelName[0] == '!'; + if (required) + ++accelName; if (accelDir) { log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\"" " via path \"%s\".", accelName, accelDir); @@ -305,8 +326,11 @@ crypto_openssl_init_engines(const char *accelName, e = ENGINE_by_id(accelName); } if (!e) { - log_warn(LD_CRYPTO, "Unable to load dynamic OpenSSL engine \"%s\".", + log_warn(LD_CRYPTO, "Unable to load %sdynamic OpenSSL engine \"%s\".", + required?"required ":"", accelName); + if (required) + return -1; } else { log_info(LD_CRYPTO, "Loaded dynamic OpenSSL engine \"%s\".", accelName); @@ -343,6 +367,7 @@ crypto_openssl_init_engines(const char *accelName, #ifdef NID_aes_256_gcm log_engine("AES-256-GCM", ENGINE_get_cipher_engine(NID_aes_256_gcm)); #endif + return 0; #endif /* defined(DISABLE_ENGINES) */ } @@ -353,7 +378,8 @@ crypto_openssl_late_init(int useAccel, const char *accelName, const char *accelDir) { if (useAccel > 0) { - crypto_openssl_init_engines(accelName, accelDir); + if (crypto_openssl_init_engines(accelName, accelDir) < 0) + return -1; } else { log_info(LD_CRYPTO, "NOT using OpenSSL engine support."); } @@ -382,7 +408,7 @@ crypto_openssl_thread_cleanup(void) void crypto_openssl_global_cleanup(void) { - #ifndef OPENSSL_1_1_API +#ifndef OPENSSL_1_1_API EVP_cleanup(); #endif #ifndef NEW_THREAD_API diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.h b/src/lib/crypt_ops/crypto_openssl_mgt.h index 8dbadfc9d2..c67ab6467c 100644 --- a/src/lib/crypt_ops/crypto_openssl_mgt.h +++ b/src/lib/crypt_ops/crypto_openssl_mgt.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -83,6 +83,6 @@ int crypto_openssl_late_init(int useAccel, const char *accelName, void crypto_openssl_thread_cleanup(void); void crypto_openssl_global_cleanup(void); -#endif /* ENABLE_OPENSSL */ +#endif /* defined(ENABLE_OPENSSL) */ #endif /* !defined(TOR_CRYPTO_OPENSSL_H) */ diff --git a/src/lib/crypt_ops/crypto_options.inc b/src/lib/crypt_ops/crypto_options.inc new file mode 100644 index 0000000000..5bee0daacd --- /dev/null +++ b/src/lib/crypt_ops/crypto_options.inc @@ -0,0 +1,19 @@ + +/** + * @file crypto_options.inc + * @brief Declare configuration options for the crypto_ops module. + **/ + +/** Holds configuration about our cryptography options. */ +BEGIN_CONF_STRUCT(crypto_options_t) + +/** Should we enable extra OpenSSL hardware acceleration (where available)? */ +CONF_VAR(HardwareAccel, BOOL, CFLG_IMMUTABLE, "0") + +/** Optional OpenSSL hardware-acceleration engine name */ +CONF_VAR(AccelName, STRING, CFLG_IMMUTABLE, NULL) + +/** Optional OpenSSL hardware-acceleration engine search directory. */ +CONF_VAR(AccelDir, FILENAME, CFLG_IMMUTABLE, NULL) + +END_CONF_STRUCT(crypto_options_t) diff --git a/src/lib/crypt_ops/crypto_options_st.h b/src/lib/crypt_ops/crypto_options_st.h new file mode 100644 index 0000000000..a453c451fe --- /dev/null +++ b/src/lib/crypt_ops/crypto_options_st.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file crypto_options_st.h + * @brief Header for lib/crypt_ops/crypto_options_st.c + **/ + +#ifndef TOR_LIB_CRYPT_OPS_CRYPTO_OPTIONS_ST_H +#define TOR_LIB_CRYPT_OPS_CRYPTO_OPTIONS_ST_H + +#include "lib/conf/confdecl.h" + +#define CONF_CONTEXT STRUCT +#include "lib/crypt_ops/crypto_options.inc" +#undef CONF_CONTEXT + +typedef struct crypto_options_t crypto_options_t; + +#endif /* !defined(TOR_LIB_CRYPT_OPS_CRYPTO_OPTIONS_ST_H) */ diff --git a/src/lib/crypt_ops/crypto_pwbox.c b/src/lib/crypt_ops/crypto_pwbox.c index a8db08f7b7..bfad27d9fc 100644 --- a/src/lib/crypt_ops/crypto_pwbox.c +++ b/src/lib/crypt_ops/crypto_pwbox.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2019, The Tor Project, Inc. */ +/* Copyright (c) 2014-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_pwbox.h b/src/lib/crypt_ops/crypto_pwbox.h index 5a26889fb2..0a85b1230a 100644 --- a/src/lib/crypt_ops/crypto_pwbox.h +++ b/src/lib/crypt_ops/crypto_pwbox.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014-2019, The Tor Project, Inc. */ +/* Copyright (c) 2014-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c index 206929d6b3..ce6f21dbb4 100644 --- a/src/lib/crypt_ops/crypto_rand.c +++ b/src/lib/crypt_ops/crypto_rand.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,7 +11,6 @@ * number generators, and working with randomness. **/ -#ifndef CRYPTO_RAND_PRIVATE #define CRYPTO_RAND_PRIVATE #include "lib/crypt_ops/crypto_rand.h" @@ -37,17 +36,18 @@ #include "lib/defs/digest_sizes.h" #include "lib/crypt_ops/crypto_digest.h" +#include "lib/ctime/di_ops.h" #ifdef ENABLE_NSS #include "lib/crypt_ops/crypto_nss_mgt.h" #endif #ifdef ENABLE_OPENSSL -DISABLE_GCC_WARNING(redundant-decls) +DISABLE_GCC_WARNING("-Wredundant-decls") #include <openssl/rand.h> #include <openssl/sha.h> -ENABLE_GCC_WARNING(redundant-decls) -#endif +ENABLE_GCC_WARNING("-Wredundant-decls") +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS #include <pk11pub.h> @@ -248,7 +248,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) (void)out; (void)out_len; return -1; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ static const char *filenames[] = { "/dev/srandom", "/dev/urandom", "/dev/random", NULL }; @@ -315,7 +315,7 @@ crypto_strongest_rand_raw(uint8_t *out, size_t out_len) } } - if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len)) + if ((out_len < sanity_min_size) || !safe_mem_is_zero((char*)out, out_len)) return 0; } @@ -419,7 +419,7 @@ crypto_seed_openssl_rng(void) else return -1; } -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS /** @@ -442,7 +442,7 @@ crypto_seed_nss_rng(void) return load_entropy_ok ? 0 : -1; } -#endif +#endif /* defined(ENABLE_NSS) */ /** * Seed the RNG for any and all crypto libraries that we're using with bytes @@ -520,7 +520,7 @@ crypto_rand_unmocked(char *to, size_t n) #undef BUFLEN } -#else +#else /* !defined(ENABLE_NSS) */ int r = RAND_bytes((unsigned char*)to, (int)n); /* We consider a PRNG failure non-survivable. Let's assert so that we get a * stack trace about where it happened. @@ -530,111 +530,14 @@ crypto_rand_unmocked(char *to, size_t n) } /** - * Return a pseudorandom integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and - * INT_MAX+1, inclusive. + * Draw an unsigned 32-bit integer uniformly at random. */ -int -crypto_rand_int(unsigned int max) -{ - unsigned int val; - unsigned int cutoff; - tor_assert(max <= ((unsigned int)INT_MAX)+1); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT_MAX - (UINT_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** - * Return a pseudorandom integer, chosen uniformly from the values i such - * that min <= i < max. - * - * <b>min</b> MUST be in range [0, <b>max</b>). - * <b>max</b> MUST be in range (min, INT_MAX]. - **/ -int -crypto_rand_int_range(unsigned int min, unsigned int max) -{ - tor_assert(min < max); - tor_assert(max <= INT_MAX); - - /* The overflow is avoided here because crypto_rand_int() returns a value - * between 0 and (max - min) inclusive. */ - return min + crypto_rand_int(max - min); -} - -/** - * As crypto_rand_int_range, but supports uint64_t. - **/ -uint64_t -crypto_rand_uint64_range(uint64_t min, uint64_t max) +uint32_t +crypto_rand_u32(void) { - tor_assert(min < max); - return min + crypto_rand_uint64(max - min); -} - -/** - * As crypto_rand_int_range, but supports time_t. - **/ -time_t -crypto_rand_time_range(time_t min, time_t max) -{ - tor_assert(min < max); - return min + (time_t)crypto_rand_uint64(max - min); -} - -/** - * Return a pseudorandom 64-bit integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. - **/ -uint64_t -crypto_rand_uint64(uint64_t max) -{ - uint64_t val; - uint64_t cutoff; - tor_assert(max < UINT64_MAX); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT64_MAX - (UINT64_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** - * Return a pseudorandom double d, chosen uniformly from the range - * 0.0 <= d < 1.0. - **/ -double -crypto_rand_double(void) -{ - /* We just use an unsigned int here; we don't really care about getting - * more than 32 bits of resolution */ - unsigned int u; - crypto_rand((char*)&u, sizeof(u)); -#if SIZEOF_INT == 4 -#define UINT_MAX_AS_DOUBLE 4294967296.0 -#elif SIZEOF_INT == 8 -#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 -#else -#error SIZEOF_INT is neither 4 nor 8 -#endif /* SIZEOF_INT == 4 || ... */ - return ((double)u) / UINT_MAX_AS_DOUBLE; + uint32_t rand; + crypto_rand((void*)&rand, sizeof(rand)); + return rand; } /** @@ -724,8 +627,6 @@ crypto_force_rand_ssleay(void) RAND_set_rand_method(default_method); return 1; } -#endif +#endif /* defined(ENABLE_OPENSSL) */ return 0; } - -#endif /* !defined(CRYPTO_RAND_PRIVATE) */ diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h index 86fa20faa3..99aff5d4a9 100644 --- a/src/lib/crypt_ops/crypto_rand.h +++ b/src/lib/crypt_ops/crypto_rand.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,6 +16,7 @@ #include "lib/cc/compat_compiler.h" #include "lib/cc/torint.h" #include "lib/testsupport/testsupport.h" +#include "lib/malloc/malloc.h" /* random numbers */ int crypto_seed_rng(void) ATTR_WUR; @@ -24,9 +25,11 @@ void crypto_rand_unmocked(char *to, size_t n); void crypto_strongest_rand(uint8_t *out, size_t out_len); MOCK_DECL(void,crypto_strongest_rand_,(uint8_t *out, size_t out_len)); int crypto_rand_int(unsigned int max); +unsigned crypto_rand_uint(unsigned limit); int crypto_rand_int_range(unsigned int min, unsigned int max); uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); time_t crypto_rand_time_range(time_t min, time_t max); +uint32_t crypto_rand_u32(void); uint64_t crypto_rand_uint64(uint64_t max); double crypto_rand_double(void); struct tor_weak_rng_t; @@ -40,6 +43,61 @@ void *smartlist_choose(const struct smartlist_t *sl); void smartlist_shuffle(struct smartlist_t *sl); int crypto_force_rand_ssleay(void); +/** + * A fast PRNG, for use when the PRNG provided by our crypto library isn't + * fast enough. This one _should_ be cryptographically strong, but + * has seen less auditing than the PRNGs in OpenSSL and NSS. Use with + * caution. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +typedef struct crypto_fast_rng_t crypto_fast_rng_t; +/** + * Number of bytes used to seed a crypto_rand_fast_t. + **/ +crypto_fast_rng_t *crypto_fast_rng_new(void); +#define CRYPTO_FAST_RNG_SEED_LEN 48 +crypto_fast_rng_t *crypto_fast_rng_new_from_seed(const uint8_t *seed); +void crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n); +void crypto_fast_rng_free_(crypto_fast_rng_t *); +#define crypto_fast_rng_free(c) \ + FREE_AND_NULL(crypto_fast_rng_t, crypto_fast_rng_free_, (c)) + +unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit); +uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit); +uint32_t crypto_fast_rng_get_u32(crypto_fast_rng_t *rng); +uint64_t crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng, + uint64_t min, uint64_t max); +double crypto_fast_rng_get_double(crypto_fast_rng_t *rng); + +/** + * Using the fast_rng <b>rng</b>, yield true with probability + * 1/<b>n</b>. Otherwise yield false. + * + * <b>n</b> must not be zero. + **/ +#define crypto_fast_rng_one_in_n(rng, n) \ + (0 == (crypto_fast_rng_get_uint((rng), (n)))) + +crypto_fast_rng_t *get_thread_fast_rng(void); + +#ifdef CRYPTO_PRIVATE +/* These are only used from crypto_init.c */ +void destroy_thread_fast_rng(void); +void crypto_rand_fast_init(void); +void crypto_rand_fast_shutdown(void); +#endif /* defined(CRYPTO_PRIVATE) */ + +#if defined(TOR_UNIT_TESTS) +/* Used for white-box testing */ +size_t crypto_fast_rng_get_bytes_used_per_stream(void); +/* For deterministic prng implementations */ +void crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng); +/* To override the prng for testing. */ +crypto_fast_rng_t *crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng); +#endif /* defined(TOR_UNIT_TESTS) */ + #ifdef CRYPTO_RAND_PRIVATE STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c new file mode 100644 index 0000000000..172ea48bdb --- /dev/null +++ b/src/lib/crypt_ops/crypto_rand_fast.c @@ -0,0 +1,438 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand_fast.c + * + * \brief A fast strong PRNG for use when our underlying cryptographic + * library's PRNG isn't fast enough. + **/ + +/* This library is currently implemented to use the same implementation + * technique as libottery, using AES-CTR-256 as our underlying stream cipher. + * It's backtracking-resistant immediately, and prediction-resistant after + * a while. + * + * Here's how it works: + * + * We generate pseudorandom bytes using AES-CTR-256. We generate BUFLEN bytes + * at a time. When we do this, we keep the first SEED_LEN bytes as the key + * and the IV for our next invocation of AES_CTR, and yield the remaining + * BUFLEN - SEED_LEN bytes to the user as they invoke the PRNG. As we yield + * bytes to the user, we clear them from the buffer. + * + * After we have refilled the buffer RESEED_AFTER times, we mix in an + * additional SEED_LEN bytes from our strong PRNG into the seed. + * + * If the user ever asks for a huge number of bytes at once, we pull SEED_LEN + * bytes from the PRNG and use them with our stream cipher to fill the user's + * request. + */ + +#define CRYPTO_PRIVATE + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/crypt_ops/crypto_cipher.h" +#include "lib/crypt_ops/crypto_digest.h" +#include "lib/crypt_ops/crypto_util.h" +#include "lib/intmath/cmp.h" +#include "lib/cc/ctassert.h" +#include "lib/malloc/map_anon.h" +#include "lib/thread/threads.h" + +#include "lib/log/util_bug.h" + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#include <string.h> + +#ifdef NOINHERIT_CAN_FAIL +#define CHECK_PID +#endif + +#ifdef CHECK_PID +#define PID_FIELD_LEN sizeof(pid_t) +#else +#define PID_FIELD_LEN 0 +#endif + +/* Alias for CRYPTO_FAST_RNG_SEED_LEN to make our code shorter. + */ +#define SEED_LEN (CRYPTO_FAST_RNG_SEED_LEN) + +/* The amount of space that we mmap for a crypto_fast_rng_t. + */ +#define MAPLEN 4096 + +/* The number of random bytes that we can yield to the user after each + * time we fill a crypto_fast_rng_t's buffer. + */ +#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN - PID_FIELD_LEN) + +/* The number of buffer refills after which we should fetch more + * entropy from crypto_strongest_rand(). + */ +#define RESEED_AFTER 16 + +/* The length of the stream cipher key we will use for the PRNG, in bytes. + */ +#define KEY_LEN (CRYPTO_FAST_RNG_SEED_LEN - CIPHER_IV_LEN) +/* The length of the stream cipher key we will use for the PRNG, in bits. + */ +#define KEY_BITS (KEY_LEN * 8) + +/* Make sure that we have a key length we can actually use with AES. */ +CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256); + +struct crypto_fast_rng_t { + /** How many more fills does this buffer have before we should mix + * in the output of crypto_strongest_rand()? + * + * This value may be negative if unit tests are enabled. If so, it + * indicates that we should never mix in extra data from + * crypto_strongest_rand(). + */ + int16_t n_till_reseed; + /** How many bytes are remaining in cbuf_t.bytes? */ + uint16_t bytes_left; +#ifdef CHECK_PID + /** Which process owns this fast_rng? If this value is zero, we do not + * need to test the owner. */ + pid_t owner; +#endif + struct cbuf_t { + /** The seed (key and IV) that we will use the next time that we refill + * cbuf_t. */ + uint8_t seed[SEED_LEN]; + /** + * Bytes that we are yielding to the user. The next byte to be + * yielded is at bytes[BUFLEN-bytes_left]; all other bytes in this + * array are set to zero. + */ + uint8_t bytes[BUFLEN]; + } buf; +}; + +/* alignof(uint8_t) should be 1, so there shouldn't be any padding in cbuf_t. + */ +CTASSERT(sizeof(struct cbuf_t) == BUFLEN+SEED_LEN); +/* We're trying to fit all of the RNG state into a nice mmapable chunk. + */ +CTASSERT(sizeof(crypto_fast_rng_t) <= MAPLEN); + +/** + * Initialize and return a new fast PRNG, using a strong random seed. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, use crypto_rand(), or wrap this in a mutex. + **/ +crypto_fast_rng_t * +crypto_fast_rng_new(void) +{ + uint8_t seed[SEED_LEN]; + crypto_strongest_rand(seed, sizeof(seed)); + crypto_fast_rng_t *result = crypto_fast_rng_new_from_seed(seed); + memwipe(seed, 0, sizeof(seed)); + return result; +} + +/** + * Initialize and return a new fast PRNG, using a seed value specified + * in <b>seed</b>. This value must be CRYPTO_FAST_RNG_SEED_LEN bytes + * long. + * + * Note that this object is NOT thread-safe. If you need a thread-safe + * prng, you should probably look at get_thread_fast_rng(). Alternatively, + * use crypto_rand(), wrap this in a mutex. + **/ +crypto_fast_rng_t * +crypto_fast_rng_new_from_seed(const uint8_t *seed) +{ + unsigned inherit = INHERIT_RES_KEEP; + /* We try to allocate this object as securely as we can, to avoid + * having it get dumped, swapped, or shared after fork. + */ + crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result), + ANONMAP_PRIVATE | ANONMAP_NOINHERIT, + &inherit); + memcpy(result->buf.seed, seed, SEED_LEN); + /* Causes an immediate refill once the user asks for data. */ + result->bytes_left = 0; + result->n_till_reseed = RESEED_AFTER; +#ifdef CHECK_PID + if (inherit == INHERIT_RES_KEEP) { + /* This value will neither be dropped nor zeroed after fork, so we need to + * check our pid to make sure we are not sharing it across a fork. This + * can be expensive if the pid value isn't cached, sadly. + */ + result->owner = getpid(); + } +#elif defined(_WIN32) + /* Windows can't fork(), so there's no need to noinherit. */ +#else + /* We decided above that noinherit would always do _something_. Assert here + * that we were correct. */ + tor_assertf(inherit != INHERIT_RES_KEEP, + "We failed to create a non-inheritable memory region, even " + "though we believed such a failure to be impossible! This is " + "probably a bug in Tor support for your platform; please report " + "it."); +#endif /* defined(CHECK_PID) || ... */ + return result; +} + +#ifdef TOR_UNIT_TESTS +/** + * Unit tests only: prevent a crypto_fast_rng_t from ever mixing in more + * entropy. + */ +void +crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng) +{ + rng->n_till_reseed = -1; +} +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Helper: create a crypto_cipher_t object from SEED_LEN bytes of + * input. The first KEY_LEN bytes are used as the stream cipher's key, + * and the remaining CIPHER_IV_LEN bytes are used as its IV. + **/ +static inline crypto_cipher_t * +cipher_from_seed(const uint8_t *seed) +{ + return crypto_cipher_new_with_iv_and_bits(seed, seed+KEY_LEN, KEY_BITS); +} + +/** + * Helper: mix additional entropy into <b>rng</b> by using our XOF to mix the + * old value for the seed with some additional bytes from + * crypto_strongest_rand(). + **/ +static void +crypto_fast_rng_add_entopy(crypto_fast_rng_t *rng) +{ + crypto_xof_t *xof = crypto_xof_new(); + crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN); + { + uint8_t seedbuf[SEED_LEN]; + crypto_strongest_rand(seedbuf, SEED_LEN); + crypto_xof_add_bytes(xof, seedbuf, SEED_LEN); + memwipe(seedbuf, 0, SEED_LEN); + } + crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN); + crypto_xof_free(xof); +} + +/** + * Helper: refill the seed bytes and output buffer of <b>rng</b>, using + * the input seed bytes as input (key and IV) for the stream cipher. + * + * If the n_till_reseed counter has reached zero, mix more random bytes into + * the seed before refilling the buffer. + **/ +static void +crypto_fast_rng_refill(crypto_fast_rng_t *rng) +{ + rng->n_till_reseed--; + if (rng->n_till_reseed == 0) { + /* It's time to reseed the RNG. */ + crypto_fast_rng_add_entopy(rng); + rng->n_till_reseed = RESEED_AFTER; + } else if (rng->n_till_reseed < 0) { +#ifdef TOR_UNIT_TESTS + /* Reseeding is disabled for testing; never do it on this prng. */ + rng->n_till_reseed = -1; +#else + /* If testing is disabled, this shouldn't be able to become negative. */ + tor_assert_unreached(); +#endif /* defined(TOR_UNIT_TESTS) */ + } + /* Now fill rng->buf with output from our stream cipher, initialized from + * that seed value. */ + crypto_cipher_t *c = cipher_from_seed(rng->buf.seed); + memset(&rng->buf, 0, sizeof(rng->buf)); + crypto_cipher_crypt_inplace(c, (char*)&rng->buf, sizeof(rng->buf)); + crypto_cipher_free(c); + + rng->bytes_left = sizeof(rng->buf.bytes); +} + +/** + * Release all storage held by <b>rng</b>. + **/ +void +crypto_fast_rng_free_(crypto_fast_rng_t *rng) +{ + if (!rng) + return; + memwipe(rng, 0, sizeof(*rng)); + tor_munmap_anonymous(rng, sizeof(*rng)); +} + +/** + * Helper: extract bytes from the PRNG, refilling it as necessary. Does not + * optimize the case when the user has asked for a huge output. + **/ +static void +crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out, + const size_t n) +{ +#ifdef CHECK_PID + if (rng->owner) { + /* Note that we only need to do this check when we have owner set: that + * is, when our attempt to block inheriting failed, and the result was + * INHERIT_RES_KEEP. + * + * If the result was INHERIT_RES_DROP, then any attempt to access the rng + * memory after forking will crash. + * + * If the result was INHERIT_RES_ZERO, then forking will set the bytes_left + * and n_till_reseed fields to zero. This function will call + * crypto_fast_rng_refill(), which will in turn reseed the PRNG. + * + * So we only need to do this test in the case when mmap_anonymous() + * returned INHERIT_KEEP. We avoid doing it needlessly, since getpid() is + * often a system call, and that can be slow. + */ + tor_assert(rng->owner == getpid()); + } +#endif /* defined(CHECK_PID) */ + + size_t bytes_to_yield = n; + + while (bytes_to_yield) { + if (rng->bytes_left == 0) + crypto_fast_rng_refill(rng); + + const size_t to_copy = MIN(rng->bytes_left, bytes_to_yield); + + tor_assert(sizeof(rng->buf.bytes) >= rng->bytes_left); + uint8_t *copy_from = rng->buf.bytes + + (sizeof(rng->buf.bytes) - rng->bytes_left); + memcpy(out, copy_from, to_copy); + memset(copy_from, 0, to_copy); + + out += to_copy; + bytes_to_yield -= to_copy; + rng->bytes_left -= to_copy; + } +} + +/** + * Extract <b>n</b> bytes from <b>rng</b> into the buffer at <b>out</b>. + **/ +void +crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n) +{ + if (PREDICT_UNLIKELY(n > BUFLEN)) { + /* The user has asked for a lot of output; generate it from a stream + * cipher seeded by the PRNG rather than by pulling it out of the PRNG + * directly. + */ + uint8_t seed[SEED_LEN]; + crypto_fast_rng_getbytes_impl(rng, seed, SEED_LEN); + crypto_cipher_t *c = cipher_from_seed(seed); + memset(out, 0, n); + crypto_cipher_crypt_inplace(c, (char*)out, n); + crypto_cipher_free(c); + memwipe(seed, 0, sizeof(seed)); + return; + } + + crypto_fast_rng_getbytes_impl(rng, out, n); +} + +#if defined(TOR_UNIT_TESTS) +/** for white-box testing: return the number of bytes that are returned from + * the user for each invocation of the stream cipher in this RNG. */ +size_t +crypto_fast_rng_get_bytes_used_per_stream(void) +{ + return BUFLEN; +} +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Thread-local instance for our fast RNG. + **/ +static tor_threadlocal_t thread_rng; + +/** + * Return a per-thread fast RNG, initializing it if necessary. + * + * You do not need to free this yourself. + * + * It is NOT safe to share this value across threads. + **/ +crypto_fast_rng_t * +get_thread_fast_rng(void) +{ + crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng); + + if (PREDICT_UNLIKELY(rng == NULL)) { + rng = crypto_fast_rng_new(); + tor_threadlocal_set(&thread_rng, rng); + } + + return rng; +} + +/** + * Used when a thread is exiting: free the per-thread fast RNG if needed. + * Invoked from the crypto subsystem's thread-cleanup code. + **/ +void +destroy_thread_fast_rng(void) +{ + crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng); + if (!rng) + return; + crypto_fast_rng_free(rng); + tor_threadlocal_set(&thread_rng, NULL); +} + +#ifdef TOR_UNIT_TESTS +/** + * Replace the current thread's rng with <b>rng</b>. For use by the + * unit tests only. Returns the previous thread rng. + **/ +crypto_fast_rng_t * +crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng) +{ + crypto_fast_rng_t *old_rng = tor_threadlocal_get(&thread_rng); + tor_threadlocal_set(&thread_rng, rng); + return old_rng; +} +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Initialize the global thread-local key that will be used to keep track + * of per-thread fast RNG instances. Called from the crypto subsystem's + * initialization code. + **/ +void +crypto_rand_fast_init(void) +{ + tor_threadlocal_init(&thread_rng); +} + +/** + * Initialize the global thread-local key that will be used to keep track + * of per-thread fast RNG instances. Called from the crypto subsystem's + * shutdown code. + **/ +void +crypto_rand_fast_shutdown(void) +{ + destroy_thread_fast_rng(); + tor_threadlocal_destroy(&thread_rng); +} diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c new file mode 100644 index 0000000000..b2516c4bdc --- /dev/null +++ b/src/lib/crypt_ops/crypto_rand_numeric.c @@ -0,0 +1,194 @@ +/** + * \file crypto_rand_numeric.c + * + * \brief Functions for retrieving uniformly distributed numbers + * from our PRNGs. + **/ + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/log/util_bug.h" + +/** + * Implementation macro: yields code that returns a uniform unbiased + * random number between 0 and limit. "type" is the type of the number to + * return; "maxval" is the largest possible value of "type"; and "fill_stmt" + * is a code snippet that fills an object named "val" with random bits. + **/ +#define IMPLEMENT_RAND_UNSIGNED(type, maxval, limit, fill_stmt) \ + do { \ + type val; \ + type cutoff; \ + tor_assert((limit) > 0); \ + \ + /* We ignore any values that are >= 'cutoff,' to avoid biasing */ \ + /* the distribution with clipping at the upper end of the type's */ \ + /* range. */ \ + cutoff = (maxval) - ((maxval)%(limit)); \ + while (1) { \ + fill_stmt; \ + if (val < cutoff) \ + return val % (limit); \ + } \ + } while (0) + +/** + * Return a pseudorandom integer chosen uniformly from the values between 0 + * and <b>limit</b>-1 inclusive. limit must be strictly greater than 0, and + * less than UINT_MAX. */ +unsigned +crypto_rand_uint(unsigned limit) +{ + tor_assert(limit < UINT_MAX); + IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit, + crypto_rand((char*)&val, sizeof(val))); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and + * INT_MAX+1, inclusive. + */ +int +crypto_rand_int(unsigned int max) +{ + /* We can't use IMPLEMENT_RAND_UNSIGNED directly, since we're trying + * to return a signed type. Instead we make sure that the range is + * reasonable for a nonnegative int, use crypto_rand_uint(), and cast. + */ + tor_assert(max <= ((unsigned int)INT_MAX)+1); + + return (int)crypto_rand_uint(max); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values i such + * that min <= i < max. + * + * <b>min</b> MUST be in range [0, <b>max</b>). + * <b>max</b> MUST be in range (min, INT_MAX]. + **/ +int +crypto_rand_int_range(unsigned int min, unsigned int max) +{ + tor_assert(min < max); + tor_assert(max <= INT_MAX); + + /* The overflow is avoided here because crypto_rand_int() returns a value + * between 0 and (max - min) inclusive. */ + return min + crypto_rand_int(max - min); +} + +/** + * As crypto_rand_int_range, but supports uint64_t. + **/ +uint64_t +crypto_rand_uint64_range(uint64_t min, uint64_t max) +{ + tor_assert(min < max); + return min + crypto_rand_uint64(max - min); +} + +/** + * As crypto_rand_int_range, but supports time_t. + **/ +time_t +crypto_rand_time_range(time_t min, time_t max) +{ + tor_assert(min < max); + return min + (time_t)crypto_rand_uint64(max - min); +} + +/** + * Return a pseudorandom 64-bit integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. + **/ +uint64_t +crypto_rand_uint64(uint64_t max) +{ + tor_assert(max < UINT64_MAX); + IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, max, + crypto_rand((char*)&val, sizeof(val))); +} + +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif /* SIZEOF_INT == 4 || ... */ + +/** + * Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + **/ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int u; + crypto_rand((char*)&u, sizeof(u)); + return ((double)u) / UINT_MAX_AS_DOUBLE; +} + +/** + * As crypto_rand_uint, but extract the result from a crypto_fast_rng_t + */ +unsigned +crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit) +{ + tor_assert(limit < UINT_MAX); + IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit, + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val))); +} + +/** + * As crypto_rand_uint64, but extract the result from a crypto_fast_rng_t. + */ +uint64_t +crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit) +{ + tor_assert(limit < UINT64_MAX); + IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, limit, + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val))); +} + +/** + * As crypto_rand_u32, but extract the result from a crypto_fast_rng_t. + */ +uint32_t +crypto_fast_rng_get_u32(crypto_fast_rng_t *rng) +{ + uint32_t val; + crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)); + return val; +} + +/** + * As crypto_rand_uint64_range(), but extract the result from a + * crypto_fast_rng_t. + */ +uint64_t +crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng, + uint64_t min, uint64_t max) +{ + /* Handle corrupted input */ + if (BUG(min >= max)) { + return min; + } + + return min + crypto_fast_rng_get_uint64(rng, max - min); +} + +/** + * As crypto_rand_get_double() but extract the result from a crypto_fast_rng_t. + */ +double +crypto_fast_rng_get_double(crypto_fast_rng_t *rng) +{ + unsigned int u; + crypto_fast_rng_getbytes(rng, (void*)&u, sizeof(u)); + return ((double)u) / UINT_MAX_AS_DOUBLE; +} + diff --git a/src/lib/crypt_ops/crypto_rsa.c b/src/lib/crypt_ops/crypto_rsa.c index 8fd8a8aa7b..195e4bbaf9 100644 --- a/src/lib/crypt_ops/crypto_rsa.c +++ b/src/lib/crypt_ops/crypto_rsa.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -59,7 +59,7 @@ crypto_get_rsa_padding(int padding) default: tor_assert(0); return -1; // LCOV_EXCL_LINE } } -#endif +#endif /* defined(ENABLE_OPENSSL) */ /** Compare the public-key components of a and b. Return non-zero iff * a==b. A NULL key is considered to be distinct from all non-NULL diff --git a/src/lib/crypt_ops/crypto_rsa.h b/src/lib/crypt_ops/crypto_rsa.h index 6d9cc8d30e..ab2e9db80d 100644 --- a/src/lib/crypt_ops/crypto_rsa.h +++ b/src/lib/crypt_ops/crypto_rsa.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -122,7 +122,7 @@ struct rsa_st *crypto_pk_get_openssl_rsa_(crypto_pk_t *env); crypto_pk_t *crypto_new_pk_from_openssl_rsa_(struct rsa_st *rsa); MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_openssl_evp_pkey_,( crypto_pk_t *env,int private)); -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS struct SECKEYPublicKeyStr; @@ -132,7 +132,7 @@ const struct SECKEYPublicKeyStr *crypto_pk_get_nss_pubkey( const crypto_pk_t *key); const struct SECKEYPrivateKeyStr *crypto_pk_get_nss_privkey( const crypto_pk_t *key); -#endif +#endif /* defined(ENABLE_NSS) */ void crypto_pk_assign_public(crypto_pk_t *dest, const crypto_pk_t *src); void crypto_pk_assign_private(crypto_pk_t *dest, const crypto_pk_t *src); @@ -143,6 +143,6 @@ struct SECItemStr; STATIC int secitem_uint_cmp(const struct SECItemStr *a, const struct SECItemStr *b); #endif -#endif +#endif /* defined(TOR_UNIT_TESTS) */ -#endif +#endif /* !defined(TOR_CRYPTO_RSA_H) */ diff --git a/src/lib/crypt_ops/crypto_rsa_nss.c b/src/lib/crypt_ops/crypto_rsa_nss.c index 7abf6716f0..66f325e868 100644 --- a/src/lib/crypt_ops/crypto_rsa_nss.c +++ b/src/lib/crypt_ops/crypto_rsa_nss.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -156,7 +156,7 @@ crypto_pk_get_openssl_evp_pkey_,(crypto_pk_t *pk, int private)) tor_free(buf); return result; } -#endif +#endif /* defined(ENABLE_OPENSSL) */ /** Allocate and return storage for a public key. The key itself will not yet * be set. @@ -645,7 +645,7 @@ crypto_pk_asn1_decode(const char *str, size_t len) return result; } -DISABLE_GCC_WARNING(unused-parameter) +DISABLE_GCC_WARNING("-Wunused-parameter") /** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the Base64 * encoding of the DER representation of the private key into the diff --git a/src/lib/crypt_ops/crypto_rsa_openssl.c b/src/lib/crypt_ops/crypto_rsa_openssl.c index 17eae24cc2..c96ee81fd3 100644 --- a/src/lib/crypt_ops/crypto_rsa_openssl.c +++ b/src/lib/crypt_ops/crypto_rsa_openssl.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,7 +16,7 @@ #include "lib/log/util_bug.h" #include "lib/fs/files.h" -DISABLE_GCC_WARNING(redundant-decls) +DISABLE_GCC_WARNING("-Wredundant-decls") #include <openssl/err.h> #include <openssl/rsa.h> @@ -27,7 +27,7 @@ DISABLE_GCC_WARNING(redundant-decls) #include <openssl/bn.h> #include <openssl/conf.h> -ENABLE_GCC_WARNING(redundant-decls) +ENABLE_GCC_WARNING("-Wredundant-decls") #include "lib/log/log.h" #include "lib/encoding/binascii.h" @@ -54,7 +54,7 @@ crypto_pk_key_is_private(const crypto_pk_t *k) const BIGNUM *p, *q; RSA_get0_factors(k->key, &p, &q); return p != NULL; /* XXX/yawning: Should we check q? */ -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ return k && k->key && k->key->p; #endif /* defined(OPENSSL_1_1_API) */ } @@ -288,7 +288,7 @@ crypto_pk_num_bits(crypto_pk_t *env) tor_assert(n != NULL); return RSA_bits(env->key); -#else /* !(defined(OPENSSL_1_1_API)) */ +#else /* !defined(OPENSSL_1_1_API) */ tor_assert(env->key->n); return BN_num_bits(env->key->n); #endif /* defined(OPENSSL_1_1_API) */ @@ -583,15 +583,15 @@ rsa_private_key_too_long(RSA *rsa, int max_bits) dmp1 = RSA_get0_dmp1(rsa); dmq1 = RSA_get0_dmq1(rsa); iqmp = RSA_get0_iqmp(rsa); -#else +#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,1)) */ /* The accessors above did not exist in openssl 1.1.0. */ p = q = dmp1 = dmq1 = iqmp = NULL; RSA_get0_key(rsa, &n, &e, &d); -#endif +#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,1) */ if (RSA_bits(rsa) > max_bits) return true; -#else +#else /* !defined(OPENSSL_1_1_API) */ n = rsa->n; e = rsa->e; p = rsa->p; @@ -600,7 +600,7 @@ rsa_private_key_too_long(RSA *rsa, int max_bits) dmp1 = rsa->dmp1; dmq1 = rsa->dmq1; iqmp = rsa->iqmp; -#endif +#endif /* defined(OPENSSL_1_1_API) */ if (n && BN_num_bits(n) > max_bits) return true; diff --git a/src/lib/crypt_ops/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c index 42276597d4..3a9ed5ef58 100644 --- a/src/lib/crypt_ops/crypto_s2k.c +++ b/src/lib/crypt_ops/crypto_s2k.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -285,7 +285,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv < 0) return S2K_FAILED; return (int)key_out_len; -#else +#else /* !defined(ENABLE_OPENSSL) */ SECItem passItem = { .type = siBuffer, .data = (unsigned char *) secret, .len = (int)secret_len }; @@ -325,7 +325,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (alg) SECOID_DestroyAlgorithmID(alg, PR_TRUE); return rv; -#endif +#endif /* defined(ENABLE_OPENSSL) */ } case S2K_TYPE_SCRYPT: { @@ -348,7 +348,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len, if (rv != 0) return S2K_FAILED; return (int)key_out_len; -#else /* !(defined(HAVE_SCRYPT)) */ +#else /* !defined(HAVE_SCRYPT) */ return S2K_NO_SCRYPT_SUPPORT; #endif /* defined(HAVE_SCRYPT) */ } @@ -380,7 +380,7 @@ secret_to_key_derivekey(uint8_t *key_out, size_t key_out_len, #ifndef HAVE_SCRYPT if (type == S2K_TYPE_SCRYPT) return S2K_NO_SCRYPT_SUPPORT; - #endif +#endif if (! legacy_format) { ++spec; diff --git a/src/lib/crypt_ops/crypto_s2k.h b/src/lib/crypt_ops/crypto_s2k.h index a16a3d781e..181a17acb1 100644 --- a/src/lib/crypt_ops/crypto_s2k.h +++ b/src/lib/crypt_ops/crypto_s2k.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h new file mode 100644 index 0000000000..2115d4fc99 --- /dev/null +++ b/src/lib/crypt_ops/crypto_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_sys.h + * \brief Declare subsystem object for the crypto module. + **/ + +#ifndef TOR_CRYPTO_SYS_H +#define TOR_CRYPTO_SYS_H + +extern const struct subsys_fns_t sys_crypto; + +#endif /* !defined(TOR_CRYPTO_SYS_H) */ diff --git a/src/lib/crypt_ops/crypto_util.c b/src/lib/crypt_ops/crypto_util.c index 67a1a9eb92..7ebb860d09 100644 --- a/src/lib/crypt_ops/crypto_util.c +++ b/src/lib/crypt_ops/crypto_util.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -10,8 +10,6 @@ * \brief Common cryptographic utilities. **/ -#define CRYPTO_UTIL_PRIVATE - #include "lib/crypt_ops/crypto_util.h" #include "lib/cc/compat_compiler.h" @@ -26,11 +24,11 @@ #include <stdlib.h> #ifdef ENABLE_OPENSSL -DISABLE_GCC_WARNING(redundant-decls) +DISABLE_GCC_WARNING("-Wredundant-decls") #include <openssl/err.h> #include <openssl/crypto.h> -ENABLE_GCC_WARNING(redundant-decls) -#endif +ENABLE_GCC_WARNING("-Wredundant-decls") +#endif /* defined(ENABLE_OPENSSL) */ #include "lib/log/log.h" #include "lib/log/util_bug.h" @@ -109,3 +107,17 @@ memwipe(void *mem, uint8_t byte, size_t sz) **/ memset(mem, byte, sz); } + +/** + * Securely all memory in <b>str</b>, then free it. + * + * As tor_free(), tolerates null pointers. + **/ +void +tor_str_wipe_and_free_(char *str) +{ + if (!str) + return; + memwipe(str, 0, strlen(str)); + tor_free_(str); +} diff --git a/src/lib/crypt_ops/crypto_util.h b/src/lib/crypt_ops/crypto_util.h index 613a1bd0dd..36ee230176 100644 --- a/src/lib/crypt_ops/crypto_util.h +++ b/src/lib/crypt_ops/crypto_util.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,8 +14,18 @@ #define TOR_CRYPTO_UTIL_H #include "lib/cc/torint.h" +#include "lib/malloc/malloc.h" /** OpenSSL-based utility functions. */ void memwipe(void *mem, uint8_t byte, size_t sz); +void tor_str_wipe_and_free_(char *str); +/** + * Securely all memory in <b>str</b>, then free it. + * + * As tor_free(), tolerates null pointers, and sets <b>str</b> to NULL. + **/ +#define tor_str_wipe_and_free(str) \ + FREE_AND_NULL(char, tor_str_wipe_and_free_, (str)) + #endif /* !defined(TOR_CRYPTO_UTIL_H) */ diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c index 0dba64d595..5162d6d8bd 100644 --- a/src/lib/crypt_ops/digestset.c +++ b/src/lib/crypt_ops/digestset.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,7 +11,7 @@ #include "lib/crypt_ops/crypto_rand.h" #include "lib/defs/digest_sizes.h" #include "lib/crypt_ops/digestset.h" -#include "siphash.h" +#include "ext/siphash.h" /* Wrap our hash function to have the signature that the bloom filter * needs. */ diff --git a/src/lib/crypt_ops/digestset.h b/src/lib/crypt_ops/digestset.h index 91d53a0542..505ac10395 100644 --- a/src/lib/crypt_ops/digestset.h +++ b/src/lib/crypt_ops/digestset.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -26,4 +26,4 @@ void digestset_add(digestset_t *set, const char *addr); int digestset_probably_contains(const digestset_t *set, const char *addr); -#endif +#endif /* !defined(TOR_DIGESTSET_H) */ diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am index 1022096fdc..7644cab412 100644 --- a/src/lib/crypt_ops/include.am +++ b/src/lib/crypt_ops/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-crypt-ops-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_crypt_ops_a_SOURCES = \ src/lib/crypt_ops/crypto_cipher.c \ src/lib/crypt_ops/crypto_curve25519.c \ @@ -17,6 +18,8 @@ src_lib_libtor_crypt_ops_a_SOURCES = \ src/lib/crypt_ops/crypto_ope.c \ src/lib/crypt_ops/crypto_pwbox.c \ src/lib/crypt_ops/crypto_rand.c \ + src/lib/crypt_ops/crypto_rand_fast.c \ + src/lib/crypt_ops/crypto_rand_numeric.c \ src/lib/crypt_ops/crypto_rsa.c \ src/lib/crypt_ops/crypto_s2k.c \ src/lib/crypt_ops/crypto_util.c \ @@ -25,12 +28,14 @@ src_lib_libtor_crypt_ops_a_SOURCES = \ if USE_NSS src_lib_libtor_crypt_ops_a_SOURCES += \ src/lib/crypt_ops/aes_nss.c \ + src/lib/crypt_ops/crypto_digest_nss.c \ src/lib/crypt_ops/crypto_dh_nss.c \ src/lib/crypt_ops/crypto_nss_mgt.c \ src/lib/crypt_ops/crypto_rsa_nss.c else src_lib_libtor_crypt_ops_a_SOURCES += \ src/lib/crypt_ops/aes_openssl.c \ + src/lib/crypt_ops/crypto_digest_openssl.c \ src/lib/crypt_ops/crypto_rsa_openssl.c endif @@ -48,6 +53,7 @@ src_lib_libtor_crypt_ops_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_crypt_ops_testing_a_CFLAGS = \ $(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/crypt_ops/aes.h \ src/lib/crypt_ops/compat_openssl.h \ @@ -62,9 +68,12 @@ noinst_HEADERS += \ src/lib/crypt_ops/crypto_nss_mgt.h \ src/lib/crypt_ops/crypto_openssl_mgt.h \ src/lib/crypt_ops/crypto_ope.h \ + src/lib/crypt_ops/crypto_options.inc \ + src/lib/crypt_ops/crypto_options_st.h \ src/lib/crypt_ops/crypto_pwbox.h \ src/lib/crypt_ops/crypto_rand.h \ src/lib/crypt_ops/crypto_rsa.h \ src/lib/crypt_ops/crypto_s2k.h \ + src/lib/crypt_ops/crypto_sys.h \ src/lib/crypt_ops/crypto_util.h \ src/lib/crypt_ops/digestset.h diff --git a/src/lib/crypt_ops/lib_crypt_ops.md b/src/lib/crypt_ops/lib_crypt_ops.md new file mode 100644 index 0000000000..4e675e4871 --- /dev/null +++ b/src/lib/crypt_ops/lib_crypt_ops.md @@ -0,0 +1,137 @@ +@dir /lib/crypt_ops +@brief lib/crypt_ops: Cryptographic operations. + +This module contains wrappers around the cryptographic libraries that we +support, and implementations for some higher-level cryptographic +constructions that we use. + +It wraps our two major cryptographic backends (OpenSSL or NSS, as configured +by the user), and also wraps other cryptographic code in src/ext. + +Generally speaking, Tor code shouldn't be calling OpenSSL or NSS +(or any other crypto library) directly. Instead, we should indirect through +one of the functions in this directory, or through \refdir{lib/tls}. + +Cryptography functionality that's available is described below. + +### RNG facilities ### + +The most basic RNG capability in Tor is the crypto_rand() family of +functions. These currently use OpenSSL's RAND_() backend, but may use +something faster in the future. + +In addition to crypto_rand(), which fills in a buffer with random +bytes, we also have functions to produce random integers in certain +ranges; to produce random hostnames; to produce random doubles, etc. + +When you're creating a long-term cryptographic secret, you might want +to use crypto_strongest_rand() instead of crypto_rand(). It takes the +operating system's entropy source and combines it with output from +crypto_rand(). This is a pure paranoia measure, but it might help us +someday. + +You can use smartlist_choose() to pick a random element from a smartlist +and smartlist_shuffle() to randomize the order of a smartlist. Both are +potentially a bit slow. + +### Cryptographic digests and related functions ### + +We treat digests as separate types based on the length of their +outputs. We support one 160-bit digest (SHA1), two 256-bit digests +(SHA256 and SHA3-256), and two 512-bit digests (SHA512 and SHA3-512). + +You should not use SHA1 for anything new. + +The crypto_digest\*() family of functions manipulates digests. You +can either compute a digest of a chunk of memory all at once using +crypto_digest(), crypto_digest256(), or crypto_digest512(). Or you +can create a crypto_digest_t object with +crypto_digest{,256,512}_new(), feed information to it in chunks using +crypto_digest_add_bytes(), and then extract the final digest using +crypto_digest_get_digest(). You can copy the state of one of these +objects using crypto_digest_dup() or crypto_digest_assign(). + +We support the HMAC hash-based message authentication code +instantiated using SHA256. See crypto_hmac_sha256. (You should not +add any HMAC users with SHA1, and HMAC is not necessary with SHA3.) + +We also support the SHA3 cousins, SHAKE128 and SHAKE256. Unlike +digests, these are extendable output functions (or XOFs) where you can +get any amount of output. Use the crypto_xof_\*() functions to access +these. + +We have several ways to derive keys from cryptographically strong secret +inputs (like diffie-hellman outputs). The old +crypto_expand_key_material_TAP() performs an ad-hoc KDF based on SHA1 -- you +shouldn't use it for implementing anything but old versions of the Tor +protocol. You can use HKDF-SHA256 (as defined in RFC5869) for more modern +protocols. Also consider SHAKE256. + +If your input is potentially weak, like a password or passphrase, use a salt +along with the secret_to_key() functions as defined in crypto_s2k.c. Prefer +scrypt over other hashing methods when possible. If you're using a password +to encrypt something, see the "boxed file storage" section below. + +Finally, in order to store objects in hash tables, Tor includes the +randomized SipHash 2-4 function. Call it via the siphash24g() function in +src/ext/siphash.h whenever you're creating a hashtable whose keys may be +manipulated by an attacker in order to DoS you with collisions. + + +### Stream ciphers ### + +You can create instances of a stream cipher using crypto_cipher_new(). +These are stateful objects of type crypto_cipher_t. Note that these +objects only support AES-128 right now; a future version should add +support for AES-128 and/or ChaCha20. + +You can encrypt/decrypt with crypto_cipher_encrypt or +crypto_cipher_decrypt. The crypto_cipher_crypt_inplace function performs +an encryption without a copy. + +Note that sensible people should not use raw stream ciphers; they should +probably be using some kind of AEAD. Sorry. + +### Public key functionality ### + +We support four public key algorithms: DH1024, RSA, Curve25519, and +Ed25519. + +We support DH1024 over two prime groups. You access these via the +crypto_dh_\*() family of functions. + +We support RSA in many bit sizes for signing and encryption. You access +it via the crypto_pk_*() family of functions. Note that a crypto_pk_t +may or may not include a private key. See the crypto_pk_* functions in +crypto.c for a full list of functions here. + +For Curve25519 functionality, see the functions and types in +crypto_curve25519.c. Curve25519 is generally suitable for when you need +a secure fast elliptic-curve diffie hellman implementation. When +designing new protocols, prefer it over DH in Z_p. + +For Ed25519 functionality, see the functions and types in +crypto_ed25519.c. Ed25519 is a generally suitable as a secure fast +elliptic curve signature method. For new protocols, prefer it over RSA +signatures. + +### Metaformats for storage ### + +When OpenSSL manages the storage of some object, we use whatever format +OpenSSL provides -- typically, some kind of PEM-wrapped base 64 encoding +that starts with "----- BEGIN CRYPTOGRAPHIC OBJECT ----". + +When we manage the storage of some cryptographic object, we prefix the +object with 32-byte NUL-padded prefix in order to avoid accidental +object confusion; see the crypto_read_tagged_contents_from_file() and +crypto_write_tagged_contents_to_file() functions for manipulating +these. The prefix is "== type: tag ==", where type describes the object +and its encoding, and tag indicates which one it is. + +### Boxed-file storage ### + +When managing keys, you frequently want to have some way to write a +secret object to disk, encrypted with a passphrase. The crypto_pwbox +and crypto_unpwbox functions do so in a way that's likely to be +readable by future versions of Tor. + diff --git a/src/lib/ctime/di_ops.c b/src/lib/ctime/di_ops.c index 89e0837ae9..d57d286990 100644 --- a/src/lib/ctime/di_ops.c +++ b/src/lib/ctime/di_ops.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2019, The Tor Project, Inc. */ +/* Copyright (c) 2011-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -72,10 +72,10 @@ tor_memcmp(const void *a, const void *b, size_t len) * actually implementation-defined in standard C. So how do we * get away with assuming it? Easy. We check.) */ #if ((-60 >> 8) != -1) -#error "According to cpp, right-shift doesn't perform sign-extension." +#error "cpp says right-shift doesn't perform sign-extension." #endif #ifndef RSHIFT_DOES_SIGN_EXTEND -#error "According to configure, right-shift doesn't perform sign-extension." +#error "configure says right-shift doesn't perform sign-extension." #endif /* If v1 == v2, equal_p is ~0, so this will leave retval @@ -145,8 +145,11 @@ tor_memeq(const void *a, const void *b, size_t sz) /* Implement di_digest256_map_t as a linked list of entries. */ struct di_digest256_map_t { + /** Pointer to the next entry in the list. */ struct di_digest256_map_t *next; + /** Key for this entry. */ uint8_t key[32]; + /** Value for this entry. */ void *val; }; @@ -276,3 +279,30 @@ select_array_member_cumulative_timei(const uint64_t *entries, int n_entries, return i_chosen; } + +/** + * If <b>s</b> is true, then copy <b>n</b> bytes from <b>src</b> to + * <b>dest</b>. Otherwise leave <b>dest</b> alone. + * + * This function behaves the same as + * + * if (s) + * memcpy(dest, src, n); + * + * except that it tries to run in the same amount of time whether <b>s</b> is + * true or not. + **/ +void +memcpy_if_true_timei(bool s, void *dest, const void *src, size_t n) +{ + // If s is true, mask will be ~0. If s is false, mask will be 0. + const char mask = (char) -(signed char)s; + + char *destp = dest; + const char *srcp = src; + for (size_t i = 0; i < n; ++i) { + *destp = (*destp & ~mask) | (*srcp & mask); + ++destp; + ++srcp; + } +} diff --git a/src/lib/ctime/di_ops.h b/src/lib/ctime/di_ops.h index 264b56a8c1..9fe2884ecc 100644 --- a/src/lib/ctime/di_ops.h +++ b/src/lib/ctime/di_ops.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,6 +16,8 @@ int tor_memcmp(const void *a, const void *b, size_t sz); int tor_memeq(const void *a, const void *b, size_t sz); +/** Perform a constant-time comparison of the <b>sz</b> bytes at <b>a</b> and + * <b>b</b>, yielding true if they are different, and false otherwise. */ #define tor_memneq(a,b,sz) (!tor_memeq((a),(b),(sz))) /** Alias for the platform's memcmp() function. This function is @@ -24,7 +26,19 @@ int tor_memeq(const void *a, const void *b, size_t sz); * implementation. */ #define fast_memcmp(a,b,c) (memcmp((a),(b),(c))) +/** Alias for the platform's memcmp() function, for use in testing equality. + * + * This function is <em>not</em> data-independent: we define this alias so + * that we can mark cases where we are deliberately using a data-dependent + * memcmp() implementation. + */ #define fast_memeq(a,b,c) (0==memcmp((a),(b),(c))) +/** Alias for the platform's memcmp() function, for use in testing inequality. + * + * This function is <em>not</em> data-independent: we define this alias so + * that we can mark cases where we are deliberately using a data-dependent + * memcmp() implementation. + */ #define fast_memneq(a,b,c) (0!=memcmp((a),(b),(c))) int safe_mem_is_zero(const void *mem, size_t sz); @@ -35,9 +49,17 @@ int safe_mem_is_zero(const void *mem, size_t sz); * * Not efficient for large maps! */ typedef struct di_digest256_map_t di_digest256_map_t; +/** + * Type for a function used to free members of a di_digest256_map_t. + **/ typedef void (*dimap_free_fn)(void *); void dimap_free_(di_digest256_map_t *map, dimap_free_fn free_fn); +/** + * @copydoc dimap_free_ + * + * Additionally, set the pointer <b>map</b> to NULL. + **/ #define dimap_free(map, free_fn) \ do { \ dimap_free_((map), (free_fn)); \ @@ -51,5 +73,6 @@ int select_array_member_cumulative_timei(const uint64_t *entries, int n_entries, uint64_t total, uint64_t rand_val); -#endif /* !defined(TOR_DI_OPS_H) */ +void memcpy_if_true_timei(bool s, void *dest, const void *src, size_t n); +#endif /* !defined(TOR_DI_OPS_H) */ diff --git a/src/lib/ctime/include.am b/src/lib/ctime/include.am index b46c43ba0c..83942ca4e0 100644 --- a/src/lib/ctime/include.am +++ b/src/lib/ctime/include.am @@ -11,6 +11,7 @@ else mulodi4_source= endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_ctime_a_SOURCES = \ $(mulodi4_source) \ src/ext/csiphash.c \ @@ -21,5 +22,6 @@ src_lib_libtor_ctime_testing_a_SOURCES = \ src_lib_libtor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@ src_lib_libtor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/ctime/di_ops.h diff --git a/src/lib/ctime/lib_ctime.md b/src/lib/ctime/lib_ctime.md new file mode 100644 index 0000000000..913199f6a5 --- /dev/null +++ b/src/lib/ctime/lib_ctime.md @@ -0,0 +1,14 @@ +@dir /lib/ctime +@brief lib/ctime: Constant-time code to avoid side-channels. + +This module contains constant-time implementations of various +data comparison and table lookup functions. We use these in preference to +memcmp() and so forth, since memcmp() can leak information about its inputs +based on how fast it returns. In general, your code should call tor_memeq() +and tor_memneq(), not memcmp(). + +We also define some _non_-constant-time wrappers for memcmp() here: Since we +consider calls to memcmp() to be in error, we require that code that actually +doesn't need to be constant-time to use the fast_memeq() / fast_memneq() / +fast_memcmp() aliases instead. + diff --git a/src/lib/defs/dh_sizes.h b/src/lib/defs/dh_sizes.h index a2ffbc51c2..bc2707b36f 100644 --- a/src/lib/defs/dh_sizes.h +++ b/src/lib/defs/dh_sizes.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,4 +19,4 @@ /** Length of our legacy DH keys. */ #define DH1024_KEY_LEN (1024/8) -#endif +#endif /* !defined(TOR_DH_SIZES_H) */ diff --git a/src/lib/defs/digest_sizes.h b/src/lib/defs/digest_sizes.h index 525e5209d6..7eef1747db 100644 --- a/src/lib/defs/digest_sizes.h +++ b/src/lib/defs/digest_sizes.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_DIGEST_SIZES_H @@ -24,4 +24,4 @@ /** Length of the output of our 64-bit optimized message digests (SHA512). */ #define DIGEST512_LEN 64 -#endif +#endif /* !defined(TOR_DIGEST_SIZES_H) */ diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am index 48ee7f29fc..84ee403771 100644 --- a/src/lib/defs/include.am +++ b/src/lib/defs/include.am @@ -1,5 +1,8 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/defs/dh_sizes.h \ src/lib/defs/digest_sizes.h \ + src/lib/defs/logging_types.h \ + src/lib/defs/time.h \ src/lib/defs/x25519_sizes.h diff --git a/src/lib/defs/lib_defs.md b/src/lib/defs/lib_defs.md new file mode 100644 index 0000000000..5762e4550b --- /dev/null +++ b/src/lib/defs/lib_defs.md @@ -0,0 +1,2 @@ +@dir /lib/defs +@brief lib/defs: Lowest-level constants, used in many places. diff --git a/src/lib/defs/logging_types.h b/src/lib/defs/logging_types.h new file mode 100644 index 0000000000..33aa46186b --- /dev/null +++ b/src/lib/defs/logging_types.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file logging_types.h + * + * \brief Global definition for types used by logging systems. + **/ + +#ifndef TOR_LOGGING_TYPES_H +#define TOR_LOGGING_TYPES_H + +/* We define this here so that it can be used both by backtrace.h and + * log.h. + */ + +/** Mask of zero or more log domains, OR'd together. */ +typedef uint64_t log_domain_mask_t; + +#endif /* !defined(TOR_LOGGING_TYPES_H) */ diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h new file mode 100644 index 0000000000..5707330795 --- /dev/null +++ b/src/lib/defs/time.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_TIME_DEFS_H +#define TOR_TIME_DEFS_H + +/** + * \file time.h + * + * \brief Definitions for timing-related constants. + **/ + +/** How many microseconds per second */ +#define TOR_USEC_PER_SEC (1000000) +/** How many nanoseconds per microsecond */ +#define TOR_NSEC_PER_USEC (1000) +/** How many nanoseconds per millisecond */ +#define TOR_NSEC_PER_MSEC (1000*1000) + +#endif /* !defined(TOR_TIME_DEFS_H) */ diff --git a/src/lib/defs/x25519_sizes.h b/src/lib/defs/x25519_sizes.h index 8933a8866b..acb08c5e6a 100644 --- a/src/lib/defs/x25519_sizes.h +++ b/src/lib/defs/x25519_sizes.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -23,14 +23,22 @@ /** Length of the result of a curve25519 handshake. */ #define CURVE25519_OUTPUT_LEN 32 +/** Length of an Ed25519 public key */ #define ED25519_PUBKEY_LEN 32 +/** Length of an Ed25519 secret key */ #define ED25519_SECKEY_LEN 64 +/** Length of the seed that is ordinarily expanded to an Ed25519 secret + * key. */ #define ED25519_SECKEY_SEED_LEN 32 +/** Length of an Ed25519 signature. */ #define ED25519_SIG_LEN 64 +/** Length of a Curve25519 key when encoded in base 64, with padding. */ #define CURVE25519_BASE64_PADDED_LEN 44 +/** Length of a Ed25519 key when encoded in base 64, without padding. */ #define ED25519_BASE64_LEN 43 +/** Length of a Ed25519 signature when encoded in base 64, without padding. */ #define ED25519_SIG_BASE64_LEN 86 -#endif +#endif /* !defined(TOR_X25519_SIZES_H) */ diff --git a/src/lib/dispatch/.may_include b/src/lib/dispatch/.may_include new file mode 100644 index 0000000000..884f4c0dbc --- /dev/null +++ b/src/lib/dispatch/.may_include @@ -0,0 +1,11 @@ +orconfig.h + +ext/tor_queue.h + +lib/cc/*.h +lib/container/*.h +lib/dispatch/*.h +lib/intmath/*.h +lib/log/*.h +lib/malloc/*.h +lib/testsupport/*.h
\ No newline at end of file diff --git a/src/lib/dispatch/dispatch.h b/src/lib/dispatch/dispatch.h new file mode 100644 index 0000000000..9c7c4833c2 --- /dev/null +++ b/src/lib/dispatch/dispatch.h @@ -0,0 +1,114 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DISPATCH_H +#define TOR_DISPATCH_H + +#include "lib/dispatch/msgtypes.h" + +/** + * \file dispatch.h + * \brief Low-level APIs for message-passing system. + * + * This module implements message dispatch based on a set of short integer + * identifiers. For a higher-level interface, see pubsub.h. + * + * Each message is represented as a generic msg_t object, and is discriminated + * by its message_id_t. Messages are delivered by a dispatch_t object, which + * delivers each message to its recipients by a configured "channel". + * + * A "channel" is a means of delivering messages. Every message_id_t must + * be associated with exactly one channel, identified by channel_id_t. + * When a channel receives messages, a callback is invoked to either process + * the messages immediately, or to cause them to be processed later. + * + * Every message_id_t has zero or more associated receiver functions set up in + * the dispatch_t object. Once the dispatch_t object is created, receivers + * can be enabled or disabled [TODO], but not added or removed. + * + * Every message_id_t has an associated datatype, identified by a + * msg_type_id_t. These datatypes can be associated with functions to + * (for example) free them, or format them for debugging. + * + * To setup a dispatch_t object, first create a dispatch_cfg_t object, and + * configure messages with their types, channels, and receivers. Then, use + * dispatch_new() with that dispatch_cfg_t to create the dispatch_t object. + * + * (We use a two-phase contruction procedure here to enable better static + * reasoning about publish/subscribe relationships.) + * + * Once you have a dispatch_t, you can queue messages on it with + * dispatch_send*(), and cause those messages to be delivered with + * dispatch_flush(). + **/ + +/** + * A "dispatcher" is the highest-level object; it handles making sure that + * messages are received and delivered properly. Only the mainloop + * should handle this type directly. + */ +typedef struct dispatch_t dispatch_t; + +struct dispatch_cfg_t; + +dispatch_t *dispatch_new(const struct dispatch_cfg_t *cfg); + +/** + * Free a dispatcher. Tor does this at exit. + */ +#define dispatch_free(d) \ + FREE_AND_NULL(dispatch_t, dispatch_free_, (d)) + +void dispatch_free_(dispatch_t *); + +int dispatch_send(dispatch_t *d, + subsys_id_t sender, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + msg_aux_data_t auxdata); + +int dispatch_send_msg(dispatch_t *d, msg_t *m); + +int dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m); + +/* Flush up to <b>max_msgs</b> currently pending messages from the + * dispatcher. Messages that are not pending when this function are + * called, are not flushed by this call. Return 0 on success, -1 on + * unrecoverable error. + */ +int dispatch_flush(dispatch_t *, channel_id_t chan, int max_msgs); + +/** + * Function callback type used to alert some other module when a channel's + * queue changes from empty to nonempty. + * + * Ex 1: To cause messages to be processed immediately on-stack, this callback + * should invoke dispatch_flush() directly. + * + * Ex 2: To cause messages to be processed very soon, from the event queue, + * this callback should schedule an event callback to run dispatch_flush(). + * + * Ex 3: To cause messages to be processed periodically, this function should + * do nothing, and a periodic event should invoke dispatch_flush(). + **/ +typedef void (*dispatch_alertfn_t)(struct dispatch_t *, + channel_id_t, void *); + +int dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan, + dispatch_alertfn_t fn, void *userdata); + +#define dispatch_free_msg(d,msg) \ + STMT_BEGIN { \ + msg_t **msg_tmp_ptr__ = &(msg); \ + dispatch_free_msg_((d), *msg_tmp_ptr__); \ + *msg_tmp_ptr__= NULL; \ + } STMT_END +void dispatch_free_msg_(const dispatch_t *d, msg_t *msg); + +char *dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg); + +#endif /* !defined(TOR_DISPATCH_H) */ diff --git a/src/lib/dispatch/dispatch_cfg.c b/src/lib/dispatch/dispatch_cfg.c new file mode 100644 index 0000000000..a54188dcaa --- /dev/null +++ b/src/lib/dispatch/dispatch_cfg.c @@ -0,0 +1,141 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_cfg.c + * \brief Create and configure a dispatch_cfg_t. + * + * A dispatch_cfg_t object is used to configure a set of messages and + * associated information before creating a dispatch_t. + */ + +#define DISPATCH_PRIVATE + +#include "orconfig.h" +#include "lib/dispatch/dispatch_cfg.h" +#include "lib/dispatch/dispatch_cfg_st.h" +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" + +#include "lib/container/smartlist.h" +#include "lib/malloc/malloc.h" + +/** + * Create and return a new dispatch_cfg_t. + **/ +dispatch_cfg_t * +dcfg_new(void) +{ + dispatch_cfg_t *cfg = tor_malloc(sizeof(dispatch_cfg_t)); + cfg->type_by_msg = smartlist_new(); + cfg->chan_by_msg = smartlist_new(); + cfg->fns_by_type = smartlist_new(); + cfg->recv_by_msg = smartlist_new(); + return cfg; +} + +/** + * Associate a message with a datatype. Return 0 on success, -1 if a + * different type was previously associated with the message ID. + **/ +int +dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg, + msg_type_id_t type) +{ + smartlist_grow(cfg->type_by_msg, msg+1); + msg_type_id_t *oldval = smartlist_get(cfg->type_by_msg, msg); + if (oldval != NULL && *oldval != type) { + return -1; + } + if (!oldval) + smartlist_set(cfg->type_by_msg, msg, tor_memdup(&type, sizeof(type))); + return 0; +} + +/** + * Associate a message with a channel. Return 0 on success, -1 if a + * different channel was previously associated with the message ID. + **/ +int +dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg, + channel_id_t chan) +{ + smartlist_grow(cfg->chan_by_msg, msg+1); + channel_id_t *oldval = smartlist_get(cfg->chan_by_msg, msg); + if (oldval != NULL && *oldval != chan) { + return -1; + } + if (!oldval) + smartlist_set(cfg->chan_by_msg, msg, tor_memdup(&chan, sizeof(chan))); + return 0; +} + +/** + * Associate a set of functions with a datatype. Return 0 on success, -1 if + * different functions were previously associated with the type. + **/ +int +dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type, + const dispatch_typefns_t *fns) +{ + smartlist_grow(cfg->fns_by_type, type+1); + dispatch_typefns_t *oldfns = smartlist_get(cfg->fns_by_type, type); + if (oldfns && (oldfns->free_fn != fns->free_fn || + oldfns->fmt_fn != fns->fmt_fn)) + return -1; + if (!oldfns) + smartlist_set(cfg->fns_by_type, type, tor_memdup(fns, sizeof(*fns))); + return 0; +} + +/** + * Associate a receiver with a message ID. Multiple receivers may be + * associated with a single messasge ID. + * + * Return 0 on success, on failure. + **/ +int +dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg, + subsys_id_t sys, recv_fn_t fn) +{ + smartlist_grow(cfg->recv_by_msg, msg+1); + smartlist_t *receivers = smartlist_get(cfg->recv_by_msg, msg); + if (!receivers) { + receivers = smartlist_new(); + smartlist_set(cfg->recv_by_msg, msg, receivers); + } + + dispatch_rcv_t *rcv = tor_malloc(sizeof(dispatch_rcv_t)); + rcv->sys = sys; + rcv->enabled = true; + rcv->fn = fn; + smartlist_add(receivers, (void*)rcv); + return 0; +} + +/** Helper: release all storage held by <b>cfg</b>. */ +void +dcfg_free_(dispatch_cfg_t *cfg) +{ + if (!cfg) + return; + + SMARTLIST_FOREACH(cfg->type_by_msg, msg_type_id_t *, id, tor_free(id)); + SMARTLIST_FOREACH(cfg->chan_by_msg, channel_id_t *, id, tor_free(id)); + SMARTLIST_FOREACH(cfg->fns_by_type, dispatch_typefns_t *, f, tor_free(f)); + smartlist_free(cfg->type_by_msg); + smartlist_free(cfg->chan_by_msg); + smartlist_free(cfg->fns_by_type); + SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, receivers) { + if (!receivers) + continue; + SMARTLIST_FOREACH(receivers, dispatch_rcv_t *, rcv, tor_free(rcv)); + smartlist_free(receivers); + } SMARTLIST_FOREACH_END(receivers); + smartlist_free(cfg->recv_by_msg); + + tor_free(cfg); +} diff --git a/src/lib/dispatch/dispatch_cfg.h b/src/lib/dispatch/dispatch_cfg.h new file mode 100644 index 0000000000..a4f1948eac --- /dev/null +++ b/src/lib/dispatch/dispatch_cfg.h @@ -0,0 +1,50 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_DISPATCH_CFG_H +#define TOR_DISPATCH_CFG_H + +/** + * @file dispatch_cfg.h + * @brief Header for distpach_cfg.c + **/ + +#include "lib/dispatch/msgtypes.h" +#include "lib/testsupport/testsupport.h" + +/** + * A "dispatch_cfg" is the configuration used to set up a dispatcher. + * It is created and accessed with a set of dcfg_* functions, and then + * used with dispatcher_new() to make the dispatcher. + */ +typedef struct dispatch_cfg_t dispatch_cfg_t; + +dispatch_cfg_t *dcfg_new(void); + +int dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg, + msg_type_id_t type); + +int dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg, + channel_id_t chan); + +int dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type, + const dispatch_typefns_t *fns); + +int dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg, + subsys_id_t sys, recv_fn_t fn); + +/** Free a dispatch_cfg_t. */ +#define dcfg_free(cfg) \ + FREE_AND_NULL(dispatch_cfg_t, dcfg_free_, (cfg)) + +void dcfg_free_(dispatch_cfg_t *cfg); + +#ifdef DISPATCH_NEW_PRIVATE +struct smartlist_t; +STATIC int max_in_u16_sl(const struct smartlist_t *sl, int dflt); +#endif + +#endif /* !defined(TOR_DISPATCH_CFG_H) */ diff --git a/src/lib/dispatch/dispatch_cfg_st.h b/src/lib/dispatch/dispatch_cfg_st.h new file mode 100644 index 0000000000..3c99adf2f7 --- /dev/null +++ b/src/lib/dispatch/dispatch_cfg_st.h @@ -0,0 +1,33 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file dispatch_cfg_st.h + * @brief Declarations for dispatch-configuration types. + **/ + +#ifndef TOR_DISPATCH_CFG_ST_H +#define TOR_DISPATCH_CFG_ST_H + +struct smartlist_t; + +/** Information needed to create a dispatcher, but in a less efficient, more + * mutable format. + * + * Nearly everybody should use the \refdir{lib/pubsub} module to configure + * dispatchers, instead of using this. */ +struct dispatch_cfg_t { + /** A list of msg_type_id_t (cast to void*), indexed by msg_t. */ + struct smartlist_t *type_by_msg; + /** A list of channel_id_t (cast to void*), indexed by msg_t. */ + struct smartlist_t *chan_by_msg; + /** A list of dispatch_rcv_t, indexed by msg_type_id_t. */ + struct smartlist_t *fns_by_type; + /** A list of dispatch_typefns_t, indexed by msg_t. */ + struct smartlist_t *recv_by_msg; +}; + +#endif /* !defined(TOR_DISPATCH_CFG_ST_H) */ diff --git a/src/lib/dispatch/dispatch_core.c b/src/lib/dispatch/dispatch_core.c new file mode 100644 index 0000000000..3d51c876a7 --- /dev/null +++ b/src/lib/dispatch/dispatch_core.c @@ -0,0 +1,260 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_core.c + * \brief Core module for sending and receiving messages. + */ + +#define DISPATCH_PRIVATE +#include "orconfig.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" +#include "lib/dispatch/dispatch_naming.h" + +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +#include <string.h> + +/** + * Use <b>d</b> to drop all storage held for <b>msg</b>. + * + * (We need the dispatcher so we know how to free the auxiliary data.) + **/ +void +dispatch_free_msg_(const dispatch_t *d, msg_t *msg) +{ + if (!msg) + return; + + d->typefns[msg->type].free_fn(msg->aux_data__); + tor_free(msg); +} + +/** + * Format the auxiliary data held by msg. + **/ +char * +dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg) +{ + if (!msg) + return NULL; + + return d->typefns[msg->type].fmt_fn(msg->aux_data__); +} + +/** + * Release all storage held by <b>d</b>. + **/ +void +dispatch_free_(dispatch_t *d) +{ + if (d == NULL) + return; + + size_t n_queues = d->n_queues; + for (size_t i = 0; i < n_queues; ++i) { + msg_t *m, *mtmp; + TOR_SIMPLEQ_FOREACH_SAFE(m, &d->queues[i].queue, next, mtmp) { + dispatch_free_msg(d, m); + } + } + + size_t n_msgs = d->n_msgs; + + for (size_t i = 0; i < n_msgs; ++i) { + tor_free(d->table[i]); + } + tor_free(d->table); + tor_free(d->typefns); + tor_free(d->queues); + + // This is the only time we will treat d->cfg as non-const. + //dispatch_cfg_free_((dispatch_items_t *) d->cfg); + + tor_free(d); +} + +/** + * Tell the dispatcher to call <b>fn</b> with <b>userdata</b> whenever + * <b>chan</b> becomes nonempty. Return 0 on success, -1 on error. + **/ +int +dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan, + dispatch_alertfn_t fn, void *userdata) +{ + if (BUG(chan >= d->n_queues)) + return -1; + + dqueue_t *q = &d->queues[chan]; + q->alert_fn = fn; + q->alert_fn_arg = userdata; + return 0; +} + +/** + * Send a message on the appropriate channel notifying that channel if + * necessary. + * + * This function takes ownership of the auxiliary data; it can't be static or + * stack-allocated, and the caller is not allowed to use it afterwards. + * + * This function does not check the various vields of the message object for + * consistency. + **/ +int +dispatch_send(dispatch_t *d, + subsys_id_t sender, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + msg_aux_data_t auxdata) +{ + if (!d->table[msg]) { + /* Fast path: nobody wants this data. */ + + d->typefns[type].free_fn(auxdata); + return 0; + } + + msg_t *m = tor_malloc(sizeof(msg_t)); + + m->sender = sender; + m->channel = channel; + m->msg = msg; + m->type = type; + memcpy(&m->aux_data__, &auxdata, sizeof(msg_aux_data_t)); + + return dispatch_send_msg(d, m); +} + +int +dispatch_send_msg(dispatch_t *d, msg_t *m) +{ + if (BUG(!d)) + goto err; + if (BUG(!m)) + goto err; + if (BUG(m->channel >= d->n_queues)) + goto err; + if (BUG(m->msg >= d->n_msgs)) + goto err; + + dtbl_entry_t *ent = d->table[m->msg]; + if (ent) { + if (BUG(m->type != ent->type)) + goto err; + if (BUG(m->channel != ent->channel)) + goto err; + } + + return dispatch_send_msg_unchecked(d, m); + err: + /* Probably it isn't safe to free m, since type could be wrong. */ + return -1; +} + +/** + * Send a message on the appropriate queue, notifying that queue if necessary. + * + * This function takes ownership of the message object and its auxiliary data; + * it can't be static or stack-allocated, and the caller isn't allowed to use + * it afterwards. + * + * This function does not check the various fields of the message object for + * consistency, and can crash if they are out of range. Only functions that + * have already constructed the message in a safe way, or checked it for + * correctness themselves, should call this function. + **/ +int +dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m) +{ + /* Find the right queue. */ + dqueue_t *q = &d->queues[m->channel]; + bool was_empty = TOR_SIMPLEQ_EMPTY(&q->queue); + + /* Append the message. */ + TOR_SIMPLEQ_INSERT_TAIL(&q->queue, m, next); + + if (debug_logging_enabled()) { + char *arg = dispatch_fmt_msg_data(d, m); + log_debug(LD_MESG, + "Queued: %s (%s) from %s, on %s.", + get_message_id_name(m->msg), + arg, + get_subsys_id_name(m->sender), + get_channel_id_name(m->channel)); + tor_free(arg); + } + + /* If we just made the queue nonempty for the first time, call the alert + * function. */ + if (was_empty) { + q->alert_fn(d, m->channel, q->alert_fn_arg); + } + + return 0; +} + +/** + * Run all of the callbacks on <b>d</b> associated with <b>m</b>. + **/ +static void +dispatcher_run_msg_cbs(const dispatch_t *d, msg_t *m) +{ + tor_assert(m->msg <= d->n_msgs); + dtbl_entry_t *ent = d->table[m->msg]; + int n_fns = ent->n_fns; + + if (debug_logging_enabled()) { + char *arg = dispatch_fmt_msg_data(d, m); + log_debug(LD_MESG, + "Delivering: %s (%s) from %s, on %s:", + get_message_id_name(m->msg), + arg, + get_subsys_id_name(m->sender), + get_channel_id_name(m->channel)); + tor_free(arg); + } + + int i; + for (i=0; i < n_fns; ++i) { + if (ent->rcv[i].enabled) { + log_debug(LD_MESG, " Delivering to %s.", + get_subsys_id_name(ent->rcv[i].sys)); + ent->rcv[i].fn(m); + } + } +} + +/** + * Run up to <b>max_msgs</b> callbacks for messages on the channel <b>ch</b> + * on the given dispatcher. Return 0 on success or recoverable failure, + * -1 on unrecoverable error. + **/ +int +dispatch_flush(dispatch_t *d, channel_id_t ch, int max_msgs) +{ + if (BUG(ch >= d->n_queues)) + return 0; + + int n_flushed = 0; + dqueue_t *q = &d->queues[ch]; + + while (n_flushed < max_msgs) { + msg_t *m = TOR_SIMPLEQ_FIRST(&q->queue); + if (!m) + break; + TOR_SIMPLEQ_REMOVE_HEAD(&q->queue, next); + dispatcher_run_msg_cbs(d, m); + dispatch_free_msg(d, m); + ++n_flushed; + } + + return 0; +} diff --git a/src/lib/dispatch/dispatch_naming.c b/src/lib/dispatch/dispatch_naming.c new file mode 100644 index 0000000000..bb49343712 --- /dev/null +++ b/src/lib/dispatch/dispatch_naming.c @@ -0,0 +1,70 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file dispatch_naming.c + * @brief Name-to-ID maps for our message dispatch system. + **/ + +#include "orconfig.h" + +#include "lib/cc/compat_compiler.h" + +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/msgtypes.h" + +#include "lib/container/namemap.h" +#include "lib/container/namemap_st.h" + +#include "lib/log/util_bug.h" +#include "lib/log/log.h" + +#include <stdlib.h> + +/** Global namemap for message IDs. */ +static namemap_t message_id_map = NAMEMAP_INIT(); +/** Global namemap for subsystem IDs. */ +static namemap_t subsys_id_map = NAMEMAP_INIT(); +/** Global namemap for channel IDs. */ +static namemap_t channel_id_map = NAMEMAP_INIT(); +/** Global namemap for message type IDs. */ +static namemap_t msg_type_id_map = NAMEMAP_INIT(); + +void +dispatch_naming_init(void) +{ +} + +#ifndef COCCI +/* Helper macro: declare functions to map IDs to and from names for a given + * type in a namemap_t. + */ +#define DECLARE_ID_MAP_FNS(type) \ + type##_id_t \ + get_##type##_id(const char *name) \ + { \ + unsigned u = namemap_get_or_create_id(&type##_id_map, name); \ + tor_assert(u != NAMEMAP_ERR); \ + tor_assert(u != ERROR_ID); \ + return (type##_id_t) u; \ + } \ + const char * \ + get_##type##_id_name(type##_id_t id) \ + { \ + return namemap_fmt_name(&type##_id_map, id); \ + } \ + size_t \ + get_num_##type##_ids(void) \ + { \ + return namemap_get_size(&type##_id_map); \ + } \ + EAT_SEMICOLON +#endif /* !defined(COCCI) */ + +DECLARE_ID_MAP_FNS(message); +DECLARE_ID_MAP_FNS(channel); +DECLARE_ID_MAP_FNS(subsys); +DECLARE_ID_MAP_FNS(msg_type); diff --git a/src/lib/dispatch/dispatch_naming.h b/src/lib/dispatch/dispatch_naming.h new file mode 100644 index 0000000000..72206d3ed5 --- /dev/null +++ b/src/lib/dispatch/dispatch_naming.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file dispatch_naming.h + * @brief Header for dispatch_naming.c + **/ + +#ifndef TOR_DISPATCH_NAMING_H +#define TOR_DISPATCH_NAMING_H + +#include "lib/dispatch/msgtypes.h" +#include <stddef.h> + +/** + * Return an existing channel ID by name, allocating the channel ID if + * if necessary. Returns ERROR_ID if we have run out of + * channels + */ +channel_id_t get_channel_id(const char *); +/** + * Return the name corresponding to a given channel ID. + **/ +const char *get_channel_id_name(channel_id_t); +/** + * Return the total number of _named_ channel IDs. + **/ +size_t get_num_channel_ids(void); + +/* As above, but for messages. */ +message_id_t get_message_id(const char *); +const char *get_message_id_name(message_id_t); +size_t get_num_message_ids(void); + +/* As above, but for subsystems */ +subsys_id_t get_subsys_id(const char *); +const char *get_subsys_id_name(subsys_id_t); +size_t get_num_subsys_ids(void); + +/* As above, but for types. Note that types additionally must be + * "defined", if any message is to use them. */ +msg_type_id_t get_msg_type_id(const char *); +const char *get_msg_type_id_name(msg_type_id_t); +size_t get_num_msg_type_ids(void); + +void dispatch_naming_init(void); + +#endif /* !defined(TOR_DISPATCH_NAMING_H) */ diff --git a/src/lib/dispatch/dispatch_new.c b/src/lib/dispatch/dispatch_new.c new file mode 100644 index 0000000000..e1dbb1c4b8 --- /dev/null +++ b/src/lib/dispatch/dispatch_new.c @@ -0,0 +1,176 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_new.c + * \brief Code to construct a dispatch_t from a dispatch_cfg_t. + **/ + +#define DISPATCH_NEW_PRIVATE +#define DISPATCH_PRIVATE +#include "orconfig.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" +#include "lib/dispatch/dispatch_cfg.h" +#include "lib/dispatch/dispatch_cfg_st.h" + +#include "lib/cc/ctassert.h" +#include "lib/intmath/cmp.h" +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +#include <string.h> + +/** Given a smartlist full of (possibly NULL) pointers to uint16_t values, + * return the largest value, or dflt if the list is empty. */ +STATIC int +max_in_u16_sl(const smartlist_t *sl, int dflt) +{ + uint16_t *maxptr = NULL; + SMARTLIST_FOREACH_BEGIN(sl, uint16_t *, u) { + if (!maxptr) + maxptr = u; + else if (u && *u > *maxptr) + maxptr = u; + } SMARTLIST_FOREACH_END(u); + + return maxptr ? *maxptr : dflt; +} + +/* The above function is only safe to call if we are sure that channel_id_t + * and msg_type_id_t are really uint16_t. They should be so defined in + * msgtypes.h, but let's be extra cautious. + */ +CTASSERT(sizeof(uint16_t) == sizeof(msg_type_id_t)); +CTASSERT(sizeof(uint16_t) == sizeof(channel_id_t)); + +/** Helper: Format an unformattable message auxiliary data item: just return a +* copy of the string <>. */ +static char * +type_fmt_nop(msg_aux_data_t arg) +{ + (void)arg; + return tor_strdup("<>"); +} + +/** Helper: Free an unfreeable message auxiliary data item: do nothing. */ +static void +type_free_nop(msg_aux_data_t arg) +{ + (void)arg; +} + +/** Type functions to use when no type functions are provided. */ +static dispatch_typefns_t nop_typefns = { + .free_fn = type_free_nop, + .fmt_fn = type_fmt_nop +}; + +/** + * Alert function to use when none is configured: do nothing. + **/ +static void +alert_fn_nop(dispatch_t *d, channel_id_t ch, void *arg) +{ + (void)d; + (void)ch; + (void)arg; +} + +/** + * Given a list of recvfn_t, create and return a new dtbl_entry_t mapping + * to each of those functions. + **/ +static dtbl_entry_t * +dtbl_entry_from_lst(smartlist_t *receivers) +{ + if (!receivers) + return NULL; + + size_t n_recv = smartlist_len(receivers); + dtbl_entry_t *ent; + ent = tor_malloc_zero(offsetof(dtbl_entry_t, rcv) + + sizeof(dispatch_rcv_t) * n_recv); + + ent->n_fns = n_recv; + + SMARTLIST_FOREACH_BEGIN(receivers, const dispatch_rcv_t *, rcv) { + memcpy(&ent->rcv[rcv_sl_idx], rcv, sizeof(*rcv)); + if (rcv->enabled) { + ++ent->n_enabled; + } + } SMARTLIST_FOREACH_END(rcv); + + return ent; +} + +/** Create and return a new dispatcher from a given dispatch_cfg_t. */ +dispatch_t * +dispatch_new(const dispatch_cfg_t *cfg) +{ + dispatch_t *d = tor_malloc_zero(sizeof(dispatch_t)); + + /* Any message that has a type or a receiver counts towards our messages */ + const size_t n_msgs = MAX(smartlist_len(cfg->type_by_msg), + smartlist_len(cfg->recv_by_msg)) + 1; + + /* Any channel that any message has counts towards the number of channels. */ + const size_t n_chans = (size_t) + MAX(1, max_in_u16_sl(cfg->chan_by_msg,0)) + 1; + + /* Any type that a message has, or that has functions, counts towards + * the number of types. */ + const size_t n_types = (size_t) MAX(max_in_u16_sl(cfg->type_by_msg,0), + smartlist_len(cfg->fns_by_type)) + 1; + + d->n_msgs = n_msgs; + d->n_queues = n_chans; + d->n_types = n_types; + + /* Initialize the array of type-functions. */ + d->typefns = tor_calloc(n_types, sizeof(dispatch_typefns_t)); + for (size_t i = 0; i < n_types; ++i) { + /* Default to no-op for everything... */ + memcpy(&d->typefns[i], &nop_typefns, sizeof(dispatch_typefns_t)); + } + SMARTLIST_FOREACH_BEGIN(cfg->fns_by_type, dispatch_typefns_t *, fns) { + /* Set the functions if they are provided. */ + if (fns) { + if (fns->free_fn) + d->typefns[fns_sl_idx].free_fn = fns->free_fn; + if (fns->fmt_fn) + d->typefns[fns_sl_idx].fmt_fn = fns->fmt_fn; + } + } SMARTLIST_FOREACH_END(fns); + + /* Initialize the message queues: one for each channel. */ + d->queues = tor_calloc(d->n_queues, sizeof(dqueue_t)); + for (size_t i = 0; i < d->n_queues; ++i) { + TOR_SIMPLEQ_INIT(&d->queues[i].queue); + d->queues[i].alert_fn = alert_fn_nop; + } + + /* Build the dispatch tables mapping message IDs to receivers. */ + d->table = tor_calloc(d->n_msgs, sizeof(dtbl_entry_t *)); + SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, rcv) { + d->table[rcv_sl_idx] = dtbl_entry_from_lst(rcv); + } SMARTLIST_FOREACH_END(rcv); + + /* Fill in the empty entries in the dispatch tables: + * types and channels for each message. */ + SMARTLIST_FOREACH_BEGIN(cfg->type_by_msg, msg_type_id_t *, type) { + if (d->table[type_sl_idx]) + d->table[type_sl_idx]->type = *type; + } SMARTLIST_FOREACH_END(type); + + SMARTLIST_FOREACH_BEGIN(cfg->chan_by_msg, channel_id_t *, chan) { + if (d->table[chan_sl_idx]) + d->table[chan_sl_idx]->channel = *chan; + } SMARTLIST_FOREACH_END(chan); + + return d; +} diff --git a/src/lib/dispatch/dispatch_st.h b/src/lib/dispatch/dispatch_st.h new file mode 100644 index 0000000000..ad5b4efc40 --- /dev/null +++ b/src/lib/dispatch/dispatch_st.h @@ -0,0 +1,108 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file dispatch_st.h + * + * \brief private structures used for the dispatcher module + */ + +#ifndef TOR_DISPATCH_ST_H +#define TOR_DISPATCH_ST_H + +#ifdef DISPATCH_PRIVATE + +#include "lib/container/smartlist.h" + +/** + * Information about the recipient of a message. + **/ +typedef struct dispatch_rcv_t { + /** The subsystem receiving a message. */ + subsys_id_t sys; + /** True iff this recipient is enabled. */ + bool enabled; + /** The function that will handle the message. */ + recv_fn_t fn; +} dispatch_rcv_t; + +/** + * Information used by a dispatcher to handle and dispatch a single message + * ID. It maps that message ID to its type, channel, and list of receiver + * functions. + * + * This structure is used when the dispatcher is running. + **/ +typedef struct dtbl_entry_t { + /** The number of enabled non-stub subscribers for this message. + * + * Note that for now, this will be the same as <b>n_fns</b>, since there is + * no way to turn these subscribers on an off yet. */ + uint16_t n_enabled; + /** The channel that handles this message. */ + channel_id_t channel; + /** The associated C type for this message. */ + msg_type_id_t type; + /** + * The number of functions pointers for subscribers that receive this + * message, in rcv. */ + uint16_t n_fns; + /** + * The recipients for this message. + */ + dispatch_rcv_t rcv[FLEXIBLE_ARRAY_MEMBER]; +} dtbl_entry_t; + +/** + * A queue of messages for a given channel, used by a live dispatcher. + */ +typedef struct dqueue_t { + /** The queue of messages itself. */ + TOR_SIMPLEQ_HEAD( , msg_t) queue; + /** A function to be called when the queue becomes nonempty. */ + dispatch_alertfn_t alert_fn; + /** An argument for the alert_fn. */ + void *alert_fn_arg; +} dqueue_t ; + +/** + * A single dispatcher for cross-module messages. + */ +struct dispatch_t { + /** + * The length of <b>table</b>: the number of message IDs that this + * dispatcher can handle. + */ + size_t n_msgs; + /** + * The length of <b>queues</b>: the number of channels that this dispatcher + * has configured. + */ + size_t n_queues; + /** + * The length of <b>typefns</b>: the number of C type IDs that this + * dispatcher has configured. + */ + size_t n_types; + /** + * An array of message queues, indexed by channel ID. + */ + dqueue_t *queues; + /** + * An array of entries about how to handle particular message types, indexed + * by message ID. + */ + dtbl_entry_t **table; + /** + * An array of function tables for manipulating types, index by message + * type ID. + **/ + dispatch_typefns_t *typefns; +}; + +#endif /* defined(DISPATCH_PRIVATE) */ + +#endif /* !defined(TOR_DISPATCH_ST_H) */ diff --git a/src/lib/dispatch/include.am b/src/lib/dispatch/include.am new file mode 100644 index 0000000000..4a0e0dfd90 --- /dev/null +++ b/src/lib/dispatch/include.am @@ -0,0 +1,27 @@ + +noinst_LIBRARIES += src/lib/libtor-dispatch.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-dispatch-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_dispatch_a_SOURCES = \ + src/lib/dispatch/dispatch_cfg.c \ + src/lib/dispatch/dispatch_core.c \ + src/lib/dispatch/dispatch_naming.c \ + src/lib/dispatch/dispatch_new.c + +src_lib_libtor_dispatch_testing_a_SOURCES = \ + $(src_lib_libtor_dispatch_a_SOURCES) +src_lib_libtor_dispatch_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_dispatch_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/dispatch/dispatch.h \ + src/lib/dispatch/dispatch_cfg.h \ + src/lib/dispatch/dispatch_cfg_st.h \ + src/lib/dispatch/dispatch_naming.h \ + src/lib/dispatch/dispatch_st.h \ + src/lib/dispatch/msgtypes.h diff --git a/src/lib/dispatch/lib_dispatch.md b/src/lib/dispatch/lib_dispatch.md new file mode 100644 index 0000000000..153ca50080 --- /dev/null +++ b/src/lib/dispatch/lib_dispatch.md @@ -0,0 +1,14 @@ +@dir /lib/dispatch +@brief lib/dispatch: In-process message delivery. + +This module provides a general in-process "message dispatch" system in which +typed messages are sent on channels. The dispatch.h header has far more +information. + +It is used by by \refdir{lib/pubsub} to implement our general +inter-module publish/subscribe system. + +This is not a fancy multi-threaded many-to-many dispatcher as you may be used +to from more sophisticated architectures: this dispatcher is intended only +for use in improving Tor's architecture. + diff --git a/src/lib/dispatch/msgtypes.h b/src/lib/dispatch/msgtypes.h new file mode 100644 index 0000000000..01d969dcb5 --- /dev/null +++ b/src/lib/dispatch/msgtypes.h @@ -0,0 +1,80 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file msgtypes.h + * \brief Types used for messages in the dispatcher code. + **/ + +#ifndef TOR_DISPATCH_MSGTYPES_H +#define TOR_DISPATCH_MSGTYPES_H + +#include <stdint.h> + +#include "ext/tor_queue.h" + +/** + * These types are aliases for subsystems, channels, and message IDs. + **/ +typedef uint16_t subsys_id_t; +typedef uint16_t channel_id_t; +typedef uint16_t message_id_t; + +/** + * This identifies a C type that can be sent along with a message. + **/ +typedef uint16_t msg_type_id_t; + +/** + * An ID value returned for *_type_t when none exists. + */ +#define ERROR_ID 65535 + +/** + * Auxiliary (untyped) data sent along with a message. + * + * We define this as a union of a pointer and a u64, so that the integer + * types will have the same range across platforms. + **/ +typedef union { + void *ptr; + uint64_t u64; +} msg_aux_data_t; + +/** + * Structure of a received message. + **/ +typedef struct msg_t { + TOR_SIMPLEQ_ENTRY(msg_t) next; + subsys_id_t sender; + channel_id_t channel; + message_id_t msg; + /** We could omit this field, since it is implicit in the message type, but + * IMO let's leave it in for safety. */ + msg_type_id_t type; + /** Untyped auxiliary data. You shouldn't have to mess with this + * directly. */ + msg_aux_data_t aux_data__; +} msg_t; + +/** + * A function that a subscriber uses to receive a message. + **/ +typedef void (*recv_fn_t)(const msg_t *m); + +/** + * Table of functions to use for a given C type. Any omitted (NULL) functions + * will be treated as no-ops. + **/ +typedef struct dispatch_typefns_t { + /** Release storage held for the auxiliary data of this type. */ + void (*free_fn)(msg_aux_data_t); + /** Format and return a newly allocated string describing the contents + * of this data element. */ + char *(*fmt_fn)(msg_aux_data_t); +} dispatch_typefns_t; + +#endif /* !defined(TOR_DISPATCH_MSGTYPES_H) */ diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include index 7c2ef36929..c9bf4b1786 100644 --- a/src/lib/encoding/.may_include +++ b/src/lib/encoding/.may_include @@ -1,5 +1,6 @@ orconfig.h lib/cc/*.h +lib/container/*.h lib/ctime/*.h lib/encoding/*.h lib/intmath/*.h diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c index bd063440d6..3e549eb8e3 100644 --- a/src/lib/encoding/binascii.c +++ b/src/lib/encoding/binascii.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -84,7 +84,7 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) } /** Implements base32 decoding as in RFC 4648. - * Returns 0 if successful, -1 otherwise. + * Return the number of bytes decoded if successful; -1 otherwise. */ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) @@ -147,7 +147,7 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) memset(tmp, 0, srclen); /* on the heap, this should be safe */ tor_free(tmp); tmp = NULL; - return 0; + return i; } #define BASE64_OPENSSL_LINELEN 64 @@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags) return enclen; } +/** Return an upper bound on the number of bytes that might be needed to hold + * the data from decoding the base64 string <b>srclen</b>. This is only an + * upper bound, since some part of the base64 string might be padding or + * space. */ +size_t +base64_decode_maxsize(size_t srclen) +{ + tor_assert(srclen < INT_MAX / 3); + + return CEIL_DIV(srclen * 3, 4); +} + /** Internal table mapping 6 bit values to the Base64 alphabet. */ static const char base64_encode_table[64] = { 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', @@ -309,8 +321,10 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, return (int) enclen; } -/** As base64_encode, but do not add any internal spaces or external padding - * to the output stream. */ +/** As base64_encode, but do not add any internal spaces, and remove external + * padding from the output stream. + * dest must be at least base64_encode_size(srclen, 0), including space for + * the removed external padding. */ int base64_encode_nopad(char *dest, size_t destlen, const uint8_t *src, size_t srclen) diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h index 7e3cc04f09..9cb03bab62 100644 --- a/src/lib/encoding/binascii.h +++ b/src/lib/encoding/binascii.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen); #define BASE64_ENCODE_MULTILINE 1 size_t base64_encode_size(size_t srclen, int flags); +size_t base64_decode_maxsize(size_t srclen); int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, int flags); int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen); @@ -57,4 +58,4 @@ size_t base32_encoded_size(size_t srclen); void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen); -#endif /* !defined(TOR_UTIL_FORMAT_H) */ +#endif /* !defined(TOR_BINASCII_H) */ diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c index 8110f3dd9c..613e4a00c6 100644 --- a/src/lib/encoding/confline.c +++ b/src/lib/encoding/confline.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -82,6 +82,19 @@ config_line_find(const config_line_t *lines, return NULL; } +/** As config_line_find(), but perform a case-insensitive comparison. */ +const config_line_t * +config_line_find_case(const config_line_t *lines, + const char *key) +{ + const config_line_t *cl; + for (cl = lines; cl; cl = cl->next) { + if (!strcasecmp(cl->key, key)) + return cl; + } + return NULL; +} + /** Auxiliary function that does all the work of config_get_lines. * <b>recursion_level</b> is the count of how many nested %includes we have. * <b>opened_lst</b> will have a list of opened files if provided. @@ -138,6 +151,8 @@ config_get_lines_aux(const char *string, config_line_t **result, int extended, if (allow_include && !strcmp(k, "%include") && handle_include) { tor_free(k); include_used = 1; + log_notice(LD_CONFIG, "Processing configuration path \"%s\" at " + "recursion level %d.", v, recursion_level); config_line_t *include_list; if (handle_include(v, recursion_level, extended, &include_list, @@ -148,9 +163,6 @@ config_get_lines_aux(const char *string, config_line_t **result, int extended, tor_free(v); return -1; } - log_notice(LD_CONFIG, "Included configuration file or " - "directory at recursion level %d: \"%s\".", - recursion_level, v); *next = include_list; if (list_last) next = &list_last->next; @@ -240,10 +252,39 @@ config_lines_dup_and_filter(const config_line_t *inp, return result; } +/** + * Given a linelist <b>inp</b> beginning with the key <b>header</b>, find the + * next line with that key, and remove that instance and all following lines + * from the list. Return the lines that were removed. Operate + * case-insensitively. + * + * For example, if the header is "H", and <b>inp</b> contains "H, A, B, H, C, + * H, D", this function will alter <b>inp</b> to contain only "H, A, B", and + * return the elements "H, C, H, D" as a separate list. + **/ +config_line_t * +config_lines_partition(config_line_t *inp, const char *header) +{ + if (BUG(inp == NULL)) + return NULL; + if (BUG(strcasecmp(inp->key, header))) + return NULL; + + /* Advance ptr until it points to the link to the next segment of this + list. */ + config_line_t **ptr = &inp->next; + while (*ptr && strcasecmp((*ptr)->key, header)) { + ptr = &(*ptr)->next; + } + config_line_t *remainder = *ptr; + *ptr = NULL; + return remainder; +} + /** Return true iff a and b contain identical keys and values in identical * order. */ int -config_lines_eq(config_line_t *a, config_line_t *b) +config_lines_eq(const config_line_t *a, const config_line_t *b) { while (a && b) { if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value)) diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h index 3d9ae8a662..ce0d6c6e17 100644 --- a/src/lib/encoding/confline.h +++ b/src/lib/encoding/confline.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -48,7 +48,10 @@ config_line_t *config_lines_dup_and_filter(const config_line_t *inp, const char *key); const config_line_t *config_line_find(const config_line_t *lines, const char *key); -int config_lines_eq(config_line_t *a, config_line_t *b); +const config_line_t *config_line_find_case(const config_line_t *lines, + const char *key); +config_line_t *config_lines_partition(config_line_t *inp, const char *header); +int config_lines_eq(const config_line_t *a, const config_line_t *b); int config_count_key(const config_line_t *a, const char *key); void config_free_lines_(config_line_t *front); #define config_free_lines(front) \ diff --git a/src/lib/encoding/cstring.c b/src/lib/encoding/cstring.c index 29d3714126..54c330fca3 100644 --- a/src/lib/encoding/cstring.c +++ b/src/lib/encoding/cstring.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/cstring.h b/src/lib/encoding/cstring.h index 904a2c9c1c..2a3f6d0fc4 100644 --- a/src/lib/encoding/cstring.h +++ b/src/lib/encoding/cstring.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am index 2d2aa3988a..48d0120bfc 100644 --- a/src/lib/encoding/include.am +++ b/src/lib/encoding/include.am @@ -4,12 +4,15 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-encoding-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_encoding_a_SOURCES = \ src/lib/encoding/binascii.c \ src/lib/encoding/confline.c \ src/lib/encoding/cstring.c \ src/lib/encoding/keyval.c \ + src/lib/encoding/kvline.c \ src/lib/encoding/pem.c \ + src/lib/encoding/qstring.c \ src/lib/encoding/time_fmt.c src_lib_libtor_encoding_testing_a_SOURCES = \ @@ -17,10 +20,13 @@ src_lib_libtor_encoding_testing_a_SOURCES = \ src_lib_libtor_encoding_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_encoding_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/encoding/binascii.h \ src/lib/encoding/confline.h \ src/lib/encoding/cstring.h \ src/lib/encoding/keyval.h \ + src/lib/encoding/kvline.h \ src/lib/encoding/pem.h \ + src/lib/encoding/qstring.h \ src/lib/encoding/time_fmt.h diff --git a/src/lib/encoding/keyval.c b/src/lib/encoding/keyval.c index c5da5a0bfc..0eb1219d43 100644 --- a/src/lib/encoding/keyval.c +++ b/src/lib/encoding/keyval.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/keyval.h b/src/lib/encoding/keyval.h index cd327b7a82..b4966b01de 100644 --- a/src/lib/encoding/keyval.h +++ b/src/lib/encoding/keyval.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,4 +14,4 @@ int string_is_key_value(int severity, const char *string); -#endif +#endif /* !defined(TOR_KEYVAL_H) */ diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c new file mode 100644 index 0000000000..5b220605d6 --- /dev/null +++ b/src/lib/encoding/kvline.c @@ -0,0 +1,297 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.c + * + * \brief Manipulating lines of key-value pairs. + **/ + +#include "orconfig.h" + +#include "lib/container/smartlist.h" +#include "lib/encoding/confline.h" +#include "lib/encoding/cstring.h" +#include "lib/encoding/kvline.h" +#include "lib/encoding/qstring.h" +#include "lib/malloc/malloc.h" +#include "lib/string/compat_ctype.h" +#include "lib/string/printf.h" +#include "lib/string/util_string.h" +#include "lib/log/escape.h" +#include "lib/log/util_bug.h" + +#include <stdbool.h> +#include <stddef.h> +#include <string.h> + +/** Return true iff we need to quote and escape the string <b>s</b> to encode + * it. + * + * kvline_can_encode_lines() also uses this (with + * <b>as_keyless_val</b> true) to check whether a key would require + * quoting. + */ +static bool +needs_escape(const char *s, bool as_keyless_val) +{ + if (as_keyless_val && *s == 0) + return true; + /* Keyless values containing '=' need to be escaped. */ + if (as_keyless_val && strchr(s, '=')) + return true; + + for (; *s; ++s) { + if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) || + *s == '\'' || *s == '\"') { + return true; + } + } + return false; +} + +/** + * Return true iff the key in <b>line</b> is not set. + **/ +static bool +line_has_no_key(const config_line_t *line) +{ + return line->key == NULL || strlen(line->key) == 0; +} + +/** + * Return true iff the value in <b>line</b> is not set. + **/ +static bool +line_has_no_val(const config_line_t *line) +{ + return line->value == NULL || strlen(line->value) == 0; +} + +/** + * Return true iff the all the lines in <b>line</b> can be encoded + * using <b>flags</b>. + **/ +static bool +kvline_can_encode_lines(const config_line_t *line, unsigned flags) +{ + for ( ; line; line = line->next) { + const bool keyless = line_has_no_key(line); + if (keyless && ! (flags & KV_OMIT_KEYS)) { + /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */ + return false; + } + + if (needs_escape(line->value, keyless) && ! (flags & (KV_QUOTED|KV_RAW))) { + /* If both KV_QUOTED and KV_RAW are false, we can't encode a + value that needs quotes. */ + return false; + } + if (!keyless && needs_escape(line->key, true)) { + /* We can't handle keys that need quoting. */ + return false; + } + } + return true; +} + +/** + * Encode a linked list of lines in <b>line</b> as a series of 'Key=Value' + * pairs, using the provided <b>flags</b> to encode it. Return a newly + * allocated string on success, or NULL on failure. + * + * If KV_QUOTED is set in <b>flags</b>, then all values that contain + * spaces or unusual characters are escaped and quoted. Otherwise, such + * values are not allowed. Mutually exclusive with KV_RAW. + * + * If KV_OMIT_KEYS is set in <b>flags</b>, then pairs with empty keys are + * allowed, and are encoded as 'Value'. Otherwise, such pairs are not + * allowed. + * + * If KV_OMIT_VALS is set in <b>flags</b>, then an empty value is + * encoded as 'Key', not as 'Key=' or 'Key=""'. Mutually exclusive with + * KV_OMIT_KEYS. + * + * If KV_RAW is set in <b>flags</b>, then don't apply any quoting to + * the value, and assume that the caller has adequately quoted it. + * (The control protocol has some quirks that make this necessary.) + * Mutually exclusive with KV_QUOTED. + * + * KV_QUOTED_QSTRING is not supported. + */ +char * +kvline_encode(const config_line_t *line, + unsigned flags) +{ + tor_assert(! (flags & KV_QUOTED_QSTRING)); + + tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) != + (KV_OMIT_KEYS|KV_OMIT_VALS)); + tor_assert((flags & (KV_QUOTED|KV_RAW)) != (KV_QUOTED|KV_RAW)); + + if (!kvline_can_encode_lines(line, flags)) + return NULL; + + smartlist_t *elements = smartlist_new(); + + for (; line; line = line->next) { + + const char *k = ""; + const char *eq = "="; + const char *v = ""; + const bool keyless = line_has_no_key(line); + bool esc = needs_escape(line->value, keyless); + char *tmp = NULL; + + if (! keyless) { + k = line->key; + } else { + eq = ""; + } + + if ((flags & KV_OMIT_VALS) && line_has_no_val(line)) { + eq = ""; + v = ""; + } else if (!(flags & KV_RAW) && esc) { + tmp = esc_for_log(line->value); + v = tmp; + } else { + v = line->value; + } + + smartlist_add_asprintf(elements, "%s%s%s", k, eq, v); + tor_free(tmp); + } + + char *result = smartlist_join_strings(elements, " ", 0, NULL); + + SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp)); + smartlist_free(elements); + + return result; +} + +/** + * Decode a <b>line</b> containing a series of space-separated 'Key=Value' + * pairs, using the provided <b>flags</b> to decode it. Return a newly + * allocated list of pairs on success, or NULL on failure. + * + * If KV_QUOTED is set in <b>flags</b>, then (double-)quoted values are + * allowed and handled as C strings. Otherwise, such values are not allowed. + * + * If KV_OMIT_KEYS is set in <b>flags</b>, then values without keys are + * allowed. Otherwise, such values are not allowed. + * + * If KV_OMIT_VALS is set in <b>flags</b>, then keys without values are + * allowed. Otherwise, such keys are not allowed. Mutually exclusive with + * KV_OMIT_KEYS. + * + * If KV_QUOTED_QSTRING is set in <b>flags</b>, then double-quoted values + * are allowed and handled as QuotedStrings per qstring.c. Do not add + * new users of this flag. + * + * KV_RAW is not supported. + */ +config_line_t * +kvline_parse(const char *line, unsigned flags) +{ + tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) != + (KV_OMIT_KEYS|KV_OMIT_VALS)); + tor_assert(!(flags & KV_RAW)); + + const char *cp = line, *cplast = NULL; + const bool omit_keys = (flags & KV_OMIT_KEYS) != 0; + const bool omit_vals = (flags & KV_OMIT_VALS) != 0; + const bool quoted = (flags & (KV_QUOTED|KV_QUOTED_QSTRING)) != 0; + const bool c_quoted = (flags & (KV_QUOTED)) != 0; + + config_line_t *result = NULL; + config_line_t **next_line = &result; + + char *key = NULL; + char *val = NULL; + + while (*cp) { + key = val = NULL; + /* skip all spaces */ + { + size_t idx = strspn(cp, " \t\r\v\n"); + cp += idx; + } + if (BUG(cp == cplast)) { + /* If we didn't parse anything since the last loop, this code is + * broken. */ + goto err; // LCOV_EXCL_LINE + } + cplast = cp; + if (! *cp) + break; /* End of string; we're done. */ + + /* Possible formats are K=V, K="V", K, V, and "V", depending on flags. */ + + /* Find where the key ends */ + if (*cp != '\"') { + size_t idx = strcspn(cp, " \t\r\v\n="); + + if (cp[idx] == '=') { + key = tor_memdup_nulterm(cp, idx); + cp += idx + 1; + } else if (omit_vals) { + key = tor_memdup_nulterm(cp, idx); + cp += idx; + goto commit; + } else { + if (!omit_keys) + goto err; + } + } + + if (*cp == '\"') { + /* The type is "V". */ + if (!quoted) + goto err; + size_t len=0; + if (c_quoted) { + cp = unescape_string(cp, &val, &len); + } else { + cp = decode_qstring(cp, strlen(cp), &val, &len); + } + if (cp == NULL || len != strlen(val)) { + // The string contains a NUL or is badly coded. + goto err; + } + } else { + size_t idx = strcspn(cp, " \t\r\v\n"); + val = tor_memdup_nulterm(cp, idx); + cp += idx; + } + + commit: + if (key && strlen(key) == 0) { + /* We don't allow empty keys. */ + goto err; + } + + *next_line = tor_malloc_zero(sizeof(config_line_t)); + (*next_line)->key = key ? key : tor_strdup(""); + (*next_line)->value = val ? val : tor_strdup(""); + next_line = &(*next_line)->next; + key = val = NULL; + } + + if (! (flags & KV_QUOTED_QSTRING)) { + if (!kvline_can_encode_lines(result, flags)) { + goto err; + } + } + return result; + + err: + tor_free(key); + tor_free(val); + config_free_lines(result); + return NULL; +} diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h new file mode 100644 index 0000000000..34c52908e3 --- /dev/null +++ b/src/lib/encoding/kvline.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file kvline.h + * + * \brief Header for kvline.c + **/ + +#ifndef TOR_KVLINE_H +#define TOR_KVLINE_H + +struct config_line_t; + +#define KV_QUOTED (1u<<0) +#define KV_OMIT_KEYS (1u<<1) +#define KV_OMIT_VALS (1u<<2) +#define KV_QUOTED_QSTRING (1u<<3) +#define KV_RAW (1u<<4) + +struct config_line_t *kvline_parse(const char *line, unsigned flags); +char *kvline_encode(const struct config_line_t *line, unsigned flags); + +#endif /* !defined(TOR_KVLINE_H) */ diff --git a/src/lib/encoding/lib_encoding.md b/src/lib/encoding/lib_encoding.md new file mode 100644 index 0000000000..66dd9d8caf --- /dev/null +++ b/src/lib/encoding/lib_encoding.md @@ -0,0 +1,6 @@ +@dir /lib/encoding +@brief lib/encoding: Encoding data in various forms, types, and transformations + +Here we have time formats (timefmt.c), quoted strings (qstring.c), C strings +(string.c) base-16/32/64 (binascii.c), and more. + diff --git a/src/lib/encoding/pem.c b/src/lib/encoding/pem.c index 51f37d0840..6c9f10e085 100644 --- a/src/lib/encoding/pem.c +++ b/src/lib/encoding/pem.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -42,7 +42,7 @@ pem_encoded_size(size_t src_len, const char *objtype) /** * PEM-encode the <b>srclen</b>-byte object at <b>src</b> into the - * <b>destlen<\b>-byte buffer at <b>dest</b>, tagging it with <b>objtype</b>. + * <b>destlen</b>-byte buffer at <b>dest</b>, tagging it with <b>objtype</b>. * Return 0 on success and -1 on failure. */ int diff --git a/src/lib/encoding/pem.h b/src/lib/encoding/pem.h index 0bbb06a794..027c31c315 100644 --- a/src/lib/encoding/pem.h +++ b/src/lib/encoding/pem.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -23,4 +23,4 @@ int pem_encode(char *dest, size_t destlen, const uint8_t *src, size_t srclen, int pem_decode(uint8_t *dest, size_t destlen, const char *src, size_t srclen, const char *objtype); -#endif +#endif /* !defined(TOR_PEM_H) */ diff --git a/src/lib/encoding/qstring.c b/src/lib/encoding/qstring.c new file mode 100644 index 0000000000..5a34924eab --- /dev/null +++ b/src/lib/encoding/qstring.c @@ -0,0 +1,90 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file qstring.c + * \brief Implement QuotedString parsing. + * + * Note that this is only used for controller authentication; do not + * create new users for this. Instead, prefer the cstring.c functions. + **/ + +#include "orconfig.h" +#include "lib/encoding/qstring.h" +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +/** If the first <b>in_len_max</b> characters in <b>start</b> contain a + * QuotedString, return the length of that + * string (as encoded, including quotes). Otherwise return -1. */ +static inline int +get_qstring_length(const char *start, size_t in_len_max, + int *chars_out) +{ + const char *cp, *end; + int chars = 0; + + if (*start != '\"') + return -1; + + cp = start+1; + end = start+in_len_max; + + /* Calculate length. */ + while (1) { + if (cp >= end) { + return -1; /* Too long. */ + } else if (*cp == '\\') { + if (++cp == end) + return -1; /* Can't escape EOS. */ + ++cp; + ++chars; + } else if (*cp == '\"') { + break; + } else { + ++cp; + ++chars; + } + } + if (chars_out) + *chars_out = chars; + return (int)(cp - start+1); +} + +/** Given a pointer to a string starting at <b>start</b> containing + * <b>in_len_max</b> characters, decode a string beginning with one double + * quote, containing any number of non-quote characters or characters escaped + * with a backslash, and ending with a final double quote. Place the resulting + * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>; + * store its length in <b>out_len</b>. On success, return a pointer to the + * character immediately following the escaped string. On failure, return + * NULL. */ +const char * +decode_qstring(const char *start, size_t in_len_max, + char **out, size_t *out_len) +{ + const char *cp, *end; + char *outp; + int len, n_chars = 0; + + len = get_qstring_length(start, in_len_max, &n_chars); + if (len<0) + return NULL; + + end = start+len-1; /* Index of last quote. */ + tor_assert(*end == '\"'); + outp = *out = tor_malloc(len+1); + *out_len = n_chars; + + cp = start+1; + while (cp < end) { + if (*cp == '\\') + ++cp; + *outp++ = *cp++; + } + *outp = '\0'; + tor_assert((outp - *out) == (int)*out_len); + + return end+1; +} diff --git a/src/lib/encoding/qstring.h b/src/lib/encoding/qstring.h new file mode 100644 index 0000000000..f19a7dad87 --- /dev/null +++ b/src/lib/encoding/qstring.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file qstring.h + * \brief Header for qstring.c + */ + +#ifndef TOR_ENCODING_QSTRING_H +#define TOR_ENCODING_QSTRING_H + +#include <stddef.h> + +const char *decode_qstring(const char *start, size_t in_len_max, + char **out, size_t *out_len); + +#endif /* !defined(TOR_ENCODING_QSTRING_H) */ diff --git a/src/lib/encoding/time_fmt.c b/src/lib/encoding/time_fmt.c index 40543d41e0..573dfaad82 100644 --- a/src/lib/encoding/time_fmt.c +++ b/src/lib/encoding/time_fmt.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/encoding/time_fmt.h b/src/lib/encoding/time_fmt.h index 0ddeca57fc..80e47c5332 100644 --- a/src/lib/encoding/time_fmt.h +++ b/src/lib/encoding/time_fmt.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -41,4 +41,4 @@ int parse_iso_time_nospace(const char *cp, time_t *t); int parse_http_time(const char *buf, struct tm *tm); int format_time_interval(char *out, size_t out_len, long interval); -#endif +#endif /* !defined(TOR_TIME_FMT_H) */ diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include index 48cc0ef088..314424545e 100644 --- a/src/lib/err/.may_include +++ b/src/lib/err/.may_include @@ -1,3 +1,6 @@ orconfig.h lib/cc/*.h +lib/defs/*.h lib/err/*.h +lib/subsys/*.h +lib/version/*.h diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c index 8606f42177..afb6b9503f 100644 --- a/src/lib/err/backtrace.c +++ b/src/lib/err/backtrace.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Tor Project, Inc. */ +/* Copyright (c) 2013-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -52,12 +52,14 @@ #include <pthread.h> #endif -#define EXPOSE_CLEAN_BACKTRACE +#include "lib/cc/ctassert.h" + +#define BACKTRACE_PRIVATE #include "lib/err/backtrace.h" -#include "lib/err/torerr.h" #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ - defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) + defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) && \ + defined(HAVE_PTHREAD_H) #define USE_BACKTRACE #endif @@ -72,15 +74,40 @@ static char bt_version[128] = ""; #ifdef USE_BACKTRACE + /** Largest stack depth to try to dump. */ #define MAX_DEPTH 256 -/** Static allocation of stack to dump. This is static so we avoid stack - * pressure. */ -static void *cb_buf[MAX_DEPTH]; +/** The size of the callback buffer, so we can clear it in unlock_cb_buf(). */ +#define SIZEOF_CB_BUF (MAX_DEPTH * sizeof(void *)) /** Protects cb_buf from concurrent access. Pthreads, since this code * is Unix-only, and since this code needs to be lowest-level. */ static pthread_mutex_t cb_buf_mutex = PTHREAD_MUTEX_INITIALIZER; +/** Lock and return a static stack pointer buffer that can hold up to + * MAX_DEPTH function pointers. */ +static void ** +lock_cb_buf(void) +{ + /* Lock the mutex first, before even declaring the buffer. */ + pthread_mutex_lock(&cb_buf_mutex); + + /** Static allocation of stack to dump. This is static so we avoid stack + * pressure. */ + static void *cb_buf[MAX_DEPTH]; + CTASSERT(SIZEOF_CB_BUF == sizeof(cb_buf)); + memset(cb_buf, 0, SIZEOF_CB_BUF); + + return cb_buf; +} + +/** Unlock the static stack pointer buffer. */ +static void +unlock_cb_buf(void **cb_buf) +{ + memset(cb_buf, 0, SIZEOF_CB_BUF); + pthread_mutex_unlock(&cb_buf_mutex); +} + /** Change a stacktrace in <b>stack</b> of depth <b>depth</b> so that it will * log the correct function from which a signal was received with context * <b>ctx</b>. (When we get a signal, the current function will not have @@ -104,7 +131,7 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx) return; stack[n] = (void*) ctx->PC_FROM_UCONTEXT; -#else /* !(defined(PC_FROM_UCONTEXT)) */ +#else /* !defined(PC_FROM_UCONTEXT) */ (void) depth; (void) ctx; (void) stack; @@ -115,14 +142,14 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx) * that with a backtrace log. Send messages via the tor_log function at * logger". */ void -log_backtrace_impl(int severity, int domain, const char *msg, +log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg, tor_log_fn logger) { size_t depth; char **symbols; size_t i; - pthread_mutex_lock(&cb_buf_mutex); + void **cb_buf = lock_cb_buf(); depth = backtrace(cb_buf, MAX_DEPTH); symbols = backtrace_symbols(cb_buf, (int)depth); @@ -140,7 +167,7 @@ log_backtrace_impl(int severity, int domain, const char *msg, raw_free(symbols); done: - pthread_mutex_unlock(&cb_buf_mutex); + unlock_cb_buf(cb_buf); } static void crash_handler(int sig, siginfo_t *si, void *ctx_) @@ -156,6 +183,8 @@ crash_handler(int sig, siginfo_t *si, void *ctx_) int n_fds, i; const int *fds = NULL; + void **cb_buf = lock_cb_buf(); + (void) si; depth = backtrace(cb_buf, MAX_DEPTH); @@ -172,7 +201,9 @@ crash_handler(int sig, siginfo_t *si, void *ctx_) for (i=0; i < n_fds; ++i) backtrace_symbols_fd(cb_buf, (int)depth, fds[i]); - abort(); + unlock_cb_buf(cb_buf); + + tor_raw_abort_(); } /** Write a backtrace to all of the emergency-error fds. */ @@ -183,20 +214,26 @@ dump_stack_symbols_to_error_fds(void) const int *fds = NULL; size_t depth; + void **cb_buf = lock_cb_buf(); + depth = backtrace(cb_buf, MAX_DEPTH); n_fds = tor_log_get_sigsafe_err_fds(&fds); for (i=0; i < n_fds; ++i) backtrace_symbols_fd(cb_buf, (int)depth, fds[i]); + + unlock_cb_buf(cb_buf); } +/* The signals that we want our backtrace handler to trap */ +static int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, + SIGIO, -1 }; + /** Install signal handlers as needed so that when we crash, we produce a * useful stack trace. Return 0 on success, -errno on failure. */ static int install_bt_handler(void) { - int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, - SIGIO, -1 }; int i, rv=0; struct sigaction sa; @@ -219,10 +256,12 @@ install_bt_handler(void) * libc has pre-loaded the symbols we need to dump things, so that later * reads won't be denied by the sandbox code */ char **symbols; + void **cb_buf = lock_cb_buf(); size_t depth = backtrace(cb_buf, MAX_DEPTH); symbols = backtrace_symbols(cb_buf, (int) depth); if (symbols) raw_free(symbols); + unlock_cb_buf(cb_buf); } return rv; @@ -232,12 +271,29 @@ install_bt_handler(void) static void remove_bt_handler(void) { + int i; + + struct sigaction sa; + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = SIG_DFL; + sigfillset(&sa.sa_mask); + + for (i = 0; trap_signals[i] >= 0; ++i) { + /* remove_bt_handler() is called on shutdown, from low-level code. + * It's not a fatal error, so we just ignore it. */ + (void)sigaction(trap_signals[i], &sa, NULL); + } + + /* cb_buf_mutex is statically initialised, so we can not destroy it. + * If we destroy it, and then re-initialise tor, all our backtraces will + * fail. */ } #endif /* defined(USE_BACKTRACE) */ #ifdef NO_BACKTRACE_IMPL void -log_backtrace_impl(int severity, int domain, const char *msg, +log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg, tor_log_fn logger) { logger(severity, domain, "%s: %s. (Stack trace not available)", diff --git a/src/lib/err/backtrace.h b/src/lib/err/backtrace.h index 48b41fca02..d02e6960b5 100644 --- a/src/lib/err/backtrace.h +++ b/src/lib/err/backtrace.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Tor Project, Inc. */ +/* Copyright (c) 2013-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_BACKTRACE_H @@ -12,11 +12,14 @@ #include "orconfig.h" #include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" +#include "lib/defs/logging_types.h" -typedef void (*tor_log_fn)(int, unsigned, const char *fmt, ...) +typedef void (*tor_log_fn)(int, log_domain_mask_t, const char *fmt, ...) CHECK_PRINTF(3,4); -void log_backtrace_impl(int severity, int domain, const char *msg, +void log_backtrace_impl(int severity, log_domain_mask_t domain, + const char *msg, tor_log_fn logger); int configure_backtrace_handler(const char *tor_version); void clean_up_backtrace_handler(void); @@ -26,11 +29,11 @@ const char *get_tor_backtrace_version(void); #define log_backtrace(sev, dom, msg) \ log_backtrace_impl((sev), (dom), (msg), tor_log) -#ifdef EXPOSE_CLEAN_BACKTRACE +#ifdef BACKTRACE_PRIVATE #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) void clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx); #endif -#endif /* defined(EXPOSE_CLEAN_BACKTRACE) */ +#endif /* defined(BACKTRACE_PRIVATE) */ #endif /* !defined(TOR_BACKTRACE_H) */ diff --git a/src/lib/err/include.am b/src/lib/err/include.am index f2a409c51e..883ac91511 100644 --- a/src/lib/err/include.am +++ b/src/lib/err/include.am @@ -5,15 +5,19 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-err-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_err_a_SOURCES = \ - src/lib/err/backtrace.c \ - src/lib/err/torerr.c + src/lib/err/backtrace.c \ + src/lib/err/torerr.c \ + src/lib/err/torerr_sys.c src_lib_libtor_err_testing_a_SOURCES = \ $(src_lib_libtor_err_a_SOURCES) src_lib_libtor_err_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/err/backtrace.h \ - src/lib/err/torerr.h + src/lib/err/torerr.h \ + src/lib/err/torerr_sys.h diff --git a/src/lib/err/lib_err.md b/src/lib/err/lib_err.md new file mode 100644 index 0000000000..cb4eba2e0d --- /dev/null +++ b/src/lib/err/lib_err.md @@ -0,0 +1,13 @@ +@dir /lib/err +@brief lib/err: Lowest-level error handling code. + +This module is responsible for generating stack traces, handling raw +assertion failures, and otherwise reporting problems that might not be +safe to report via the regular logging module. + +There are three kinds of users for the functions in this module: + * Code that needs a way to assert(), but which cannot use the regular + `tor_assert()` macros in logging module. + * Code that needs signal-safe error reporting. + * Higher-level error handling code. + diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c index 6b5224273a..2de75c0be4 100644 --- a/src/lib/err/torerr.c +++ b/src/lib/err/torerr.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -110,6 +110,14 @@ tor_log_get_sigsafe_err_fds(const int **out) * Update the list of fds that get errors from inside a signal handler or * other emergency condition. Ignore any beyond the first * TOR_SIGSAFE_LOG_MAX_FDS. + * + * These fds must remain open even after the log module has shut down. (And + * they should remain open even while logs are being reconfigured.) Therefore, + * any fds closed by the log module should be dup()ed, and the duplicate fd + * should be given to the err module in fds. In particular, the log module + * closes the file log fds, but does not close the stdio log fds. + * + * If fds is NULL or n is 0, clears the list of error fds. */ void tor_log_set_sigsafe_err_fds(const int *fds, int n) @@ -118,8 +126,52 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n) n = TOR_SIGSAFE_LOG_MAX_FDS; } - memcpy(sigsafe_log_fds, fds, n * sizeof(int)); - n_sigsafe_log_fds = n; + /* Clear the entire array. This code mitigates against some race conditions, + * but there are still some races here: + * - err logs are disabled while the array is cleared, and + * - a thread can read the old value of n_sigsafe_log_fds, then read a + * partially written array. + * We could fix these races using atomics, but atomics use the err module. */ + n_sigsafe_log_fds = 0; + memset(sigsafe_log_fds, 0, sizeof(sigsafe_log_fds)); + if (fds && n > 0) { + memcpy(sigsafe_log_fds, fds, n * sizeof(int)); + n_sigsafe_log_fds = n; + } +} + +/** + * Reset the list of emergency error fds to its default. + */ +void +tor_log_reset_sigsafe_err_fds(void) +{ + int fds[] = { STDERR_FILENO }; + tor_log_set_sigsafe_err_fds(fds, 1); +} + +/** + * Flush the list of fds that get errors from inside a signal handler or + * other emergency condition. These fds are shared with the logging code: + * flushing them also flushes the log buffers. + * + * This function is safe to call during signal handlers. + */ +void +tor_log_flush_sigsafe_err_fds(void) +{ + /* If we don't have fsync() in unistd.h, we can't flush the logs. */ +#ifdef HAVE_FSYNC + int n_fds, i; + const int *fds = NULL; + + n_fds = tor_log_get_sigsafe_err_fds(&fds); + for (i = 0; i < n_fds; ++i) { + /* This function is called on error and on shutdown, so we don't log, or + * take any other action, if fsync() fails. */ + (void)fsync(fds[i]); + } +#endif /* defined(HAVE_FSYNC) */ } /** @@ -161,6 +213,18 @@ tor_raw_assertion_failed_msg_(const char *file, int line, const char *expr, tor_log_err_sigsafe_write("\n"); } +/** + * Call the abort() function to kill the current process with a fatal + * error. But first, flush the raw error file descriptors, so error messages + * are written before process termination. + **/ +void +tor_raw_abort_(void) +{ + tor_log_flush_sigsafe_err_fds(); + abort(); +} + /* As format_{hex,dex}_number_sigsafe, but takes a <b>radix</b> argument * in range 2..16 inclusive. */ static int @@ -195,7 +259,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len, unsigned digit = (unsigned) (x % radix); if (cp <= buf) { /* Not tor_assert(); see above. */ - abort(); + tor_raw_abort_(); } --cp; *cp = "0123456789ABCDEF"[digit]; @@ -204,7 +268,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len, /* NOT tor_assert; see above. */ if (cp != buf) { - abort(); // LCOV_EXCL_LINE + tor_raw_abort_(); // LCOV_EXCL_LINE } return len; @@ -224,8 +288,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len, * does not guarantee that an int is wider than a char (an int must be at * least 16 bits but it is permitted for a char to be that wide as well), we * can't assume a signed int is sufficient to accommodate an unsigned char. - * Thus, format_helper_exit_status() will still need to emit any require '-' - * on its own. + * Thus, callers will still need to add any required '-' to the final string. * * For most purposes, you'd want to use tor_snprintf("%x") instead of this * function; it's designed to be used in code paths where you can't call diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h index 6ae91fbe85..ce1b049c47 100644 --- a/src/lib/err/torerr.h +++ b/src/lib/err/torerr.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,13 +20,13 @@ #define raw_assert(expr) STMT_BEGIN \ if (!(expr)) { \ tor_raw_assertion_failed_msg_(__FILE__, __LINE__, #expr, NULL); \ - abort(); \ + tor_raw_abort_(); \ } \ STMT_END #define raw_assert_unreached(expr) raw_assert(0) #define raw_assert_unreached_msg(msg) STMT_BEGIN \ tor_raw_assertion_failed_msg_(__FILE__, __LINE__, "0", (msg)); \ - abort(); \ + tor_raw_abort_(); \ STMT_END void tor_raw_assertion_failed_msg_(const char *file, int line, @@ -39,9 +39,13 @@ void tor_raw_assertion_failed_msg_(const char *file, int line, void tor_log_err_sigsafe(const char *m, ...); int tor_log_get_sigsafe_err_fds(const int **out); void tor_log_set_sigsafe_err_fds(const int *fds, int n); +void tor_log_reset_sigsafe_err_fds(void); +void tor_log_flush_sigsafe_err_fds(void); void tor_log_sigsafe_err_set_granularity(int ms); +void tor_raw_abort_(void) ATTR_NORETURN; + int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len); int format_dec_number_sigsafe(unsigned long x, char *buf, int max_len); -#endif /* !defined(TOR_TORLOG_H) */ +#endif /* !defined(TOR_TORERR_H) */ diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c new file mode 100644 index 0000000000..8ee1521f3b --- /dev/null +++ b/src/lib/err/torerr_sys.c @@ -0,0 +1,45 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.c + * \brief Subsystem object for the error handling subsystem. + **/ + +#include "orconfig.h" +#include "lib/err/backtrace.h" +#include "lib/err/torerr.h" +#include "lib/err/torerr_sys.h" +#include "lib/subsys/subsys.h" +#include "lib/version/torversion.h" + +#include <stddef.h> + +static int +subsys_torerr_initialize(void) +{ + if (configure_backtrace_handler(get_version()) < 0) + return -1; + tor_log_reset_sigsafe_err_fds(); + + return 0; +} +static void +subsys_torerr_shutdown(void) +{ + /* Stop handling signals with backtraces, then flush the logs. */ + clean_up_backtrace_handler(); + tor_log_flush_sigsafe_err_fds(); +} + +const subsys_fns_t sys_torerr = { + .name = "err", + SUBSYS_DECLARE_LOCATION(), + /* Low-level error handling is a diagnostic feature, we want it to init + * right after windows process security, and shutdown last. + * (Security never shuts down.) */ + .level = -99, + .supported = true, + .initialize = subsys_torerr_initialize, + .shutdown = subsys_torerr_shutdown +}; diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h new file mode 100644 index 0000000000..b86ccd2790 --- /dev/null +++ b/src/lib/err/torerr_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file torerr_sys.h + * \brief Declare subsystem object for torerr.c + **/ + +#ifndef TOR_TORERR_SYS_H +#define TOR_TORERR_SYS_H + +extern const struct subsys_fns_t sys_torerr; + +#endif /* !defined(TOR_TORERR_SYS_H) */ diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include index 30af508914..54aa75fbff 100644 --- a/src/lib/evloop/.may_include +++ b/src/lib/evloop/.may_include @@ -8,9 +8,10 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h lib/time/*.h -src/ext/timeouts/timeout.c -tor_queue.h
\ No newline at end of file +ext/timeouts/timeout.c +ext/tor_queue.h
\ No newline at end of file diff --git a/src/lib/evloop/compat_libevent.c b/src/lib/evloop/compat_libevent.c index 939d77f857..0fd247d331 100644 --- a/src/lib/evloop/compat_libevent.c +++ b/src/lib/evloop/compat_libevent.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2019, The Tor Project, Inc. */ +/* Copyright (c) 2009-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -130,7 +130,7 @@ rescan_mainloop_cb(evutil_socket_t fd, short events, void *arg) /** Initialize the Libevent library and set up the event base. */ void -tor_libevent_initialize(tor_libevent_cfg *torcfg) +tor_libevent_initialize(tor_libevent_cfg_t *torcfg) { tor_assert(the_event_base == NULL); /* some paths below don't use torcfg, so avoid unused variable warnings */ @@ -432,7 +432,7 @@ mainloop_event_activate(mainloop_event_t *event) * * If the event is scheduled for a different time, cancel it and run * after this delay instead. If the event is currently pending to run - * <em>now</b>, has no effect. + * <b>now</b>, has no effect. * * Do not call this function with <b>tv</b> == NULL -- use * mainloop_event_activate() instead. diff --git a/src/lib/evloop/compat_libevent.h b/src/lib/evloop/compat_libevent.h index 92724c369c..277ba3add6 100644 --- a/src/lib/evloop/compat_libevent.h +++ b/src/lib/evloop/compat_libevent.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2009-2019, The Tor Project, Inc. */ +/* Copyright (c) 2009-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -61,15 +61,15 @@ void mainloop_event_free_(mainloop_event_t *event); /** Defines a configuration for using libevent with Tor: passed as an argument * to tor_libevent_initialize() to describe how we want to set up. */ -typedef struct tor_libevent_cfg { +typedef struct tor_libevent_cfg_t { /** How many CPUs should we use (not currently useful). */ int num_cpus; /** How many milliseconds should we allow between updating bandwidth limits? * (Not currently useful). */ int msec_per_tick; -} tor_libevent_cfg; +} tor_libevent_cfg_t; -void tor_libevent_initialize(tor_libevent_cfg *cfg); +void tor_libevent_initialize(tor_libevent_cfg_t *cfg); bool tor_libevent_is_initialized(void); MOCK_DECL(struct event_base *, tor_libevent_get_base, (void)); const char *tor_libevent_get_method(void); diff --git a/src/lib/evloop/evloop_sys.c b/src/lib/evloop/evloop_sys.c new file mode 100644 index 0000000000..b639810c23 --- /dev/null +++ b/src/lib/evloop/evloop_sys.c @@ -0,0 +1,50 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file evloop_sys.c + * @brief Subsystem definition for the event loop module + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/evloop/evloop_sys.h" +#include "lib/log/log.h" + +static int +subsys_evloop_initialize(void) +{ + if (tor_init_libevent_rng() < 0) { + log_warn(LD_NET, "Problem initializing libevent RNG."); + return -1; + } + return 0; +} + +static void +subsys_evloop_postfork(void) +{ +#ifdef TOR_UNIT_TESTS + tor_libevent_postfork(); +#endif +} + +static void +subsys_evloop_shutdown(void) +{ + tor_libevent_free_all(); +} + +const struct subsys_fns_t sys_evloop = { + .name = "evloop", + SUBSYS_DECLARE_LOCATION(), + .supported = true, + .level = -20, + .initialize = subsys_evloop_initialize, + .shutdown = subsys_evloop_shutdown, + .postfork = subsys_evloop_postfork, +}; diff --git a/src/lib/evloop/evloop_sys.h b/src/lib/evloop/evloop_sys.h new file mode 100644 index 0000000000..a37440e7a6 --- /dev/null +++ b/src/lib/evloop/evloop_sys.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file evloop_sys.h + * @brief Declare subsystem object for the event loop module. + **/ + +#ifndef TOR_LIB_EVLOOP_EVLOOP_SYS_H +#define TOR_LIB_EVLOOP_EVLOOP_SYS_H + +extern const struct subsys_fns_t sys_evloop; + +#endif /* !defined(TOR_LIB_EVLOOP_EVLOOP_SYS_H) */ diff --git a/src/lib/evloop/include.am b/src/lib/evloop/include.am index 6b0076272a..41cd2f45c5 100644 --- a/src/lib/evloop/include.am +++ b/src/lib/evloop/include.am @@ -5,21 +5,24 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-evloop-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_evloop_a_SOURCES = \ src/lib/evloop/compat_libevent.c \ + src/lib/evloop/evloop_sys.c \ src/lib/evloop/procmon.c \ src/lib/evloop/timers.c \ src/lib/evloop/token_bucket.c \ src/lib/evloop/workqueue.c - src_lib_libtor_evloop_testing_a_SOURCES = \ $(src_lib_libtor_evloop_a_SOURCES) src_lib_libtor_evloop_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_evloop_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/evloop/compat_libevent.h \ + src/lib/evloop/evloop_sys.h \ src/lib/evloop/procmon.h \ src/lib/evloop/timers.h \ src/lib/evloop/token_bucket.h \ diff --git a/src/lib/evloop/lib_evloop.md b/src/lib/evloop/lib_evloop.md new file mode 100644 index 0000000000..830be88148 --- /dev/null +++ b/src/lib/evloop/lib_evloop.md @@ -0,0 +1,7 @@ +@dir /lib/evloop +@brief lib/evloop: Low-level event loop. + +This modules has tools to manage the [libevent](https://libevent.org/) event +loop and related functionality, in order to implement asynchronous +networking, timers, periodic events, and other scheduling tasks. + diff --git a/src/lib/evloop/procmon.c b/src/lib/evloop/procmon.c index 52469fa5fc..718c7d4777 100644 --- a/src/lib/evloop/procmon.c +++ b/src/lib/evloop/procmon.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2019, The Tor Project, Inc. */ +/* Copyright (c) 2011-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -165,8 +165,8 @@ tor_validate_process_specifier(const char *process_spec, return parse_process_specifier(process_spec, &ppspec, msg); } -/* DOCDOC poll_interval_tv */ -static const struct timeval poll_interval_tv = {15, 0}; +/* We check this often for presence of owning controller process. */ +static const struct timeval poll_interval_tv = {15, 0}; // 15 seconds. /** Create a process-termination monitor for the process specifier * given in <b>process_spec</b>. Return a newly allocated @@ -303,7 +303,7 @@ tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_) tor_free(errmsg); } } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /* Unix makes this part easy, if a bit racy. */ its_dead_jim = kill(procmon->pid, 0); its_dead_jim = its_dead_jim && (errno == ESRCH); diff --git a/src/lib/evloop/procmon.h b/src/lib/evloop/procmon.h index 6caae5be86..28f443da18 100644 --- a/src/lib/evloop/procmon.h +++ b/src/lib/evloop/procmon.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2019, The Tor Project, Inc. */ +/* Copyright (c) 2011-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/evloop/time_periodic.md b/src/lib/evloop/time_periodic.md new file mode 100644 index 0000000000..8b3589d9db --- /dev/null +++ b/src/lib/evloop/time_periodic.md @@ -0,0 +1,76 @@ + +@page time_periodic Time and periodic events in Tor + +### What time is it? ### + +We have several notions of the current time in Tor. + +The *wallclock time* is available from time(NULL) with +second-granularity and tor_gettimeofday() with microsecond +granularity. It corresponds most closely to "the current time and date". + +The *monotonic time* is available with the set of monotime_\* +functions declared in compat_time.h. Unlike the wallclock time, it +can only move forward. It does not necessarily correspond to a real +world time, and it is not portable between systems. + +The *coarse monotonic time* is available from the set of +monotime_coarse_\* functions in compat_time.h. It is the same as +monotime_\* on some platforms. On others, it gives a monotonic timer +with less precision, but which it's more efficient to access. + +### Cached views of time. ### + +On some systems (like Linux), many time functions use a VDSO to avoid +the overhead of a system call. But on other systems, gettimeofday() +and time() can be costly enough that you wouldn't want to call them +tens of thousands of times. To get a recent, but not especially +accurate, view of the current time, see approx_time() and +tor_gettimeofday_cached(). + + +### Parsing and encoding time values ### + +Tor has functions to parse and format time in these formats: + + - RFC1123 format. ("Fri, 29 Sep 2006 15:54:20 GMT"). For this, + use format_rfc1123_time() and parse_rfc1123_time. + + - ISO8601 format. ("2006-10-29 10:57:20") For this, use + format_local_iso_time() and format_iso_time(). We also support the + variant format "2006-10-29T10:57:20" with format_iso_time_nospace(), and + "2006-10-29T10:57:20.123456" with format_iso_time_nospace_usec(). + + - HTTP format collections (preferably "Mon, 25 Jul 2016 04:01:11 + GMT" or possibly "Wed Jun 30 21:49:08 1993" or even "25-Jul-16 + 04:01:11 GMT"). For this, use parse_http_time(). Don't generate anything + but the first format. + +Some of these functions use struct tm. You can use the standard +tor_localtime_r() and tor_gmtime_r() to wrap these in a safe way. We +also have a tor_timegm() function. + +### Scheduling events ### + +The main way to schedule a not-too-frequent periodic event with +respect to the Tor mainloop is via the mechanism in periodic.c. +There's a big table of periodic_events in mainloop.c, each of which gets +invoked on its own schedule. You should not expect more than about +one second of accuracy with these timers. + +You can create an independent timer using libevent directly, or using +the periodic_timer_new() function. But you should avoid doing this +for per-connection or per-circuit timers: Libevent's internal timer +implementation uses a min-heap, and those tend to start scaling poorly +once you have a few thousand entries. + +If you need to create a large number of fine-grained timers for some +purpose, you should consider the mechanism in src/common/timers.c, +which is optimized for the case where you have a large number of +timers with not-too-long duration, many of which will be deleted +before they actually expire. These timers should be reasonably +accurate within a handful of milliseconds -- possibly better on some +platforms. (The timers.c module uses William Ahern's timeout.c +implementation as its backend, which is based on a hierarchical timing +wheel algorithm. It's cool stuff; check it out.) + diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c index e46d2635a8..11418e93fd 100644 --- a/src/lib/evloop/timers.c +++ b/src/lib/evloop/timers.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2019, The Tor Project, Inc. */ +/* Copyright (c) 2016-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,7 +11,7 @@ * The main advantage of tor_timer_t over using libevent's timers is that * they're way more efficient if we need to have thousands or millions of * them. For more information, see - * http://www.25thandclement.com/~william/projects/timeout.c.html + * https://www.25thandclement.com/~william/projects/timeout.c.html * * Periodic timers are available in the backend, but I've turned them off. * We can turn them back on if needed. @@ -48,7 +48,7 @@ #include <winsock2.h> #endif -struct timeout_cb { +struct timeout_cb_t { timer_cb_fn_t cb; void *arg; }; @@ -56,19 +56,21 @@ struct timeout_cb { /* * These definitions are for timeouts.c and timeouts.h. */ -#ifdef __GNUC__ +#ifdef COCCI +#define TIMEOUT_PUBLIC +#elif defined(__GNUC__) /* We're not exposing any of the functions outside this file. */ #define TIMEOUT_PUBLIC __attribute__((__unused__)) static #else /* We're not exposing any of the functions outside this file. */ #define TIMEOUT_PUBLIC static -#endif /* defined(__GNUC__) */ +#endif /* defined(COCCI) || ... */ /* We're not using periodic events. */ #define TIMEOUT_DISABLE_INTERVALS /* We always know the global_timeouts object, so we don't need each timeout * to keep a pointer to it. */ #define TIMEOUT_DISABLE_RELATIVE_ACCESS -/* We're providing our own struct timeout_cb. */ +/* We're providing our own struct timeout_cb_t. */ #define TIMEOUT_CB_OVERRIDE /* We're going to support timers that are pretty far out in advance. Making * this big can be inefficient, but having a significant number of timers @@ -80,7 +82,8 @@ struct timeout_cb { * use 32-bit math. */ #define WHEEL_BIT 5 #endif -#include "src/ext/timeouts/timeout.c" + +#include "ext/timeouts/timeout.c" static struct timeouts *global_timeouts = NULL; static struct mainloop_event_t *global_timer_event = NULL; diff --git a/src/lib/evloop/timers.h b/src/lib/evloop/timers.h index 7595554204..dd55446121 100644 --- a/src/lib/evloop/timers.h +++ b/src/lib/evloop/timers.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016-2019, The Tor Project, Inc. */ +/* Copyright (c) 2016-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,6 +13,7 @@ #include "lib/testsupport/testsupport.h" struct monotime_t; +struct timeval; typedef struct timeout tor_timer_t; typedef void (*timer_cb_fn_t)(tor_timer_t *, void *, const struct monotime_t *); diff --git a/src/lib/evloop/token_bucket.c b/src/lib/evloop/token_bucket.c index ee6d631e3b..a2b330fddb 100644 --- a/src/lib/evloop/token_bucket.c +++ b/src/lib/evloop/token_bucket.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -256,3 +256,55 @@ token_bucket_rw_dec(token_bucket_rw_t *bucket, flags |= TB_WRITE; return flags; } + +/** Initialize a token bucket in <b>bucket</b>, set up to allow <b>rate</b> + * per second, with a maximum burst of <b>burst</b>. The bucket is created + * such that <b>now_ts</b> is the current timestamp. The bucket starts out + * full. */ +void +token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst, uint32_t now_ts) +{ + memset(bucket, 0, sizeof(token_bucket_ctr_t)); + token_bucket_ctr_adjust(bucket, rate, burst); + token_bucket_ctr_reset(bucket, now_ts); +} + +/** Change the configured rate and burst of the given token bucket object in + * <b>bucket</b>. */ +void +token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst) +{ + token_bucket_cfg_init(&bucket->cfg, rate, burst); + token_bucket_raw_adjust(&bucket->counter, &bucket->cfg); +} + +/** Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>. */ +void +token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts) +{ + token_bucket_raw_reset(&bucket->counter, &bucket->cfg); + bucket->last_refilled_at_timestamp = now_ts; +} + +/** Refill <b>bucket</b> as appropriate, given that the current timestamp is + * <b>now_ts</b>. */ +void +token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts) +{ + const uint32_t elapsed_ticks = + (now_ts - bucket->last_refilled_at_timestamp); + if (elapsed_ticks > UINT32_MAX-(300*1000)) { + /* Either about 48 days have passed since the last refill, or the + * monotonic clock has somehow moved backwards. (We're looking at you, + * Windows.). We accept up to a 5 minute jump backwards as + * "unremarkable". + */ + return; + } + + token_bucket_raw_refill_steps(&bucket->counter, &bucket->cfg, + elapsed_ticks); + bucket->last_refilled_at_timestamp = now_ts; +} diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h index 9398d2baa3..460dad23e4 100644 --- a/src/lib/evloop/token_bucket.h +++ b/src/lib/evloop/token_bucket.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2019, The Tor Project, Inc. */ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -103,6 +103,35 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket) return token_bucket_raw_get(&bucket->write_bucket); } +/** + * A specialized bucket containing a single counter. + */ + +typedef struct token_bucket_ctr_t { + token_bucket_cfg_t cfg; + token_bucket_raw_t counter; + uint32_t last_refilled_at_timestamp; +} token_bucket_ctr_t; + +void token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst, uint32_t now_ts); +void token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate, + uint32_t burst); +void token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts); +void token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts); + +static inline bool +token_bucket_ctr_dec(token_bucket_ctr_t *bucket, ssize_t n) +{ + return token_bucket_raw_dec(&bucket->counter, n); +} + +static inline size_t +token_bucket_ctr_get(const token_bucket_ctr_t *bucket) +{ + return token_bucket_raw_get(&bucket->counter); +} + #ifdef TOKEN_BUCKET_PRIVATE /* To avoid making the rates too small, we consider units of "steps", @@ -112,6 +141,6 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket) STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate); -#endif +#endif /* defined(TOKEN_BUCKET_PRIVATE) */ -#endif /* TOR_TOKEN_BUCKET_H */ +#endif /* !defined(TOR_TOKEN_BUCKET_H) */ diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c index 931f65e710..603dddd5a3 100644 --- a/src/lib/evloop/workqueue.c +++ b/src/lib/evloop/workqueue.c @@ -15,7 +15,7 @@ * * The main thread informs the worker threads of pending work by using a * condition variable. The workers inform the main process of completed work - * by using an alert_sockets_t object, as implemented in compat_threads.c. + * by using an alert_sockets_t object, as implemented in net/alertsock.c. * * The main thread can also queue an "update" that will be handled by all the * workers. This is useful for updating state that all the workers share. @@ -36,7 +36,7 @@ #include "lib/net/socket.h" #include "lib/thread/threads.h" -#include "tor_queue.h" +#include "ext/tor_queue.h" #include <event2/event.h> #include <string.h> @@ -44,13 +44,13 @@ #define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW #define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1) -TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_s); +TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_t); typedef struct work_tailq_t work_tailq_t; -struct threadpool_s { +struct threadpool_t { /** An array of pointers to workerthread_t: one for each running worker * thread. */ - struct workerthread_s **threads; + struct workerthread_t **threads; /** Condition variable that we wait on when we have no work, and which * gets signaled when our queue becomes nonempty. */ @@ -59,9 +59,6 @@ struct threadpool_s { * <b>p</b> is work[p]. */ work_tailq_t work[WORKQUEUE_N_PRIORITIES]; - /** Weak RNG, used to decide when to ignore priority. */ - tor_weak_rng_t weak_rng; - /** The current 'update generation' of the threadpool. Any thread that is * at an earlier generation needs to run the update function. */ unsigned generation; @@ -95,14 +92,14 @@ struct threadpool_s { /** Number of bits needed to hold all legal values of workqueue_priority_t */ #define WORKQUEUE_PRIORITY_BITS 2 -struct workqueue_entry_s { +struct workqueue_entry_t { /** The next workqueue_entry_t that's pending on the same thread or * reply queue. */ - TOR_TAILQ_ENTRY(workqueue_entry_s) next_work; + TOR_TAILQ_ENTRY(workqueue_entry_t) next_work; /** The threadpool to which this workqueue_entry_t was assigned. This field * is set when the workqueue_entry_t is created, and won't be cleared until * after it's handled in the main thread. */ - struct threadpool_s *on_pool; + struct threadpool_t *on_pool; /** True iff this entry is waiting for a worker to start processing it. */ uint8_t pending; /** Priority of this entry. */ @@ -115,22 +112,22 @@ struct workqueue_entry_s { void *arg; }; -struct replyqueue_s { +struct replyqueue_t { /** Mutex to protect the answers field */ tor_mutex_t lock; /** Doubly-linked list of answers that the reply queue needs to handle. */ - TOR_TAILQ_HEAD(, workqueue_entry_s) answers; + TOR_TAILQ_HEAD(, workqueue_entry_t) answers; /** Mechanism to wake up the main thread when it is receiving answers. */ alert_sockets_t alert; }; /** A worker thread represents a single thread in a thread pool. */ -typedef struct workerthread_s { +typedef struct workerthread_t { /** Which thread it this? In range 0..in_pool->n_threads-1 */ int index; /** The pool this thread is a part of. */ - struct threadpool_s *in_pool; + struct threadpool_t *in_pool; /** User-supplied state field that we pass to the worker functions of each * work item. */ void *state; @@ -238,7 +235,7 @@ worker_thread_extract_next_work(workerthread_t *thread) this_queue = &pool->work[i]; if (!TOR_TAILQ_EMPTY(this_queue)) { queue = this_queue; - if (! tor_weak_random_one_in_n(&pool->weak_rng, + if (! crypto_fast_rng_one_in_n(get_thread_fast_rng(), thread->lower_priority_chance)) { /* Usually we'll just break now, so that we can get out of the loop * and use the queue where we found work. But with a small @@ -555,11 +552,6 @@ threadpool_new(int n_threads, for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) { TOR_TAILQ_INIT(&pool->work[i]); } - { - unsigned seed; - crypto_rand((void*)&seed, sizeof(seed)); - tor_init_weak_random(&pool->weak_rng, seed); - } pool->new_thread_state_fn = new_thread_state_fn; pool->new_thread_state_arg = arg; @@ -622,8 +614,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg) tp->reply_cb(tp); } -/** Register the threadpool <b>tp</b>'s reply queue with the libevent - * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after +/** Register the threadpool <b>tp</b>'s reply queue with Tor's global + * libevent mainloop. If <b>cb</b> is provided, it is run after * each time there is work to process from the reply queue. Return 0 on * success, -1 on failure. */ diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h index 333a3f6dde..43cfebf788 100644 --- a/src/lib/evloop/workqueue.h +++ b/src/lib/evloop/workqueue.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Tor Project, Inc. */ +/* Copyright (c) 2013-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,12 +13,12 @@ /** A replyqueue is used to tell the main thread about the outcome of * work that we queued for the workers. */ -typedef struct replyqueue_s replyqueue_t; +typedef struct replyqueue_t replyqueue_t; /** A thread-pool manages starting threads and passing work to them. */ -typedef struct threadpool_s threadpool_t; +typedef struct threadpool_t threadpool_t; /** A workqueue entry represents a request that has been passed to a thread * pool. */ -typedef struct workqueue_entry_s workqueue_entry_t; +typedef struct workqueue_entry_t workqueue_entry_t; /** Possible return value from a work function: */ typedef enum workqueue_reply_t { @@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp); replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); void replyqueue_process(replyqueue_t *queue); -struct event_base; int threadpool_register_reply_event(threadpool_t *tp, void (*cb)(threadpool_t *tp)); diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c index d723d04d2a..56e3818f5c 100644 --- a/src/lib/fdio/fdio.c +++ b/src/lib/fdio/fdio.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,16 +17,21 @@ #ifdef _WIN32 #include <windows.h> #endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif #include "lib/fdio/fdio.h" #include "lib/cc/torint.h" #include "lib/err/torerr.h" #include <stdlib.h> +#include <stdio.h> -/** @{ */ -/** Some old versions of Unix didn't define constants for these values, +/* Some old versions of Unix didn't define constants for these values, * and instead expect you to say 0, 1, or 2. */ + +/** @cond */ #ifndef SEEK_SET #define SEEK_SET 0 #endif @@ -36,7 +41,7 @@ #ifndef SEEK_END #define SEEK_END 2 #endif -/** @} */ +/** @endcond */ /** Return the position of <b>fd</b> with respect to the start of the file. */ off_t diff --git a/src/lib/fdio/fdio.h b/src/lib/fdio/fdio.h index 8395af353b..99bc33c64b 100644 --- a/src/lib/fdio/fdio.h +++ b/src/lib/fdio/fdio.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,6 +13,9 @@ #define TOR_FDIO_H #include <stddef.h> +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif off_t tor_fd_getpos(int fd); int tor_fd_setpos(int fd, off_t pos); diff --git a/src/lib/fdio/include.am b/src/lib/fdio/include.am index 6c18f00a0d..545bbc929e 100644 --- a/src/lib/fdio/include.am +++ b/src/lib/fdio/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-fdio-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_fdio_a_SOURCES = \ src/lib/fdio/fdio.c @@ -13,5 +14,6 @@ src_lib_libtor_fdio_testing_a_SOURCES = \ src_lib_libtor_fdio_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_fdio_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/fdio/fdio.h diff --git a/src/lib/fdio/lib_fdio.md b/src/lib/fdio/lib_fdio.md new file mode 100644 index 0000000000..9fe4b4d2be --- /dev/null +++ b/src/lib/fdio/lib_fdio.md @@ -0,0 +1,5 @@ +@dir /lib/fdio +@brief lib/fdio: Code to read/write on file descriptors. + +(This module also handles sockets, on platforms where a socket is not a kind +of fd.) diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include index b1e49fc891..c192e6181c 100644 --- a/src/lib/fs/.may_include +++ b/src/lib/fs/.may_include @@ -13,4 +13,4 @@ lib/malloc/*.h lib/memarea/*.h lib/sandbox/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/fs/conffile.c b/src/lib/fs/conffile.c index 0d5d56b335..9583093c12 100644 --- a/src/lib/fs/conffile.c +++ b/src/lib/fs/conffile.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -152,6 +152,7 @@ config_process_include(const char *path, int recursion_level, int extended, int rv = -1; SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) { + log_notice(LD_CONFIG, "Including configuration file \"%s\".", config_file); config_line_t *included_config = NULL; config_line_t *included_config_last = NULL; if (config_get_included_config(config_file, recursion_level, extended, diff --git a/src/lib/fs/conffile.h b/src/lib/fs/conffile.h index 7af9119dbb..cbfe4ec7c9 100644 --- a/src/lib/fs/conffile.h +++ b/src/lib/fs/conffile.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_CONFFILE_H @@ -20,4 +20,4 @@ int config_get_lines_include(const char *string, struct config_line_t **result, int extended, int *has_include, struct smartlist_t *opened_lst); -#endif /* !defined(TOR_CONFLINE_H) */ +#endif /* !defined(TOR_CONFFILE_H) */ diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c index 3c31e00d99..3432df0299 100644 --- a/src/lib/fs/dir.c +++ b/src/lib/fs/dir.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -262,7 +262,7 @@ check_private_dir,(const char *dirname, cpd_check_t check, } } close(fd); -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ /* Win32 case: we can't open() a directory. */ (void)effective_user; @@ -347,7 +347,7 @@ tor_listdir, (const char *dirname)) } FindClose(handle); tor_free(pattern); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ const char *prot_dname = sandbox_intern_string(dirname); DIR *d; struct dirent *de; diff --git a/src/lib/fs/dir.h b/src/lib/fs/dir.h index 826bc2dfc5..c4ab430891 100644 --- a/src/lib/fs/dir.h +++ b/src/lib/fs/dir.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_DIR_H @@ -30,4 +30,4 @@ MOCK_DECL(int, check_private_dir, (const char *dirname, cpd_check_t check, MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname)); -#endif +#endif /* !defined(TOR_DIR_H) */ diff --git a/src/lib/fs/files.c b/src/lib/fs/files.c index ec7dbca0e9..a0b5a40aac 100644 --- a/src/lib/fs/files.c +++ b/src/lib/fs/files.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h index 52c94c914f..a109cd6248 100644 --- a/src/lib/fs/files.h +++ b/src/lib/fs/files.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -27,7 +27,7 @@ #ifdef HAVE_SYS_STAT_H #include <sys/stat.h> #endif -#endif +#endif /* defined(_WIN32) */ #ifndef O_BINARY #define O_BINARY 0 @@ -108,7 +108,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, * Tor is built for unit tests, or when Tor is built on an operating system * without its own getdelim(). */ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); -#endif +#endif /* !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS) */ #ifdef HAVE_GETDELIM /** @@ -123,10 +123,10 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); */ #define tor_getdelim(lineptr, n, delim, stream) \ getdelim((lineptr), (n), (delim), (stream)) -#else +#else /* !defined(HAVE_GETDELIM) */ #define tor_getdelim(lineptr, n, delim, stream) \ compat_getdelim_((lineptr), (n), (delim), (stream)) -#endif +#endif /* defined(HAVE_GETDELIM) */ #ifdef HAVE_GETLINE /** @@ -137,9 +137,9 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream); */ #define tor_getline(lineptr, n, stream) \ getline((lineptr), (n), (stream)) -#else +#else /* !defined(HAVE_GETLINE) */ #define tor_getline(lineptr, n, stream) \ tor_getdelim((lineptr), (n), '\n', (stream)) -#endif +#endif /* defined(HAVE_GETLINE) */ -#endif +#endif /* !defined(TOR_FS_H) */ diff --git a/src/lib/fs/freespace.c b/src/lib/fs/freespace.c index ee0f93073d..511f2a0b98 100644 --- a/src/lib/fs/freespace.c +++ b/src/lib/fs/freespace.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/include.am b/src/lib/fs/include.am index f33e4d6430..493db8f044 100644 --- a/src/lib/fs/include.am +++ b/src/lib/fs/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-fs-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_fs_a_SOURCES = \ src/lib/fs/conffile.c \ src/lib/fs/dir.c \ @@ -25,6 +26,7 @@ src_lib_libtor_fs_testing_a_SOURCES = \ src_lib_libtor_fs_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_fs_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/fs/conffile.h \ src/lib/fs/dir.h \ diff --git a/src/lib/fs/lib_fs.md b/src/lib/fs/lib_fs.md new file mode 100644 index 0000000000..3b5b0ac7d5 --- /dev/null +++ b/src/lib/fs/lib_fs.md @@ -0,0 +1,9 @@ +@dir /lib/fs +@brief lib/fs: Files, filenames, directories, etc. + +This module is mostly a set of compatibility wrappers around +operating-system-specific filesystem access. + +It also contains a set of convenience functions for safely writing to files, +creating directories, and so on. + diff --git a/src/lib/fs/lockfile.c b/src/lib/fs/lockfile.c index 933ff1e02f..c081f57a5d 100644 --- a/src/lib/fs/lockfile.c +++ b/src/lib/fs/lockfile.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/lockfile.h b/src/lib/fs/lockfile.h index 8aeee4cc7f..91effd701d 100644 --- a/src/lib/fs/lockfile.h +++ b/src/lib/fs/lockfile.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,4 +17,4 @@ tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking, int *locked_out); void tor_lockfile_unlock(tor_lockfile_t *lockfile); -#endif +#endif /* !defined(TOR_LOCKFILE_H) */ diff --git a/src/lib/fs/mmap.c b/src/lib/fs/mmap.c index daaee1f9b1..cc1c40b7ab 100644 --- a/src/lib/fs/mmap.c +++ b/src/lib/fs/mmap.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -42,8 +42,8 @@ * failure, return NULL. Sets errno properly, using ERANGE to mean * "empty file". Must only be called on trusted Tor-owned files, as changing * the underlying file's size causes unspecified behavior. */ -tor_mmap_t * -tor_mmap_file(const char *filename) +MOCK_IMPL(tor_mmap_t *, +tor_mmap_file,(const char *filename)) { int fd; /* router file */ char *string; @@ -111,8 +111,8 @@ tor_mmap_file(const char *filename) } /** Release storage held for a memory mapping; returns 0 on success, * or -1 on failure (and logs a warning). */ -int -tor_munmap_file(tor_mmap_t *handle) +MOCK_IMPL(int, +tor_munmap_file,(tor_mmap_t *handle)) { int res; @@ -132,8 +132,8 @@ tor_munmap_file(tor_mmap_t *handle) return res; } #elif defined(_WIN32) -tor_mmap_t * -tor_mmap_file(const char *filename) +MOCK_IMPL(tor_mmap_t *, +tor_mmap_file,(const char *filename)) { TCHAR tfilename[MAX_PATH]= {0}; tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t)); @@ -213,8 +213,8 @@ tor_mmap_file(const char *filename) } /* Unmap the file, and return 0 for success or -1 for failure */ -int -tor_munmap_file(tor_mmap_t *handle) +MOCK_IMPL(int, +tor_munmap_file,(tor_mmap_t *handle)) { if (handle == NULL) return 0; @@ -237,4 +237,4 @@ tor_munmap_file(tor_mmap_t *handle) } #else #error "cannot implement tor_mmap_file" -#endif /* defined(HAVE_MMAP) || ... || ... */ +#endif /* defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN) || ... */ diff --git a/src/lib/fs/mmap.h b/src/lib/fs/mmap.h index 18fb18a13c..e142bd78c3 100644 --- a/src/lib/fs/mmap.h +++ b/src/lib/fs/mmap.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,6 +13,7 @@ #define TOR_MMAP_H #include "lib/cc/compat_compiler.h" +#include "lib/testsupport/testsupport.h" #include <stddef.h> #ifdef _WIN32 @@ -35,7 +36,7 @@ typedef struct tor_mmap_t { } tor_mmap_t; -tor_mmap_t *tor_mmap_file(const char *filename); -int tor_munmap_file(tor_mmap_t *handle); +MOCK_DECL(tor_mmap_t *, tor_mmap_file, (const char *filename)); +MOCK_DECL(int, tor_munmap_file, (tor_mmap_t *handle)); -#endif +#endif /* !defined(TOR_MMAP_H) */ diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c index b3ef61979d..0d57be4b06 100644 --- a/src/lib/fs/path.c +++ b/src/lib/fs/path.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -72,7 +72,7 @@ expand_filename(const char *filename) * Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/ */ return tor_strdup(filename); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ if (*filename == '~') { char *home, *result=NULL; const char *rest; @@ -102,7 +102,7 @@ expand_filename(const char *filename) } tor_free(username); rest = slash ? (slash+1) : ""; -#else /* !(defined(HAVE_PWD_H)) */ +#else /* !defined(HAVE_PWD_H) */ log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h"); return tor_strdup(filename); #endif /* defined(HAVE_PWD_H) */ @@ -153,7 +153,7 @@ clean_fname_for_stat(char *name) return; name[len-1]='\0'; } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ (void)name; #endif /* defined(_WIN32) */ } @@ -233,7 +233,7 @@ alloc_getcwd(void) raw_free(cwd); // alias for free to avoid tripping check-spaces. } return result; -#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */ +#else /* !defined(HAVE_GET_CURRENT_DIR_NAME) */ size_t size = 1024; char *buf = NULL; char *ptr = NULL; @@ -255,9 +255,10 @@ alloc_getcwd(void) #endif /* !defined(_WIN32) */ /** Expand possibly relative path <b>fname</b> to an absolute path. - * Return a newly allocated string, possibly equal to <b>fname</b>. */ + * Return a newly allocated string, which may be a duplicate of <b>fname</b>. + */ char * -make_path_absolute(char *fname) +make_path_absolute(const char *fname) { #ifdef _WIN32 char *absfname_malloced = _fullpath(NULL, fname, 1); @@ -268,7 +269,7 @@ make_path_absolute(char *fname) if (absfname_malloced) raw_free(absfname_malloced); return absfname; -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ char *absfname = NULL, *path = NULL; tor_assert(fname); diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h index 4675ac84e8..f0e253c556 100644 --- a/src/lib/fs/path.h +++ b/src/lib/fs/path.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -25,6 +25,6 @@ char *expand_filename(const char *filename); int path_is_relative(const char *filename); void clean_fname_for_stat(char *name); int get_parent_directory(char *fname); -char *make_path_absolute(char *fname); +char *make_path_absolute(const char *fname); -#endif +#endif /* !defined(TOR_PATH_H) */ diff --git a/src/lib/fs/storagedir.c b/src/lib/fs/storagedir.c index 2caddf1ad9..5e4f9ee257 100644 --- a/src/lib/fs/storagedir.c +++ b/src/lib/fs/storagedir.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* Copyright (c) 2017-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/storagedir.h b/src/lib/fs/storagedir.h index 7e6633a0bb..9997550b8f 100644 --- a/src/lib/fs/storagedir.h +++ b/src/lib/fs/storagedir.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* Copyright (c) 2017-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,7 +15,7 @@ typedef struct storage_dir_t storage_dir_t; struct config_line_t; -struct sandbox_cfg_elem; +struct sandbox_cfg_elem_t; struct tor_mmap_t; struct smartlist_t; @@ -25,7 +25,7 @@ void storage_dir_free_(storage_dir_t *d); FREE_AND_NULL(storage_dir_t, storage_dir_free_, (d)) int storage_dir_register_with_sandbox(storage_dir_t *d, - struct sandbox_cfg_elem **cfg); + struct sandbox_cfg_elem_t **cfg); const struct smartlist_t *storage_dir_list(storage_dir_t *d); uint64_t storage_dir_get_usage(storage_dir_t *d); struct tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname); diff --git a/src/lib/fs/userdb.c b/src/lib/fs/userdb.c index 95205c670e..40fc4dae97 100644 --- a/src/lib/fs/userdb.c +++ b/src/lib/fs/userdb.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/userdb.h b/src/lib/fs/userdb.h index 5c39794873..4341237c5f 100644 --- a/src/lib/fs/userdb.h +++ b/src/lib/fs/userdb.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -21,6 +21,6 @@ struct passwd; const struct passwd *tor_getpwnam(const char *username); const struct passwd *tor_getpwuid(uid_t uid); char *get_user_homedir(const char *username); -#endif +#endif /* !defined(_WIN32) */ -#endif +#endif /* !defined(TOR_USERDB_H) */ diff --git a/src/lib/fs/winlib.c b/src/lib/fs/winlib.c index b7302bd4ca..65ccdae40b 100644 --- a/src/lib/fs/winlib.c +++ b/src/lib/fs/winlib.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/fs/winlib.h b/src/lib/fs/winlib.h index 64a22439e5..27837ac46e 100644 --- a/src/lib/fs/winlib.h +++ b/src/lib/fs/winlib.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,6 +17,6 @@ #include <tchar.h> HANDLE load_windows_system_library(const TCHAR *library_name); -#endif +#endif /* defined(_WIN32) */ -#endif +#endif /* !defined(TOR_WINLIB_H) */ diff --git a/src/lib/geoip/country.h b/src/lib/geoip/country.h index 9a8911d494..feab554a16 100644 --- a/src/lib/geoip/country.h +++ b/src/lib/geoip/country.h @@ -1,9 +1,14 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * @file country.h + * @brief Country type for geoip. + **/ + #ifndef TOR_COUNTRY_H #define TOR_COUNTRY_H @@ -11,6 +16,7 @@ /** A signed integer representing a country code. */ typedef int16_t country_t; +/** Maximum value for country_t. */ #define COUNTRY_MAX INT16_MAX -#endif +#endif /* !defined(TOR_COUNTRY_H) */ diff --git a/src/lib/geoip/geoip.c b/src/lib/geoip/geoip.c index 70b1c2dc8c..ee03d5baa1 100644 --- a/src/lib/geoip/geoip.c +++ b/src/lib/geoip/geoip.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -70,12 +70,18 @@ static smartlist_t *geoip_countries = NULL; * The index is encoded in the pointer, and 1 is added so that NULL can mean * not found. */ static strmap_t *country_idxplus1_by_lc_code = NULL; -/** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted - * by their respective ip_low. */ -static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL; - -/** SHA1 digest of the GeoIP files to include in extra-info descriptors. */ +/** List of all known geoip_ipv4_entry_t sorted + * by their respective ip_low values. */ +static smartlist_t *geoip_ipv4_entries = NULL; +/** List of all known geoip_ipv6_entry_t, sorted by their respective + * ip_low values. */ +static smartlist_t *geoip_ipv6_entries = NULL; + +/** SHA1 digest of the IPv4 GeoIP file to include in extra-info + * descriptors. */ static char geoip_digest[DIGEST_LEN]; +/** SHA1 digest of the IPv6 GeoIP file to include in extra-info + * descriptors. */ static char geoip6_digest[DIGEST_LEN]; /** Return a list of geoip_country_t for all known countries. */ diff --git a/src/lib/geoip/geoip.h b/src/lib/geoip/geoip.h index f872ebd25f..2fc7fae754 100644 --- a/src/lib/geoip/geoip.h +++ b/src/lib/geoip/geoip.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -31,6 +31,7 @@ int geoip_get_country_by_ipv6(const struct in6_addr *addr); /** A per-country GeoIP record. */ typedef struct geoip_country_t { + /** A nul-terminated two-letter country-code. */ char countrycode[3]; } geoip_country_t; diff --git a/src/lib/geoip/include.am b/src/lib/geoip/include.am index 9710d75ac7..ea426d14bc 100644 --- a/src/lib/geoip/include.am +++ b/src/lib/geoip/include.am @@ -4,6 +4,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-geoip-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_geoip_a_SOURCES = \ src/lib/geoip/geoip.c @@ -12,6 +13,7 @@ src_lib_libtor_geoip_testing_a_SOURCES = \ src_lib_libtor_geoip_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_geoip_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/geoip/geoip.h \ src/lib/geoip/country.h diff --git a/src/lib/geoip/lib_geoip.md b/src/lib/geoip/lib_geoip.md new file mode 100644 index 0000000000..a3ee39d574 --- /dev/null +++ b/src/lib/geoip/lib_geoip.md @@ -0,0 +1,3 @@ +@dir /lib/geoip +@brief lib/geoip: IP-to-country mapping + diff --git a/src/lib/intmath/addsub.c b/src/lib/intmath/addsub.c index 12146f4e72..44613417fa 100644 --- a/src/lib/intmath/addsub.c +++ b/src/lib/intmath/addsub.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/addsub.h b/src/lib/intmath/addsub.h index 83efa82919..bca911103b 100644 --- a/src/lib/intmath/addsub.h +++ b/src/lib/intmath/addsub.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,4 +16,4 @@ uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b); -#endif /* !defined(TOR_INTMATH_MULDIV_H) */ +#endif /* !defined(TOR_INTMATH_ADDSUB_H) */ diff --git a/src/lib/intmath/bits.c b/src/lib/intmath/bits.c index 2158790e3f..dace9ffe18 100644 --- a/src/lib/intmath/bits.c +++ b/src/lib/intmath/bits.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/bits.h b/src/lib/intmath/bits.h index c1483a18b8..687651ba35 100644 --- a/src/lib/intmath/bits.h +++ b/src/lib/intmath/bits.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h index d0b0e8b954..4e6c2b649a 100644 --- a/src/lib/intmath/cmp.h +++ b/src/lib/intmath/cmp.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -36,4 +36,7 @@ ((v) > (max)) ? (max) : \ (v) ) +/** Give the absolute value of <b>x</b>, independent of its type. */ +#define ABS(x) ( ((x)<0) ? -(x) : (x) ) + #endif /* !defined(TOR_INTMATH_CMP_H) */ diff --git a/src/lib/intmath/include.am b/src/lib/intmath/include.am index 45ee3bd53b..155ffa145a 100644 --- a/src/lib/intmath/include.am +++ b/src/lib/intmath/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-intmath-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_intmath_a_SOURCES = \ src/lib/intmath/addsub.c \ src/lib/intmath/bits.c \ @@ -16,6 +17,7 @@ src_lib_libtor_intmath_testing_a_SOURCES = \ src_lib_libtor_intmath_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_intmath_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/intmath/addsub.h \ src/lib/intmath/cmp.h \ diff --git a/src/lib/intmath/lib_intmath.md b/src/lib/intmath/lib_intmath.md new file mode 100644 index 0000000000..4446b715cb --- /dev/null +++ b/src/lib/intmath/lib_intmath.md @@ -0,0 +1,2 @@ +@dir /lib/intmath +@brief lib/intmath: Integer mathematics. diff --git a/src/lib/intmath/logic.h b/src/lib/intmath/logic.h index a4cecd69cc..b5fb79f66e 100644 --- a/src/lib/intmath/logic.h +++ b/src/lib/intmath/logic.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,4 +17,4 @@ /** Macro: true if two values have different boolean values. */ #define bool_neq(a,b) (!(a)!=!(b)) -#endif +#endif /* !defined(HAVE_TOR_LOGIC_H) */ diff --git a/src/lib/intmath/muldiv.c b/src/lib/intmath/muldiv.c index 6a292db7ba..875cf1bbf2 100644 --- a/src/lib/intmath/muldiv.c +++ b/src/lib/intmath/muldiv.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -69,6 +69,20 @@ gcd64(uint64_t a, uint64_t b) return a; } +/** Return the unsigned integer product of <b>a</b> and <b>b</b>. If overflow + * is detected, return UINT64_MAX instead. */ +uint64_t +tor_mul_u64_nowrap(uint64_t a, uint64_t b) +{ + if (a == 0 || b == 0) { + return 0; + } else if (PREDICT_UNLIKELY(UINT64_MAX / a < b)) { + return UINT64_MAX; + } else { + return a*b; + } +} + /* Given a fraction *<b>numer</b> / *<b>denom</b>, simplify it. * Requires that the denominator is greater than 0. */ void diff --git a/src/lib/intmath/muldiv.h b/src/lib/intmath/muldiv.h index 64500b6dce..43700cf1dc 100644 --- a/src/lib/intmath/muldiv.h +++ b/src/lib/intmath/muldiv.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -18,6 +18,8 @@ unsigned round_to_next_multiple_of(unsigned number, unsigned divisor); uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor); uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor); +uint64_t tor_mul_u64_nowrap(uint64_t a, uint64_t b); + void simplify_fraction64(uint64_t *numer, uint64_t *denom); /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b> diff --git a/src/lib/intmath/weakrng.c b/src/lib/intmath/weakrng.c index 99c9252c2b..a29a6a086b 100644 --- a/src/lib/intmath/weakrng.c +++ b/src/lib/intmath/weakrng.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/intmath/weakrng.h b/src/lib/intmath/weakrng.h index e26bf58cbb..0394e05f79 100644 --- a/src/lib/intmath/weakrng.h +++ b/src/lib/intmath/weakrng.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,8 +19,11 @@ typedef struct tor_weak_rng_t { uint32_t state; } tor_weak_rng_t; +#ifndef COCCI #define TOR_WEAK_RNG_INIT {383745623} +#endif #define TOR_WEAK_RANDOM_MAX (INT_MAX) + void tor_init_weak_random(tor_weak_rng_t *weak_rng, unsigned seed); int32_t tor_weak_random(tor_weak_rng_t *weak_rng); int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top); @@ -28,4 +31,4 @@ int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top); * <b>n</b> */ #define tor_weak_random_one_in_n(rng, n) (0==tor_weak_random_range((rng),(n))) -#endif +#endif /* !defined(TOR_WEAKRNG_H) */ diff --git a/src/lib/lib.md b/src/lib/lib.md new file mode 100644 index 0000000000..4f77a4c1d0 --- /dev/null +++ b/src/lib/lib.md @@ -0,0 +1,131 @@ +@dir /lib +@brief lib: low-level functionality. + +The "lib" directory contains low-level functionality. In general, this +code is not necessarily Tor-specific, but is instead possibly useful for +other applications. + +The modules in `lib` are currently well-factored: each one depends +only on lower-level modules. You can see an up-to-date list of the +modules, sorted from lowest to highest level, by running +`./scripts/maint/practracker/includes.py --toposort`. + +As of this writing, the library modules are (from lowest to highest +level): + + - \refdir{lib/cc} -- Macros for managing the C compiler and + language. + + - \refdir{lib/version} -- Holds the current version of Tor. + + - \refdir{lib/testsupport} -- Helpers for making + test-only code, and test mocking support. + + - \refdir{lib/defs} -- Lowest-level constants. + + - \refdir{lib/subsys} -- Types used for declaring a + "subsystem". (_A subsystem is a module with support for initialization, + shutdown, configuration, and so on._) + + - \refdir{lib/conf} -- For declaring configuration options. + + - \refdir{lib/arch} -- For handling differences in CPU + architecture. + + - \refdir{lib/err} -- Lowest-level error handling code. + + - \refdir{lib/malloc} -- Memory management. + management. + + - \refdir{lib/intmath} -- Integer mathematics. + + - \refdir{lib/fdio} -- For + reading and writing n file descriptors. + + - \refdir{lib/lock} -- Simple locking support. + (_Lower-level than the rest of the threading code._) + + - \refdir{lib/ctime} -- Constant-time code to avoid + side-channels. + + - \refdir{lib/string} -- Low-level string manipulation. + + - \refdir{lib/wallclock} -- + For inspecting and manipulating the current (UTC) time. + + - \refdir{lib/osinfo} -- For inspecting the OS version + and capabilities. + + - \refdir{lib/smartlist_core} -- The bare-bones + pieces of our dynamic array ("smartlist") implementation. + + - \refdir{lib/log} -- Log messages to files, syslogs, etc. + + - \refdir{lib/container} -- General purpose containers, + including dynamic arrays ("smartlists"), hashtables, bit arrays, + etc. + + - \refdir{lib/trace} -- A general-purpose API + function-tracing functionality Tor. (_Currently not much used._) + + - \refdir{lib/thread} -- Mid-level Threading. + + - \refdir{lib/term} -- Terminal manipulation + (like reading a password from the user). + + - \refdir{lib/memarea} -- A fast + "arena" style allocator, where the data is freed all at once. + + - \refdir{lib/encoding} -- Encoding + data in various formats, datatypes, and transformations. + + - \refdir{lib/dispatch} -- A general-purpose in-process + message delivery system. + + - \refdir{lib/sandbox} -- Our Linux seccomp2 sandbox + implementation. + + - \refdir{lib/pubsub} -- A publish/subscribe message passing system. + + - \refdir{lib/fs} -- Files, filenames, directories, etc. + + - \refdir{lib/confmgt} -- Parse, encode, and manipulate onfiguration files. + + - \refdir{lib/crypt_ops} -- Cryptographic operations. + + - \refdir{lib/meminfo} -- Functions for inspecting our + memory usage, if the malloc implementation exposes that to us. + + - \refdir{lib/time} -- Higher level time functions, including + fine-gained and monotonic timers. + + - \refdir{lib/math} -- Floating-point mathematical utilities. + + - \refdir{lib/buf} -- An efficient byte queue. + + - \refdir{lib/net} -- Networking code, including address + manipulation, compatibility wrappers, etc. + + - \refdir{lib/compress} -- Wraps several compression libraries. + + - \refdir{lib/geoip} -- IP-to-country mapping. + + - \refdir{lib/tls} -- TLS library wrappers. + + - \refdir{lib/evloop} -- Low-level event-loop. + + - \refdir{lib/process} -- Launch and manage subprocesses. + +### What belongs in lib? + +In general, if you can imagine some program wanting the functionality +you're writing, even if that program had nothing to do with Tor, your +functionality belongs in lib. + +If it falls into one of the existing "lib" categories, your +functionality belongs in lib. + +If you are using platform-specific `ifdef`s to manage compatibility +issues among platforms, you should probably consider whether you can +put your code into lib. + diff --git a/src/lib/llharden/.may_include b/src/lib/llharden/.may_include new file mode 100644 index 0000000000..038237dadf --- /dev/null +++ b/src/lib/llharden/.may_include @@ -0,0 +1,3 @@ +lib/llharden/*.h +lib/subsys/*.h +orconfig.h diff --git a/src/lib/llharden/include.am b/src/lib/llharden/include.am new file mode 100644 index 0000000000..0a4788c7dc --- /dev/null +++ b/src/lib/llharden/include.am @@ -0,0 +1,19 @@ + +noinst_LIBRARIES += src/lib/libtor-llharden.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-llharden-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_llharden_a_SOURCES = \ + src/lib/llharden/winprocess_sys.c + +src_lib_libtor_llharden_testing_a_SOURCES = \ + $(src_lib_libtor_llharden_a_SOURCES) +src_lib_libtor_llharden_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_llharden_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/llharden/winprocess_sys.h diff --git a/src/lib/llharden/lib_llharden.md b/src/lib/llharden/lib_llharden.md new file mode 100644 index 0000000000..69e9af5327 --- /dev/null +++ b/src/lib/llharden/lib_llharden.md @@ -0,0 +1,6 @@ +@dir /lib/llharden +@brief lib/llharden: low-level unconditional process hardening + +This module contains process hardening code that we want to run before any +other code, including configuration. It needs to be self-contained, since +nothing else will be initialized at this point. diff --git a/src/lib/llharden/winprocess_sys.c b/src/lib/llharden/winprocess_sys.c new file mode 100644 index 0000000000..f2c88d8c75 --- /dev/null +++ b/src/lib/llharden/winprocess_sys.c @@ -0,0 +1,67 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.c + * \brief Subsystem object for windows process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/llharden/winprocess_sys.h" + +#include <stdbool.h> +#include <stddef.h> + +#ifdef _WIN32 +#include <windows.h> + +#define WINPROCESS_SYS_ENABLED true + +static int +subsys_winprocess_initialize(void) +{ +#ifndef HeapEnableTerminationOnCorruption +#define HeapEnableTerminationOnCorruption 1 +#endif + + /* On heap corruption, just give up; don't try to play along. */ + HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0); + + /* SetProcessDEPPolicy is only supported on 32-bit Windows. + * (On 64-bit Windows it always fails, and some compilers don't like the + * PSETDEP cast.) + * 32-bit Windows defines _WIN32. + * 64-bit Windows defines _WIN32 and _WIN64. */ +#ifndef _WIN64 + /* Call SetProcessDEPPolicy to permanently enable DEP. + The function will not resolve on earlier versions of Windows, + and failure is not dangerous. */ + HMODULE hMod = GetModuleHandleA("Kernel32.dll"); + if (hMod) { + typedef BOOL (WINAPI *PSETDEP)(DWORD); + PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod, + "SetProcessDEPPolicy"); + if (setdeppolicy) { + /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */ + setdeppolicy(3); + } + } +#endif /* !defined(_WIN64) */ + + return 0; +} +#else /* !defined(_WIN32) */ +#define WINPROCESS_SYS_ENABLED false +#define subsys_winprocess_initialize NULL +#endif /* defined(_WIN32) */ + +const subsys_fns_t sys_winprocess = { + .name = "winprocess", + SUBSYS_DECLARE_LOCATION(), + /* HeapEnableTerminationOnCorruption and setdeppolicy() are security + * features, we want them to run first. */ + .level = -100, + .supported = WINPROCESS_SYS_ENABLED, + .initialize = subsys_winprocess_initialize, +}; diff --git a/src/lib/llharden/winprocess_sys.h b/src/lib/llharden/winprocess_sys.h new file mode 100644 index 0000000000..bece1b3da9 --- /dev/null +++ b/src/lib/llharden/winprocess_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file winprocess_sys.h + * \brief Declare subsystem object for winprocess.c + **/ + +#ifndef TOR_WINPROCESS_SYS_H +#define TOR_WINPROCESS_SYS_H + +extern const struct subsys_fns_t sys_winprocess; + +#endif /* !defined(TOR_WINPROCESS_SYS_H) */ diff --git a/src/lib/lock/compat_mutex.c b/src/lib/lock/compat_mutex.c index 4ad5929715..b0084a3484 100644 --- a/src/lib/lock/compat_mutex.c +++ b/src/lib/lock/compat_mutex.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,7 +29,15 @@ tor_mutex_new_nonrecursive(void) tor_mutex_init_nonrecursive(m); return m; } -/** Release all storage and system resources held by <b>m</b>. */ +/** Release all storage and system resources held by <b>m</b>. + * + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ void tor_mutex_free_(tor_mutex_t *m) { diff --git a/src/lib/lock/compat_mutex.h b/src/lib/lock/compat_mutex.h index b63ce24024..5631993cc4 100644 --- a/src/lib/lock/compat_mutex.h +++ b/src/lib/lock/compat_mutex.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -48,7 +48,7 @@ typedef struct tor_mutex_t { #else /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */ int _unused; -#endif /* defined(USE_WIN32_MUTEX) || ... */ +#endif /* defined(USE_WIN32_THREADS) || ... */ } tor_mutex_t; tor_mutex_t *tor_mutex_new(void); @@ -58,6 +58,11 @@ void tor_mutex_init_nonrecursive(tor_mutex_t *m); void tor_mutex_acquire(tor_mutex_t *m); void tor_mutex_release(tor_mutex_t *m); void tor_mutex_free_(tor_mutex_t *m); +/** + * @copydoc tor_mutex_free_ + * + * Additionally, set the pointer <b>m</b> to NULL. + **/ #define tor_mutex_free(m) FREE_AND_NULL(tor_mutex_t, tor_mutex_free_, (m)) void tor_mutex_uninit(tor_mutex_t *m); diff --git a/src/lib/lock/compat_mutex_pthreads.c b/src/lib/lock/compat_mutex_pthreads.c index ee5f520cd0..ac83c42a47 100644 --- a/src/lib/lock/compat_mutex_pthreads.c +++ b/src/lib/lock/compat_mutex_pthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,8 +17,14 @@ * "recursive" mutexes (i.e., once we can re-lock if we're already holding * them.) */ static pthread_mutexattr_t attr_recursive; +/** + * True iff <b>attr_recursive</b> has been initialized. + **/ static int attr_initialized = 0; +/** + * Initialize the locking module, if it is not already initialized. + **/ void tor_locking_init(void) { @@ -88,12 +94,26 @@ tor_mutex_release(tor_mutex_t *m) } /** Clean up the mutex <b>m</b> so that it no longer uses any system * resources. Does not free <b>m</b>. This function must only be called on - * mutexes from tor_mutex_init(). */ + * mutexes from tor_mutex_init(). + * + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ void tor_mutex_uninit(tor_mutex_t *m) { int err; raw_assert(m); + /* If the mutex is already locked, wait until after it is unlocked to destroy + * it. Locking and releasing the mutex makes undefined behaviour less likely, + * but does not prevent it. Another thread can lock the mutex between release + * and destroy. */ + tor_mutex_acquire(m); + tor_mutex_release(m); err = pthread_mutex_destroy(&m->mutex); if (PREDICT_UNLIKELY(err)) { // LCOV_EXCL_START diff --git a/src/lib/lock/compat_mutex_winthreads.c b/src/lib/lock/compat_mutex_winthreads.c index b0f5999e42..5fe6870a93 100644 --- a/src/lib/lock/compat_mutex_winthreads.c +++ b/src/lib/lock/compat_mutex_winthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/lock/include.am b/src/lib/lock/include.am index 4e6f444347..1475b9911b 100644 --- a/src/lib/lock/include.am +++ b/src/lib/lock/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-lock-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_lock_a_SOURCES = \ src/lib/lock/compat_mutex.c @@ -20,5 +21,6 @@ src_lib_libtor_lock_testing_a_SOURCES = \ src_lib_libtor_lock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_lock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/lock/compat_mutex.h diff --git a/src/lib/lock/lib_lock.md b/src/lib/lock/lib_lock.md new file mode 100644 index 0000000000..6f6727bfc2 --- /dev/null +++ b/src/lib/lock/lib_lock.md @@ -0,0 +1,6 @@ +@dir /lib/lock +@brief lib/lock: Simple locking support. + +This module is more low-level than the rest of the threading code, since it +is needed by more intermediate-level modules. + diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include index 852173aab3..54d96324db 100644 --- a/src/lib/log/.may_include +++ b/src/lib/log/.may_include @@ -1,6 +1,7 @@ orconfig.h lib/cc/*.h +lib/defs/*.h lib/smartlist_core/*.h lib/err/*.h lib/fdio/*.h @@ -9,7 +10,7 @@ lib/lock/*.h lib/log/*.h lib/malloc/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h +lib/version/*.h lib/wallclock/*.h - -micro-revision.i
\ No newline at end of file diff --git a/src/lib/log/escape.c b/src/lib/log/escape.c index 6ca01c6963..88ca52a78c 100644 --- a/src/lib/log/escape.c +++ b/src/lib/log/escape.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/escape.h b/src/lib/log/escape.h index 2f726186c5..0d1eaef4c1 100644 --- a/src/lib/log/escape.h +++ b/src/lib/log/escape.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,4 +20,4 @@ char *esc_for_log(const char *string) ATTR_MALLOC; char *esc_for_log_len(const char *chars, size_t n) ATTR_MALLOC; const char *escaped(const char *string); -#endif /* !defined(TOR_TORLOG_H) */ +#endif /* !defined(TOR_ESCAPE_H) */ diff --git a/src/lib/log/include.am b/src/lib/log/include.am index 4a6c9b3686..5b9f7113ba 100644 --- a/src/lib/log/include.am +++ b/src/lib/log/include.am @@ -5,11 +5,12 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-log-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_log_a_SOURCES = \ src/lib/log/escape.c \ - src/lib/log/git_revision.c \ src/lib/log/ratelim.c \ src/lib/log/log.c \ + src/lib/log/log_sys.c \ src/lib/log/util_bug.c if WIN32 @@ -21,16 +22,11 @@ src_lib_libtor_log_testing_a_SOURCES = \ src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) -# Declare that these object files depend on micro-revision.i. Without this -# rule, we could try to build them before micro-revision.i was created. -src/lib/log/git_revision.$(OBJEXT) \ - src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \ - micro-revision.i - +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/log/escape.h \ - src/lib/log/git_revision.h \ src/lib/log/ratelim.h \ src/lib/log/log.h \ + src/lib/log/log_sys.h \ src/lib/log/util_bug.h \ src/lib/log/win32err.h diff --git a/src/lib/log/lib_log.md b/src/lib/log/lib_log.md new file mode 100644 index 0000000000..8740d6a02f --- /dev/null +++ b/src/lib/log/lib_log.md @@ -0,0 +1,10 @@ +@dir /lib/log +@brief lib/log: Log messages to files, syslogs, etc. + +You can think of this as the logical "midpoint" of the +\refdir{lib} code": much of the higher-level code is higher-level +_because_ it uses the logging module, and much of the lower-level code is +specifically written to avoid having to log, because the logging module +depends on it. + + diff --git a/src/lib/log/log.c b/src/lib/log/log.c index a9ad38fb25..9ee87c0668 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -32,7 +32,8 @@ #define LOG_PRIVATE #include "lib/log/log.h" -#include "lib/log/git_revision.h" +#include "lib/log/log_sys.h" +#include "lib/version/git_revision.h" #include "lib/log/ratelim.h" #include "lib/lock/compat_mutex.h" #include "lib/smartlist_core/smartlist_core.h" @@ -48,15 +49,12 @@ #include "lib/wallclock/approx_time.h" #include "lib/wallclock/time_to_tm.h" #include "lib/fdio/fdio.h" +#include "lib/cc/ctassert.h" #ifdef HAVE_ANDROID_LOG_H #include <android/log.h> #endif // HAVE_ANDROID_LOG_H. -/** Given a severity, yields an index into log_severity_list_t.masks to use - * for that severity. */ -#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR) - /** @{ */ /** The string we stick at the end of a log message when it is too long, * and its length. */ @@ -153,7 +151,7 @@ severity_to_android_log_priority(int severity) // LCOV_EXCL_STOP } } -#endif // HAVE_ANDROID_LOG_H. +#endif /* defined(HAVE_ANDROID_LOG_H) */ /** A mutex to guard changes to logfiles and logging. */ static tor_mutex_t log_mutex; @@ -223,6 +221,7 @@ int log_global_min_severity_ = LOG_NOTICE; static void delete_log(logfile_t *victim); static void close_log(logfile_t *victim); +static void close_log_sigsafe(logfile_t *victim); static char *domain_to_string(log_domain_mask_t domain, char *buf, size_t buflen); @@ -277,8 +276,8 @@ static int log_time_granularity = 1; /** Define log time granularity for all logs to be <b>granularity_msec</b> * milliseconds. */ -void -set_log_time_granularity(int granularity_msec) +MOCK_IMPL(void, +set_log_time_granularity,(int granularity_msec)) { log_time_granularity = granularity_msec; tor_log_sigsafe_err_set_granularity(granularity_msec); @@ -524,7 +523,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len, * pass them, and some very old ones do not detect overflow so well. * Regrettably, they call their maximum line length MAXLINE. */ #if MAXLINE < 64 -#warn "MAXLINE is a very low number; it might not be from syslog.h after all" +#warning "MAXLINE is very low; it might not be from syslog.h." #endif char *m = msg_after_prefix; if (msg_len >= MAXLINE) @@ -533,7 +532,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len, if (m != msg_after_prefix) { tor_free(m); } -#else /* !(defined(MAXLINE)) */ +#else /* !defined(MAXLINE) */ /* We have syslog but not MAXLINE. That's promising! */ syslog(severity, "%s", msg_after_prefix); #endif /* defined(MAXLINE) */ @@ -585,8 +584,7 @@ logv,(int severity, log_domain_mask_t domain, const char *funcname, /* check that severity is sane. Overrunning the masks array leads to * interesting and hard to diagnose effects */ raw_assert(severity >= LOG_ERR && severity <= LOG_DEBUG); - /* check that we've initialised the log mutex before we try to lock it */ - raw_assert(log_mutex_initialized); + LOCK_LOGS(); if ((! (domain & LD_NOCB)) && pending_cb_messages @@ -630,6 +628,10 @@ void tor_log(int severity, log_domain_mask_t domain, const char *format, ...) { va_list ap; + + /* check that domain is composed of known domains and flags */ + raw_assert((domain & (LD_ALL_DOMAINS|LD_ALL_FLAGS)) == domain); + if (severity > log_global_min_severity_) return; va_start(ap,format); @@ -663,18 +665,21 @@ tor_log_update_sigsafe_err_fds(void) const logfile_t *lf; int found_real_stderr = 0; + /* The fds are the file descriptors of tor's stdout, stderr, and file + * logs. The log and err modules flush these fds during their shutdowns. */ int fds[TOR_SIGSAFE_LOG_MAX_FDS]; int n_fds; LOCK_LOGS(); /* Reserve the first one for stderr. This is safe because when we daemonize, - * we dup2 /dev/null to stderr, */ + * we dup2 /dev/null to stderr. */ fds[0] = STDERR_FILENO; n_fds = 1; for (lf = logfiles; lf; lf = lf->next) { - /* Don't try callback to the control port, or syslogs: We can't - * do them from a signal handler. Don't try stdout: we always do stderr. + /* Don't try callback to the control port, syslogs, android logs, or any + * other non-file descriptor log: We can't call arbitrary functions from a + * signal handler. */ if (lf->is_temporary || logfile_is_external(lf) || lf->seems_dead || lf->fd < 0) @@ -683,10 +688,12 @@ tor_log_update_sigsafe_err_fds(void) (LD_BUG|LD_GENERAL)) { if (lf->fd == STDERR_FILENO) found_real_stderr = 1; - /* Avoid duplicates */ + /* Avoid duplicates by checking the log module fd against fds */ if (int_array_contains(fds, n_fds, lf->fd)) continue; - fds[n_fds++] = lf->fd; + /* Update fds using the log module's fd */ + fds[n_fds] = lf->fd; + n_fds++; if (n_fds == TOR_SIGSAFE_LOG_MAX_FDS) break; } @@ -694,9 +701,13 @@ tor_log_update_sigsafe_err_fds(void) if (!found_real_stderr && int_array_contains(fds, n_fds, STDOUT_FILENO)) { - /* Don't use a virtual stderr when we're also logging to stdout. */ + /* Don't use a virtual stderr when we're also logging to stdout. + * If we reached max_fds logs, we'll now have (max_fds - 1) logs. + * That's ok, max_fds is large enough that most tor instances don't exceed + * it. */ raw_assert(n_fds >= 2); /* Don't tor_assert inside log fns */ - fds[0] = fds[--n_fds]; + --n_fds; + fds[0] = fds[n_fds]; } UNLOCK_LOGS(); @@ -727,7 +738,7 @@ tor_log_get_logfile_names(smartlist_t *out) /** Implementation of the log_fn backend, used when we have * variadic macros. All arguments are as for log_fn, except for - * <b>fn</b>, which is the name of the calling functions. */ + * <b>fn</b>, which is the name of the calling function. */ void log_fn_(int severity, log_domain_mask_t domain, const char *fn, const char *format, ...) @@ -804,16 +815,42 @@ logs_free_all(void) } /* We _could_ destroy the log mutex here, but that would screw up any logs - * that happened between here and the end of execution. */ + * that happened between here and the end of execution. + * If tor is re-initialized, log_mutex_initialized will still be 1. So we + * won't trigger any undefined behaviour by trying to re-initialize the + * log mutex. */ +} + +/** Flush the signal-safe log files. + * + * This function is safe to call from a signal handler. It is currenly called + * by the BUG() macros, when terminating the process on an abnormal condition. + */ +void +logs_flush_sigsafe(void) +{ + /* If we don't have fsync() in unistd.h, we can't flush the logs. */ +#ifdef HAVE_FSYNC + logfile_t *victim, *next; + /* We can't LOCK_LOGS() in a signal handler, because it may call + * signal-unsafe functions. And we can't deallocate memory, either. */ + next = logfiles; + logfiles = NULL; + while (next) { + victim = next; + next = next->next; + if (victim->needs_close) { + /* We can't do anything useful if the flush fails. */ + (void)fsync(victim->fd); + } + } +#endif /* defined(HAVE_FSYNC) */ } /** Remove and free the log entry <b>victim</b> from the linked-list * logfiles (it is probably present, but it might not be due to thread * racing issues). After this function is called, the caller shouldn't * refer to <b>victim</b> anymore. - * - * Long-term, we need to do something about races in the log subsystem - * in general. See bug 222 for more details. */ static void delete_log(logfile_t *victim) @@ -833,13 +870,26 @@ delete_log(logfile_t *victim) } /** Helper: release system resources (but not memory) held by a single - * logfile_t. */ + * signal-safe logfile_t. If the log's resources can not be released in + * a signal handler, does nothing. */ static void -close_log(logfile_t *victim) +close_log_sigsafe(logfile_t *victim) { if (victim->needs_close && victim->fd >= 0) { + /* We can't do anything useful here if close() fails: we're shutting + * down logging, and the err module only does fatal errors. */ close(victim->fd); victim->fd = -1; + } +} + +/** Helper: release system resources (but not memory) held by a single + * logfile_t. */ +static void +close_log(logfile_t *victim) +{ + if (victim->needs_close) { + close_log_sigsafe(victim); } else if (victim->is_syslog) { #ifdef HAVE_SYSLOG_H if (--syslog_count == 0) { @@ -863,15 +913,15 @@ set_log_severity_config(int loglevelMin, int loglevelMax, raw_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG); memset(severity_out, 0, sizeof(log_severity_list_t)); for (i = loglevelMin; i >= loglevelMax; --i) { - severity_out->masks[SEVERITY_MASK_IDX(i)] = ~0u; + severity_out->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS; } } /** Add a log handler named <b>name</b> to send all messages in <b>severity</b> * to <b>fd</b>. Copies <b>severity</b>. Helper: does no locking. */ -static void -add_stream_log_impl(const log_severity_list_t *severity, - const char *name, int fd) +MOCK_IMPL(STATIC void, +add_stream_log_impl,(const log_severity_list_t *severity, + const char *name, int fd)) { logfile_t *lf; lf = tor_malloc_zero(sizeof(logfile_t)); @@ -927,18 +977,16 @@ logs_set_domain_logging(int enabled) UNLOCK_LOGS(); } -/** Add a log handler to receive messages during startup (before the real - * logs are initialized). +/** Add a log handler to accept messages when no other log is configured. */ void -add_temp_log(int min_severity) +add_default_log(int min_severity) { log_severity_list_t *s = tor_malloc_zero(sizeof(log_severity_list_t)); set_log_severity_config(min_severity, LOG_ERR, s); LOCK_LOGS(); - add_stream_log_impl(s, "<temp>", fileno(stdout)); + add_stream_log_impl(s, "<default>", fileno(stdout)); tor_free(s); - logfiles->is_temporary = 1; UNLOCK_LOGS(); } @@ -1020,7 +1068,7 @@ flush_pending_log_callbacks(void) do { SMARTLIST_FOREACH_BEGIN(messages, pending_log_message_t *, msg) { const int severity = msg->severity; - const int domain = msg->domain; + const log_domain_mask_t domain = msg->domain; for (lf = logfiles; lf; lf = lf->next) { if (! lf->callback || lf->seems_dead || ! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) { @@ -1081,8 +1129,7 @@ flush_log_messages_from_startup(void) UNLOCK_LOGS(); } -/** Close any log handlers added by add_temp_log() or marked by - * mark_logs_temp(). */ +/** Close any log handlers marked by mark_logs_temp(). */ void close_temp_logs(void) { @@ -1134,10 +1181,10 @@ mark_logs_temp(void) * opening the logfile failed, -1 is returned and errno is set appropriately * (by open(2)). Takes ownership of fd. */ -int -add_file_log(const log_severity_list_t *severity, - const char *filename, - int fd) +MOCK_IMPL(int, +add_file_log,(const log_severity_list_t *severity, + const char *filename, + int fd)) { logfile_t *lf; @@ -1231,7 +1278,7 @@ add_android_log(const log_severity_list_t *severity, UNLOCK_LOGS(); return 0; } -#endif // HAVE_ANDROID_LOG_H. +#endif /* defined(HAVE_ANDROID_LOG_H) */ /** If <b>level</b> is a valid log severity, return the corresponding * numeric value. Otherwise, return -1. */ @@ -1267,9 +1314,16 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", "MESG", + NULL }; +CTASSERT(ARRAY_LENGTH(domain_list) == N_LOGGING_DOMAINS + 1); + +CTASSERT(HIGHEST_RESERVED_LD_DOMAIN_ < LD_ALL_DOMAINS); +CTASSERT(LD_ALL_DOMAINS < LOWEST_RESERVED_LD_FLAG_); +CTASSERT(LOWEST_RESERVED_LD_FLAG_ < LD_ALL_FLAGS); + /** Return a bitmask for the log domain for which <b>domain</b> is the name, * or 0 if there is no such name. */ static log_domain_mask_t @@ -1278,7 +1332,7 @@ parse_log_domain(const char *domain) int i; for (i=0; domain_list[i]; ++i) { if (!strcasecmp(domain, domain_list[i])) - return (1u<<i); + return (UINT64_C(1)<<i); } return 0; } @@ -1350,7 +1404,7 @@ parse_log_severity_config(const char **cfg_ptr, const char *dash, *space; char *sev_lo, *sev_hi; int low, high, i; - log_domain_mask_t domains = ~0u; + log_domain_mask_t domains = LD_ALL_DOMAINS; if (*cfg == '[') { int err = 0; @@ -1368,9 +1422,9 @@ parse_log_severity_config(const char **cfg_ptr, tor_free(domains_str); SMARTLIST_FOREACH_BEGIN(domains_list, const char *, domain) { if (!strcmp(domain, "*")) { - domains = ~0u; + domains = LD_ALL_DOMAINS; } else { - int d; + log_domain_mask_t d; int negate=0; if (*domain == '~') { negate = 1; @@ -1464,7 +1518,7 @@ switch_logs_debug(void) LOCK_LOGS(); for (lf = logfiles; lf; lf=lf->next) { for (i = LOG_DEBUG; i >= LOG_ERR; --i) - lf->severities->masks[SEVERITY_MASK_IDX(i)] = ~0u; + lf->severities->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS; } log_global_min_severity_ = get_min_log_level(); UNLOCK_LOGS(); diff --git a/src/lib/log/log.h b/src/lib/log/log.h index d7a5070610..aafbf9be2f 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001, Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,20 +11,24 @@ **/ #ifndef TOR_TORLOG_H +#define TOR_TORLOG_H #include <stdarg.h> #include "lib/cc/torint.h" #include "lib/cc/compat_compiler.h" +#include "lib/defs/logging_types.h" #include "lib/testsupport/testsupport.h" #ifdef HAVE_SYSLOG_H #include <syslog.h> #define LOG_WARN LOG_WARNING #if LOG_DEBUG < LOG_ERR +#ifndef COCCI #error "Your syslog.h thinks high numbers are more important. " \ "We aren't prepared to deal with that." #endif -#else /* !(defined(HAVE_SYSLOG_H)) */ +#endif /* LOG_DEBUG < LOG_ERR */ +#else /* !defined(HAVE_SYSLOG_H) */ /* Note: Syslog's logging code refers to priorities, with 0 being the most * important. Thus, all our comparisons needed to be reversed when we added * syslog support. @@ -55,75 +59,92 @@ /* Logging domains */ /** Catch-all for miscellaneous events and fatal errors. */ -#define LD_GENERAL (1u<<0) +#define LD_GENERAL (UINT64_C(1)<<0) /** The cryptography subsystem. */ -#define LD_CRYPTO (1u<<1) +#define LD_CRYPTO (UINT64_C(1)<<1) /** Networking. */ -#define LD_NET (1u<<2) +#define LD_NET (UINT64_C(1)<<2) /** Parsing and acting on our configuration. */ -#define LD_CONFIG (1u<<3) +#define LD_CONFIG (UINT64_C(1)<<3) /** Reading and writing from the filesystem. */ -#define LD_FS (1u<<4) +#define LD_FS (UINT64_C(1)<<4) /** Other servers' (non)compliance with the Tor protocol. */ -#define LD_PROTOCOL (1u<<5) +#define LD_PROTOCOL (UINT64_C(1)<<5) /** Memory management. */ -#define LD_MM (1u<<6) +#define LD_MM (UINT64_C(1)<<6) /** HTTP implementation. */ -#define LD_HTTP (1u<<7) +#define LD_HTTP (UINT64_C(1)<<7) /** Application (socks) requests. */ -#define LD_APP (1u<<8) +#define LD_APP (UINT64_C(1)<<8) /** Communication via the controller protocol. */ -#define LD_CONTROL (1u<<9) +#define LD_CONTROL (UINT64_C(1)<<9) /** Building, using, and managing circuits. */ -#define LD_CIRC (1u<<10) +#define LD_CIRC (UINT64_C(1)<<10) /** Hidden services. */ -#define LD_REND (1u<<11) +#define LD_REND (UINT64_C(1)<<11) /** Internal errors in this Tor process. */ -#define LD_BUG (1u<<12) +#define LD_BUG (UINT64_C(1)<<12) /** Learning and using information about Tor servers. */ -#define LD_DIR (1u<<13) +#define LD_DIR (UINT64_C(1)<<13) /** Learning and using information about Tor servers. */ -#define LD_DIRSERV (1u<<14) +#define LD_DIRSERV (UINT64_C(1)<<14) /** Onion routing protocol. */ -#define LD_OR (1u<<15) +#define LD_OR (UINT64_C(1)<<15) /** Generic edge-connection functionality. */ -#define LD_EDGE (1u<<16) +#define LD_EDGE (UINT64_C(1)<<16) #define LD_EXIT LD_EDGE /** Bandwidth accounting. */ -#define LD_ACCT (1u<<17) +#define LD_ACCT (UINT64_C(1)<<17) /** Router history */ -#define LD_HIST (1u<<18) +#define LD_HIST (UINT64_C(1)<<18) /** OR handshaking */ -#define LD_HANDSHAKE (1u<<19) +#define LD_HANDSHAKE (UINT64_C(1)<<19) /** Heartbeat messages */ -#define LD_HEARTBEAT (1u<<20) +#define LD_HEARTBEAT (UINT64_C(1)<<20) /** Abstract channel_t code */ -#define LD_CHANNEL (1u<<21) +#define LD_CHANNEL (UINT64_C(1)<<21) /** Scheduler */ -#define LD_SCHED (1u<<22) +#define LD_SCHED (UINT64_C(1)<<22) /** Guard nodes */ -#define LD_GUARD (1u<<23) +#define LD_GUARD (UINT64_C(1)<<23) /** Generation and application of consensus diffs. */ -#define LD_CONSDIFF (1u<<24) +#define LD_CONSDIFF (UINT64_C(1)<<24) /** Denial of Service mitigation. */ -#define LD_DOS (1u<<25) -/** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 26 +#define LD_DOS (UINT64_C(1)<<25) +/** Processes */ +#define LD_PROCESS (UINT64_C(1)<<26) +/** Pluggable Transports. */ +#define LD_PT (UINT64_C(1)<<27) +/** Bootstrap tracker. */ +#define LD_BTRACK (UINT64_C(1)<<28) +/** Message-passing backend. */ +#define LD_MESG (UINT64_C(1)<<29) -/** This log message is not safe to send to a callback-based logger - * immediately. Used as a flag, not a log domain. */ -#define LD_NOCB (1u<<31) -/** This log message should not include a function name, even if it otherwise - * would. Used as a flag, not a log domain. */ -#define LD_NOFUNCNAME (1u<<30) +/** The number of log domains. */ +#define N_LOGGING_DOMAINS 30 +/** The highest log domain */ +#define HIGHEST_RESERVED_LD_DOMAIN_ (UINT64_C(1)<<(N_LOGGING_DOMAINS - 1)) +/** All log domains. */ +#define LD_ALL_DOMAINS ((~(UINT64_C(0)))>>(64 - N_LOGGING_DOMAINS)) + +/** The number of log flags. */ +#define N_LOGGING_FLAGS 3 +/** First bit that is reserved in log_domain_mask_t for non-domain flags. */ +#define LOWEST_RESERVED_LD_FLAG_ (UINT64_C(1)<<(64 - N_LOGGING_FLAGS)) +/** All log flags. */ +#define LD_ALL_FLAGS ((~(UINT64_C(0)))<<(64 - N_LOGGING_FLAGS)) #ifdef TOR_UNIT_TESTS /** This log message should not be intercepted by mock_saving_logv */ -#define LD_NO_MOCK (1u<<29) +#define LD_NO_MOCK (UINT64_C(1)<<61) #endif -/** Mask of zero or more log domains, OR'd together. */ -typedef uint32_t log_domain_mask_t; +/** This log message is not safe to send to a callback-based logger + * immediately. Used as a flag, not a log domain. */ +#define LD_NOCB (UINT64_C(1)<<62) +/** This log message should not include a function name, even if it otherwise + * would. Used as a flag, not a log domain. */ +#define LD_NOFUNCNAME (UINT64_C(1)<<63) /** Configures which severities are logged for each logging domain for a given * log target. */ @@ -134,7 +155,8 @@ typedef struct log_severity_list_t { } log_severity_list_t; /** Callback type used for add_callback_log. */ -typedef void (*log_callback)(int severity, uint32_t domain, const char *msg); +typedef void (*log_callback)(int severity, log_domain_mask_t domain, + const char *msg); void init_logging(int disable_startup_queue); int parse_log_level(const char *level); @@ -143,11 +165,11 @@ int parse_log_severity_config(const char **cfg, log_severity_list_t *severity_out); void set_log_severity_config(int minSeverity, int maxSeverity, log_severity_list_t *severity_out); -void add_stream_log(const log_severity_list_t *severity, const char *name, - int fd); -int add_file_log(const log_severity_list_t *severity, - const char *filename, - int fd); +void add_stream_log(const log_severity_list_t *severity, + const char *name, int fd); +MOCK_DECL(int, add_file_log,(const log_severity_list_t *severity, + const char *filename, + int fd)); #ifdef HAVE_SYSLOG_H int add_syslog_log(const log_severity_list_t *severity, @@ -164,7 +186,8 @@ void logs_set_domain_logging(int enabled); int get_min_log_level(void); void switch_logs_debug(void); void logs_free_all(void); -void add_temp_log(int min_severity); +void logs_flush_sigsafe(void); +void add_default_log(int min_severity); void close_temp_logs(void); void rollback_log_changes(void); void mark_logs_temp(void); @@ -173,7 +196,7 @@ void change_callback_log_severity(int loglevelMin, int loglevelMax, void flush_pending_log_callbacks(void); void flush_log_messages_from_startup(void); void log_set_application_name(const char *name); -void set_log_time_granularity(int granularity_msec); +MOCK_DECL(void, set_log_time_granularity,(int granularity_msec)); void truncate_logs(void); void tor_log(int severity, log_domain_mask_t domain, const char *format, ...) @@ -186,6 +209,21 @@ void tor_log_get_logfile_names(struct smartlist_t *out); extern int log_global_min_severity_; +#ifdef TOR_COVERAGE +/* For coverage builds, we try to avoid our log_debug optimization, since it + * can have weird effects on internal macro coverage. */ +#define debug_logging_enabled() (1) +#else +static inline bool debug_logging_enabled(void); +/** + * Return true iff debug logging is enabled for at least one domain. + */ +static inline bool debug_logging_enabled(void) +{ + return PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG); +} +#endif /* defined(TOR_COVERAGE) */ + void log_fn_(int severity, log_domain_mask_t domain, const char *funcname, const char *format, ...) CHECK_PRINTF(4,5); @@ -215,8 +253,8 @@ void tor_log_string(int severity, log_domain_mask_t domain, log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, args) #define log_debug(domain, args...) \ STMT_BEGIN \ - if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG)) \ - log_fn_(LOG_DEBUG, domain, __FUNCTION__, args); \ + if (debug_logging_enabled()) \ + log_fn_(LOG_DEBUG, domain, __FUNCTION__, args); \ STMT_END #define log_info(domain, args...) \ log_fn_(LOG_INFO, domain, __FUNCTION__, args) @@ -233,8 +271,8 @@ void tor_log_string(int severity, log_domain_mask_t domain, #define log_debug(domain, args, ...) \ STMT_BEGIN \ - if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG)) \ - log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__); \ + if (debug_logging_enabled()) \ + log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__); \ STMT_END #define log_info(domain, args,...) \ log_fn_(LOG_INFO, domain, __FUNCTION__, args, ##__VA_ARGS__) @@ -270,7 +308,14 @@ extern const log_domain_mask_t LD_GENERAL_; MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, const char *funcname, const char *suffix, const char *format, va_list ap) CHECK_PRINTF(5,0)); +MOCK_DECL(STATIC void, add_stream_log_impl,( + const log_severity_list_t *severity, const char *name, int fd)); +#endif /* defined(LOG_PRIVATE) */ + +#if defined(LOG_PRIVATE) || defined(TOR_UNIT_TESTS) +/** Given a severity, yields an index into log_severity_list_t.masks to use + * for that severity. */ +#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR) #endif -# define TOR_TORLOG_H #endif /* !defined(TOR_TORLOG_H) */ diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c new file mode 100644 index 0000000000..021c05d3e6 --- /dev/null +++ b/src/lib/log/log_sys.c @@ -0,0 +1,38 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.c + * \brief Setup and tear down the logging module. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/log/escape.h" +#include "lib/log/log.h" +#include "lib/log/log_sys.h" + +static int +subsys_logging_initialize(void) +{ + init_logging(0); + return 0; +} + +static void +subsys_logging_shutdown(void) +{ + logs_free_all(); + escaped(NULL); +} + +const subsys_fns_t sys_logging = { + .name = "log", + SUBSYS_DECLARE_LOCATION(), + .supported = true, + /* Logging depends on threads, approx time, raw logging, and security. + * Most other lib modules depend on logging. */ + .level = -90, + .initialize = subsys_logging_initialize, + .shutdown = subsys_logging_shutdown, +}; diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h new file mode 100644 index 0000000000..523c2e5008 --- /dev/null +++ b/src/lib/log/log_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file log_sys.h + * \brief Declare subsystem object for the logging module. + **/ + +#ifndef TOR_LOG_SYS_H +#define TOR_LOG_SYS_H + +extern const struct subsys_fns_t sys_logging; + +#endif /* !defined(TOR_LOG_SYS_H) */ diff --git a/src/lib/log/ratelim.c b/src/lib/log/ratelim.c index 5eec742aa7..ac401fb398 100644 --- a/src/lib/log/ratelim.c +++ b/src/lib/log/ratelim.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/log/ratelim.h b/src/lib/log/ratelim.h index 48edd7c849..e9b55d40dc 100644 --- a/src/lib/log/ratelim.h +++ b/src/lib/log/ratelim.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -45,9 +45,11 @@ typedef struct ratelim_t { int n_calls_since_last_time; } ratelim_t; +#ifndef COCCI #define RATELIM_INIT(r) { (r), 0, 0 } +#endif #define RATELIM_TOOMANY (16*1000*1000) char *rate_limit_log(ratelim_t *lim, time_t now); -#endif +#endif /* !defined(TOR_RATELIM_H) */ diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c index c65a91ae9e..d698ddd8a6 100644 --- a/src/lib/log/util_bug.c +++ b/src/lib/log/util_bug.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,6 +11,7 @@ #include "lib/log/util_bug.h" #include "lib/log/log.h" #include "lib/err/backtrace.h" +#include "lib/err/torerr.h" #ifdef TOR_UNIT_TESTS #include "lib/smartlist_core/smartlist_core.h" #include "lib/smartlist_core/smartlist_foreach.h" @@ -63,7 +64,7 @@ tor_set_failed_assertion_callback(void (*fn)(void)) { failed_assertion_cb = fn; } -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ #define capturing_bugs() (0) #define add_captured_bug(s) do { } while (0) #endif /* defined(TOR_UNIT_TESTS) */ @@ -71,24 +72,42 @@ tor_set_failed_assertion_callback(void (*fn)(void)) /** Helper for tor_assert: report the assertion failure. */ void tor_assertion_failed_(const char *fname, unsigned int line, - const char *func, const char *expr) + const char *func, const char *expr, + const char *fmt, ...) { - char buf[256]; + char *buf = NULL; + char *extra = NULL; + va_list ap; + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" +#endif + if (fmt) { + va_start(ap,fmt); + tor_vasprintf(&extra, fmt, ap); + va_end(ap); + } +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.", fname, line, func, expr); - tor_snprintf(buf, sizeof(buf), - "Assertion %s failed in %s at %s:%u", - expr, func, fname, line); + tor_asprintf(&buf, "Assertion %s failed in %s at %s:%u: %s", + expr, func, fname, line, extra ? extra : ""); + tor_free(extra); log_backtrace(LOG_ERR, LD_BUG, buf); + tor_free(buf); } /** Helper for tor_assert_nonfatal: report the assertion failure. */ void tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, - int once) + int once, const char *fmt, ...) { - char buf[256]; + char *buf = NULL; const char *once_str = once ? " (Future instances of this warning will be silenced.)": ""; if (! expr) { @@ -98,7 +117,7 @@ tor_bug_occurred_(const char *fname, unsigned int line, } log_warn(LD_BUG, "%s:%u: %s: This line should not have been reached.%s", fname, line, func, once_str); - tor_snprintf(buf, sizeof(buf), + tor_asprintf(&buf, "Line unexpectedly reached at %s at %s:%u", func, fname, line); } else { @@ -106,13 +125,32 @@ tor_bug_occurred_(const char *fname, unsigned int line, add_captured_bug(expr); return; } + + va_list ap; + char *extra = NULL; + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" +#endif + if (fmt) { + va_start(ap,fmt); + tor_vasprintf(&extra, fmt, ap); + va_end(ap); + } +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s", fname, line, func, expr, once_str); - tor_snprintf(buf, sizeof(buf), - "Non-fatal assertion %s failed in %s at %s:%u", - expr, func, fname, line); + tor_asprintf(&buf, "Non-fatal assertion %s failed in %s at %s:%u%s%s", + expr, func, fname, line, fmt ? " : " : "", + extra ? extra : ""); + tor_free(extra); } log_backtrace(LOG_WARN, LD_BUG, buf); + tor_free(buf); #ifdef TOR_UNIT_TESTS if (failed_assertion_cb) { @@ -122,16 +160,18 @@ tor_bug_occurred_(const char *fname, unsigned int line, } /** - * Call the abort() function to kill the current process with a fatal - * error. + * Call the tor_raw_abort_() function to close raw logs, then kill the current + * process with a fatal error. But first, close the file-based log file + * descriptors, so error messages are written before process termination. * * (This is a separate function so that we declare it in util_bug.h without - * including stdlib in all the users of util_bug.h) + * including torerr.h in all the users of util_bug.h) **/ void tor_abort_(void) { - abort(); + logs_flush_sigsafe(); + tor_raw_abort_(); } #ifdef _WIN32 diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h index 17e8d0c5a7..684dc7c6dd 100644 --- a/src/lib/log/util_bug.h +++ b/src/lib/log/util_bug.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -80,10 +80,10 @@ tor__assert_tmp_value__; \ } ) #define ASSERT_PREDICT_LIKELY_(e) ASSERT_PREDICT_UNLIKELY_(e) -#else +#else /* !(defined(TOR_UNIT_TESTS) && defined(__GNUC__)) */ #define ASSERT_PREDICT_UNLIKELY_(e) PREDICT_UNLIKELY(e) #define ASSERT_PREDICT_LIKELY_(e) PREDICT_LIKELY(e) -#endif +#endif /* defined(TOR_UNIT_TESTS) && defined(__GNUC__) */ /* Sometimes we don't want to use assertions during branch coverage tests; it * leads to tons of unreached branches which in reality are only assertions we @@ -92,21 +92,28 @@ #define tor_assert(a) STMT_BEGIN \ (void)(a); \ STMT_END -#else +#define tor_assertf(a, fmt, ...) STMT_BEGIN \ + (void)(a); \ + (void)(fmt); \ + STMT_END +#else /* !(defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_T...)) */ /** Like assert(3), but send assertion failures to the log as well as to * stderr. */ -#define tor_assert(expr) STMT_BEGIN \ +#define tor_assert(expr) tor_assertf(expr, NULL) + +#define tor_assertf(expr, fmt, ...) STMT_BEGIN \ if (ASSERT_PREDICT_LIKELY_(expr)) { \ } else { \ - tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr); \ - tor_abort_(); \ + tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr, \ + fmt, ##__VA_ARGS__); \ + tor_abort_(); \ } STMT_END #endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */ #define tor_assert_unreached() \ STMT_BEGIN { \ tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, \ - "line should be unreached"); \ + "line should be unreached", NULL); \ tor_abort_(); \ } STMT_END @@ -124,7 +131,9 @@ #undef BUG // Coverity defines this in global headers; let's override it. This is a // magic coverity-only preprocessor thing. +#ifndef COCCI #nodef BUG(x) (x) +#endif #endif /* defined(__COVERITY__) */ #if defined(__COVERITY__) || defined(__clang_analyzer__) @@ -133,37 +142,58 @@ #define ALL_BUGS_ARE_FATAL #endif +/** Define ALL_BUGS_ARE_FATAL if you want Tor to crash when any problem comes + * up, so you can get a coredump and track things down. */ #ifdef ALL_BUGS_ARE_FATAL #define tor_assert_nonfatal_unreached() tor_assert(0) #define tor_assert_nonfatal(cond) tor_assert((cond)) +#define tor_assertf_nonfatal(cond, fmt, ...) \ + tor_assertf(cond, fmt, ##__VA_ARGS__) #define tor_assert_nonfatal_unreached_once() tor_assert(0) #define tor_assert_nonfatal_once(cond) tor_assert((cond)) #define BUG(cond) \ (ASSERT_PREDICT_UNLIKELY_(cond) ? \ - (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")"), \ + (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",NULL), \ tor_abort_(), 1) \ : 0) +#ifndef COCCI +#define IF_BUG_ONCE(cond) if (BUG(cond)) +#endif #elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) #define tor_assert_nonfatal_unreached() STMT_NIL #define tor_assert_nonfatal(cond) ((void)(cond)) +#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN \ + (void)cond; \ + (void)fmt; \ + STMT_END #define tor_assert_nonfatal_unreached_once() STMT_NIL #define tor_assert_nonfatal_once(cond) ((void)(cond)) #define BUG(cond) (ASSERT_PREDICT_UNLIKELY_(cond) ? 1 : 0) +#ifndef COCCI +#define IF_BUG_ONCE(cond) if (BUG(cond)) +#endif #else /* Normal case, !ALL_BUGS_ARE_FATAL, !DISABLE_ASSERTS_IN_UNIT_TESTS */ #define tor_assert_nonfatal_unreached() STMT_BEGIN \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0, NULL); \ STMT_END #define tor_assert_nonfatal(cond) STMT_BEGIN \ if (ASSERT_PREDICT_LIKELY_(cond)) { \ } else { \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0, NULL);\ + } \ + STMT_END +#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN \ + if (ASSERT_PREDICT_UNLIKELY_(cond)) { \ + } else { \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0, \ + fmt, ##__VA_ARGS__); \ } \ STMT_END #define tor_assert_nonfatal_unreached_once() STMT_BEGIN \ static int warning_logged__ = 0; \ if (!warning_logged__) { \ warning_logged__ = 1; \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1, NULL); \ } \ STMT_END #define tor_assert_nonfatal_once(cond) STMT_BEGIN \ @@ -171,15 +201,15 @@ if (ASSERT_PREDICT_LIKELY_(cond)) { \ } else if (!warning_logged__) { \ warning_logged__ = 1; \ - tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1); \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1, NULL);\ } \ STMT_END #define BUG(cond) \ (ASSERT_PREDICT_UNLIKELY_(cond) ? \ - (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \ + (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0,NULL),1) \ : 0) -#endif /* defined(ALL_BUGS_ARE_FATAL) || ... */ +#ifndef COCCI #ifdef __GNUC__ #define IF_BUG_ONCE__(cond,var) \ if (( { \ @@ -188,26 +218,28 @@ if (bool_result && !var) { \ var = 1; \ tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, \ - "!("#cond")", 1); \ + ("!("#cond")"), 1, NULL); \ } \ bool_result; } )) -#else /* !(defined(__GNUC__)) */ +#else /* !defined(__GNUC__) */ #define IF_BUG_ONCE__(cond,var) \ static int var = 0; \ if ((cond) ? \ (var ? 1 : \ (var=1, \ tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, \ - "!("#cond")", 1), \ + ("!("#cond")"), 1, NULL), \ 1)) \ : 0) #endif /* defined(__GNUC__) */ +#endif /* !defined(COCCI) */ + #define IF_BUG_ONCE_VARNAME_(a) \ warning_logged_on_ ## a ## __ #define IF_BUG_ONCE_VARNAME__(a) \ IF_BUG_ONCE_VARNAME_(a) -/** This macro behaves as 'if (bug(x))', except that it only logs its +/** This macro behaves as 'if (BUG(x))', except that it only logs its * warning once, no matter how many times it triggers. */ @@ -215,6 +247,8 @@ IF_BUG_ONCE__(ASSERT_PREDICT_UNLIKELY_(cond), \ IF_BUG_ONCE_VARNAME__(__LINE__)) +#endif /* defined(ALL_BUGS_ARE_FATAL) || ... */ + /** * Use this macro after a nonfatal assertion, and before a case statement * where you would want to fall through. @@ -226,16 +260,23 @@ #define FALLTHROUGH_UNLESS_ALL_BUGS_ARE_FATAL FALLTHROUGH #endif -/** Define this if you want Tor to crash when any problem comes up, - * so you can get a coredump and track things down. */ -// #define tor_fragile_assert() tor_assert_unreached(0) +/** In older code, we used tor_fragile_assert() to mark optional failure + * points. At these points, we could make some debug builds fail. + * (But release builds would continue.) + * + * To get the same behaviour in recent tor versions, define + * ALL_BUGS_ARE_FATAL, and use any non-fatal assertion or *BUG() macro. + */ #define tor_fragile_assert() tor_assert_nonfatal_unreached_once() void tor_assertion_failed_(const char *fname, unsigned int line, - const char *func, const char *expr); + const char *func, const char *expr, + const char *fmt, ...) + CHECK_PRINTF(5,6); void tor_bug_occurred_(const char *fname, unsigned int line, const char *func, const char *expr, - int once); + int once, const char *fmt, ...) + CHECK_PRINTF(6,7); void tor_abort_(void) ATTR_NORETURN; diff --git a/src/lib/log/win32err.c b/src/lib/log/win32err.c index dc45cb4c3d..8136813aab 100644 --- a/src/lib/log/win32err.c +++ b/src/lib/log/win32err.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -47,7 +47,7 @@ format_win32_error(DWORD err) result = tor_malloc(len); wcstombs(result,str,len); result[len-1] = '\0'; -#else /* !(defined(UNICODE)) */ +#else /* !defined(UNICODE) */ result = tor_strdup(str); #endif /* defined(UNICODE) */ } else { diff --git a/src/lib/log/win32err.h b/src/lib/log/win32err.h index 33413dfd15..5c1386a64d 100644 --- a/src/lib/log/win32err.h +++ b/src/lib/log/win32err.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,4 +19,4 @@ char *format_win32_error(DWORD err); #endif -#endif +#endif /* !defined(TOR_WIN32ERR_H) */ diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include index cc62bb1013..7686bf862a 100644 --- a/src/lib/malloc/.may_include +++ b/src/lib/malloc/.may_include @@ -3,4 +3,4 @@ orconfig.h lib/cc/*.h lib/err/*.h lib/malloc/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am index 502cc1c6b7..b74292bc6e 100644 --- a/src/lib/malloc/include.am +++ b/src/lib/malloc/include.am @@ -5,8 +5,10 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-malloc-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_malloc_a_SOURCES = \ - src/lib/malloc/malloc.c + src/lib/malloc/malloc.c \ + src/lib/malloc/map_anon.c if USE_OPENBSD_MALLOC src_lib_libtor_malloc_a_SOURCES += src/ext/OpenBSD_malloc_Linux.c @@ -17,5 +19,7 @@ src_lib_libtor_malloc_testing_a_SOURCES = \ src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ - src/lib/malloc/malloc.h + src/lib/malloc/malloc.h \ + src/lib/malloc/map_anon.h diff --git a/src/lib/malloc/lib_malloc.md b/src/lib/malloc/lib_malloc.md new file mode 100644 index 0000000000..ff61722f02 --- /dev/null +++ b/src/lib/malloc/lib_malloc.md @@ -0,0 +1,76 @@ +@dir /lib/malloc +@brief lib/malloc: Wrappers and utilities for memory management. + + +Tor imposes a few light wrappers over C's native malloc and free +functions, to improve convenience, and to allow wholescale replacement +of malloc and free as needed. + +You should never use 'malloc', 'calloc', 'realloc, or 'free' on their +own; always use the variants prefixed with 'tor_'. +They are the same as the standard C functions, with the following +exceptions: + + * `tor_free(NULL)` is a no-op. + * `tor_free()` is a macro that takes an lvalue as an argument and sets it to + NULL after freeing it. To avoid this behavior, you can use `tor_free_()` + instead. + * tor_malloc() and friends fail with an assertion if they are asked to + allocate a value so large that it is probably an underflow. + * It is always safe to `tor_malloc(0)`, regardless of whether your libc + allows it. + * `tor_malloc()`, `tor_realloc()`, and friends are never allowed to fail. + Instead, Tor will die with an assertion. This means that you never + need to check their return values. See the next subsection for + information on why we think this is a good idea. + +We define additional general-purpose memory allocation functions as well: + + * `tor_malloc_zero(x)` behaves as `calloc(1, x)`, except the it makes clear + the intent to allocate a single zeroed-out value. + * `tor_reallocarray(x,y)` behaves as the OpenBSD reallocarray function. + Use it for cases when you need to realloc() in a multiplication-safe + way. + +And specific-purpose functions as well: + + * `tor_strdup()` and `tor_strndup()` behaves as the underlying libc + functions, but use `tor_malloc()` instead of the underlying function. + * `tor_memdup()` copies a chunk of memory of a given size. + * `tor_memdup_nulterm()` copies a chunk of memory of a given size, then + NUL-terminates it just to be safe. + +#### Why assert on allocation failure? + +Why don't we allow `tor_malloc()` and its allies to return NULL? + +First, it's error-prone. Many programmers forget to check for NULL return +values, and testing for `malloc()` failures is a major pain. + +Second, it's not necessarily a great way to handle OOM conditions. It's +probably better (we think) to have a memory target where we dynamically free +things ahead of time in order to stay under the target. Trying to respond to +an OOM at the point of `tor_malloc()` failure, on the other hand, would involve +a rare operation invoked from deep in the call stack. (Again, that's +error-prone and hard to debug.) + +Third, thanks to the rise of Linux and other operating systems that allow +memory to be overcommitted, you can't actually ever rely on getting a NULL +from `malloc()` when you're out of memory; instead you have to use an approach +closer to tracking the total memory usage. + +#### Conventions for your own allocation functions. + +Whenever you create a new type, the convention is to give it a pair of +`x_new()` and `x_free_()` functions, named after the type. + +Calling `x_free(NULL)` should always be a no-op. + +There should additionally be an `x_free()` macro, defined in terms of +`x_free_()`. This macro should set its lvalue to NULL. You can define it +using the FREE_AND_NULL macro, as follows: + +``` +#define x_free(ptr) FREE_AND_NULL(x_t, x_free_, (ptr)) +``` + diff --git a/src/lib/malloc/malloc.c b/src/lib/malloc/malloc.c index 8628acfc97..9c9d600260 100644 --- a/src/lib/malloc/malloc.c +++ b/src/lib/malloc/malloc.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h index ef6b509ca4..80e8091adc 100644 --- a/src/lib/malloc/malloc.h +++ b/src/lib/malloc/malloc.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -48,12 +48,12 @@ void tor_free_(void *mem); raw_free(*tor_free__tmpvar); \ *tor_free__tmpvar=NULL; \ STMT_END -#else +#else /* !defined(__GNUC__) */ #define tor_free(p) STMT_BEGIN \ raw_free(p); \ (p)=NULL; \ STMT_END -#endif +#endif /* defined(__GNUC__) */ #define tor_malloc(size) tor_malloc_(size) #define tor_malloc_zero(size) tor_malloc_zero_(size) diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c new file mode 100644 index 0000000000..628966012a --- /dev/null +++ b/src/lib/malloc/map_anon.c @@ -0,0 +1,271 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file map_anon.c + * \brief Manage anonymous mappings. + **/ + +#include "orconfig.h" +#include "lib/malloc/map_anon.h" +#include "lib/malloc/malloc.h" +#include "lib/err/torerr.h" + +#ifdef HAVE_SYS_MMAN_H +#include <sys/mman.h> +#endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_MACH_VM_INHERIT_H +#include <mach/vm_inherit.h> +#endif + +#ifdef _WIN32 +#include <windows.h> +#endif + +#include <string.h> +#include <errno.h> + +/** + * Macro to get the high bytes of a size_t, if there are high bytes. + * Windows needs this; other operating systems define a size_t that does + * what it should. + */ +#if SIZEOF_SIZE_T > 4 +#define HIGH_SIZE_T_BYTES(sz) ((sz) >> 32) +#else +#define HIGH_SIZE_T_BYTES(sz) (0) +#endif + +/* Here we define a MINHERIT macro that is minherit() or madvise(), depending + * on what we actually want. + * + * If there's a flag that sets pages to zero after fork, we define FLAG_ZERO + * to be that flag. If there's a flag unmaps pages after fork, we define + * FLAG_NOINHERIT to be that flag. + */ +#if defined(HAVE_MINHERIT) +#define MINHERIT minherit + +#ifdef INHERIT_ZERO +#define FLAG_ZERO INHERIT_ZERO +#elif defined(MAP_INHERIT_ZERO) +#define FLAG_ZERO MAP_INHERIT_ZERO +#endif +#ifdef INHERIT_NONE +#define FLAG_NOINHERIT INHERIT_NONE +#elif defined(VM_INHERIT_NONE) +#define FLAG_NOINHERIT VM_INHERIT_NONE +#elif defined(MAP_INHERIT_NONE) +#define FLAG_NOINHERIT MAP_INHERIT_NONE +#endif /* defined(INHERIT_NONE) || ... */ + +#elif defined(HAVE_MADVISE) + +#define MINHERIT madvise + +#ifdef MADV_WIPEONFORK +#define FLAG_ZERO MADV_WIPEONFORK +#endif +#ifdef MADV_DONTFORK +#define FLAG_NOINHERIT MADV_DONTFORK +#endif + +#endif /* defined(HAVE_MINHERIT) || ... */ + +#if defined(HAVE_MINHERIT) && !defined(FLAG_ZERO) && !defined(FLAG_NOINHERIT) +#warning "minherit() is defined, but FLAG_ZERO/NOINHERIT are not." +#warning "This is probably a bug in Tor's support for this platform." +#endif + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being swapped + * to disk. Return 0 on success or if the facility is not available on this + * OS; return -1 on failure. + */ +static int +lock_mem(void *mem, size_t sz) +{ +#ifdef _WIN32 + return VirtualLock(mem, sz) ? 0 : -1; +#elif defined(HAVE_MLOCK) + return mlock(mem, sz); +#else + (void) mem; + (void) sz; + + return 0; +#endif /* defined(_WIN32) || ... */ +} + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from appearing in + * a core dump. Return 0 on success or if the facility is not available on + * this OS; return -1 on failure. + */ +static int +nodump_mem(void *mem, size_t sz) +{ +#if defined(MADV_DONTDUMP) + int rv = madvise(mem, sz, MADV_DONTDUMP); + if (rv == 0) { + return 0; + } else if (errno == ENOSYS || errno == EINVAL) { + return 0; // syscall not supported, or flag not supported. + } else { + tor_log_err_sigsafe("Unexpected error from madvise: ", + strerror(errno), + NULL); + return -1; + } +#else /* !defined(MADV_DONTDUMP) */ + (void) mem; + (void) sz; + return 0; +#endif /* defined(MADV_DONTDUMP) */ +} + +/** + * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being + * accessible in child processes -- ideally by having them set to 0 after a + * fork, and if that doesn't work, by having them unmapped after a fork. + * Return 0 on success or if the facility is not available on this OS; return + * -1 on failure. + * + * If we successfully make the memory uninheritable, adjust the value of + * *<b>inherit_result_out</b>. + */ +static int +noinherit_mem(void *mem, size_t sz, inherit_res_t *inherit_result_out) +{ +#ifdef FLAG_ZERO + int r = MINHERIT(mem, sz, FLAG_ZERO); + if (r == 0) { + *inherit_result_out = INHERIT_RES_ZERO; + return 0; + } +#endif /* defined(FLAG_ZERO) */ + +#ifdef FLAG_NOINHERIT + int r2 = MINHERIT(mem, sz, FLAG_NOINHERIT); + if (r2 == 0) { + *inherit_result_out = INHERIT_RES_DROP; + return 0; + } +#endif /* defined(FLAG_NOINHERIT) */ + +#if defined(FLAG_ZERO) || defined(FLAG_NOINHERIT) + /* At least one operation was tried, and neither succeeded. */ + + if (errno == ENOSYS || errno == EINVAL) { + /* Syscall not supported, or flag not supported. */ + return 0; + } else { + tor_log_err_sigsafe("Unexpected error from minherit: ", + strerror(errno), + NULL); + return -1; + } +#else /* !(defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)) */ + (void)inherit_result_out; + (void)mem; + (void)sz; + return 0; +#endif /* defined(FLAG_ZERO) || defined(FLAG_NOINHERIT) */ +} + +/** + * Return a new anonymous memory mapping that holds <b>sz</b> bytes. + * + * Memory mappings are unlike the results from malloc() in that they are + * handled separately by the operating system, and as such can have different + * kernel-level flags set on them. + * + * The "flags" argument may be zero or more of ANONMAP_PRIVATE and + * ANONMAP_NOINHERIT. + * + * Memory returned from this function must be released with + * tor_munmap_anonymous(). + * + * If <b>inherit_result_out</b> is non-NULL, set it to one of + * INHERIT_RES_KEEP, INHERIT_RES_DROP, or INHERIT_RES_ZERO, depending on the + * properties of the returned memory. + * + * [Note: OS people use the word "anonymous" here to mean that the memory + * isn't associated with any file. This has *nothing* to do with the kind of + * anonymity that Tor is trying to provide.] + */ +void * +tor_mmap_anonymous(size_t sz, unsigned flags, + inherit_res_t *inherit_result_out) +{ + void *ptr; + inherit_res_t itmp=0; + if (inherit_result_out == NULL) { + inherit_result_out = &itmp; + } + *inherit_result_out = INHERIT_RES_KEEP; + +#if defined(_WIN32) + HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE, + NULL, /*attributes*/ + PAGE_READWRITE, + HIGH_SIZE_T_BYTES(sz), + sz & 0xffffffff, + NULL /* name */); + raw_assert(mapping != NULL); + ptr = MapViewOfFile(mapping, FILE_MAP_WRITE, + 0, 0, /* Offset */ + 0 /* Extend to end of mapping */); + raw_assert(ptr); + CloseHandle(mapping); /* mapped view holds a reference */ +#elif defined(HAVE_SYS_MMAN_H) + ptr = mmap(NULL, sz, + PROT_READ|PROT_WRITE, + MAP_ANON|MAP_PRIVATE, + -1, 0); + raw_assert(ptr != MAP_FAILED); + raw_assert(ptr != NULL); +#else + ptr = tor_malloc_zero(sz); +#endif /* defined(_WIN32) || ... */ + + if (flags & ANONMAP_PRIVATE) { + int lock_result = lock_mem(ptr, sz); + raw_assert(lock_result == 0); + int nodump_result = nodump_mem(ptr, sz); + raw_assert(nodump_result == 0); + } + + if (flags & ANONMAP_NOINHERIT) { + int noinherit_result = noinherit_mem(ptr, sz, inherit_result_out); + raw_assert(noinherit_result == 0); + } + + return ptr; +} + +/** + * Release <b>sz</b> bytes of memory that were previously mapped at + * <b>mapping</b> by tor_mmap_anonymous(). + **/ +void +tor_munmap_anonymous(void *mapping, size_t sz) +{ + if (!mapping) + return; + +#if defined(_WIN32) + (void)sz; + UnmapViewOfFile(mapping); +#elif defined(HAVE_SYS_MMAN_H) + munmap(mapping, sz); +#else + (void)sz; + tor_free(mapping); +#endif /* defined(_WIN32) || ... */ +} diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h new file mode 100644 index 0000000000..0354668d65 --- /dev/null +++ b/src/lib/malloc/map_anon.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file map_anon.h + * \brief Headers for map_anon.c + **/ + +#ifndef TOR_MAP_ANON_H +#define TOR_MAP_ANON_H + +#include "lib/malloc/malloc.h" +#include <stddef.h> + +/** + * When this flag is specified, try to prevent the mapping from being + * swapped or dumped. + * + * In some operating systems, this flag is not implemented. + */ +#define ANONMAP_PRIVATE (1u<<0) +/** + * When this flag is specified, try to prevent the mapping from being + * inherited after a fork(). In some operating systems, trying to access it + * afterwards will cause its contents to be zero. In others, trying to access + * it afterwards will cause a crash. + * + * In some operating systems, this flag is not implemented at all. + */ +#define ANONMAP_NOINHERIT (1u<<1) + +typedef enum { + /** Possible value for inherit_result_out: the memory will be kept + * by any child process. */ + INHERIT_RES_KEEP=0, + /** Possible value for inherit_result_out: the memory will be dropped in the + * child process. Attempting to access it will likely cause a segfault. */ + INHERIT_RES_DROP, + /** Possible value for inherit_result_out: the memory will be cleared in + * the child process. */ + INHERIT_RES_ZERO +} inherit_res_t; + +/* Here we define the NOINHERIT_CAN_FAIL macro if and only if + * it's possible that ANONMAP_NOINHERIT might yield inheritable memory. + */ +#ifdef _WIN32 +/* Windows can't fork, so NOINHERIT is never needed. */ +#elif defined(HAVE_MINHERIT) +/* minherit() will always have a working MAP_INHERIT_NONE or MAP_INHERIT_ZERO. + * NOINHERIT should always work. + */ +#elif defined(HAVE_MADVISE) +/* madvise() sometimes has neither MADV_DONTFORK and MADV_WIPEONFORK. + * We need to be ready for the possibility it failed. + * + * (Linux added DONTFORK in 2.6.16 and WIPEONFORK in 4.14. If we someday + * require 2.6.16 or later, we can assume that DONTFORK will work.) + */ +#define NOINHERIT_CAN_FAIL +#else +#define NOINHERIT_CAN_FAIL +#endif /* defined(_WIN32) || ... */ + +void *tor_mmap_anonymous(size_t sz, unsigned flags, + inherit_res_t *inherit_result_out); +void tor_munmap_anonymous(void *mapping, size_t sz); + +#endif /* !defined(TOR_MAP_ANON_H) */ diff --git a/src/lib/math/.may_include b/src/lib/math/.may_include index 1fd26864dc..f8bc264a5f 100644 --- a/src/lib/math/.may_include +++ b/src/lib/math/.may_include @@ -3,3 +3,5 @@ orconfig.h lib/cc/*.h lib/log/*.h lib/math/*.h +lib/testsupport/*.h +lib/crypt_ops/*.h diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c index eafad358c3..c09555209e 100644 --- a/src/lib/math/fp.c +++ b/src/lib/math/fp.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -74,8 +74,8 @@ clamp_double_to_int64(double number) branches that are not taken. */ #define PROBLEMATIC_FLOAT_CONVERSION_WARNING -DISABLE_GCC_WARNING(float-conversion) -#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */ +DISABLE_GCC_WARNING("-Wfloat-conversion") +#endif /* (defined(MINGW_ANY)||defined(__FreeBSD__)) && GCC_VERSION >= 409 */ /* With clang 4.0 we apparently run into "double promotion" warnings here, @@ -84,7 +84,7 @@ DISABLE_GCC_WARNING(float-conversion) #if defined(__clang__) #if __has_warning("-Wdouble-promotion") #define PROBLEMATIC_DOUBLE_PROMOTION_WARNING -DISABLE_GCC_WARNING(double-promotion) +DISABLE_GCC_WARNING("-Wdouble-promotion") #endif #endif /* defined(__clang__) */ @@ -115,9 +115,29 @@ DISABLE_GCC_WARNING(double-promotion) return signbit(number) ? INT64_MIN : INT64_MAX; #ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING -ENABLE_GCC_WARNING(double-promotion) +ENABLE_GCC_WARNING("-Wdouble-promotion") #endif #ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING -ENABLE_GCC_WARNING(float-conversion) +ENABLE_GCC_WARNING("-Wfloat-conversion") +#endif +} + +/* isinf() wrapper for tor */ +int +tor_isinf(double x) +{ + /* Same as above, work around the "double promotion" warnings */ +#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING +DISABLE_GCC_WARNING("-Wfloat-conversion") +#endif +#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING +DISABLE_GCC_WARNING("-Wdouble-promotion") +#endif + return isinf(x); +#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING +ENABLE_GCC_WARNING("-Wdouble-promotion") +#endif +#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING +ENABLE_GCC_WARNING("-Wfloat-conversion") #endif } diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h index 6f07152e92..0a7a685485 100644 --- a/src/lib/math/fp.h +++ b/src/lib/math/fp.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,5 +19,6 @@ double tor_mathlog(double d) ATTR_CONST; long tor_lround(double d) ATTR_CONST; int64_t tor_llround(double d) ATTR_CONST; int64_t clamp_double_to_int64(double number); +int tor_isinf(double x); -#endif +#endif /* !defined(TOR_FP_H) */ diff --git a/src/lib/math/include.am b/src/lib/math/include.am index b088b3f3cc..b2ca280f47 100644 --- a/src/lib/math/include.am +++ b/src/lib/math/include.am @@ -5,16 +5,19 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-math-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_math_a_SOURCES = \ src/lib/math/fp.c \ - src/lib/math/laplace.c - + src/lib/math/laplace.c \ + src/lib/math/prob_distr.c src_lib_libtor_math_testing_a_SOURCES = \ $(src_lib_libtor_math_a_SOURCES) src_lib_libtor_math_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/math/fp.h \ - src/lib/math/laplace.h + src/lib/math/laplace.h \ + src/lib/math/prob_distr.h diff --git a/src/lib/math/laplace.c b/src/lib/math/laplace.c index 302edb20b8..a0e67384e6 100644 --- a/src/lib/math/laplace.c +++ b/src/lib/math/laplace.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,7 +29,7 @@ sample_laplace_distribution(double mu, double b, double p) tor_assert(p >= 0.0 && p < 1.0); /* This is the "inverse cumulative distribution function" from: - * http://en.wikipedia.org/wiki/Laplace_distribution */ + * https://en.wikipedia.org/wiki/Laplace_distribution */ if (p <= 0.0) { /* Avoid taking log(0.0) == -INFINITY, as some processors or compiler * options can cause the program to trap. */ diff --git a/src/lib/math/laplace.h b/src/lib/math/laplace.h index e8651e5197..e0dd166bbd 100644 --- a/src/lib/math/laplace.h +++ b/src/lib/math/laplace.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,4 +19,4 @@ int64_t sample_laplace_distribution(double mu, double b, double p); int64_t add_laplace_noise(int64_t signal, double random, double delta_f, double epsilon); -#endif +#endif /* !defined(TOR_LAPLACE_H) */ diff --git a/src/lib/math/lib_math.md b/src/lib/math/lib_math.md new file mode 100644 index 0000000000..9cc256d24b --- /dev/null +++ b/src/lib/math/lib_math.md @@ -0,0 +1,6 @@ +@dir /lib/math +@brief lib/math: Floating-point math utilities. + +This module includes a bunch of floating-point compatibility code, and +implementations for several probability distributions. + diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c new file mode 100644 index 0000000000..31d485120e --- /dev/null +++ b/src/lib/math/prob_distr.c @@ -0,0 +1,1692 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file prob_distr.c + * + * \brief + * Implements various probability distributions. + * Almost all code is courtesy of Riastradh. + * + * \details + * Here are some details that might help you understand this file: + * + * - Throughout this file, `eps' means the largest relative error of a + * correctly rounded floating-point operation, which in binary64 + * floating-point arithmetic is 2^-53. Here the relative error of a + * true value x from a computed value y is |x - y|/|x|. This + * definition of epsilon is conventional for numerical analysts when + * writing error analyses. (If your libm doesn't provide correctly + * rounded exp and log, their relative error is usually below 2*2^-53 + * and probably closer to 1.1*2^-53 instead.) + * + * The C constant DBL_EPSILON is actually twice this, and should + * perhaps rather be named ulp(1) -- that is, it is the distance from + * 1 to the next greater floating-point number, which is usually of + * more interest to programmers and hardware engineers. + * + * Since this file is concerned mainly with error bounds rather than + * with low-level bit-hacking of floating-point numbers, we adopt the + * numerical analysts' definition in the comments, though we do use + * DBL_EPSILON in a handful of places where it is convenient to use + * some function of eps = DBL_EPSILON/2 in a case analysis. + * + * - In various functions (e.g. sample_log_logistic()) we jump through hoops so + * that we can use reals closer to 0 than closer to 1, since we achieve much + * greater accuracy for floating point numbers near 0. In particular, we can + * represent differences as small as 10^-300 for numbers near 0, but of no + * less than 10^-16 for numbers near 1. + **/ + +#define PROB_DISTR_PRIVATE + +#include "orconfig.h" + +#include "lib/math/prob_distr.h" + +#include "lib/crypt_ops/crypto_rand.h" +#include "lib/cc/ctassert.h" +#include "lib/log/util_bug.h" + +#include <float.h> +#include <math.h> +#include <stddef.h> + +#ifndef COCCI +/** Declare a function that downcasts from a generic dist struct to the actual + * subtype probablity distribution it represents. */ +#define DECLARE_PROB_DISTR_DOWNCAST_FN(name) \ + static inline \ + const struct name##_t * \ + dist_to_const_##name(const struct dist_t *obj) { \ + tor_assert(obj->ops == &name##_ops); \ + return SUBTYPE_P(obj, struct name ## _t, base); \ + } +DECLARE_PROB_DISTR_DOWNCAST_FN(uniform) +DECLARE_PROB_DISTR_DOWNCAST_FN(geometric) +DECLARE_PROB_DISTR_DOWNCAST_FN(logistic) +DECLARE_PROB_DISTR_DOWNCAST_FN(log_logistic) +DECLARE_PROB_DISTR_DOWNCAST_FN(genpareto) +DECLARE_PROB_DISTR_DOWNCAST_FN(weibull) +#endif /* !defined(COCCI) */ + +/** + * Count number of one bits in 32-bit word. + */ +static unsigned +bitcount32(uint32_t x) +{ + + /* Count two-bit groups. */ + x -= (x >> 1) & UINT32_C(0x55555555); + + /* Count four-bit groups. */ + x = ((x >> 2) & UINT32_C(0x33333333)) + (x & UINT32_C(0x33333333)); + + /* Count eight-bit groups. */ + x = (x + (x >> 4)) & UINT32_C(0x0f0f0f0f); + + /* Sum all eight-bit groups, and extract the sum. */ + return (x * UINT32_C(0x01010101)) >> 24; +} + +/** + * Count leading zeros in 32-bit word. + */ +static unsigned +clz32(uint32_t x) +{ + + /* Round up to a power of two. */ + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; + + /* Subtract count of one bits from 32. */ + return (32 - bitcount32(x)); +} + +/* + * Some lemmas that will be used throughout this file to prove various error + * bounds: + * + * Lemma 1. If |d| <= 1/2, then 1/(1 + d) <= 2. + * + * Proof. If 0 <= d <= 1/2, then 1 + d >= 1, so that 1/(1 + d) <= 1. + * If -1/2 <= d <= 0, then 1 + d >= 1/2, so that 1/(1 + d) <= 2. QED. + * + * Lemma 2. If b = a*(1 + d)/(1 + d') for |d'| < 1/2 and nonzero a, b, + * then b = a*(1 + e) for |e| <= 2|d' - d|. + * + * Proof. |a - b|/|a| + * = |a - a*(1 + d)/(1 + d')|/|a| + * = |1 - (1 + d)/(1 + d')| + * = |(1 + d' - 1 - d)/(1 + d')| + * = |(d' - d)/(1 + d')| + * <= 2|d' - d|, by Lemma 1, + * + * QED. + * + * Lemma 3. For |d|, |d'| < 1/4, + * + * |log((1 + d)/(1 + d'))| <= 4|d - d'|. + * + * Proof. Write + * + * log((1 + d)/(1 + d')) + * = log(1 + (1 + d)/(1 + d') - 1) + * = log(1 + (1 + d - 1 - d')/(1 + d') + * = log(1 + (d - d')/(1 + d')). + * + * By Lemma 1, |(d - d')/(1 + d')| < 2|d' - d| < 1, so the Taylor + * series of log(1 + x) converges absolutely for (d - d')/(1 + d'), + * and thus we have + * + * |log(1 + (d - d')/(1 + d'))| + * = |\sum_{n=1}^\infty ((d - d')/(1 + d'))^n/n| + * <= \sum_{n=1}^\infty |(d - d')/(1 + d')|^n/n + * <= \sum_{n=1}^\infty |2(d' - d)|^n/n + * <= \sum_{n=1}^\infty |2(d' - d)|^n + * = 1/(1 - |2(d' - d)|) + * <= 4|d' - d|, + * + * QED. + * + * Lemma 4. If 1/e <= 1 + x <= e, then + * + * log(1 + (1 + d) x) = (1 + d') log(1 + x) + * + * for |d'| < 8|d|. + * + * Proof. Write + * + * log(1 + (1 + d) x) + * = log(1 + x + x*d) + * = log((1 + x) (1 + x + x*d)/(1 + x)) + * = log(1 + x) + log((1 + x + x*d)/(1 + x)) + * = log(1 + x) (1 + log((1 + x + x*d)/(1 + x))/log(1 + x)). + * + * The relative error is bounded by + * + * |log((1 + x + x*d)/(1 + x))/log(1 + x)| + * <= 4|x + x*d - x|/|log(1 + x)|, by Lemma 3, + * = 4|x*d|/|log(1 + x)| + * < 8|d|, + * + * since in this range 0 < 1 - 1/e < x/log(1 + x) <= e - 1 < 2. QED. + */ + +/** + * Compute the logistic function: f(x) = 1/(1 + e^{-x}) = e^x/(1 + e^x). + * Maps a log-odds-space probability in [-infinity, +infinity] into a + * direct-space probability in [0,1]. Inverse of logit. + * + * Ill-conditioned for large x; the identity logistic(-x) = 1 - + * logistic(x) and the function logistichalf(x) = logistic(x) - 1/2 may + * help to rearrange a computation. + * + * This implementation gives relative error bounded by 7 eps. + */ +STATIC double +logistic(double x) +{ + if (x <= log(DBL_EPSILON/2)) { + /* + * If x <= log(DBL_EPSILON/2) = log(eps), then e^x <= eps. In this case + * we will approximate the logistic() function with e^x because the + * relative error is less than eps. Here is a calculation of the + * relative error between the logistic() function and e^x and a proof + * that it's less than eps: + * + * |e^x - e^x/(1 + e^x)|/|e^x/(1 + e^x)| + * <= |1 - 1/(1 + e^x)|*|1 + e^x| + * = |e^x/(1 + e^x)|*|1 + e^x| + * = |e^x| + * <= eps. + */ + return exp(x); /* return e^x */ + } else if (x <= -log(DBL_EPSILON/2)) { + /* + * e^{-x} > 0, so 1 + e^{-x} > 1, and 0 < 1/(1 + + * e^{-x}) < 1; further, since e^{-x} < 1 + e^{-x}, we + * also have 0 < 1/(1 + e^{-x}) < 1. Thus, if exp has + * relative error d0, + has relative error d1, and / + * has relative error d2, then we get + * + * (1 + d2)/[(1 + (1 + d0) e^{-x})(1 + d1)] + * = (1 + d0)/[1 + e^{-x} + d0 e^{-x} + * + d1 + d1 e^{-x} + d0 d1 e^{-x}] + * = (1 + d0)/[(1 + e^{-x}) + * * (1 + d0 e^{-x}/(1 + e^{-x}) + * + d1/(1 + e^{-x}) + * + d0 d1 e^{-x}/(1 + e^{-x}))]. + * = (1 + d0)/[(1 + e^{-x})(1 + d')] + * = [1/(1 + e^{-x})] (1 + d0)/(1 + d') + * + * where + * + * d' = d0 e^{-x}/(1 + e^{-x}) + * + d1/(1 + e^{-x}) + * + d0 d1 e^{-x}/(1 + e^{-x}). + * + * By Lemma 2 this relative error is bounded by + * + * 2|d0 - d'| + * = 2|d0 - d0 e^{-x}/(1 + e^{-x}) + * - d1/(1 + e^{-x}) + * - d0 d1 e^{-x}/(1 + e^{-x})| + * <= 2|d0| + 2|d0 e^{-x}/(1 + e^{-x})| + * + 2|d1/(1 + e^{-x})| + * + 2|d0 d1 e^{-x}/(1 + e^{-x})| + * <= 2|d0| + 2|d0| + 2|d1| + 2|d0 d1| + * <= 4|d0| + 2|d1| + 2|d0 d1| + * <= 6 eps + 2 eps^2. + */ + return 1/(1 + exp(-x)); + } else { + /* + * e^{-x} <= eps, so the relative error of 1 from 1/(1 + * + e^{-x}) is + * + * |1/(1 + e^{-x}) - 1|/|1/(1 + e^{-x})| + * = |e^{-x}/(1 + e^{-x})|/|1/(1 + e^{-x})| + * = |e^{-x}| + * <= eps. + * + * This computation avoids an intermediate overflow + * exception, although the effect on the result is + * harmless. + * + * XXX Should maybe raise inexact here. + */ + return 1; + } +} + +/** + * Compute the logit function: log p/(1 - p). Defined on [0,1]. Maps + * a direct-space probability in [0,1] to a log-odds-space probability + * in [-infinity, +infinity]. Inverse of logistic. + * + * Ill-conditioned near 1/2 and 1; the identity logit(1 - p) = + * -logit(p) and the function logithalf(p0) = logit(1/2 + p0) may help + * to rearrange a computation for p in [1/(1 + e), 1 - 1/(1 + e)]. + * + * This implementation gives relative error bounded by 10 eps. + */ +STATIC double +logit(double p) +{ + + /* logistic(-1) <= p <= logistic(+1) */ + if (1/(1 + exp(1)) <= p && p <= 1/(1 + exp(-1))) { + /* + * For inputs near 1/2, we want to compute log1p(near + * 0) rather than log(near 1), so write this as: + * + * log(p/(1 - p)) = -log((1 - p)/p) + * = -log(1 + (1 - p)/p - 1) + * = -log(1 + (1 - p - p)/p) + * = -log(1 + (1 - 2p)/p). + * + * Since p = 2p/2 <= 1 <= 2*2p = 4p, the floating-point + * evaluation of 1 - 2p is exact; the only error arises + * from division and log1p. First, note that if + * logistic(-1) <= p <= logistic(+1), (1 - 2p)/p lies + * in the bounds of Lemma 4. + * + * If division has relative error d0 and log1p has + * relative error d1, the outcome is + * + * -(1 + d1) log(1 + (1 - 2p) (1 + d0)/p) + * = -(1 + d1) (1 + d') log(1 + (1 - 2p)/p) + * = -(1 + d1 + d' + d1 d') log(1 + (1 - 2p)/p). + * + * where |d'| < 8|d0| by Lemma 4. The relative error + * is then bounded by + * + * |d1 + d' + d1 d'| + * <= |d1| + 8|d0| + 8|d1 d0| + * <= 9 eps + 8 eps^2. + */ + return -log1p((1 - 2*p)/p); + } else { + /* + * For inputs near 0, although 1 - p may be rounded to + * 1, it doesn't matter much because the magnitude of + * the result is so much larger. For inputs near 1, we + * can compute 1 - p exactly, although the precision on + * the input is limited so we won't ever get more than + * about 700 for the output. + * + * If - has relative error d0, / has relative error d1, + * and log has relative error d2, then + * + * (1 + d2) log((1 + d0) p/[(1 - p)(1 + d1)]) + * = (1 + d2) [log(p/(1 - p)) + log((1 + d0)/(1 + d1))] + * = log(p/(1 - p)) + d2 log(p/(1 - p)) + * + (1 + d2) log((1 + d0)/(1 + d1)) + * = log(p/(1 - p))*[1 + d2 + + * + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))] + * + * Since 0 <= p < logistic(-1) or logistic(+1) < p <= + * 1, we have |log(p/(1 - p))| > 1. Hence this error + * is bounded by + * + * |d2 + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))| + * <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1)) + * / log(p/(1 - p))| + * <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))| + * <= |d2| + 4|(1 + d2) (d0 - d1)|, by Lemma 3, + * <= |d2| + 4|d0 - d1 + d2 d0 - d1 d0| + * <= |d2| + 4|d0| + 4|d1| + 4|d2 d0| + 4|d1 d0| + * <= 9 eps + 8 eps^2. + */ + return log(p/(1 - p)); + } +} + +/** + * Compute the logit function, translated in input by 1/2: logithalf(p) + * = logit(1/2 + p). Defined on [-1/2, 1/2]. Inverse of logistichalf. + * + * Ill-conditioned near +/-1/2. If |p0| > 1/2 - 1/(1 + e), it may be + * better to compute 1/2 + p0 or -1/2 - p0 and to use logit instead. + * This implementation gives relative error bounded by 34 eps. + */ +STATIC double +logithalf(double p0) +{ + + if (fabs(p0) <= 0.5 - 1/(1 + exp(1))) { + /* + * logit(1/2 + p0) + * = log((1/2 + p0)/(1 - (1/2 + p0))) + * = log((1/2 + p0)/(1/2 - p0)) + * = log(1 + (1/2 + p0)/(1/2 - p0) - 1) + * = log(1 + (1/2 + p0 - (1/2 - p0))/(1/2 - p0)) + * = log(1 + (1/2 + p0 - 1/2 + p0)/(1/2 - p0)) + * = log(1 + 2 p0/(1/2 - p0)) + * + * If the error of subtraction is d0, the error of + * division is d1, and the error of log1p is d2, then + * what we compute is + * + * (1 + d2) log(1 + (1 + d1) 2 p0/[(1 + d0) (1/2 - p0)]) + * = (1 + d2) log(1 + (1 + d') 2 p0/(1/2 - p0)) + * = (1 + d2) (1 + d'') log(1 + 2 p0/(1/2 - p0)) + * = (1 + d2 + d'' + d2 d'') log(1 + 2 p0/(1/2 - p0)), + * + * where |d'| < 2|d0 - d1| <= 4 eps by Lemma 2, and + * |d''| < 8|d'| < 32 eps by Lemma 4 since + * + * 1/e <= 1 + 2*p0/(1/2 - p0) <= e + * + * when |p0| <= 1/2 - 1/(1 + e). Hence the relative + * error is bounded by + * + * |d2 + d'' + d2 d''| + * <= |d2| + |d''| + |d2 d''| + * <= |d1| + 32 |d0| + 32 |d1 d0| + * <= 33 eps + 32 eps^2. + */ + return log1p(2*p0/(0.5 - p0)); + } else { + /* + * We have a choice of computing logit(1/2 + p0) or + * -logit(1 - (1/2 + p0)) = -logit(1/2 - p0). It + * doesn't matter which way we do this: either way, + * since 1/2 p0 <= 1/2 <= 2 p0, the sum and difference + * are computed exactly. So let's do the one that + * skips the final negation. + * + * The result is + * + * (1 + d1) log((1 + d0) (1/2 + p0)/[(1 + d2) (1/2 - p0)]) + * = (1 + d1) (1 + log((1 + d0)/(1 + d2)) + * / log((1/2 + p0)/(1/2 - p0))) + * * log((1/2 + p0)/(1/2 - p0)) + * = (1 + d') log((1/2 + p0)/(1/2 - p0)) + * = (1 + d') logit(1/2 + p0) + * + * where + * + * d' = d1 + log((1 + d0)/(1 + d2))/logit(1/2 + p0) + * + d1 log((1 + d0)/(1 + d2))/logit(1/2 + p0). + * + * For |p| > 1/2 - 1/(1 + e), logit(1/2 + p0) > 1. + * Provided |d0|, |d2| < 1/4, by Lemma 3 we have + * + * |log((1 + d0)/(1 + d2))| <= 4|d0 - d2|. + * + * Hence the relative error is bounded by + * + * |d'| <= |d1| + 4|d0 - d2| + 4|d1| |d0 - d2| + * <= |d1| + 4|d0| + 4|d2| + 4|d1 d0| + 4|d1 d2| + * <= 9 eps + 8 eps^2. + */ + return log((0.5 + p0)/(0.5 - p0)); + } +} + +/* + * The following random_uniform_01 is tailored for IEEE 754 binary64 + * floating-point or smaller. It can be adapted to larger + * floating-point formats like i387 80-bit or IEEE 754 binary128, but + * it may require sampling more bits. + */ +CTASSERT(FLT_RADIX == 2); +CTASSERT(-DBL_MIN_EXP <= 1021); +CTASSERT(DBL_MANT_DIG <= 53); + +/** + * Draw a floating-point number in [0, 1] with uniform distribution. + * + * Note that the probability of returning 0 is less than 2^-1074, so + * callers need not check for it. However, callers that cannot handle + * rounding to 1 must deal with that, because it occurs with + * probability 2^-54, which is small but nonnegligible. + */ +STATIC double +random_uniform_01(void) +{ + uint32_t z, x, hi, lo; + double s; + + /* + * Draw an exponent, geometrically distributed, but give up if + * we get a run of more than 1088 zeros, which really means the + * system is broken. + */ + z = 0; + while ((x = crypto_fast_rng_get_u32(get_thread_fast_rng())) == 0) { + if (z >= 1088) + /* Your bit sampler is broken. Go home. */ + return 0; + z += 32; + } + z += clz32(x); + + /* + * Pick 32-bit halves of an odd normalized significand. + * Picking it odd breaks ties in the subsequent rounding, which + * occur only with measure zero in the uniform distribution on + * [0, 1]. + */ + hi = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x80000000); + lo = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x00000001); + + /* Round to nearest scaled significand in [2^63, 2^64]. */ + s = hi*(double)4294967296 + lo; + + /* Rescale into [1/2, 1] and apply exponent in one swell foop. */ + return s * ldexp(1, -(64 + z)); +} + +/*******************************************************************/ + +/* Functions for specific probability distributions start here: */ + +/* + * Logistic(mu, sigma) distribution, supported on (-infinity,+infinity) + * + * This is the uniform distribution on [0,1] mapped into log-odds + * space, scaled by sigma and translated by mu. + * + * pdf(x) = e^{-(x - mu)/sigma} sigma (1 + e^{-(x - mu)/sigma})^2 + * cdf(x) = 1/(1 + e^{-(x - mu)/sigma}) = logistic((x - mu)/sigma) + * sf(x) = 1 - cdf(x) = 1 - logistic((x - mu)/sigma = logistic(-(x - mu)/sigma) + * icdf(p) = mu + sigma log p/(1 - p) = mu + sigma logit(p) + * isf(p) = mu + sigma log (1 - p)/p = mu - sigma logit(p) + */ + +/** + * Compute the CDF of the Logistic(mu, sigma) distribution: the + * logistic function. Well-conditioned for negative inputs and small + * positive inputs; ill-conditioned for large positive inputs. + */ +STATIC double +cdf_logistic(double x, double mu, double sigma) +{ + return logistic((x - mu)/sigma); +} + +/** + * Compute the SF of the Logistic(mu, sigma) distribution: the logistic + * function reflected over the y axis. Well-conditioned for positive + * inputs and small negative inputs; ill-conditioned for large negative + * inputs. + */ +STATIC double +sf_logistic(double x, double mu, double sigma) +{ + return logistic(-(x - mu)/sigma); +} + +/** + * Compute the inverse of the CDF of the Logistic(mu, sigma) + * distribution: the logit function. Well-conditioned near 0; + * ill-conditioned near 1/2 and 1. + */ +STATIC double +icdf_logistic(double p, double mu, double sigma) +{ + return mu + sigma*logit(p); +} + +/** + * Compute the inverse of the SF of the Logistic(mu, sigma) + * distribution: the -logit function. Well-conditioned near 0; + * ill-conditioned near 1/2 and 1. + */ +STATIC double +isf_logistic(double p, double mu, double sigma) +{ + return mu - sigma*logit(p); +} + +/* + * LogLogistic(alpha, beta) distribution, supported on (0, +infinity). + * + * This is the uniform distribution on [0,1] mapped into odds space, + * scaled by positive alpha and shaped by positive beta. + * + * Equivalent to computing exp of a Logistic(log alpha, 1/beta) sample. + * (Name arises because the pdf has LogLogistic(x; alpha, beta) = + * Logistic(log x; log alpha, 1/beta) and mathematicians got their + * covariance contravariant.) + * + * pdf(x) = (beta/alpha) (x/alpha)^{beta - 1}/(1 + (x/alpha)^beta)^2 + * = (1/e^mu sigma) (x/e^mu)^{1/sigma - 1} / + * (1 + (x/e^mu)^{1/sigma})^2 + * cdf(x) = 1/(1 + (x/alpha)^-beta) = 1/(1 + (x/e^mu)^{-1/sigma}) + * = 1/(1 + (e^{log x}/e^mu)^{-1/sigma}) + * = 1/(1 + (e^{log x - mu})^{-1/sigma}) + * = 1/(1 + e^{-(log x - mu)/sigma}) + * = logistic((log x - mu)/sigma) + * = logistic((log x - log alpha)/(1/beta)) + * sf(x) = 1 - 1/(1 + (x/alpha)^-beta) + * = (x/alpha)^-beta/(1 + (x/alpha)^-beta) + * = 1/((x/alpha)^beta + 1) + * = 1/(1 + (x/alpha)^beta) + * icdf(p) = alpha (p/(1 - p))^{1/beta} + * = alpha e^{logit(p)/beta} + * = e^{mu + sigma logit(p)} + * isf(p) = alpha ((1 - p)/p)^{1/beta} + * = alpha e^{-logit(p)/beta} + * = e^{mu - sigma logit(p)} + */ + +/** + * Compute the CDF of the LogLogistic(alpha, beta) distribution. + * Well-conditioned for all x and alpha, and the condition number + * + * -beta/[1 + (x/alpha)^{-beta}] + * + * grows linearly with beta. + * + * Loosely, the relative error of this implementation is bounded by + * + * 4 eps + 2 eps^2 + O(beta eps), + * + * so don't bother trying this for beta anywhere near as large as + * 1/eps, around which point it levels off at 1. + */ +STATIC double +cdf_log_logistic(double x, double alpha, double beta) +{ + /* + * Let d0 be the error of x/alpha; d1, of pow; d2, of +; and + * d3, of the final quotient. The exponentiation gives + * + * ((1 + d0) x/alpha)^{-beta} + * = (x/alpha)^{-beta} (1 + d0)^{-beta} + * = (x/alpha)^{-beta} (1 + (1 + d0)^{-beta} - 1) + * = (x/alpha)^{-beta} (1 + d') + * + * where d' = (1 + d0)^{-beta} - 1. If y = (x/alpha)^{-beta}, + * the denominator is + * + * (1 + d2) (1 + (1 + d1) (1 + d') y) + * = (1 + d2) (1 + y + (d1 + d' + d1 d') y) + * = 1 + y + (1 + d2) (d1 + d' + d1 d') y + * = (1 + y) (1 + (1 + d2) (d1 + d' + d1 d') y/(1 + y)) + * = (1 + y) (1 + d''), + * + * where d'' = (1 + d2) (d1 + d' + d1 d') y/(1 + y). The + * final result is + * + * (1 + d3) / [(1 + d2) (1 + d'') (1 + y)] + * = (1 + d''') / (1 + y) + * + * for |d'''| <= 2|d3 - d''| by Lemma 2 as long as |d''| < 1/2 + * (which may not be the case for very large beta). This + * relative error is therefore bounded by + * + * |d'''| + * <= 2|d3 - d''| + * <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d') y/(1 + y)| + * <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d')| + * = 2|d3| + 2|d1 + d' + d1 d' + d2 d1 + d2 d' + d2 d1 d'| + * <= 2|d3| + 2|d1| + 2|d'| + 2|d1 d'| + 2|d2 d1| + 2|d2 d'| + * + 2|d2 d1 d'| + * <= 4 eps + 2 eps^2 + (2 + 2 eps + 2 eps^2) |d'|. + * + * Roughly, |d'| = |(1 + d0)^{-beta} - 1| grows like beta eps, + * until it levels off at 1. + */ + return 1/(1 + pow(x/alpha, -beta)); +} + +/** + * Compute the SF of the LogLogistic(alpha, beta) distribution. + * Well-conditioned for all x and alpha, and the condition number + * + * beta/[1 + (x/alpha)^beta] + * + * grows linearly with beta. + * + * Loosely, the relative error of this implementation is bounded by + * + * 4 eps + 2 eps^2 + O(beta eps) + * + * so don't bother trying this for beta anywhere near as large as + * 1/eps, beyond which point it grows unbounded. + */ +STATIC double +sf_log_logistic(double x, double alpha, double beta) +{ + /* + * The error analysis here is essentially the same as in + * cdf_log_logistic, except that rather than levelling off at + * 1, |(1 + d0)^beta - 1| grows unbounded. + */ + return 1/(1 + pow(x/alpha, beta)); +} + +/** + * Compute the inverse of the CDF of the LogLogistic(alpha, beta) + * distribution. Ill-conditioned for p near 1 and beta near 0 with + * condition number 1/[beta (1 - p)]. + */ +STATIC double +icdf_log_logistic(double p, double alpha, double beta) +{ + return alpha*pow(p/(1 - p), 1/beta); +} + +/** + * Compute the inverse of the SF of the LogLogistic(alpha, beta) + * distribution. Ill-conditioned for p near 1 and for large beta, with + * condition number -1/[beta (1 - p)]. + */ +STATIC double +isf_log_logistic(double p, double alpha, double beta) +{ + return alpha*pow((1 - p)/p, 1/beta); +} + +/* + * Weibull(lambda, k) distribution, supported on (0, +infinity). + * + * pdf(x) = (k/lambda) (x/lambda)^{k - 1} e^{-(x/lambda)^k} + * cdf(x) = 1 - e^{-(x/lambda)^k} + * icdf(p) = lambda * (-log (1 - p))^{1/k} + * sf(x) = e^{-(x/lambda)^k} + * isf(p) = lambda * (-log p)^{1/k} + */ + +/** + * Compute the CDF of the Weibull(lambda, k) distribution. + * Well-conditioned for small x and k, and for large lambda -- + * condition number + * + * -k (x/lambda)^k exp(-(x/lambda)^k)/[exp(-(x/lambda)^k) - 1] + * + * grows linearly with k, x^k, and lambda^{-k}. + */ +STATIC double +cdf_weibull(double x, double lambda, double k) +{ + return -expm1(-pow(x/lambda, k)); +} + +/** + * Compute the SF of the Weibull(lambda, k) distribution. + * Well-conditioned for small x and k, and for large lambda -- + * condition number + * + * -k (x/lambda)^k + * + * grows linearly with k, x^k, and lambda^{-k}. + */ +STATIC double +sf_weibull(double x, double lambda, double k) +{ + return exp(-pow(x/lambda, k)); +} + +/** + * Compute the inverse of the CDF of the Weibull(lambda, k) + * distribution. Ill-conditioned for p near 1, and for k near 0; + * condition number is + * + * (p/(1 - p))/(k log(1 - p)). + */ +STATIC double +icdf_weibull(double p, double lambda, double k) +{ + return lambda*pow(-log1p(-p), 1/k); +} + +/** + * Compute the inverse of the SF of the Weibull(lambda, k) + * distribution. Ill-conditioned for p near 0, and for k near 0; + * condition number is + * + * 1/(k log(p)). + */ +STATIC double +isf_weibull(double p, double lambda, double k) +{ + return lambda*pow(-log(p), 1/k); +} + +/* + * GeneralizedPareto(mu, sigma, xi), supported on (mu, +infinity) for + * nonnegative xi, or (mu, mu - sigma/xi) for negative xi. + * + * Samples: + * = mu - sigma log U, if xi = 0; + * = mu + sigma (U^{-xi} - 1)/xi = mu + sigma*expm1(-xi log U)/xi, if xi =/= 0, + * where U is uniform on (0,1]. + * = mu + sigma (e^{xi X} - 1)/xi, + * where X has standard exponential distribution. + * + * pdf(x) = sigma^{-1} (1 + xi (x - mu)/sigma)^{-(1 + 1/xi)} + * cdf(x) = 1 - (1 + xi (x - mu)/sigma)^{-1/xi} + * = 1 - e^{-log(1 + xi (x - mu)/sigma)/xi} + * --> 1 - e^{-(x - mu)/sigma} as xi --> 0 + * sf(x) = (1 + xi (x - mu)/sigma)^{-1/xi} + * --> e^{-(x - mu)/sigma} as xi --> 0 + * icdf(p) = mu + sigma*(p^{-xi} - 1)/xi + * = mu + sigma*expm1(-xi log p)/xi + * --> mu + sigma*log p as xi --> 0 + * isf(p) = mu + sigma*((1 - p)^{xi} - 1)/xi + * = mu + sigma*expm1(-xi log1p(-p))/xi + * --> mu + sigma*log1p(-p) as xi --> 0 + */ + +/** + * Compute the CDF of the GeneralizedPareto(mu, sigma, xi) + * distribution. Well-conditioned everywhere. For standard + * distribution (mu=0, sigma=1), condition number + * + * (x/(1 + x xi)) / ((1 + x xi)^{1/xi} - 1) + * + * is bounded by 1, attained only at x = 0. + */ +STATIC double +cdf_genpareto(double x, double mu, double sigma, double xi) +{ + double x_0 = (x - mu)/sigma; + + /* + * log(1 + xi x_0)/xi + * = (-1/xi) \sum_{n=1}^infinity (-xi x_0)^n/n + * = (-1/xi) (-xi x_0 + \sum_{n=2}^infinity (-xi x_0)^n/n) + * = x_0 - (1/xi) \sum_{n=2}^infinity (-xi x_0)^n/n + * = x_0 - x_0 \sum_{n=2}^infinity (-xi x_0)^{n-1}/n + * = x_0 (1 - d), + * + * where d = \sum_{n=2}^infinity (-xi x_0)^{n-1}/n. If |xi| < + * eps/4|x_0|, then + * + * |d| <= \sum_{n=2}^infinity (eps/4)^{n-1}/n + * <= \sum_{n=2}^infinity (eps/4)^{n-1} + * = \sum_{n=1}^infinity (eps/4)^n + * = (eps/4) \sum_{n=0}^infinity (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2 + * + * for any 0 < eps < 2. Thus, the relative error of x_0 from + * log(1 + xi x_0)/xi is bounded by eps. + */ + if (fabs(xi) < 1e-17/x_0) + return -expm1(-x_0); + else + return -expm1(-log1p(xi*x_0)/xi); +} + +/** + * Compute the SF of the GeneralizedPareto(mu, sigma, xi) distribution. + * For standard distribution (mu=0, sigma=1), ill-conditioned for xi + * near 0; condition number + * + * -x (1 + x xi)^{(-1 - xi)/xi}/(1 + x xi)^{-1/xi} + * = -x (1 + x xi)^{-1/xi - 1}/(1 + x xi)^{-1/xi} + * = -(x/(1 + x xi)) (1 + x xi)^{-1/xi}/(1 + x xi)^{-1/xi} + * = -x/(1 + x xi) + * + * is bounded by 1/xi. + */ +STATIC double +sf_genpareto(double x, double mu, double sigma, double xi) +{ + double x_0 = (x - mu)/sigma; + + if (fabs(xi) < 1e-17/x_0) + return exp(-x_0); + else + return exp(-log1p(xi*x_0)/xi); +} + +/** + * Compute the inverse of the CDF of the GeneralizedPareto(mu, sigma, + * xi) distribution. Ill-conditioned for p near 1; condition number is + * + * xi (p/(1 - p))/(1 - (1 - p)^xi) + */ +STATIC double +icdf_genpareto(double p, double mu, double sigma, double xi) +{ + /* + * To compute f(xi) = (U^{-xi} - 1)/xi = (e^{-xi log U} - 1)/xi + * for xi near zero (note f(xi) --> -log U as xi --> 0), write + * the absolutely convergent Taylor expansion + * + * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^infinity (-xi log U)^n/n! + * = -log U + (1/xi)*\sum_{n=2}^infinity (-xi log U)^n/n! + * = -log U + \sum_{n=2}^infinity xi^{n-1} (-log U)^n/n! + * = -log U - log U \sum_{n=2}^infinity (-xi log U)^{n-1}/n! + * = -log U (1 + \sum_{n=2}^infinity (-xi log U)^{n-1}/n!). + * + * Let d = \sum_{n=2}^infinity (-xi log U)^{n-1}/n!. What do we + * lose if we discard it and use -log U as an approximation to + * f(xi)? If |xi| < eps/-4log U, then + * + * |d| <= \sum_{n=2}^infinity |xi log U|^{n-1}/n! + * <= \sum_{n=2}^infinity (eps/4)^{n-1}/n! + * <= \sum_{n=1}^infinity (eps/4)^n + * = (eps/4) \sum_{n=0}^infinity (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2, + * + * for any 0 < eps < 2. Hence, as long as |xi| < eps/-2log U, + * f(xi) = -log U (1 + d) for |d| <= eps/2. |d| is the + * relative error of f(xi) from -log U; from this bound, the + * relative error of -log U from f(xi) is at most (eps/2)/(1 - + * eps/2) = eps/2 + (eps/2)^2 + (eps/2)^3 + ... < eps for 0 < + * eps < 1. Since -log U < 1000 for all U in (0, 1] in + * binary64 floating-point, we can safely cut xi off at 1e-20 < + * eps/4000 and attain <1ulp error from series truncation. + */ + if (fabs(xi) <= 1e-20) + return mu - sigma*log1p(-p); + else + return mu + sigma*expm1(-xi*log1p(-p))/xi; +} + +/** + * Compute the inverse of the SF of the GeneralizedPareto(mu, sigma, + * xi) distribution. Ill-conditioned for p near 1; conditon number is + * + * -xi/(1 - p^{-xi}) + */ +STATIC double +isf_genpareto(double p, double mu, double sigma, double xi) +{ + if (fabs(xi) <= 1e-20) + return mu - sigma*log(p); + else + return mu + sigma*expm1(-xi*log(p))/xi; +} + +/*******************************************************************/ + +/** + * Deterministic samplers, parametrized by uniform integer and (0,1] + * samples. No guarantees are made about _which_ mapping from the + * integer and (0,1] samples these use; all that is guaranteed is the + * distribution of the outputs conditioned on a uniform distribution on + * the inputs. The automatic tests in test_prob_distr.c double-check + * the particular mappings we use. + * + * Beware: Unlike random_uniform_01(), these are not guaranteed to be + * supported on all possible outputs. See Ilya Mironov, `On the + * Significance of the Least Significant Bits for Differential + * Privacy', for an example of what can go wrong if you try to use + * these to conceal information from an adversary but you expose the + * specific full-precision floating-point values. + * + * Note: None of these samplers use rejection sampling; they are all + * essentially inverse-CDF transforms with tweaks. If you were to add, + * say, a Gamma sampler with the Marsaglia-Tsang method, you would have + * to parametrize it by a potentially infinite stream of uniform (and + * perhaps normal) samples rather than a fixed number, which doesn't + * make for quite as nice automatic testing as for these. + */ + +/** + * Deterministically sample from the interval [a, b], indexed by a + * uniform random floating-point number p0 in (0, 1]. + * + * Note that even if p0 is nonzero, the result may be equal to a, if + * ulp(a)/2 is nonnegligible, e.g. if a = 1. For maximum resolution, + * arrange |a| <= |b|. + */ +STATIC double +sample_uniform_interval(double p0, double a, double b) +{ + /* + * XXX Prove that the distribution is, in fact, uniform on + * [a,b], particularly around p0 = 1, or at least has very + * small deviation from uniform, quantified appropriately + * (e.g., like in Monahan 1984, or by KL divergence). It + * almost certainly does but it would be nice to quantify the + * error. + */ + if ((a <= 0 && 0 <= b) || (b <= 0 && 0 <= a)) { + /* + * When ab < 0, (1 - t) a + t b is monotonic, since for + * a <= b it is a sum of nondecreasing functions of t, + * and for b <= a, of nonincreasing functions of t. + * Further, clearly at 0 and 1 it attains a and b, + * respectively. Hence it is bounded within [a, b]. + */ + return (1 - p0)*a + p0*b; + } else { + /* + * a + (b - a) t is monotonic -- it is obviously a + * nondecreasing function of t for a <= b. Further, it + * attains a at 0, and while it may overshoot b at 1, + * we have a + * + * Theorem. If 0 <= t < 1, then the floating-point + * evaluation of a + (b - a) t is bounded in [a, b]. + * + * Lemma 1. If 0 <= t < 1 is a floating-point number, + * then for any normal floating-point number x except + * the smallest in magnitude, |round(x*t)| < |x|. + * + * Proof. WLOG, assume x >= 0. Since the rounding + * function and t |---> x*t are nondecreasing, their + * composition t |---> round(x*t) is also + * nondecreasing, so it suffices to consider the + * largest floating-point number below 1, in particular + * t = 1 - ulp(1)/2. + * + * Case I: If x is a power of two, then the next + * floating-point number below x is x - ulp(x)/2 = x - + * x*ulp(1)/2 = x*(1 - ulp(1)/2) = x*t, so, since x*t + * is a floating-point number, multiplication is exact, + * and thus round(x*t) = x*t < x. + * + * Case II: If x is not a power of two, then the + * greatest lower bound of real numbers rounded to x is + * x - ulp(x)/2 = x - ulp(T(x))/2 = x - T(x)*ulp(1)/2, + * where T(X) is the largest power of two below x. + * Anything below this bound is rounded to a + * floating-point number smaller than x, and x*t = x*(1 + * - ulp(1)/2) = x - x*ulp(1)/2 < x - T(x)*ulp(1)/2 + * since T(x) < x, so round(x*t) < x*t < x. QED. + * + * Lemma 2. If x and y are subnormal, then round(x + + * y) = x + y. + * + * Proof. It is a matter of adding the significands, + * since if we treat subnormals as having an implicit + * zero bit before the `binary' point, their exponents + * are all the same. There is at most one carry/borrow + * bit, which can always be acommodated either in a + * subnormal, or, at largest, in the implicit one bit + * of a normal. + * + * Lemma 3. Let x and y be floating-point numbers. If + * round(x - y) is subnormal or zero, then it is equal + * to x - y. + * + * Proof. Case I (equal): round(x - y) = 0 iff x = y; + * hence if round(x - y) = 0, then round(x - y) = 0 = x + * - y. + * + * Case II (subnormal/subnormal): If x and y are both + * subnormal, this follows directly from Lemma 2. + * + * Case IIIa (normal/subnormal): If x is normal and y + * is subnormal, then x and y must share sign, or else + * x - y would be larger than x and thus rounded to + * normal. If s is the smallest normal positive + * floating-point number, |x| < 2s since by + * construction 2s - |y| is normal for all subnormal y. + * This means that x and y must have the same exponent, + * so the difference is the difference of significands, + * which is exact. + * + * Case IIIb (subnormal/normal): Same as case IIIa for + * -(y - x). + * + * Case IV (normal/normal): If x and y are both normal, + * then they must share sign, or else x - y would be + * larger than x and thus rounded to normal. Note that + * |y| < 2|x|, for if |y| >= 2|x|, then |x| - |y| <= + * -|x| but -|x| is normal like x. Also, |x|/2 < |y|: + * if |x|/2 is subnormal, it must hold because y is + * normal; if |x|/2 is normal, then |x|/2 >= s, so + * since |x| - |y| < s, + * + * |x|/2 = |x| - |x|/2 <= |x| - s <= |y|; + * + * that is, |x|/2 < |y| < 2|x|, so by the Sterbenz + * lemma, round(x - y) = x - y. QED. + * + * Proof of theorem. WLOG, assume 0 <= a <= b. Since + * round(a + round(round(b - a)*t) is nondecreasing in + * t and attains a at 0, the lower end of the bound is + * trivial; we must show the upper end of the bound + * strictly. It suffices to show this for the largest + * floating-point number below 1, namely 1 - ulp(1)/2. + * + * Case I: round(b - a) is normal. Then it is at most + * the smallest floating-point number above b - a. By + * Lemma 1, round(round(b - a)*t) < round(b - a). + * Since the inequality is strict, and since + * round(round(b - a)*t) is a floating-point number + * below round(b - a), and since there are no + * floating-point numbers between b - a and round(b - + * a), we must have round(round(b - a)*t) < b - a. + * Then since y |---> round(a + y) is nondecreasing, we + * must have + * + * round(a + round(round(b - a)*t)) + * <= round(a + (b - a)) + * = round(b) = b. + * + * Case II: round(b - a) is subnormal. In this case, + * Lemma 1 falls apart -- we are not guaranteed the + * strict inequality. However, by Lemma 3, the + * difference is exact: round(b - a) = b - a. Thus, + * + * round(a + round(round(b - a)*t)) + * <= round(a + round((b - a)*t)) + * <= round(a + (b - a)) + * = round(b) + * = b, + * + * QED. + */ + + /* p0 is restricted to [0,1], but we use >= to silence -Wfloat-equal. */ + if (p0 >= 1) + return b; + return a + (b - a)*p0; + } +} + +/** + * Deterministically sample from the standard logistic distribution, + * indexed by a uniform random 32-bit integer s and uniform random + * floating-point numbers t and p0 in (0, 1]. + */ +STATIC double +sample_logistic(uint32_t s, double t, double p0) +{ + double sign = (s & 1) ? -1 : +1; + double r; + + /* + * We carve up the interval (0, 1) into subregions to compute + * the inverse CDF precisely: + * + * A = (0, 1/(1 + e)] ---> (-infinity, -1] + * B = [1/(1 + e), 1/2] ---> [-1, 0] + * C = [1/2, 1 - 1/(1 + e)] ---> [0, 1] + * D = [1 - 1/(1 + e), 1) ---> [1, +infinity) + * + * Cases D and C are mirror images of cases A and B, + * respectively, so we choose between them by the sign chosen + * by a fair coin toss. We choose between cases A and B by a + * coin toss weighted by + * + * 2/(1 + e) = 1 - [1/2 - 1/(1 + e)]/(1/2): + * + * if it comes up heads, scale p0 into a uniform (0, 1/(1 + e)] + * sample p; if it comes up tails, scale p0 into a uniform (0, + * 1/2 - 1/(1 + e)] sample and compute the inverse CDF of p = + * 1/2 - p0. + */ + if (t <= 2/(1 + exp(1))) { + /* p uniform in (0, 1/(1 + e)], represented by p. */ + p0 /= 1 + exp(1); + r = logit(p0); + } else { + /* + * p uniform in [1/(1 + e), 1/2), actually represented + * by p0 = 1/2 - p uniform in (0, 1/2 - 1/(1 + e)], so + * that p = 1/2 - p. + */ + p0 *= 0.5 - 1/(1 + exp(1)); + r = logithalf(p0); + } + + /* + * We have chosen from the negative half of the standard + * logistic distribution, which is symmetric with the positive + * half. Now use the sign to choose uniformly between them. + */ + return sign*r; +} + +/** + * Deterministically sample from the logistic distribution scaled by + * sigma and translated by mu. + */ +static double +sample_logistic_locscale(uint32_t s, double t, double p0, double mu, + double sigma) +{ + + return mu + sigma*sample_logistic(s, t, p0); +} + +/** + * Deterministically sample from the standard log-logistic + * distribution, indexed by a uniform random 32-bit integer s and a + * uniform random floating-point number p0 in (0, 1]. + */ +STATIC double +sample_log_logistic(uint32_t s, double p0) +{ + + /* + * Carve up the interval (0, 1) into (0, 1/2] and [1/2, 1); the + * condition numbers of the icdf and the isf coincide at 1/2. + */ + p0 *= 0.5; + if ((s & 1) == 0) { + /* p = p0 in (0, 1/2] */ + return p0/(1 - p0); + } else { + /* p = 1 - p0 in [1/2, 1) */ + return (1 - p0)/p0; + } +} + +/** + * Deterministically sample from the log-logistic distribution with + * scale alpha and shape beta. + */ +static double +sample_log_logistic_scaleshape(uint32_t s, double p0, double alpha, + double beta) +{ + double x = sample_log_logistic(s, p0); + + return alpha*pow(x, 1/beta); +} + +/** + * Deterministically sample from the standard exponential distribution, + * indexed by a uniform random 32-bit integer s and a uniform random + * floating-point number p0 in (0, 1]. + */ +static double +sample_exponential(uint32_t s, double p0) +{ + /* + * We would like to evaluate log(p) for p near 0, and log1p(-p) + * for p near 1. Simply carve the interval into (0, 1/2] and + * [1/2, 1) by a fair coin toss. + */ + p0 *= 0.5; + if ((s & 1) == 0) + /* p = p0 in (0, 1/2] */ + return -log(p0); + else + /* p = 1 - p0 in [1/2, 1) */ + return -log1p(-p0); +} + +/** + * Deterministically sample from a Weibull distribution with scale + * lambda and shape k -- just an exponential with a shape parameter in + * addition to a scale parameter. (Yes, lambda really is the scale, + * _not_ the rate.) + */ +STATIC double +sample_weibull(uint32_t s, double p0, double lambda, double k) +{ + + return lambda*pow(sample_exponential(s, p0), 1/k); +} + +/** + * Deterministically sample from the generalized Pareto distribution + * with shape xi, indexed by a uniform random 32-bit integer s and a + * uniform random floating-point number p0 in (0, 1]. + */ +STATIC double +sample_genpareto(uint32_t s, double p0, double xi) +{ + double x = sample_exponential(s, p0); + + /* + * Write f(xi) = (e^{xi x} - 1)/xi for xi near zero as the + * absolutely convergent Taylor series + * + * f(x) = (1/xi) (xi x + \sum_{n=2}^infinity (xi x)^n/n!) + * = x + (1/xi) \sum_{n=2}^infinity (xi x)^n/n! + * = x + \sum_{n=2}^infinity xi^{n-1} x^n/n! + * = x + x \sum_{n=2}^infinity (xi x)^{n-1}/n! + * = x (1 + \sum_{n=2}^infinity (xi x)^{n-1}/n!). + * + * d = \sum_{n=2}^infinity (xi x)^{n-1}/n! is the relative error + * of f(x) from x. If |xi| < eps/4x, then + * + * |d| <= \sum_{n=2}^infinity |xi x|^{n-1}/n! + * <= \sum_{n=2}^infinity (eps/4)^{n-1}/n! + * <= \sum_{n=1}^infinity (eps/4) + * = (eps/4) \sum_{n=0}^infinity (eps/4)^n + * = (eps/4)/(1 - eps/4) + * < eps/2, + * + * for any 0 < eps < 2. Hence, as long as |xi| < eps/2x, f(xi) + * = x (1 + d) for |d| <= eps/2, so x = f(xi) (1 + d') for |d'| + * <= eps. What bound should we use for x? + * + * - If x is exponentially distributed, x > 200 with + * probability below e^{-200} << 2^{-256}, i.e. never. + * + * - If x is computed by -log(U) for U in (0, 1], x is + * guaranteed to be below 1000 in IEEE 754 binary64 + * floating-point. + * + * We can safely cut xi off at 1e-20 < eps/4000 and attain an + * error bounded by 0.5 ulp for this expression. + */ + return (fabs(xi) < 1e-20 ? x : expm1(xi*x)/xi); +} + +/** + * Deterministically sample from a generalized Pareto distribution with + * shape xi, scaled by sigma and translated by mu. + */ +static double +sample_genpareto_locscale(uint32_t s, double p0, double mu, double sigma, + double xi) +{ + + return mu + sigma*sample_genpareto(s, p0, xi); +} + +/** + * Deterministically sample from the geometric distribution with + * per-trial success probability p. + **/ +// clang-format off +/* + * XXX Quantify the error (KL divergence?) of this + * ceiling-of-exponential sampler from a true geometric distribution, + * which we could get by rejection sampling. Relevant papers: + * + * John F. Monahan, `Accuracy in Random Number Generation', + * Mathematics of Computation 45(172), October 1984, pp. 559--568. +https://pdfs.semanticscholar.org/aca6/74b96da1df77b2224e8cfc5dd6d61a471632.pdf + * Karl Bringmann and Tobias Friedrich, `Exact and Efficient + * Generation of Geometric Random Variates and Random Graphs', in + * Proceedings of the 40th International Colloaquium on Automata, + * Languages, and Programming -- ICALP 2013, Springer LNCS 7965, + * pp.267--278. + * https://doi.org/10.1007/978-3-642-39206-1_23 + * https://people.mpi-inf.mpg.de/~kbringma/paper/2013ICALP-1.pdf + */ +// clang-format on +static double +sample_geometric(uint32_t s, double p0, double p) +{ + double x = sample_exponential(s, p0); + + /* This is actually a check against 1, but we do >= so that the compiler + does not raise a -Wfloat-equal */ + if (p >= 1) + return 1; + + return ceil(-x/log1p(-p)); +} + +/*******************************************************************/ + +/** Public API for probability distributions: + * + * These are wrapper functions on top of the various probability distribution + * operations using the generic <b>dist</b> structure. + + * These are the functions that should be used by consumers of this API. + */ + +/** Returns the name of the distribution in <b>dist</b>. */ +const char * +dist_name(const struct dist_t *dist) +{ + return dist->ops->name; +} + +/* Sample a value from <b>dist</b> and return it. */ +double +dist_sample(const struct dist_t *dist) +{ + return dist->ops->sample(dist); +} + +/** Compute the CDF of <b>dist</b> at <b>x</b>. */ +double +dist_cdf(const struct dist_t *dist, double x) +{ + return dist->ops->cdf(dist, x); +} + +/** Compute the SF (Survival function) of <b>dist</b> at <b>x</b>. */ +double +dist_sf(const struct dist_t *dist, double x) +{ + return dist->ops->sf(dist, x); +} + +/** Compute the iCDF (Inverse CDF) of <b>dist</b> at <b>x</b>. */ +double +dist_icdf(const struct dist_t *dist, double p) +{ + return dist->ops->icdf(dist, p); +} + +/** Compute the iSF (Inverse Survival function) of <b>dist</b> at <b>x</b>. */ +double +dist_isf(const struct dist_t *dist, double p) +{ + return dist->ops->isf(dist, p); +} + +/** Functions for uniform distribution */ + +static double +uniform_sample(const struct dist_t *dist) +{ + const struct uniform_t *U = dist_to_const_uniform(dist); + double p0 = random_uniform_01(); + + return sample_uniform_interval(p0, U->a, U->b); +} + +static double +uniform_cdf(const struct dist_t *dist, double x) +{ + const struct uniform_t *U = dist_to_const_uniform(dist); + if (x < U->a) + return 0; + else if (x < U->b) + return (x - U->a)/(U->b - U->a); + else + return 1; +} + +static double +uniform_sf(const struct dist_t *dist, double x) +{ + const struct uniform_t *U = dist_to_const_uniform(dist); + + if (x > U->b) + return 0; + else if (x > U->a) + return (U->b - x)/(U->b - U->a); + else + return 1; +} + +static double +uniform_icdf(const struct dist_t *dist, double p) +{ + const struct uniform_t *U = dist_to_const_uniform(dist); + double w = U->b - U->a; + + return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p))); +} + +static double +uniform_isf(const struct dist_t *dist, double p) +{ + const struct uniform_t *U = dist_to_const_uniform(dist); + double w = U->b - U->a; + + return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p))); +} + +const struct dist_ops_t uniform_ops = { + .name = "uniform", + .sample = uniform_sample, + .cdf = uniform_cdf, + .sf = uniform_sf, + .icdf = uniform_icdf, + .isf = uniform_isf, +}; + +/*******************************************************************/ + +/** Private functions for each probability distribution. */ + +/** Functions for logistic distribution: */ + +static double +logistic_sample(const struct dist_t *dist) +{ + const struct logistic_t *L = dist_to_const_logistic(dist); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); + double t = random_uniform_01(); + double p0 = random_uniform_01(); + + return sample_logistic_locscale(s, t, p0, L->mu, L->sigma); +} + +static double +logistic_cdf(const struct dist_t *dist, double x) +{ + const struct logistic_t *L = dist_to_const_logistic(dist); + return cdf_logistic(x, L->mu, L->sigma); +} + +static double +logistic_sf(const struct dist_t *dist, double x) +{ + const struct logistic_t *L = dist_to_const_logistic(dist); + return sf_logistic(x, L->mu, L->sigma); +} + +static double +logistic_icdf(const struct dist_t *dist, double p) +{ + const struct logistic_t *L = dist_to_const_logistic(dist); + return icdf_logistic(p, L->mu, L->sigma); +} + +static double +logistic_isf(const struct dist_t *dist, double p) +{ + const struct logistic_t *L = dist_to_const_logistic(dist); + return isf_logistic(p, L->mu, L->sigma); +} + +const struct dist_ops_t logistic_ops = { + .name = "logistic", + .sample = logistic_sample, + .cdf = logistic_cdf, + .sf = logistic_sf, + .icdf = logistic_icdf, + .isf = logistic_isf, +}; + +/** Functions for log-logistic distribution: */ + +static double +log_logistic_sample(const struct dist_t *dist) +{ + const struct log_logistic_t *LL = dist_to_const_log_logistic(dist); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); + double p0 = random_uniform_01(); + + return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta); +} + +static double +log_logistic_cdf(const struct dist_t *dist, double x) +{ + const struct log_logistic_t *LL = dist_to_const_log_logistic(dist); + return cdf_log_logistic(x, LL->alpha, LL->beta); +} + +static double +log_logistic_sf(const struct dist_t *dist, double x) +{ + const struct log_logistic_t *LL = dist_to_const_log_logistic(dist); + return sf_log_logistic(x, LL->alpha, LL->beta); +} + +static double +log_logistic_icdf(const struct dist_t *dist, double p) +{ + const struct log_logistic_t *LL = dist_to_const_log_logistic(dist); + return icdf_log_logistic(p, LL->alpha, LL->beta); +} + +static double +log_logistic_isf(const struct dist_t *dist, double p) +{ + const struct log_logistic_t *LL = dist_to_const_log_logistic(dist); + return isf_log_logistic(p, LL->alpha, LL->beta); +} + +const struct dist_ops_t log_logistic_ops = { + .name = "log logistic", + .sample = log_logistic_sample, + .cdf = log_logistic_cdf, + .sf = log_logistic_sf, + .icdf = log_logistic_icdf, + .isf = log_logistic_isf, +}; + +/** Functions for Weibull distribution */ + +static double +weibull_sample(const struct dist_t *dist) +{ + const struct weibull_t *W = dist_to_const_weibull(dist); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); + double p0 = random_uniform_01(); + + return sample_weibull(s, p0, W->lambda, W->k); +} + +static double +weibull_cdf(const struct dist_t *dist, double x) +{ + const struct weibull_t *W = dist_to_const_weibull(dist); + return cdf_weibull(x, W->lambda, W->k); +} + +static double +weibull_sf(const struct dist_t *dist, double x) +{ + const struct weibull_t *W = dist_to_const_weibull(dist); + return sf_weibull(x, W->lambda, W->k); +} + +static double +weibull_icdf(const struct dist_t *dist, double p) +{ + const struct weibull_t *W = dist_to_const_weibull(dist); + return icdf_weibull(p, W->lambda, W->k); +} + +static double +weibull_isf(const struct dist_t *dist, double p) +{ + const struct weibull_t *W = dist_to_const_weibull(dist); + return isf_weibull(p, W->lambda, W->k); +} + +const struct dist_ops_t weibull_ops = { + .name = "Weibull", + .sample = weibull_sample, + .cdf = weibull_cdf, + .sf = weibull_sf, + .icdf = weibull_icdf, + .isf = weibull_isf, +}; + +/** Functions for generalized Pareto distributions */ + +static double +genpareto_sample(const struct dist_t *dist) +{ + const struct genpareto_t *GP = dist_to_const_genpareto(dist); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); + double p0 = random_uniform_01(); + + return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_cdf(const struct dist_t *dist, double x) +{ + const struct genpareto_t *GP = dist_to_const_genpareto(dist); + return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_sf(const struct dist_t *dist, double x) +{ + const struct genpareto_t *GP = dist_to_const_genpareto(dist); + return sf_genpareto(x, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_icdf(const struct dist_t *dist, double p) +{ + const struct genpareto_t *GP = dist_to_const_genpareto(dist); + return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi); +} + +static double +genpareto_isf(const struct dist_t *dist, double p) +{ + const struct genpareto_t *GP = dist_to_const_genpareto(dist); + return isf_genpareto(p, GP->mu, GP->sigma, GP->xi); +} + +const struct dist_ops_t genpareto_ops = { + .name = "generalized Pareto", + .sample = genpareto_sample, + .cdf = genpareto_cdf, + .sf = genpareto_sf, + .icdf = genpareto_icdf, + .isf = genpareto_isf, +}; + +/** Functions for geometric distribution on number of trials before success */ + +static double +geometric_sample(const struct dist_t *dist) +{ + const struct geometric_t *G = dist_to_const_geometric(dist); + uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng()); + double p0 = random_uniform_01(); + + return sample_geometric(s, p0, G->p); +} + +static double +geometric_cdf(const struct dist_t *dist, double x) +{ + const struct geometric_t *G = dist_to_const_geometric(dist); + + if (x < 1) + return 0; + /* 1 - (1 - p)^floor(x) = 1 - e^{floor(x) log(1 - p)} */ + return -expm1(floor(x)*log1p(-G->p)); +} + +static double +geometric_sf(const struct dist_t *dist, double x) +{ + const struct geometric_t *G = dist_to_const_geometric(dist); + + if (x < 1) + return 0; + /* (1 - p)^floor(x) = e^{ceil(x) log(1 - p)} */ + return exp(floor(x)*log1p(-G->p)); +} + +static double +geometric_icdf(const struct dist_t *dist, double p) +{ + const struct geometric_t *G = dist_to_const_geometric(dist); + + return log1p(-p)/log1p(-G->p); +} + +static double +geometric_isf(const struct dist_t *dist, double p) +{ + const struct geometric_t *G = dist_to_const_geometric(dist); + + return log(p)/log1p(-G->p); +} + +const struct dist_ops_t geometric_ops = { + .name = "geometric (1-based)", + .sample = geometric_sample, + .cdf = geometric_cdf, + .sf = geometric_sf, + .icdf = geometric_icdf, + .isf = geometric_isf, +}; diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h new file mode 100644 index 0000000000..a036073b93 --- /dev/null +++ b/src/lib/math/prob_distr.h @@ -0,0 +1,253 @@ + +/** + * \file prob_distr.h + * + * \brief Header for prob_distr.c + **/ + +#ifndef TOR_PROB_DISTR_H +#define TOR_PROB_DISTR_H + +#include "lib/cc/compat_compiler.h" +#include "lib/cc/torint.h" +#include "lib/testsupport/testsupport.h" + +/** + * Container for distribution parameters for sampling, CDF, &c. + */ +struct dist_t { + const struct dist_ops_t *ops; +}; + +/** + * Untyped initializer element for struct dist_t using the specified + * struct dist_ops_t pointer. Don't actually use this directly -- use + * the type-specific macro built out of DIST_BASE_TYPED below -- but if + * you did use this directly, it would be something like: + * + * struct weibull mydist = { + * DIST_BASE(&weibull_ops), + * .lambda = ..., + * .k = ..., + * }; + * + * Note there is NO COMPILER FEEDBACK if you accidentally do something + * like + * + * struct geometric mydist = { + * DIST_BASE(&weibull_ops), + * ... + * }; + */ +#define DIST_BASE(OPS) { .ops = (OPS) } + +/** A compile-time type-checking macro for use with DIST_BASE_TYPED. + * + * This macro works by checking that &OBJ is a pointer type that is the same + * type (except for qualifiers) as (const TYPE *)&OBJ. It's a C constraint + * violation (which requires a diagnostic) if two pointers are different types + * and are subtracted. The sizeof() forces compile-time evaluation, and the + * multiplication by zero is to discard the result of the sizeof() from the + * expression. + * + * We define this conditionally to suppress false positives from + * Coverity, which gets confused by the sizeof business. + */ +#ifdef __COVERITY__ +#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) 0 +#else +#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \ + (0*sizeof(&(OBJ) - (const TYPE *)&(OBJ))) +#endif /* defined(__COVERITY__) */ + +/** +* Typed initializer element for struct dist_t using the specified struct +* dist_ops_t pointer. Don't actually use this directly -- use a +* type-specific macro built out of it -- but if you did use this +* directly, it would be something like: +* +* struct weibull mydist = { +* DIST_BASE_TYPED(&weibull_ops, mydist, struct weibull_t), +* .lambda = ..., +* .k = ..., +* }; +* +* If you want to define a distribution type, define a canonical set of +* operations and define a type-specific initializer element like so: +* +* struct foo_t { +* struct dist_t base; +* int omega; +* double tau; +* double phi; +* }; +* +* struct dist_ops_t foo_ops = ...; +* +* #define FOO(OBJ) DIST_BASE_TYPED(&foo_ops, OBJ, struct foo_t) +* +* Then users can do: +* +* struct foo_t mydist = { +* FOO(mydist), +* .omega = ..., +* .tau = ..., +* .phi = ..., +* }; +* +* If you accidentally write +* +* struct bar_t mydist = { +* FOO(mydist), +* ... +* }; +* +* then the compiler will report a type mismatch in the sizeof +* expression, which otherwise evaporates at runtime. +*/ +#define DIST_BASE_TYPED(OPS, OBJ, TYPE) \ + DIST_BASE((OPS) + TYPE_CHECK_OBJ(OPS,OBJ,TYPE)) + +/** + * Generic operations on distributions. These simply defer to the + * corresponding dist_ops_t function. In the parlance of C++, these call + * virtual member functions. + */ +const char *dist_name(const struct dist_t *); +double dist_sample(const struct dist_t *); +double dist_cdf(const struct dist_t *, double x); +double dist_sf(const struct dist_t *, double x); +double dist_icdf(const struct dist_t *, double p); +double dist_isf(const struct dist_t *, double p); + +/** + * Set of operations on a potentially parametric family of + * distributions. In the parlance of C++, this would be called a + * `vtable' and the members are virtual member functions. + */ +struct dist_ops_t { + const char *name; + double (*sample)(const struct dist_t *); + double (*cdf)(const struct dist_t *, double x); + double (*sf)(const struct dist_t *, double x); + double (*icdf)(const struct dist_t *, double p); + double (*isf)(const struct dist_t *, double p); +}; + +/* Geometric distribution on positive number of trials before first success */ + +struct geometric_t { + struct dist_t base; + double p; /* success probability */ +}; + +extern const struct dist_ops_t geometric_ops; + +#define GEOMETRIC(OBJ) \ + DIST_BASE_TYPED(&geometric_ops, OBJ, struct geometric_t) + +/* Pareto distribution */ + +struct genpareto_t { + struct dist_t base; + double mu; + double sigma; + double xi; +}; + +extern const struct dist_ops_t genpareto_ops; + +#define GENPARETO(OBJ) \ + DIST_BASE_TYPED(&genpareto_ops, OBJ, struct genpareto_t) + +/* Weibull distribution */ + +struct weibull_t { + struct dist_t base; + double lambda; + double k; +}; + +extern const struct dist_ops_t weibull_ops; + +#define WEIBULL(OBJ) \ + DIST_BASE_TYPED(&weibull_ops, OBJ, struct weibull_t) + +/* Log-logistic distribution */ + +struct log_logistic_t { + struct dist_t base; + double alpha; + double beta; +}; + +extern const struct dist_ops_t log_logistic_ops; + +#define LOG_LOGISTIC(OBJ) \ + DIST_BASE_TYPED(&log_logistic_ops, OBJ, struct log_logistic_t) + +/* Logistic distribution */ + +struct logistic_t { + struct dist_t base; + double mu; + double sigma; +}; + +extern const struct dist_ops_t logistic_ops; + +#define LOGISTIC(OBJ) \ + DIST_BASE_TYPED(&logistic_ops, OBJ, struct logistic_t) + +/* Uniform distribution */ + +struct uniform_t { + struct dist_t base; + double a; + double b; +}; + +extern const struct dist_ops_t uniform_ops; + +#define UNIFORM(OBJ) \ + DIST_BASE_TYPED(&uniform_ops, OBJ, struct uniform_t) + +/** Only by unittests */ + +#ifdef PROB_DISTR_PRIVATE + +STATIC double logithalf(double p0); +STATIC double logit(double p); + +STATIC double random_uniform_01(void); + +STATIC double logistic(double x); +STATIC double cdf_logistic(double x, double mu, double sigma); +STATIC double sf_logistic(double x, double mu, double sigma); +STATIC double icdf_logistic(double p, double mu, double sigma); +STATIC double isf_logistic(double p, double mu, double sigma); +STATIC double sample_logistic(uint32_t s, double t, double p0); + +STATIC double cdf_log_logistic(double x, double alpha, double beta); +STATIC double sf_log_logistic(double x, double alpha, double beta); +STATIC double icdf_log_logistic(double p, double alpha, double beta); +STATIC double isf_log_logistic(double p, double alpha, double beta); +STATIC double sample_log_logistic(uint32_t s, double p0); + +STATIC double cdf_weibull(double x, double lambda, double k); +STATIC double sf_weibull(double x, double lambda, double k); +STATIC double icdf_weibull(double p, double lambda, double k); +STATIC double isf_weibull(double p, double lambda, double k); +STATIC double sample_weibull(uint32_t s, double p0, double lambda, double k); + +STATIC double sample_uniform_interval(double p0, double a, double b); + +STATIC double cdf_genpareto(double x, double mu, double sigma, double xi); +STATIC double sf_genpareto(double x, double mu, double sigma, double xi); +STATIC double icdf_genpareto(double p, double mu, double sigma, double xi); +STATIC double isf_genpareto(double p, double mu, double sigma, double xi); +STATIC double sample_genpareto(uint32_t s, double p0, double xi); + +#endif /* defined(PROB_DISTR_PRIVATE) */ + +#endif /* !defined(TOR_PROB_DISTR_H) */ diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include index 814652a93c..a1edaf2231 100644 --- a/src/lib/memarea/.may_include +++ b/src/lib/memarea/.may_include @@ -1,7 +1,7 @@ orconfig.h lib/arch/*.h lib/cc/*.h -lib/container/*.h lib/log/*.h lib/malloc/*.h lib/memarea/*.h +lib/smartlist_core/*.h
\ No newline at end of file diff --git a/src/lib/memarea/include.am b/src/lib/memarea/include.am index 94343dcead..83fb99ec73 100644 --- a/src/lib/memarea/include.am +++ b/src/lib/memarea/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-memarea-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_memarea_a_SOURCES = \ src/lib/memarea/memarea.c @@ -13,5 +14,6 @@ src_lib_libtor_memarea_testing_a_SOURCES = \ src_lib_libtor_memarea_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_memarea_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/memarea/memarea.h diff --git a/src/lib/memarea/lib_memarea.md b/src/lib/memarea/lib_memarea.md new file mode 100644 index 0000000000..fe5cb8293f --- /dev/null +++ b/src/lib/memarea/lib_memarea.md @@ -0,0 +1,28 @@ +@dir /lib/memarea +@brief lib/memarea: A fast arena-style allocator. + +This module has a fast "arena" style allocator, where memory is freed all at +once. This kind of allocation is very fast and avoids fragmentation, at the +expense of requiring all the data to be freed at the same time. We use this +for parsing and diff calculations. + +It's often handy to allocate a large number of tiny objects, all of which +need to disappear at the same time. You can do this in tor using the +memarea.c abstraction, which uses a set of grow-only buffers for allocation, +and only supports a single "free" operation at the end. + +Using memareas also helps you avoid memory fragmentation. You see, some libc +malloc implementations perform badly on the case where a large number of +small temporary objects are allocated at the same time as a few long-lived +objects of similar size. But if you use tor_malloc() for the long-lived ones +and a memarea for the temporary object, the malloc implementation is likelier +to do better. + +To create a new memarea, use `memarea_new()`. To drop all the storage from a +memarea, and invalidate its pointers, use `memarea_drop_all()`. + +The allocation functions `memarea_alloc()`, `memarea_alloc_zero()`, +`memarea_memdup()`, `memarea_strdup()`, and `memarea_strndup()` are analogous +to the similarly-named malloc() functions. There is intentionally no +`memarea_free()` or `memarea_realloc()`. + diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c index 486673116c..4d26c20eeb 100644 --- a/src/lib/memarea/memarea.c +++ b/src/lib/memarea/memarea.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2019, The Tor Project, Inc. */ +/* Copyright (c) 2008-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,7 +16,8 @@ #include "lib/arch/bytes.h" #include "lib/cc/torint.h" -#include "lib/container/smartlist.h" +#include "lib/smartlist_core/smartlist_core.h" +#include "lib/smartlist_core/smartlist_foreach.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" @@ -38,7 +39,7 @@ #elif MEMAREA_ALIGN == 8 #define MEMAREA_ALIGN_MASK ((uintptr_t)7) #else -#error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff." +#error "void* is neither 4 nor 8 bytes long." #endif /* MEMAREA_ALIGN == 4 || ... */ #if defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER) @@ -67,7 +68,7 @@ uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]); \ tor_assert(sent_val == SENTINEL_VAL); \ STMT_END -#else /* !(defined(USE_SENTINELS)) */ +#else /* !defined(USE_SENTINELS) */ #define SENTINEL_LEN 0 #define SET_SENTINEL(chunk) STMT_NIL #define CHECK_SENTINEL(chunk) STMT_NIL @@ -314,7 +315,7 @@ memarea_assert_ok(memarea_t *area) } } -#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */ +#else /* defined(DISABLE_MEMORY_SENTINELS) */ struct memarea_t { smartlist_t *pieces; diff --git a/src/lib/memarea/memarea.h b/src/lib/memarea/memarea.h index 9c23cf62e9..8b5e63e6b3 100644 --- a/src/lib/memarea/memarea.h +++ b/src/lib/memarea/memarea.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2008-2019, The Tor Project, Inc. */ +/* Copyright (c) 2008-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,6 +16,9 @@ typedef struct memarea_t memarea_t; memarea_t *memarea_new(void); void memarea_drop_all_(memarea_t *area); +/** @copydoc memarea_drop_all_ + * + * Additionally, set <b>area</b> to NULL. */ #define memarea_drop_all(area) \ do { \ memarea_drop_all_(area); \ diff --git a/src/lib/meminfo/include.am b/src/lib/meminfo/include.am index d1fdde6313..12c1bff72d 100644 --- a/src/lib/meminfo/include.am +++ b/src/lib/meminfo/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-meminfo-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_meminfo_a_SOURCES = \ src/lib/meminfo/meminfo.c @@ -13,5 +14,6 @@ src_lib_libtor_meminfo_testing_a_SOURCES = \ src_lib_libtor_meminfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_meminfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/meminfo/meminfo.h diff --git a/src/lib/meminfo/lib_meminfo.md b/src/lib/meminfo/lib_meminfo.md new file mode 100644 index 0000000000..87f509d648 --- /dev/null +++ b/src/lib/meminfo/lib_meminfo.md @@ -0,0 +1,5 @@ +@dir /lib/meminfo +@brief lib/meminfo: Inspecting malloc() usage. + +Only available when malloc() provides mallinfo() or something similar. + diff --git a/src/lib/meminfo/meminfo.c b/src/lib/meminfo/meminfo.c index bc3eef8419..b7d991e410 100644 --- a/src/lib/meminfo/meminfo.c +++ b/src/lib/meminfo/meminfo.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/meminfo/meminfo.h b/src/lib/meminfo/meminfo.h index 6f29fe0ea8..36a85c133d 100644 --- a/src/lib/meminfo/meminfo.h +++ b/src/lib/meminfo/meminfo.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,4 +17,4 @@ MOCK_DECL(int, get_total_system_memory, (size_t *mem_out)); -#endif +#endif /* !defined(TOR_MEMINFO_H) */ diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include index 13b209bbed..6e9af9737a 100644 --- a/src/lib/net/.may_include +++ b/src/lib/net/.may_include @@ -1,8 +1,9 @@ orconfig.h -siphash.h -ht.h +ext/siphash.h +ext/ht.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h @@ -11,5 +12,7 @@ lib/lock/*.h lib/log/*.h lib/net/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h -lib/malloc/*.h
\ No newline at end of file +lib/malloc/*.h +lib/smartlist_core/*.h diff --git a/src/lib/net/address.c b/src/lib/net/address.c index 69004ddb0e..6d46f9b955 100644 --- a/src/lib/net/address.c +++ b/src/lib/net/address.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -40,6 +40,7 @@ #include "lib/net/address.h" #include "lib/net/socket.h" +#include "lib/cc/ctassert.h" #include "lib/container/smartlist.h" #include "lib/ctime/di_ops.h" #include "lib/log/log.h" @@ -52,7 +53,7 @@ #include "lib/string/printf.h" #include "lib/string/util_string.h" -#include "siphash.h" +#include "ext/siphash.h" #ifdef HAVE_SYS_TIME_H #include <sys/time.h> @@ -96,8 +97,9 @@ * work correctly. Bail out here if we've found a platform where AF_UNSPEC * isn't 0. */ #if AF_UNSPEC != 0 -#error We rely on AF_UNSPEC being 0. Let us know about your platform, please! +#error "We rely on AF_UNSPEC being 0. Yours isn't. Please tell us more!" #endif +CTASSERT(AF_UNSPEC == 0); /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a * sockaddr object in *<b>sa_out</b> of object size <b>len</b>. If not enough @@ -337,7 +339,7 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate) break; case AF_INET6: /* Shortest addr [ :: ] + \0 */ - if (len < (3 + (decorate ? 2u : 0u))) + if (len < (3u + (decorate ? 2 : 0))) return NULL; if (decorate) @@ -371,7 +373,8 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate) * * If <b>accept_regular</b> is set and the address is in neither recognized * reverse lookup hostname format, try parsing the address as a regular - * IPv4 or IPv6 address too. + * IPv4 or IPv6 address too. This mode will accept IPv6 addresses with or + * without square brackets. */ int tor_addr_parse_PTR_name(tor_addr_t *result, const char *address, @@ -605,7 +608,8 @@ tor_addr_parse_mask_ports(const char *s, family = AF_INET; tor_addr_from_ipv4h(addr_out, 0); } else if (flags & TAPMP_STAR_IPV6_ONLY) { - static char nil_bytes[16] = { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 }; + static uint8_t nil_bytes[16] = + { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 }; family = AF_INET6; tor_addr_from_ipv6_bytes(addr_out, nil_bytes); } else { @@ -626,7 +630,7 @@ tor_addr_parse_mask_ports(const char *s, tor_addr_from_ipv4h(addr_out, 0); any_flag = 1; } else if (!strcmp(address, "*6") && (flags & TAPMP_EXTENDED_STAR)) { - static char nil_bytes[16] = { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 }; + static uint8_t nil_bytes[16] = { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 }; family = AF_INET6; tor_addr_from_ipv6_bytes(addr_out, nil_bytes); any_flag = 1; @@ -814,8 +818,12 @@ tor_addr_is_loopback(const tor_addr_t *addr) /* Is addr valid? * Checks that addr is non-NULL and not tor_addr_is_null(). - * If for_listening is true, IPv4 addr 0.0.0.0 is allowed. - * It means "bind to all addresses on the local machine". */ + * If for_listening is true, all IPv4 and IPv6 addresses are valid, including + * 0.0.0.0 (for IPv4) and :: (for IPv6). When listening, these addresses mean + * "bind to all addresses on the local machine". + * Otherwise, 0.0.0.0 and :: are invalid, because they are null addresses. + * All unspecified and unix addresses are invalid, regardless of for_listening. + */ int tor_addr_is_valid(const tor_addr_t *addr, int for_listening) { @@ -824,10 +832,11 @@ tor_addr_is_valid(const tor_addr_t *addr, int for_listening) return 0; } - /* Only allow IPv4 0.0.0.0 for_listening. */ - if (for_listening && addr->family == AF_INET - && tor_addr_to_ipv4h(addr) == 0) { - return 1; + /* Allow all IPv4 and IPv6 addresses, when for_listening is true */ + if (for_listening) { + if (addr->family == AF_INET || addr->family == AF_INET6) { + return 1; + } } /* Otherwise, the address is valid if it's not tor_addr_is_null() */ @@ -879,7 +888,7 @@ tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr) /** Set <b>dest</b> to equal the IPv6 address in the 16 bytes at * <b>ipv6_bytes</b>. */ void -tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *ipv6_bytes) +tor_addr_from_ipv6_bytes(tor_addr_t *dest, const uint8_t *ipv6_bytes) { tor_assert(dest); tor_assert(ipv6_bytes); @@ -892,7 +901,21 @@ tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *ipv6_bytes) void tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6) { - tor_addr_from_ipv6_bytes(dest, (const char*)in6->s6_addr); + tor_addr_from_ipv6_bytes(dest, in6->s6_addr); +} + +/** Set the 16 bytes at <b>dest</b> to equal the IPv6 address <b>src</b>. + * <b>src</b> must be an IPv6 address, if it is not, log a warning, and clear + * <b>dest</b>. */ +void +tor_addr_copy_ipv6_bytes(uint8_t *dest, const tor_addr_t *src) +{ + tor_assert(dest); + tor_assert(src); + memset(dest, 0, 16); + IF_BUG_ONCE(src->family != AF_INET6) + return; + memcpy(dest, src->addr.in6_addr.s6_addr, 16); } /** Copy a tor_addr_t from <b>src</b> to <b>dest</b>. @@ -1166,57 +1189,137 @@ fmt_addr_impl(const tor_addr_t *addr, int decorate) const char * fmt_addrport(const tor_addr_t *addr, uint16_t port) { - /* Add space for a colon and up to 5 digits. */ - static char buf[TOR_ADDR_BUF_LEN + 6]; + static char buf[TOR_ADDRPORT_BUF_LEN]; tor_snprintf(buf, sizeof(buf), "%s:%u", fmt_and_decorate_addr(addr), port); return buf; } /** Like fmt_addr(), but takes <b>addr</b> as a host-order IPv4 * addresses. Also not thread-safe, also clobbers its return buffer on - * repeated calls. */ + * repeated calls. Clean internal buffer and return empty string on failure. */ const char * fmt_addr32(uint32_t addr) { static char buf[INET_NTOA_BUF_LEN]; struct in_addr in; + int success; + in.s_addr = htonl(addr); - tor_inet_ntoa(&in, buf, sizeof(buf)); + + success = tor_inet_ntoa(&in, buf, sizeof(buf)); + tor_assertf_nonfatal(success >= 0, + "Failed to convert IP 0x%08X (HBO) to string", addr); + + IF_BUG_ONCE(success < 0) { + memset(buf, 0, INET_NTOA_BUF_LEN); + } + return buf; } +/** Return a string representing the family of <b>addr</b>. + * + * This string is a string constant, and must not be freed. + * This function is thread-safe. + */ +const char * +fmt_addr_family(const tor_addr_t *addr) +{ + static int default_bug_once = 0; + + IF_BUG_ONCE(!addr) + return "NULL pointer"; + + switch (tor_addr_family(addr)) { + case AF_INET6: + return "IPv6"; + case AF_INET: + return "IPv4"; + case AF_UNIX: + return "UNIX socket"; + case AF_UNSPEC: + return "unspecified"; + default: + if (!default_bug_once) { + log_warn(LD_BUG, "Called with unknown address family %d", + (int)tor_addr_family(addr)); + default_bug_once = 1; + } + return "unknown"; + } + //return "(unreachable code)"; +} + /** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>. The string - * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by - * square brackets. + * may be an IPv4 address, or an IPv6 address surrounded by square brackets. * - * Return an address family on success, or -1 if an invalid address string is - * provided. */ -int -tor_addr_parse(tor_addr_t *addr, const char *src) + * If <b>allow_ipv6_without_brackets</b> is true, also allow IPv6 addresses + * without brackets. + * + * Always rejects IPv4 addresses with brackets. + * + * Returns an address family on success, or -1 if an invalid address string is + * provided. */ +static int +tor_addr_parse_impl(tor_addr_t *addr, const char *src, + bool allow_ipv6_without_brackets) { /* Holds substring of IPv6 address after removing square brackets */ char *tmp = NULL; - int result; + int result = -1; struct in_addr in_tmp; struct in6_addr in6_tmp; + int brackets_detected = 0; + tor_assert(addr && src); - if (src[0] == '[' && src[1]) + + size_t len = strlen(src); + + if (len && src[0] == '[' && src[len - 1] == ']') { + brackets_detected = 1; src = tmp = tor_strndup(src+1, strlen(src)-2); + } - if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { - result = AF_INET6; - tor_addr_from_in6(addr, &in6_tmp); - } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { - result = AF_INET; - tor_addr_from_in(addr, &in_tmp); - } else { - result = -1; + /* Try to parse an IPv6 address if it has brackets, or if IPv6 addresses + * without brackets are allowed */ + if (brackets_detected || allow_ipv6_without_brackets) { + if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) { + result = AF_INET6; + tor_addr_from_in6(addr, &in6_tmp); + } + } + + /* Try to parse an IPv4 address without brackets */ + if (!brackets_detected) { + if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) { + result = AF_INET; + tor_addr_from_in(addr, &in_tmp); + } + } + + /* Clear the address on error, to avoid returning uninitialised or partly + * parsed data. + */ + if (result == -1) { + memset(addr, 0, sizeof(tor_addr_t)); } tor_free(tmp); return result; } +/** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>. The string + * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by + * square brackets. + * + * Returns an address family on success, or -1 if an invalid address string is + * provided. */ +int +tor_addr_parse(tor_addr_t *addr, const char *src) +{ + return tor_addr_parse_impl(addr, src, 1); +} + #ifdef HAVE_IFADDRS_TO_SMARTLIST /* * Convert a linked list consisting of <b>ifaddrs</b> structures @@ -1352,7 +1455,7 @@ get_interface_addresses_win32(int severity, sa_family_t family) /* This is defined on Mac OS X */ #ifndef _SIZEOF_ADDR_IFREQ -#define _SIZEOF_ADDR_IFREQ sizeof +#define _SIZEOF_ADDR_IFREQ(x) sizeof(x) #endif /* Free ifc->ifc_buf safely. */ @@ -1371,10 +1474,10 @@ ifconf_free_ifc_buf(struct ifconf *ifc) * into smartlist of <b>tor_addr_t</b> structures. */ STATIC smartlist_t * -ifreq_to_smartlist(char *buf, size_t buflen) +ifreq_to_smartlist(const uint8_t *buf, size_t buflen) { smartlist_t *result = smartlist_new(); - char *end = buf + buflen; + const uint8_t *end = buf + buflen; /* These acrobatics are due to alignment issues which trigger * undefined behaviour traps on OSX. */ @@ -1448,7 +1551,7 @@ get_interface_addresses_ioctl(int severity, sa_family_t family) /* Ensure we have least IFREQ_SIZE bytes unused at the end. Otherwise, we * don't know if we got everything during ioctl. */ } while (mult * IFREQ_SIZE - ifc.ifc_len <= IFREQ_SIZE); - result = ifreq_to_smartlist(ifc.ifc_buf, ifc.ifc_len); + result = ifreq_to_smartlist((const uint8_t *)ifc.ifc_buf, ifc.ifc_len); done: if (fd >= 0) @@ -1601,11 +1704,15 @@ get_interface_address6,(int severity, sa_family_t family, tor_addr_t *addr)) * Ideally, we want the default route, see #12377 for details */ SMARTLIST_FOREACH_BEGIN(addrs, tor_addr_t *, a) { tor_addr_copy(addr, a); + const bool is_internal = tor_addr_is_internal(a, 0); rv = 0; + log_debug(LD_NET, "Found %s interface address '%s'", + (is_internal ? "internal" : "external"), fmt_addr(addr)); + /* If we found a non-internal address, declare success. Otherwise, * keep looking. */ - if (!tor_addr_is_internal(a, 0)) + if (!is_internal) break; } SMARTLIST_FOREACH_END(a); @@ -1709,6 +1816,11 @@ get_interface_address6_list,(int severity, * form "ip" or "ip:0". Otherwise, accept those forms, and set * *<b>port_out</b> to <b>default_port</b>. * + * This function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * * Return 0 on success, -1 on failure. */ int tor_addr_port_parse(int severity, const char *addrport, @@ -1718,6 +1830,7 @@ tor_addr_port_parse(int severity, const char *addrport, int retval = -1; int r; char *addr_tmp = NULL; + bool has_port; tor_assert(addrport); tor_assert(address_out); @@ -1727,28 +1840,47 @@ tor_addr_port_parse(int severity, const char *addrport, if (r < 0) goto done; - if (!*port_out) { + has_port = !! *port_out; + /* If there's no port, use the default port, or fail if there is no default + */ + if (!has_port) { if (default_port >= 0) *port_out = default_port; else goto done; } - /* make sure that address_out is an IP address */ - if (tor_addr_parse(address_out, addr_tmp) < 0) + /* Make sure that address_out is an IP address. + * If there is no port in addrport, allow IPv6 addresses without brackets. */ + if (tor_addr_parse_impl(address_out, addr_tmp, !has_port) < 0) goto done; retval = 0; done: + /* Clear the address and port on error, to avoid returning uninitialised or + * partly parsed data. + */ + if (retval == -1) { + memset(address_out, 0, sizeof(tor_addr_t)); + *port_out = 0; + } tor_free(addr_tmp); return retval; } /** Given an address of the form "host[:port]", try to divide it into its host - * and port portions, setting *<b>address_out</b> to a newly allocated string - * holding the address portion and *<b>port_out</b> to the port (or 0 if no - * port is given). Return 0 on success, -1 on failure. */ + * and port portions. + * + * Like tor_addr_port_parse(), this function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * + * Sets *<b>address_out</b> to a newly allocated string holding the address + * portion, and *<b>port_out</b> to the port (or 0 if no port is given). + * + * Return 0 on success, -1 on failure. */ int tor_addr_port_split(int severity, const char *addrport, char **address_out, uint16_t *port_out) @@ -1757,8 +1889,11 @@ tor_addr_port_split(int severity, const char *addrport, tor_assert(addrport); tor_assert(address_out); tor_assert(port_out); + /* We need to check for IPv6 manually because the logic below doesn't - * do a good job on IPv6 addresses that lack a port. */ + * do a good job on IPv6 addresses that lack a port. + * If an IPv6 address without square brackets is ambiguous, it gets parsed + * here as an address, rather than address:port. */ if (tor_addr_parse(&a_tmp, addrport) == AF_INET6) { *port_out = 0; *address_out = tor_strdup(addrport); @@ -1798,8 +1933,7 @@ tor_addr_port_split(int severity, const char *addrport, tor_free(address_); } - if (port_out) - *port_out = ok ? ((uint16_t) port_) : 0; + *port_out = ok ? ((uint16_t) port_) : 0; return ok ? 0 : -1; } @@ -1875,17 +2009,24 @@ parse_port_range(const char *port, uint16_t *port_min_out, } /** Given a host-order <b>addr</b>, call tor_inet_ntop() on it - * and return a strdup of the resulting address. + * and return a strdup of the resulting address. Return NULL if + * tor_inet_ntop() fails. */ char * tor_dup_ip(uint32_t addr) { + const char *ip_str; char buf[TOR_ADDR_BUF_LEN]; struct in_addr in; in.s_addr = htonl(addr); - tor_inet_ntop(AF_INET, &in, buf, sizeof(buf)); - return tor_strdup(buf); + ip_str = tor_inet_ntop(AF_INET, &in, buf, sizeof(buf)); + + tor_assertf_nonfatal(ip_str, "Failed to duplicate IP %08X", addr); + if (ip_str) + return tor_strdup(buf); + + return NULL; } /** @@ -1934,7 +2075,7 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port) return ap; } -/** Return true iff <a>a</b> and <b>b</b> are the same address and port */ +/** Return true iff <b>a</b> and <b>b</b> are the same address and port */ int tor_addr_port_eq(const tor_addr_port_t *a, const tor_addr_port_t *b) @@ -2018,8 +2159,12 @@ string_is_valid_nonrfc_hostname(const char *string) smartlist_split_string(components,string,".",0,0); - if (BUG(smartlist_len(components) == 0)) - return 0; // LCOV_EXCL_LINE should be impossible given the earlier checks. + if (BUG(smartlist_len(components) == 0)) { + // LCOV_EXCL_START should be impossible given the earlier checks. + smartlist_free(components); + return 0; + // LCOV_EXCL_STOP + } /* Allow a single terminating '.' used rarely to indicate domains * are FQDNs rather than relative. */ diff --git a/src/lib/net/address.h b/src/lib/net/address.h index 9b826c8359..e5016ee4fe 100644 --- a/src/lib/net/address.h +++ b/src/lib/net/address.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -62,6 +62,7 @@ typedef uint8_t maskbits_t; struct in_addr; + /** Holds an IPv4 or IPv6 address. (Uses less memory than struct * sockaddr_storage.) */ typedef struct tor_addr_t @@ -103,6 +104,10 @@ int tor_addr_from_sockaddr(tor_addr_t *a, const struct sockaddr *sa, uint16_t *port_out); void tor_addr_make_unspec(tor_addr_t *a); void tor_addr_make_null(tor_addr_t *a, sa_family_t family); +#define tor_addr_port_make_null(addr, port, family) \ + (void)(tor_addr_make_null(addr, family), (port) = 0) +#define tor_addr_port_make_null_ap(ap, family) \ + tor_addr_port_make_null(&(ap)->addr, (ap)->port, family) char *tor_sockaddr_to_str(const struct sockaddr *sa); /** Return an in6_addr* equivalent to <b>a</b>, or NULL if <b>a</b> is not @@ -133,6 +138,7 @@ tor_addr_to_in6_assert(const tor_addr_t *a) * Requires that <b>x</b> is actually an IPv6 address. */ #define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6_assert(x)) + /** Given an IPv6 address <b>x</b>, yield it as an array of uint32_t. * * Requires that <b>x</b> is actually an IPv6 address. @@ -146,6 +152,7 @@ tor_addr_to_ipv4n(const tor_addr_t *a) { return a->family == AF_INET ? a->addr.in_addr.s_addr : 0; } + /** Return an IPv4 address in host order for <b>a</b>, or 0 if * <b>a</b> is not an IPv4 address. */ static inline uint32_t @@ -153,10 +160,11 @@ tor_addr_to_ipv4h(const tor_addr_t *a) { return ntohl(tor_addr_to_ipv4n(a)); } + /** Given an IPv6 address, return its mapped IPv4 address in host order, or * 0 if <b>a</b> is not an IPv6 address. * - * (Does not check whether the address is really a mapped address */ + * (Does not check whether the address is really a mapped address.) */ static inline uint32_t tor_addr_to_mapped_ipv4h(const tor_addr_t *a) { @@ -165,21 +173,21 @@ tor_addr_to_mapped_ipv4h(const tor_addr_t *a) // Work around an incorrect NULL pointer dereference warning in // "clang --analyze" due to limited analysis depth addr32 = tor_addr_to_in6_addr32(a); - // To improve performance, wrap this assertion in: - // #if !defined(__clang_analyzer__) || PARANOIA tor_assert(addr32); return ntohl(addr32[3]); } else { return 0; } } + /** Return the address family of <b>a</b>. Possible values are: - * AF_INET6, AF_INET, AF_UNSPEC. */ + * AF_INET6, AF_INET, AF_UNSPEC, AF_UNIX. */ static inline sa_family_t tor_addr_family(const tor_addr_t *a) { return a->family; } + /** Return an in_addr* equivalent to <b>a</b>, or NULL if <b>a</b> is not * an IPv4 address. */ static inline const struct in_addr * @@ -187,6 +195,7 @@ tor_addr_to_in(const tor_addr_t *a) { return a->family == AF_INET ? &a->addr.in_addr : NULL; } + /** Return true iff <b>a</b> is an IPv4 address equal to the host-ordered * address in <b>u</b>. */ static inline int @@ -204,24 +213,39 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u) */ #define TOR_ADDR_BUF_LEN 48 +/** Length of a buffer containing an IP address along with a port number and + * a seperating colon. + * + * This allows enough space for + * "[ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]:12345", + * plus a terminating NUL. + */ +#define TOR_ADDRPORT_BUF_LEN (TOR_ADDR_BUF_LEN + 6) + char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC; /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6 * addresses. */ #define fmt_addr(a) fmt_addr_impl((a), 0) + /** Wrapper function of fmt_addr_impl(). It decorates IPv6 * addresses. */ #define fmt_and_decorate_addr(a) fmt_addr_impl((a), 1) + const char *fmt_addr_impl(const tor_addr_t *addr, int decorate); const char *fmt_addrport(const tor_addr_t *addr, uint16_t port); -const char * fmt_addr32(uint32_t addr); +#define fmt_addrport_ap(ap) fmt_addrport(&(ap)->addr, (ap)->port) +const char *fmt_addr32(uint32_t addr); +const char *fmt_addr_family(const tor_addr_t *addr); MOCK_DECL(int,get_interface_address6,(int severity, sa_family_t family, tor_addr_t *addr)); + struct smartlist_t; -void interface_address6_list_free_(struct smartlist_t * addrs);// XXXX +void interface_address6_list_free_(struct smartlist_t * addrs); #define interface_address6_list_free(addrs) \ FREE_AND_NULL(struct smartlist_t, interface_address6_list_free_, (addrs)) + MOCK_DECL(struct smartlist_t *,get_interface_address6_list,(int severity, sa_family_t family, int include_internal)); @@ -246,6 +270,7 @@ int tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, uint64_t tor_addr_hash(const tor_addr_t *addr); struct sipkey; uint64_t tor_addr_keyed_hash(const struct sipkey *key, const tor_addr_t *addr); + int tor_addr_is_v4(const tor_addr_t *addr); int tor_addr_is_internal_(const tor_addr_t *ip, int for_listening, const char *filename, int lineno); @@ -276,21 +301,25 @@ int tor_addr_parse_PTR_name(tor_addr_t *result, const char *address, int tor_addr_parse_mask_ports(const char *s, unsigned flags, tor_addr_t *addr_out, maskbits_t *mask_out, uint16_t *port_min_out, uint16_t *port_max_out); + const char * tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate); int tor_addr_parse(tor_addr_t *addr, const char *src); void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src); void tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src); + void tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr); /** Set <b>dest</b> to the IPv4 address encoded in <b>v4addr</b> in host * order. */ #define tor_addr_from_ipv4h(dest, v4addr) \ tor_addr_from_ipv4n((dest), htonl(v4addr)) -void tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *bytes); +void tor_addr_from_ipv6_bytes(tor_addr_t *dest, const uint8_t *bytes); /** Set <b>dest</b> to the IPv4 address incoded in <b>in</b>. */ #define tor_addr_from_in(dest, in) \ tor_addr_from_ipv4n((dest), (in)->s_addr); void tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6); +void tor_addr_copy_ipv6_bytes(uint8_t *dest, const tor_addr_t *src); + int tor_addr_is_null(const tor_addr_t *addr); int tor_addr_is_loopback(const tor_addr_t *addr); @@ -299,6 +328,7 @@ int tor_addr_is_valid_ipv4n(uint32_t v4n_addr, int for_listening); #define tor_addr_is_valid_ipv4h(v4h_addr, for_listening) \ tor_addr_is_valid_ipv4n(htonl(v4h_addr), (for_listening)) int tor_port_is_valid(uint16_t port, int for_listening); + /* Are addr and port both valid? */ #define tor_addr_port_is_valid(addr, port, for_listening) \ (tor_addr_is_valid((addr), (for_listening)) && \ @@ -329,9 +359,11 @@ int parse_port_range(const char *port, uint16_t *port_min_out, uint16_t *port_max_out); int addr_mask_get_bits(uint32_t mask); char *tor_dup_ip(uint32_t addr) ATTR_MALLOC; + MOCK_DECL(int,get_interface_address,(int severity, uint32_t *addr)); #define interface_address_list_free(lst)\ interface_address6_list_free(lst) + /** Return a smartlist of the IPv4 addresses of all interfaces on the server. * Excludes loopback and multicast addresses. Only includes internal addresses * if include_internal is true. (Note that a relay behind NAT may use an @@ -377,8 +409,8 @@ STATIC struct smartlist_t *get_interface_addresses_win32(int severity, #endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */ #ifdef HAVE_IFCONF_TO_SMARTLIST -STATIC struct smartlist_t *ifreq_to_smartlist(char *ifr, - size_t buflen); +STATIC struct smartlist_t *ifreq_to_smartlist(const uint8_t *ifr, + size_t buflen); STATIC struct smartlist_t *get_interface_addresses_ioctl(int severity, sa_family_t family); #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */ diff --git a/src/lib/net/alertsock.c b/src/lib/net/alertsock.c index cc59d7d893..537fdcaee4 100644 --- a/src/lib/net/alertsock.c +++ b/src/lib/net/alertsock.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/alertsock.h b/src/lib/net/alertsock.h index c45f42be81..dab4273cf1 100644 --- a/src/lib/net/alertsock.h +++ b/src/lib/net/alertsock.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -42,4 +42,4 @@ typedef struct alert_sockets_t { int alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags); void alert_sockets_close(alert_sockets_t *socks); -#endif +#endif /* !defined(TOR_ALERTSOCK_H) */ diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 3eb0a033d5..4dbf491e1a 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,7 +11,7 @@ #define BUFFERS_PRIVATE #include "lib/net/buffers_net.h" -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/net/nettypes.h" @@ -22,6 +22,10 @@ #include <stdlib.h> +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + #ifdef PARANOIA /** Helper: If PARANOIA is defined, assert that the buffer in local variable * <b>buf</b> is well-formed. */ @@ -30,27 +34,36 @@ #define check() STMT_NIL #endif /* defined(PARANOIA) */ -/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into +/** Read up to <b>at_most</b> bytes from the file descriptor <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set - * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, - * and the number of bytes read otherwise. */ + * *<b>reached_eof</b> to 1. Uses <b>tor_socket_recv()</b> iff <b>is_socket</b> + * is true, otherwise it uses <b>read()</b>. Return -1 on error (and sets + * *<b>error</b> to errno), 0 on eof or blocking, and the number of bytes read + * otherwise. */ static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, - int *reached_eof, int *socket_error) + int *reached_eof, int *error, bool is_socket) { ssize_t read_result; if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) at_most = CHUNK_REMAINING_CAPACITY(chunk); - read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (is_socket) + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + else + read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most); if (read_result < 0) { - int e = tor_socket_errno(fd); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) - log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); + log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?", + is_socket ? "recv" : "read"); #endif - *socket_error = e; + if (error) + *error = e; return -1; } return 0; /* would block. */ @@ -63,21 +76,22 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, chunk->datalen += read_result; log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result, (int)buf->datalen); - tor_assert(read_result < INT_MAX); + tor_assert(read_result <= BUF_MAX_LEN); return (int)read_result; } } -/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most - * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 - * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on - * error; else return the number of bytes read. +/** Read from file descriptor <b>fd</b>, writing onto end of <b>buf</b>. Read + * at most <b>at_most</b> bytes, growing the buffer as necessary. If recv() + * returns 0 (because of EOF), set *<b>reached_eof</b> to 1 and return 0. + * Return -1 on error; else return the number of bytes read. */ /* XXXX indicate "read blocked" somehow? */ -int -buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, - int *reached_eof, - int *socket_error) +static int +buf_read_from_fd(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error, + bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. @@ -87,11 +101,11 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, check(); tor_assert(reached_eof); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); - if (BUG(buf->datalen >= INT_MAX)) + if (BUG(buf->datalen > BUF_MAX_LEN)) return -1; - if (BUG(buf->datalen >= INT_MAX - at_most)) + if (BUG(buf->datalen > BUF_MAX_LEN - at_most)) return -1; while (at_most > total_read) { @@ -108,11 +122,12 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + r = read_to_chunk(buf, chunk, fd, readlen, + reached_eof, socket_error, is_socket); check(); if (r < 0) return r; /* Error */ - tor_assert(total_read+r < INT_MAX); + tor_assert(total_read+r <= BUF_MAX_LEN); total_read += r; if ((size_t)r < readlen) { /* eof, block, or no more to read. */ break; @@ -122,22 +137,27 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, } /** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk - * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>. On success, deduct - * the bytes written from *<b>buf_flushlen</b>. Return the number of bytes - * written on success, 0 on blocking, -1 on failure. + * <b>chunk</b> of buffer <b>buf</b> onto file descriptor <b>fd</b>. On + * success, deduct the bytes written from *<b>buf_flushlen</b>. Return the + * number of bytes written on success, 0 on blocking, -1 on failure. */ static inline int -flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, - size_t *buf_flushlen) +flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen, bool is_socket) { ssize_t write_result; if (sz > chunk->datalen) sz = chunk->datalen; - write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (is_socket) + write_result = tor_socket_send(fd, chunk->data, sz, 0); + else + write_result = write(fd, chunk->data, sz); if (write_result < 0) { - int e = tor_socket_errno(s); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) @@ -150,20 +170,20 @@ flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, } else { *buf_flushlen -= write_result; buf_drain(buf, write_result); - tor_assert(write_result < INT_MAX); + tor_assert(write_result <= BUF_MAX_LEN); return (int)write_result; } } -/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most +/** Write data from <b>buf</b> to the file descriptor <b>fd</b>. Write at most * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by * the number of bytes actually written, and remove the written bytes * from the buffer. Return the number of bytes written on success, * -1 on failure. Return 0 if write() would block. */ -int -buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, - size_t *buf_flushlen) +static int +buf_flush_to_fd(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen, bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. @@ -171,7 +191,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, int r; size_t flushed = 0; tor_assert(buf_flushlen); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(*buf_flushlen > buf->datalen)) { *buf_flushlen = buf->datalen; } @@ -188,7 +208,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket); check(); if (r < 0) return r; @@ -197,6 +217,58 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */ break; } - tor_assert(flushed < INT_MAX); + tor_assert(flushed <= BUF_MAX_LEN); return (int)flushed; } + +/** Write data from <b>buf</b> to the socket <b>s</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); +} + +/** Read from socket <b>s</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); +} + +/** Write data from <b>buf</b> to the pipe <b>fd</b>. Write at most + * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_pipe(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false); +} + +/** Read from pipe <b>fd</b>, writing onto end of <b>buf</b>. Read at most + * <b>at_most</b> bytes, growing the buffer as necessary. If read() returns 0 + * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_pipe(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false); +} diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h index 5f69bebedf..a45c23a273 100644 --- a/src/lib/net/buffers_net.h +++ b/src/lib/net/buffers_net.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most, int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz, size_t *buf_flushlen); -#endif /* !defined(TOR_BUFFERS_H) */ +int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error); + +int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen); + +#endif /* !defined(TOR_BUFFERS_NET_H) */ diff --git a/src/lib/net/gethostname.c b/src/lib/net/gethostname.c index e54a1ea16e..001d95391d 100644 --- a/src/lib/net/gethostname.c +++ b/src/lib/net/gethostname.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/net/gethostname.h b/src/lib/net/gethostname.h index 69b0528bc0..90f8056779 100644 --- a/src/lib/net/gethostname.h +++ b/src/lib/net/gethostname.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,4 +16,4 @@ MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen)); -#endif +#endif /* !defined(TOR_GETHOSTNAME_H) */ diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c index 1a2406ce5f..d50ac2440c 100644 --- a/src/lib/net/inaddr.c +++ b/src/lib/net/inaddr.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,7 +11,9 @@ #include "lib/net/inaddr.h" #include "lib/cc/torint.h" +#include "lib/container/smartlist.h" #include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" #include "lib/net/inaddr_st.h" #include "lib/string/compat_ctype.h" #include "lib/string/compat_string.h" @@ -35,12 +37,31 @@ * (Like inet_aton(str,addr), but works on Windows and Solaris.) */ int -tor_inet_aton(const char *str, struct in_addr* addr) +tor_inet_aton(const char *str, struct in_addr *addr) { - unsigned a,b,c,d; + unsigned a, b, c, d; char more; - if (tor_sscanf(str, "%3u.%3u.%3u.%3u%c", &a,&b,&c,&d,&more) != 4) + bool is_octal = false; + smartlist_t *sl = NULL; + + if (tor_sscanf(str, "%3u.%3u.%3u.%3u%c", &a, &b, &c, &d, &more) != 4) + return 0; + + /* Parse the octets and check them for leading zeros. */ + sl = smartlist_new(); + smartlist_split_string(sl, str, ".", 0, 0); + SMARTLIST_FOREACH(sl, const char *, octet, { + is_octal = (strlen(octet) > 1 && octet[0] == '0'); + if (is_octal) { + break; + } + }); + SMARTLIST_FOREACH(sl, char *, octet, tor_free(octet)); + smartlist_free(sl); + + if (is_octal) return 0; + if (a > 255) return 0; if (b > 255) return 0; if (c > 255) return 0; @@ -168,6 +189,13 @@ tor_inet_pton(int af, const char *src, void *dst) if (af == AF_INET) { return tor_inet_aton(src, dst); } else if (af == AF_INET6) { + ssize_t len = strlen(src); + + /* Reject if src has needless trailing ':'. */ + if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') { + return 0; + } + struct in6_addr *out = dst; uint16_t words[8]; int gapPos = -1, i, setWords=0; @@ -207,7 +235,6 @@ tor_inet_pton(int af, const char *src, void *dst) return 0; if (TOR_ISXDIGIT(*src)) { char *next; - ssize_t len; long r = strtol(src, &next, 16); if (next == NULL || next == src) { /* The 'next == src' error case can happen on versions of openbsd diff --git a/src/lib/net/inaddr.h b/src/lib/net/inaddr.h index 36352b65ea..8d6766eb5d 100644 --- a/src/lib/net/inaddr.h +++ b/src/lib/net/inaddr.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -24,4 +24,4 @@ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len); const char *tor_inet_ntop(int af, const void *src, char *dst, size_t len); int tor_inet_pton(int af, const char *src, void *dst); -#endif +#endif /* !defined(TOR_INADDR_H) */ diff --git a/src/lib/net/inaddr_st.h b/src/lib/net/inaddr_st.h index 806f2c096a..b9ee2b86cf 100644 --- a/src/lib/net/inaddr_st.h +++ b/src/lib/net/inaddr_st.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -104,4 +104,4 @@ struct sockaddr_in6 { }; #endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */ -#endif /* TOR_INADDR_ST_H */ +#endif /* !defined(TOR_INADDR_ST_H) */ diff --git a/src/lib/net/include.am b/src/lib/net/include.am index ff0967e786..485019f4b7 100644 --- a/src/lib/net/include.am +++ b/src/lib/net/include.am @@ -5,12 +5,14 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-net-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_net_a_SOURCES = \ src/lib/net/address.c \ src/lib/net/alertsock.c \ src/lib/net/buffers_net.c \ src/lib/net/gethostname.c \ src/lib/net/inaddr.c \ + src/lib/net/network_sys.c \ src/lib/net/resolve.c \ src/lib/net/socket.c \ src/lib/net/socketpair.c @@ -20,6 +22,7 @@ src_lib_libtor_net_testing_a_SOURCES = \ src_lib_libtor_net_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_net_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/net/address.h \ src/lib/net/alertsock.h \ @@ -28,6 +31,7 @@ noinst_HEADERS += \ src/lib/net/inaddr.h \ src/lib/net/inaddr_st.h \ src/lib/net/nettypes.h \ + src/lib/net/network_sys.h \ src/lib/net/resolve.h \ src/lib/net/socket.h \ src/lib/net/socketpair.h \ diff --git a/src/lib/net/lib_net.md b/src/lib/net/lib_net.md new file mode 100644 index 0000000000..b61878d827 --- /dev/null +++ b/src/lib/net/lib_net.md @@ -0,0 +1,6 @@ +@dir /lib/net +@brief lib/net: Low-level network-related code. + +This module includes address manipulation, compatibility wrappers, +convenience functions, and so on. + diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h index 6209bbe18a..953673d4c3 100644 --- a/src/lib/net/nettypes.h +++ b/src/lib/net/nettypes.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -31,7 +31,7 @@ typedef int socklen_t; #define TOR_SOCKET_T_FORMAT "%"PRIuPTR #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET) #define TOR_INVALID_SOCKET INVALID_SOCKET -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /** Type used for a network socket. */ #define tor_socket_t int #define TOR_SOCKET_T_FORMAT "%d" @@ -41,4 +41,4 @@ typedef int socklen_t; #define TOR_INVALID_SOCKET (-1) #endif /* defined(_WIN32) */ -#endif +#endif /* !defined(TOR_NET_TYPES_H) */ diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c new file mode 100644 index 0000000000..e95c3ba819 --- /dev/null +++ b/src/lib/net/network_sys.c @@ -0,0 +1,47 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file network_sys.c + * \brief Subsystem object for networking setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/net/network_sys.h" +#include "lib/net/resolve.h" +#include "lib/net/socket.h" + +#ifdef _WIN32 +#include <winsock2.h> +#include <windows.h> +#endif + +static int +subsys_network_initialize(void) +{ + if (network_init() < 0) + return -1; + + return 0; +} + +static void +subsys_network_shutdown(void) +{ +#ifdef _WIN32 + WSACleanup(); +#endif + tor_free_getaddrinfo_cache(); +} + +const subsys_fns_t sys_network = { + .name = "network", + SUBSYS_DECLARE_LOCATION(), + /* Network depends on logging, and a lot of other modules depend on network. + */ + .level = -55, + .supported = true, + .initialize = subsys_network_initialize, + .shutdown = subsys_network_shutdown, +}; diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h new file mode 100644 index 0000000000..734533c7e8 --- /dev/null +++ b/src/lib/net/network_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file network_sys.h + * \brief Declare subsystem object for the network module. + **/ + +#ifndef TOR_NETWORK_SYS_H +#define TOR_NETWORK_SYS_H + +extern const struct subsys_fns_t sys_network; + +#endif /* !defined(TOR_NETWORK_SYS_H) */ diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c index 8cee29df37..68a8c01ef4 100644 --- a/src/lib/net/resolve.c +++ b/src/lib/net/resolve.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -8,6 +8,7 @@ * \brief Use the libc DNS resolver to convert hostnames into addresses. **/ +#define RESOLVE_PRIVATE #include "lib/net/resolve.h" #include "lib/net/address.h" @@ -16,8 +17,8 @@ #include "lib/string/parse_int.h" #include "lib/string/util_string.h" -#include "siphash.h" -#include "ht.h" +#include "ext/siphash.h" +#include "ext/ht.h" #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> @@ -35,6 +36,8 @@ * *<b>addr</b> to the proper IP address, in host byte order. Returns 0 * on success, -1 on failure; 1 on transient failure. * + * This function only accepts IPv4 addresses. + * * (This function exists because standard windows gethostbyname * doesn't treat raw IP addresses properly.) */ @@ -45,6 +48,11 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) tor_addr_t myaddr; int ret; + if (BUG(!addr)) + return -1; + + *addr = 0; + if ((ret = tor_addr_lookup(name, AF_INET, &myaddr))) return ret; @@ -56,183 +64,257 @@ tor_lookup_hostname,(const char *name, uint32_t *addr)) return -1; } -/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set - * *<b>addr</b> to the proper IP address and family. The <b>family</b> - * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a - * <i>preferred</i> family, though another one may be returned if only one - * family is implemented for this address. +#ifdef HAVE_GETADDRINFO + +/* Host lookup helper for tor_addr_lookup(), when getaddrinfo() is + * available on this system. * - * Return 0 on success, -1 on failure; 1 on transient failure. + * See tor_addr_lookup() for details. */ -MOCK_IMPL(int, -tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) +MOCK_IMPL(STATIC int, +tor_addr_lookup_host_impl,(const char *name, + uint16_t family, + tor_addr_t *addr)) { - /* Perhaps eventually this should be replaced by a tor_getaddrinfo or - * something. - */ - struct in_addr iaddr; - struct in6_addr iaddr6; - tor_assert(name); - tor_assert(addr); - tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC); - if (!*name) { - /* Empty address is an error. */ - return -1; - } else if (tor_inet_pton(AF_INET, name, &iaddr)) { - /* It's an IPv4 IP. */ - if (family == AF_INET6) - return -1; - tor_addr_from_in(addr, &iaddr); - return 0; - } else if (tor_inet_pton(AF_INET6, name, &iaddr6)) { - if (family == AF_INET) - return -1; - tor_addr_from_in6(addr, &iaddr6); - return 0; - } else { -#ifdef HAVE_GETADDRINFO - int err; - struct addrinfo *res=NULL, *res_p; - struct addrinfo *best=NULL; - struct addrinfo hints; - int result = -1; - memset(&hints, 0, sizeof(hints)); - hints.ai_family = family; - hints.ai_socktype = SOCK_STREAM; - err = tor_getaddrinfo(name, NULL, &hints, &res); - /* The check for 'res' here shouldn't be necessary, but it makes static - * analysis tools happy. */ - if (!err && res) { - best = NULL; - for (res_p = res; res_p; res_p = res_p->ai_next) { - if (family == AF_UNSPEC) { - if (res_p->ai_family == AF_INET) { - best = res_p; - break; - } else if (res_p->ai_family == AF_INET6 && !best) { - best = res_p; - } - } else if (family == res_p->ai_family) { + int err; + struct addrinfo *res=NULL, *res_p; + struct addrinfo *best=NULL; + struct addrinfo hints; + int result = -1; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = family; + hints.ai_socktype = SOCK_STREAM; + err = tor_getaddrinfo(name, NULL, &hints, &res); + /* The check for 'res' here shouldn't be necessary, but it makes static + * analysis tools happy. */ + if (!err && res) { + best = NULL; + for (res_p = res; res_p; res_p = res_p->ai_next) { + if (family == AF_UNSPEC) { + if (res_p->ai_family == AF_INET) { best = res_p; break; + } else if (res_p->ai_family == AF_INET6 && !best) { + best = res_p; } + } else if (family == res_p->ai_family) { + best = res_p; + break; } - if (!best) - best = res; - if (best->ai_family == AF_INET) { - tor_addr_from_in(addr, - &((struct sockaddr_in*)best->ai_addr)->sin_addr); - result = 0; - } else if (best->ai_family == AF_INET6) { - tor_addr_from_in6(addr, - &((struct sockaddr_in6*)best->ai_addr)->sin6_addr); - result = 0; - } - tor_freeaddrinfo(res); - return result; } - return (err == EAI_AGAIN) ? 1 : -1; -#else /* !(defined(HAVE_GETADDRINFO)) */ - struct hostent *ent; - int err; + if (!best) + best = res; + if (best->ai_family == AF_INET) { + tor_addr_from_in(addr, + &((struct sockaddr_in*)best->ai_addr)->sin_addr); + result = 0; + } else if (best->ai_family == AF_INET6) { + tor_addr_from_in6(addr, + &((struct sockaddr_in6*)best->ai_addr)->sin6_addr); + result = 0; + } + tor_freeaddrinfo(res); + return result; + } + return (err == EAI_AGAIN) ? 1 : -1; +} + +#else /* !defined(HAVE_GETADDRINFO) */ + +/* Host lookup helper for tor_addr_lookup(), which calls gethostbyname(). + * Used when getaddrinfo() is not available on this system. + * + * See tor_addr_lookup() for details. + */ +MOCK_IMPL(STATIC int, +tor_addr_lookup_host_impl,(const char *name, + uint16_t family, + tor_addr_t *addr)) +{ + (void) family; + struct hostent *ent; + int err; #ifdef HAVE_GETHOSTBYNAME_R_6_ARG - char buf[2048]; - struct hostent hostent; - int r; - r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err); + char buf[2048]; + struct hostent hostent; + int r; + r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err); #elif defined(HAVE_GETHOSTBYNAME_R_5_ARG) - char buf[2048]; - struct hostent hostent; - ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err); + char buf[2048]; + struct hostent hostent; + ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err); #elif defined(HAVE_GETHOSTBYNAME_R_3_ARG) - struct hostent_data data; - struct hostent hent; - memset(&data, 0, sizeof(data)); - err = gethostbyname_r(name, &hent, &data); - ent = err ? NULL : &hent; + struct hostent_data data; + struct hostent hent; + memset(&data, 0, sizeof(data)); + err = gethostbyname_r(name, &hent, &data); + ent = err ? NULL : &hent; #else - ent = gethostbyname(name); + ent = gethostbyname(name); #ifdef _WIN32 - err = WSAGetLastError(); + err = WSAGetLastError(); #else - err = h_errno; -#endif + err = h_errno; +#endif /* defined(_WIN32) */ #endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */ - if (ent) { - if (ent->h_addrtype == AF_INET) { - tor_addr_from_in(addr, (struct in_addr*) ent->h_addr); - } else if (ent->h_addrtype == AF_INET6) { - tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr); - } else { - tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type - } - return 0; + if (ent) { + if (ent->h_addrtype == AF_INET) { + tor_addr_from_in(addr, (struct in_addr*) ent->h_addr); + } else if (ent->h_addrtype == AF_INET6) { + tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr); + } else { + tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type } + return 0; + } #ifdef _WIN32 - return (err == WSATRY_AGAIN) ? 1 : -1; + return (err == WSATRY_AGAIN) ? 1 : -1; #else - return (err == TRY_AGAIN) ? 1 : -1; + return (err == TRY_AGAIN) ? 1 : -1; #endif +} #endif /* defined(HAVE_GETADDRINFO) */ + +/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set + * *<b>addr</b> to the proper IP address and family. The <b>family</b> + * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a + * <i>preferred</i> family, though another one may be returned if only one + * family is implemented for this address. + * + * Like tor_addr_parse(), this function accepts IPv6 addresses with or without + * square brackets. + * + * Return 0 on success, -1 on failure; 1 on transient failure. + */ +MOCK_IMPL(int, +tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr)) +{ + /* Perhaps eventually this should be replaced by a tor_getaddrinfo or + * something. + */ + int parsed_family = 0; + int result = -1; + + tor_assert(name); + tor_assert(addr); + tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC); + + if (!*name) { + /* Empty address is an error. */ + goto permfail; + } + + /* Is it an IP address? */ + parsed_family = tor_addr_parse(addr, name); + + if (parsed_family >= 0) { + /* If the IP address family matches, or was unspecified */ + if (parsed_family == family || family == AF_UNSPEC) { + goto success; + } else { + goto permfail; + } + } else { + /* Clear the address after a failed tor_addr_parse(). */ + memset(addr, 0, sizeof(tor_addr_t)); + result = tor_addr_lookup_host_impl(name, family, addr); + goto done; } + + /* If we weren't successful, and haven't already set the result, + * assume it's a permanent failure */ + permfail: + result = -1; + goto done; + success: + result = 0; + + /* We have set the result, now it's time to clean up */ + done: + if (result) { + /* Clear the address on error */ + memset(addr, 0, sizeof(tor_addr_t)); + } + return result; } /** Parse an address or address-port combination from <b>s</b>, resolve the * address as needed, and put the result in <b>addr_out</b> and (optionally) - * <b>port_out</b>. Return 0 on success, negative on failure. */ + * <b>port_out</b>. + * + * Like tor_addr_port_parse(), this function accepts: + * - IPv6 address and port, when the IPv6 address is in square brackets, + * - IPv6 address with square brackets, + * - IPv6 address without square brackets. + * + * Return 0 on success, negative on failure. */ int tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out) { - const char *port; tor_addr_t addr; - uint16_t portval; + uint16_t portval = 0; char *tmp = NULL; + int rv = 0; + int result; tor_assert(s); tor_assert(addr_out); s = eat_whitespace(s); - if (*s == '[') { - port = strstr(s, "]"); - if (!port) - goto err; - tmp = tor_strndup(s+1, port-(s+1)); - port = port+1; - if (*port == ':') - port++; - else - port = NULL; - } else { - port = strchr(s, ':'); - if (port) - tmp = tor_strndup(s, port-s); - else - tmp = tor_strdup(s); - if (port) - ++port; + /* Try parsing s as an address:port first, so we don't have to duplicate + * the logic that rejects IPv6:Port with no square brackets. */ + rv = tor_addr_port_parse(LOG_WARN, s, &addr, &portval, 0); + /* That was easy, no DNS required. */ + if (rv == 0) + goto success; + + /* Now let's check for malformed IPv6 addresses and ports: + * tor_addr_port_parse() requires squared brackes if there is a port, + * and we want tor_addr_port_lookup() to have the same requirement. + * But we strip the port using tor_addr_port_split(), so tor_addr_lookup() + * only sees the address, and will accept it without square brackets. */ + int family = tor_addr_parse(&addr, s); + /* If tor_addr_parse() succeeds where tor_addr_port_parse() failed, we need + * to reject this address as malformed. */ + if (family >= 0) { + /* Double-check it's an IPv6 address. If not, we have a parsing bug. + */ + tor_assertf_nonfatal(family == AF_INET6, + "Wrong family: %d (should be IPv6: %d) which " + "failed IP:port parsing, but passed IP parsing. " + "input string: '%s'; parsed address: '%s'.", + family, AF_INET6, s, fmt_addr(&addr)); + goto err; } - if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0) + /* Now we have a hostname. Let's split off the port, if any. */ + rv = tor_addr_port_split(LOG_WARN, s, &tmp, &portval); + if (rv < 0) goto err; - tor_free(tmp); - if (port) { - portval = (int) tor_parse_long(port, 10, 1, 65535, NULL, NULL); - if (!portval) - goto err; - } else { - portval = 0; - } + /* And feed the hostname to the lookup function. */ + if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0) + goto err; + success: if (port_out) *port_out = portval; tor_addr_copy(addr_out, &addr); + result = 0; + goto done; - return 0; err: + /* Clear the address and port on error */ + memset(addr_out, 0, sizeof(tor_addr_t)); + if (port_out) + *port_out = 0; + result = -1; + + /* We have set the result, now it's time to clean up */ + done: tor_free(tmp); - return -1; + return result; } #ifdef USE_SANDBOX_GETADDRINFO @@ -290,11 +372,11 @@ static HT_HEAD(getaddrinfo_cache, cached_getaddrinfo_item_t) HT_PROTOTYPE(getaddrinfo_cache, cached_getaddrinfo_item_t, node, cached_getaddrinfo_item_hash, - cached_getaddrinfo_items_eq) + cached_getaddrinfo_items_eq); HT_GENERATE2(getaddrinfo_cache, cached_getaddrinfo_item_t, node, cached_getaddrinfo_item_hash, cached_getaddrinfo_items_eq, - 0.6, tor_reallocarray_, tor_free_) + 0.6, tor_reallocarray_, tor_free_); /** If true, don't try to cache getaddrinfo results. */ static int sandbox_getaddrinfo_cache_disabled = 0; @@ -421,4 +503,13 @@ tor_make_getaddrinfo_cache_active(void) { sandbox_getaddrinfo_is_active = 1; } -#endif +#else /* !defined(USE_SANDBOX_GETADDRINFO) */ +void +sandbox_disable_getaddrinfo_cache(void) +{ +} +void +tor_make_getaddrinfo_cache_active(void) +{ +} +#endif /* defined(USE_SANDBOX_GETADDRINFO) */ diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h index 47a283c81c..ef3d9fa176 100644 --- a/src/lib/net/resolve.h +++ b/src/lib/net/resolve.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -24,12 +24,18 @@ struct tor_addr_t; +/* + * Primary lookup functions. + */ MOCK_DECL(int, tor_lookup_hostname,(const char *name, uint32_t *addr)); MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family, struct tor_addr_t *addr_out)); int tor_addr_port_lookup(const char *s, struct tor_addr_t *addr_out, uint16_t *port_out); +/* + * Sandbox helpers + */ struct addrinfo; #ifdef USE_SANDBOX_GETADDRINFO /** Pre-calls getaddrinfo in order to pre-record result. */ @@ -42,8 +48,7 @@ int tor_getaddrinfo(const char *name, const char *servname, struct addrinfo **res); void tor_freeaddrinfo(struct addrinfo *addrinfo); void tor_free_getaddrinfo_cache(void); -void tor_make_getaddrinfo_cache_active(void); -#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */ +#else /* !defined(USE_SANDBOX_GETADDRINFO) */ #define tor_getaddrinfo(name, servname, hints, res) \ getaddrinfo((name),(servname), (hints),(res)) #define tor_add_addrinfo(name) \ @@ -54,5 +59,15 @@ void tor_make_getaddrinfo_cache_active(void); #endif /* defined(USE_SANDBOX_GETADDRINFO) */ void sandbox_disable_getaddrinfo_cache(void); +void tor_make_getaddrinfo_cache_active(void); +/* + * Internal resolver wrapper; exposed for mocking. + */ +#ifdef RESOLVE_PRIVATE +MOCK_DECL(STATIC int, tor_addr_lookup_host_impl, (const char *name, + uint16_t family, + struct tor_addr_t *addr)); #endif + +#endif /* !defined(TOR_RESOLVE_H) */ diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c index fba90b7506..adc060a735 100644 --- a/src/lib/net/socket.c +++ b/src/lib/net/socket.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -9,7 +9,6 @@ * sockets. **/ -#define SOCKET_PRIVATE #include "lib/net/socket.h" #include "lib/net/socketpair.h" #include "lib/net/address.h" @@ -31,6 +30,9 @@ #endif #include <stddef.h> #include <string.h> +#ifdef __FreeBSD__ +#include <sys/sysctl.h> +#endif /** Called before we make any calls to network-related functions. * (Some operating systems require their network libraries to be @@ -60,6 +62,32 @@ network_init(void) return 0; } +/** + * Warn the user if any system network parameters should be changed. + */ +void +check_network_configuration(bool server_mode) +{ +#ifdef __FreeBSD__ + if (server_mode) { + int random_id_state; + size_t state_size = sizeof(random_id_state); + + if (sysctlbyname("net.inet.ip.random_id", &random_id_state, + &state_size, NULL, 0)) { + log_warn(LD_CONFIG, + "Failed to figure out if IP ids are randomized."); + } else if (random_id_state == 0) { + log_warn(LD_CONFIG, "Looks like IP ids are not randomized. " + "Please consider setting the net.inet.ip.random_id sysctl, " + "so your relay makes it harder to figure out how busy it is."); + } + } +#else /* !defined(__FreeBSD__) */ + (void) server_mode; +#endif /* defined(__FreeBSD__) */ +} + /* When set_max_file_sockets() is called, update this with the max file * descriptor value so we can use it to check the limit when opening a new * socket. Default value is what Debian sets as the default hard limit. */ @@ -177,7 +205,7 @@ mark_socket_closed(tor_socket_t s) bitarray_clear(open_sockets, s); } } -#else /* !(defined(DEBUG_SOCKET_COUNTING)) */ +#else /* !defined(DEBUG_SOCKET_COUNTING) */ #define mark_socket_open(s) ((void) (s)) #define mark_socket_closed(s) ((void) (s)) #endif /* defined(DEBUG_SOCKET_COUNTING) */ @@ -279,7 +307,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol, return TOR_INVALID_SOCKET; } } -#else /* !(defined(FD_CLOEXEC)) */ +#else /* !defined(FD_CLOEXEC) */ (void)cloexec; #endif /* defined(FD_CLOEXEC) */ @@ -389,7 +417,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr, return TOR_INVALID_SOCKET; } } -#else /* !(defined(FD_CLOEXEC)) */ +#else /* !defined(FD_CLOEXEC) */ (void)cloexec; #endif /* defined(FD_CLOEXEC) */ @@ -429,7 +457,9 @@ get_n_open_sockets(void) * localhost is inaccessible (for example, if the networking * stack is down). And even if it succeeds, the socket pair will not * be able to read while localhost is down later (the socket pair may - * even close, depending on OS-specific timeouts). + * even close, depending on OS-specific timeouts). The socket pair + * should work on IPv4-only, IPv6-only, and dual-stack systems, as long + * as they have the standard localhost addresses. * * Returns 0 on success and -errno on failure; do not rely on the value * of errno or WSAGetLastError(). @@ -456,11 +486,11 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) r = socketpair(family, type, protocol, fd); if (r < 0) return -errno; -#else +#else /* !(defined(HAVE_SOCKETPAIR) && !defined(_WIN32)) */ r = tor_ersatz_socketpair(family, type, protocol, fd); if (r < 0) return -r; -#endif +#endif /* defined(HAVE_SOCKETPAIR) && !defined(_WIN32) */ #if defined(FD_CLOEXEC) if (SOCKET_OK(fd[0])) { diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h index 0909619510..46735fdef0 100644 --- a/src/lib/net/socket.h +++ b/src/lib/net/socket.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock); int set_socket_nonblocking(tor_socket_t socket); int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); int network_init(void); +void check_network_configuration(bool server_mode); int get_max_sockets(void); void set_max_sockets(int); @@ -91,7 +92,7 @@ ssize_t read_all_from_socket(tor_socket_t fd, char *buf, size_t count); #define ERRNO_IS_EINTR(e) ((e) == WSAEINTR || 0) int tor_socket_errno(tor_socket_t sock); const char *tor_socket_strerror(int e); -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ #define SOCK_ERRNO(e) e #if EAGAIN == EWOULDBLOCK /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */ @@ -115,4 +116,4 @@ const char *tor_socket_strerror(int e); #define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b #endif -#endif +#endif /* !defined(TOR_SOCKET_H) */ diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c index 10eb749735..d4310020cb 100644 --- a/src/lib/net/socketpair.c +++ b/src/lib/net/socketpair.c @@ -1,6 +1,11 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ + +/** + * @file socketpair.c + * @brief Replacement socketpair() for systems that lack it + **/ #include "lib/cc/torint.h" #include "lib/net/socketpair.h" @@ -22,11 +27,11 @@ #include <windows.h> #define socket_errno() (WSAGetLastError()) #define SOCKET_EPROTONOSUPPORT WSAEPROTONOSUPPORT -#else +#else /* !defined(_WIN32) */ #define closesocket(x) close(x) #define socket_errno() (errno) #define SOCKET_EPROTONOSUPPORT EPROTONOSUPPORT -#endif +#endif /* defined(_WIN32) */ #ifdef NEED_ERSATZ_SOCKETPAIR @@ -105,7 +110,12 @@ sockaddr_eq(struct sockaddr *sa1, struct sockaddr *sa2) /** * Helper used to implement socketpair on systems that lack it, by * making a direct connection to localhost. - */ + * + * See tor_socketpair() for details. + * + * The direct connection defaults to IPv4, but falls back to IPv6 if + * IPv4 is not supported. + **/ int tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]) { diff --git a/src/lib/net/socketpair.h b/src/lib/net/socketpair.h index 6be0803881..b07016ab94 100644 --- a/src/lib/net/socketpair.h +++ b/src/lib/net/socketpair.h @@ -1,11 +1,16 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_SOCKETPAIR_H #define TOR_SOCKETPAIR_H +/** + * @file socketpair.h + * @brief Header for socketpair.c + **/ + #include "orconfig.h" #include "lib/testsupport/testsupport.h" #include "lib/net/nettypes.h" @@ -16,4 +21,4 @@ int tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2]); #endif -#endif +#endif /* !defined(TOR_SOCKETPAIR_H) */ diff --git a/src/lib/net/socks5_status.h b/src/lib/net/socks5_status.h index e55242ce66..90c4305d29 100644 --- a/src/lib/net/socks5_status.h +++ b/src/lib/net/socks5_status.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -27,6 +27,17 @@ typedef enum { SOCKS5_TTL_EXPIRED = 0x06, SOCKS5_COMMAND_NOT_SUPPORTED = 0x07, SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08, + + /* Extended error code (see prop304). Only used if the SocksPort flag + * "ExtendedErrors" is set. */ + SOCKS5_HS_NOT_FOUND = 0xF0, + SOCKS5_HS_IS_INVALID = 0xF1, + SOCKS5_HS_INTRO_FAILED = 0xF2, + SOCKS5_HS_REND_FAILED = 0xF3, + SOCKS5_HS_MISSING_CLIENT_AUTH = 0xF4, + SOCKS5_HS_BAD_CLIENT_AUTH = 0xF5, + SOCKS5_HS_BAD_ADDRESS = 0xF6, + SOCKS5_HS_INTRO_TIMEDOUT = 0xF7, } socks5_reply_status_t; -#endif +#endif /* !defined(TOR_SOCKS5_STATUS_H) */ diff --git a/src/lib/osinfo/include.am b/src/lib/osinfo/include.am index 16c5812604..84bd7feb00 100644 --- a/src/lib/osinfo/include.am +++ b/src/lib/osinfo/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-osinfo-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_osinfo_a_SOURCES = \ src/lib/osinfo/uname.c @@ -13,5 +14,6 @@ src_lib_libtor_osinfo_testing_a_SOURCES = \ src_lib_libtor_osinfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_osinfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/osinfo/uname.h diff --git a/src/lib/osinfo/lib_osinfo.md b/src/lib/osinfo/lib_osinfo.md new file mode 100644 index 0000000000..0678ecc21e --- /dev/null +++ b/src/lib/osinfo/lib_osinfo.md @@ -0,0 +1,8 @@ +@dir /lib/osinfo +@brief lib/osinfo: For inspecting the OS version and capabilities. + +In general, we use this module when we're telling the user what operating +system they are running. We shouldn't make decisions based on the output of +these checks: instead, we should have more specific checks, either at compile +time or run time, based on the observed system behavior. + diff --git a/src/lib/osinfo/uname.c b/src/lib/osinfo/uname.c index 2b37ff136c..f7f5ede307 100644 --- a/src/lib/osinfo/uname.c +++ b/src/lib/osinfo/uname.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -27,6 +27,40 @@ static char uname_result[256]; /** True iff uname_result is set. */ static int uname_result_is_set = 0; +#ifdef _WIN32 +/** Table to map claimed windows versions into human-readable windows + * versions. */ +static struct { + unsigned major; + unsigned minor; + const char *client_version; + const char *server_version; +} win_version_table[] = { + /* This table must be sorted in descending order. + * Sources: + * https://en.wikipedia.org/wiki/List_of_Microsoft_Windows_versions + * https://docs.microsoft.com/en-us/windows/desktop/api/winnt/ + * ns-winnt-_osversioninfoexa#remarks + */ + /* Windows Server 2019 is indistinguishable from Windows Server 2016 + * using GetVersionEx(). + { 10, 0, NULL, "Windows Server 2019" }, */ + // clang-format off + { 10, 0, "Windows 10", "Windows Server 2016" }, + { 6, 3, "Windows 8.1", "Windows Server 2012 R2" }, + { 6, 2, "Windows 8", "Windows Server 2012" }, + { 6, 1, "Windows 7", "Windows Server 2008 R2" }, + { 6, 0, "Windows Vista", "Windows Server 2008" }, + { 5, 2, "Windows XP Professional", "Windows Server 2003" }, + /* Windows XP did not have a server version, but we need something here */ + { 5, 1, "Windows XP", "Windows XP Server" }, + { 5, 0, "Windows 2000 Professional", "Windows 2000 Server" }, + /* Earlier versions are not supported by GetVersionEx(). */ + { 0, 0, NULL, NULL } + // clang-format on +}; +#endif /* defined(_WIN32) */ + /** Return a pointer to a description of our platform. */ MOCK_IMPL(const char *, @@ -49,31 +83,6 @@ get_uname,(void)) int is_client = 0; int is_server = 0; const char *plat = NULL; - static struct { - unsigned major; unsigned minor; - const char *client_version; const char *server_version; - } win_version_table[] = { - /* This table must be sorted in descending order. - * Sources: - * https://en.wikipedia.org/wiki/List_of_Microsoft_Windows_versions - * https://docs.microsoft.com/en-us/windows/desktop/api/winnt/ - * ns-winnt-_osversioninfoexa#remarks - */ - /* Windows Server 2019 is indistinguishable from Windows Server 2016 - * using GetVersionEx(). - { 10, 0, NULL, "Windows Server 2019" }, */ - { 10, 0, "Windows 10", "Windows Server 2016" }, - { 6, 3, "Windows 8.1", "Windows Server 2012 R2" }, - { 6, 2, "Windows 8", "Windows Server 2012" }, - { 6, 1, "Windows 7", "Windows Server 2008 R2" }, - { 6, 0, "Windows Vista", "Windows Server 2008" }, - { 5, 2, "Windows XP Professional", "Windows Server 2003" }, - /* Windows XP did not have a server version, but we need something here */ - { 5, 1, "Windows XP", "Windows XP Server" }, - { 5, 0, "Windows 2000 Professional", "Windows 2000 Server" }, - /* Earlier versions are not supported by GetVersionEx(). */ - { 0, 0, NULL, NULL } - }; memset(&info, 0, sizeof(info)); info.dwOSVersionInfoSize = sizeof(info); if (! GetVersionEx((LPOSVERSIONINFO)&info)) { @@ -137,7 +146,7 @@ get_uname,(void)) if (!is_server && !is_client) { strlcat(uname_result, " [client or server]", sizeof(uname_result)); } -#else /* !(defined(_WIN32)) */ +#else /* !defined(_WIN32) */ /* LCOV_EXCL_START -- can't provoke uname failure */ strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); /* LCOV_EXCL_STOP */ diff --git a/src/lib/osinfo/uname.h b/src/lib/osinfo/uname.h index fcce629074..c6b1f43235 100644 --- a/src/lib/osinfo/uname.h +++ b/src/lib/osinfo/uname.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,4 +15,4 @@ MOCK_DECL(const char *, get_uname,(void)); -#endif +#endif /* !defined(HAVE_TOR_UNAME_H) */ diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index 05414d2a96..ce1b6ecf59 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -1,17 +1,20 @@ orconfig.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/ctime/*.h lib/err/*.h -lib/intmath/*.h +lib/evloop/*.h lib/fs/*.h +lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/net/*.h lib/process/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h -ht.h
\ No newline at end of file +ext/ht.h diff --git a/src/lib/process/daemon.c b/src/lib/process/daemon.c index 3b90bef671..b3b98a297e 100644 --- a/src/lib/process/daemon.c +++ b/src/lib/process/daemon.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -165,7 +165,7 @@ finish_daemon(const char *desired_cwd) return 0; } -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ /* defined(_WIN32) */ int start_daemon(void) diff --git a/src/lib/process/daemon.h b/src/lib/process/daemon.h index 20920e0aae..23f3117898 100644 --- a/src/lib/process/daemon.h +++ b/src/lib/process/daemon.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -18,4 +18,4 @@ int finish_daemon(const char *desired_cwd); bool start_daemon_has_been_called(void); -#endif +#endif /* !defined(TOR_DAEMON_H) */ diff --git a/src/lib/process/env.c b/src/lib/process/env.c index 0060200ba1..517b655a5e 100644 --- a/src/lib/process/env.c +++ b/src/lib/process/env.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,7 +16,6 @@ #include "lib/container/smartlist.h" #include "lib/log/util_bug.h" #include "lib/log/log.h" -#include "lib/malloc/malloc.h" #ifdef HAVE_UNISTD_H #include <unistd.h> @@ -47,7 +46,7 @@ get_environment(void) * when we do a mostly-static build on OSX 10.7, the resulting binary won't * work on OSX 10.6. */ return *_NSGetEnviron(); -#else /* !(defined(HAVE__NSGETENVIRON)) */ +#else /* !defined(HAVE__NSGETENVIRON) */ return environ; #endif /* defined(HAVE__NSGETENVIRON) */ } diff --git a/src/lib/process/env.h b/src/lib/process/env.h index 15d59351e0..7838dcaa90 100644 --- a/src/lib/process/env.h +++ b/src/lib/process/env.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -38,4 +38,4 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars, const char *new_var, void (*free_old)(void*), int free_p); -#endif +#endif /* !defined(TOR_ENV_H) */ diff --git a/src/lib/process/include.am b/src/lib/process/include.am index c6cc3a6699..18876b3f54 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -5,13 +5,17 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-process-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_process_a_SOURCES = \ src/lib/process/daemon.c \ src/lib/process/env.c \ src/lib/process/pidfile.c \ + src/lib/process/process.c \ + src/lib/process/process_sys.c \ + src/lib/process/process_unix.c \ + src/lib/process/process_win32.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ - src/lib/process/subprocess.c \ src/lib/process/waitpid.c src_lib_libtor_process_testing_a_SOURCES = \ @@ -19,11 +23,15 @@ src_lib_libtor_process_testing_a_SOURCES = \ src_lib_libtor_process_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_process_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/process/daemon.h \ src/lib/process/env.h \ src/lib/process/pidfile.h \ + src/lib/process/process.h \ + src/lib/process/process_sys.h \ + src/lib/process/process_unix.h \ + src/lib/process/process_win32.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ - src/lib/process/subprocess.h \ src/lib/process/waitpid.h diff --git a/src/lib/process/lib_process.md b/src/lib/process/lib_process.md new file mode 100644 index 0000000000..354129e70e --- /dev/null +++ b/src/lib/process/lib_process.md @@ -0,0 +1,2 @@ +@dir /lib/process +@brief lib/process: Launch and manage subprocesses. diff --git a/src/lib/process/pidfile.c b/src/lib/process/pidfile.c index 1b9d1c6d25..e7d9d2c47a 100644 --- a/src/lib/process/pidfile.c +++ b/src/lib/process/pidfile.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/pidfile.h b/src/lib/process/pidfile.h index dfeb39e046..d04302df3a 100644 --- a/src/lib/process/pidfile.h +++ b/src/lib/process/pidfile.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,4 +13,4 @@ int write_pidfile(const char *filename); -#endif +#endif /* !defined(TOR_PIDFILE_H) */ diff --git a/src/lib/process/process.c b/src/lib/process/process.c new file mode 100644 index 0000000000..12c1f9a772 --- /dev/null +++ b/src/lib/process/process.c @@ -0,0 +1,798 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.c + * \brief Module for working with other processes. + **/ + +#define PROCESS_PRIVATE +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/process_win32.h" +#include "lib/process/env.h" + +#ifdef HAVE_STDDEF_H +#include <stddef.h> +#endif + +/** A list of all <b>process_t</b> instances currently allocated. */ +static smartlist_t *processes; + +/** + * Boolean. If true, then Tor may call execve or CreateProcess via + * tor_spawn_background. + **/ +static int may_spawn_background_process = 1; + +/** Structure to represent a child process. */ +struct process_t { + /** Process status. */ + process_status_t status; + + /** Which protocol is the process using? */ + process_protocol_t protocol; + + /** Which function to call when we have data ready from stdout? */ + process_read_callback_t stdout_read_callback; + + /** Which function to call when we have data ready from stderr? */ + process_read_callback_t stderr_read_callback; + + /** Which function call when our process terminated? */ + process_exit_callback_t exit_callback; + + /** Our exit code when the process have terminated. */ + process_exit_code_t exit_code; + + /** Name of the command we want to execute (for example: /bin/ls). */ + char *command; + + /** The arguments used for the new process. The format here is one argument + * per element of the smartlist_t. On Windows these arguments are combined + * together using the <b>tor_join_win_cmdline</b> function. On Unix the + * process name (argv[0]) and the trailing NULL is added automatically before + * the process is executed. */ + smartlist_t *arguments; + + /** The environment used for the new process. */ + smartlist_t *environment; + + /** Buffer to store data from stdout when it is read. */ + buf_t *stdout_buffer; + + /** Buffer to store data from stderr when it is read. */ + buf_t *stderr_buffer; + + /** Buffer to store data to stdin before it is written. */ + buf_t *stdin_buffer; + + /** Do we need to store some custom data with the process? */ + void *data; + +#ifndef _WIN32 + /** Our Unix process handle. */ + process_unix_t *unix_process; +#else + /** Our Win32 process handle. */ + process_win32_t *win32_process; +#endif /* !defined(_WIN32) */ +}; + +/** Convert a given process status in <b>status</b> to its string + * representation. */ +const char * +process_status_to_string(process_status_t status) +{ + switch (status) { + case PROCESS_STATUS_NOT_RUNNING: + return "not running"; + case PROCESS_STATUS_RUNNING: + return "running"; + case PROCESS_STATUS_ERROR: + return "error"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Convert a given process protocol in <b>protocol</b> to its string + * representation. */ +const char * +process_protocol_to_string(process_protocol_t protocol) +{ + switch (protocol) { + case PROCESS_PROTOCOL_LINE: + return "Line"; + case PROCESS_PROTOCOL_RAW: + return "Raw"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** + * Turn off may_spawn_background_process, so that all future calls to + * tor_spawn_background are guaranteed to fail. + **/ +void +tor_disable_spawning_background_processes(void) +{ + may_spawn_background_process = 0; +} + +/** Initialize the Process subsystem. This function initializes the Process + * subsystem's global state. For cleaning up, <b>process_free_all()</b> should + * be called. */ +void +process_init(void) +{ + processes = smartlist_new(); + +#ifdef _WIN32 + process_win32_init(); +#endif +} + +/** Free up all resources that is handled by the Process subsystem. Note that + * this call does not terminate already running processes. */ +void +process_free_all(void) +{ +#ifdef _WIN32 + process_win32_deinit(); +#endif + + SMARTLIST_FOREACH(processes, process_t *, x, process_free(x)); + smartlist_free(processes); +} + +/** Get a list of all processes. This function returns a smartlist of + * <b>process_t</b> containing all the currently allocated processes. */ +const smartlist_t * +process_get_all_processes(void) +{ + return processes; +} + +/** Allocate and initialize a new process. This function returns a newly + * allocated and initialized process data, which can be used to configure and + * later run a subprocess of Tor. Use the various <b>process_set_*()</b> + * methods to configure it and run the process using <b>process_exec()</b>. Use + * <b>command</b> to specify the path to the command to run. You can either + * specify an absolute path to the command or relative where Tor will use the + * underlying operating system's functionality for finding the command to run. + * */ +process_t * +process_new(const char *command) +{ + tor_assert(command); + + process_t *process; + process = tor_malloc_zero(sizeof(process_t)); + + /* Set our command. */ + process->command = tor_strdup(command); + + /* By default we are not running. */ + process->status = PROCESS_STATUS_NOT_RUNNING; + + /* Prepare process environment. */ + process->arguments = smartlist_new(); + process->environment = smartlist_new(); + + /* Prepare the buffers. */ + process->stdout_buffer = buf_new(); + process->stderr_buffer = buf_new(); + process->stdin_buffer = buf_new(); + +#ifndef _WIN32 + /* Prepare our Unix process handle. */ + process->unix_process = process_unix_new(); +#else + /* Prepare our Win32 process handle. */ + process->win32_process = process_win32_new(); +#endif /* !defined(_WIN32) */ + + smartlist_add(processes, process); + + return process; +} + +/** Deallocate the given process in <b>process</b>. */ +void +process_free_(process_t *process) +{ + if (! process) + return; + + /* Cleanup parameters. */ + tor_free(process->command); + + /* Cleanup arguments and environment. */ + SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x)); + smartlist_free(process->arguments); + + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + + /* Cleanup the buffers. */ + buf_free(process->stdout_buffer); + buf_free(process->stderr_buffer); + buf_free(process->stdin_buffer); + +#ifndef _WIN32 + /* Cleanup our Unix process handle. */ + process_unix_free(process->unix_process); +#else + /* Cleanup our Win32 process handle. */ + process_win32_free(process->win32_process); +#endif /* !defined(_WIN32) */ + + smartlist_remove(processes, process); + + tor_free(process); +} + +/** Execute the given process. This function executes the given process as a + * subprocess of Tor. Returns <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_exec(process_t *process) +{ + tor_assert(process); + + if (BUG(may_spawn_background_process == 0)) + return PROCESS_STATUS_ERROR; + + process_status_t status = PROCESS_STATUS_NOT_RUNNING; + + log_info(LD_PROCESS, "Starting new process: %s", process->command); + +#ifndef _WIN32 + status = process_unix_exec(process); +#else + status = process_win32_exec(process); +#endif + + /* Update our state. */ + process_set_status(process, status); + + if (status != PROCESS_STATUS_RUNNING) { + log_warn(LD_PROCESS, "Failed to start process: %s", + process_get_command(process)); + } + + return status; +} + +/** Terminate the given process. Returns true on success, + * otherwise false. */ +bool +process_terminate(process_t *process) +{ + tor_assert(process); + + /* Terminating a non-running process isn't going to work. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + log_debug(LD_PROCESS, "Terminating process"); + +#ifndef _WIN32 + return process_unix_terminate(process); +#else + return process_win32_terminate(process); +#endif +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_get_pid(process_t *process) +{ + tor_assert(process); + +#ifndef _WIN32 + return process_unix_get_pid(process); +#else + return process_win32_get_pid(process); +#endif +} + +/** Set the callback function for output from the child process's standard out + * handle. This function sets the callback function which is called every time + * the child process have written output to its standard out file handle. + * + * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stdout_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stdout_read_callback = callback; +} + +/** Set the callback function for output from the child process's standard + * error handle. This function sets the callback function which is called + * every time the child process have written output to its standard error file + * handle. + * + * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stderr_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stderr_read_callback = callback; +} + +/** Set the callback function for process exit notification. The + * <b>callback</b> function will be called every time your child process have + * terminated. */ +void +process_set_exit_callback(process_t *process, + process_exit_callback_t callback) +{ + tor_assert(process); + process->exit_callback = callback; +} + +/** Get the current command of the given process. */ +const char * +process_get_command(const process_t *process) +{ + tor_assert(process); + return process->command; +} + +void +process_set_protocol(process_t *process, process_protocol_t protocol) +{ + tor_assert(process); + process->protocol = protocol; +} + +/** Get the currently used protocol of the given process. */ +process_protocol_t +process_get_protocol(const process_t *process) +{ + tor_assert(process); + return process->protocol; +} + +/** Set opague pointer to data. This function allows you to store a pointer to + * your own data in the given process. Use <b>process_get_data()</b> in the + * various callback functions to retrieve the data again. + * + * Note that the given process does NOT take ownership of the data and you are + * responsible for freeing up any resources allocated by the given data. + * */ +void +process_set_data(process_t *process, void *data) +{ + tor_assert(process); + process->data = data; +} + +/** Get the opaque pointer to callback data from the given process. This + * function allows you get the data you stored with <b>process_set_data()</b> + * in the different callback functions. */ +void * +process_get_data(const process_t *process) +{ + tor_assert(process); + return process->data; +} + +/** Set the status of a given process. */ +void +process_set_status(process_t *process, process_status_t status) +{ + tor_assert(process); + process->status = status; +} + +/** Get the status of the given process. */ +process_status_t +process_get_status(const process_t *process) +{ + tor_assert(process); + return process->status; +} + +/** Append an argument to the list of arguments in the given process. */ +void +process_append_argument(process_t *process, const char *argument) +{ + tor_assert(process); + tor_assert(argument); + + smartlist_add(process->arguments, tor_strdup(argument)); +} + +/** Returns a list of arguments (excluding the command itself) from the + * given process. */ +const smartlist_t * +process_get_arguments(const process_t *process) +{ + tor_assert(process); + return process->arguments; +} + +/** Returns a newly allocated Unix style argument vector. Use <b>tor_free()</b> + * to deallocate it after use. */ +char ** +process_get_argv(const process_t *process) +{ + tor_assert(process); + + /** Generate a Unix style process argument vector from our process's + * arguments smartlist_t. */ + char **argv = NULL; + + char *filename = process->command; + const smartlist_t *arguments = process->arguments; + const size_t size = smartlist_len(arguments); + + /* Make space for the process filename as argv[0] and a trailing NULL. */ + argv = tor_malloc_zero(sizeof(char *) * (size + 2)); + + /* Set our filename as first argument. */ + argv[0] = filename; + + /* Put in the rest of the values from arguments. */ + SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) { + tor_assert(arg_val != NULL); + + argv[arg_val_sl_idx + 1] = arg_val; + } SMARTLIST_FOREACH_END(arg_val); + + return argv; +} + +/** This function clears the internal environment and copies over every string + * from <b>env</b> as the new environment. */ +void +process_reset_environment(process_t *process, const smartlist_t *env) +{ + tor_assert(process); + tor_assert(env); + + /* Cleanup old environment. */ + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + process->environment = smartlist_new(); + + SMARTLIST_FOREACH(env, char *, x, + smartlist_add(process->environment, tor_strdup(x))); +} + +/** Set the given <b>key</b>/<b>value</b> pair as environment variable in the + * given process. */ +void +process_set_environment(process_t *process, + const char *key, + const char *value) +{ + tor_assert(process); + tor_assert(key); + tor_assert(value); + + smartlist_add_asprintf(process->environment, "%s=%s", key, value); +} + +/** Returns a newly allocated <b>process_environment_t</b> containing the + * environment variables for the given process. */ +process_environment_t * +process_get_environment(const process_t *process) +{ + tor_assert(process); + return process_environment_make(process->environment); +} + +#ifndef _WIN32 +/** Get the internal handle for the Unix backend. */ +process_unix_t * +process_get_unix_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->unix_process); + return process->unix_process; +} +#else /* defined(_WIN32) */ +/** Get the internal handle for Windows backend. */ +process_win32_t * +process_get_win32_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->win32_process); + return process->win32_process; +} +#endif /* !defined(_WIN32) */ + +/** Write <b>size</b> bytes of <b>data</b> to the given process's standard + * input. */ +void +process_write(process_t *process, + const uint8_t *data, size_t size) +{ + tor_assert(process); + tor_assert(data); + + buf_add(process->stdin_buffer, (char *)data, size); + process_write_stdin(process, process->stdin_buffer); +} + +/** As tor_vsnprintf(), but write the data to the given process's standard + * input. */ +void +process_vprintf(process_t *process, + const char *format, va_list args) +{ + tor_assert(process); + tor_assert(format); + + int size; + char *data; + + size = tor_vasprintf(&data, format, args); + tor_assert(data != NULL); + process_write(process, (uint8_t *)data, size); + tor_free(data); +} + +/** As tor_snprintf(), but write the data to the given process's standard + * input. */ +void +process_printf(process_t *process, + const char *format, ...) +{ + tor_assert(process); + tor_assert(format); + + va_list ap; + va_start(ap, format); + process_vprintf(process, format, ap); + va_end(ap); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard output + * handle. */ +void +process_notify_event_stdout(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stdout(process, process->stdout_buffer); + + if (ret > 0) + process_read_data(process, + process->stdout_buffer, + process->stdout_read_callback); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard error + * handle. */ +void +process_notify_event_stderr(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stderr(process, process->stderr_buffer); + + if (ret > 0) + process_read_data(process, + process->stderr_buffer, + process->stderr_read_callback); +} + +/** This function is called by the Process backend when a given process is + * allowed to begin writing data to the standard input of the child process. */ +void +process_notify_event_stdin(process_t *process) +{ + tor_assert(process); + + process_write_stdin(process, process->stdin_buffer); +} + +/** This function is called by the Process backend when a given process have + * terminated. The exit status code is passed in <b>exit_code</b>. We mark the + * process as no longer running and calls the <b>exit_callback</b> with + * information about the process termination. The given <b>process</b> is + * free'd iff the exit_callback returns true. */ +void +process_notify_event_exit(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_debug(LD_PROCESS, + "Process terminated with exit code: %"PRIu64, exit_code); + + /* Update our state. */ + process_set_status(process, PROCESS_STATUS_NOT_RUNNING); + process->exit_code = exit_code; + + /* Call our exit callback, if it exists. */ + bool free_process_handle = false; + + /* The exit callback will tell us if we should process_free() our handle. */ + if (process->exit_callback) + free_process_handle = process->exit_callback(process, exit_code); + + if (free_process_handle) + process_free(process); +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard out handle. Returns the number of + * bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + return process_unix_read_stdout(process, buffer); +#else + return process_win32_read_stdout(process, buffer); +#endif +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard error handle. Returns the number + * of bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + return process_unix_read_stderr(process, buffer); +#else + return process_win32_read_stderr(process, buffer); +#endif +} + +/** This function calls the backend function for the given process whenever + * there is data to be written to the backends' file handles. */ +MOCK_IMPL(STATIC void, process_write_stdin, + (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + +#ifndef _WIN32 + process_unix_write(process, buffer); +#else + process_win32_write(process, buffer); +#endif +} + +/** This function calls the protocol handlers based on the value of + * <b>process_get_protocol(process)</b>. Currently we call + * <b>process_read_buffer()</b> for <b>PROCESS_PROTOCOL_RAW</b> and + * <b>process_read_lines()</b> for <b>PROCESS_PROTOCOL_LINE</b>. */ +STATIC void +process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + switch (process_get_protocol(process)) { + case PROCESS_PROTOCOL_RAW: + process_read_buffer(process, buffer, callback); + break; + case PROCESS_PROTOCOL_LINE: + process_read_lines(process, buffer, callback); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return; + /* LCOV_EXCL_STOP */ + } +} + +/** This function takes the content of the given <b>buffer</b> and passes it to + * the given <b>callback</b> function, but ensures that an additional zero byte + * is added to the end of the data such that the given callback implementation + * can threat the content as a ASCIIZ string. */ +STATIC void +process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer); + + /* We allocate an extra byte for the zero byte in the end. */ + char *data = tor_malloc_zero(size + 1); + + buf_get_bytes(buffer, data, size); + log_debug(LD_PROCESS, "Read data from process"); + + if (callback) + callback(process, data, size); + + tor_free(data); +} + +/** This function tries to extract complete lines from the given <b>buffer</b> + * and calls the given <b>callback</b> function whenever it has a complete + * line. Before calling <b>callback</b> we remove the trailing "\n" or "\r\n" + * from the line. If we are unable to extract a complete line we leave the data + * in the buffer for next call. */ +STATIC void +process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer) + 1; + size_t line_size = 0; + char *data = tor_malloc_zero(size); + int ret; + + while (true) { + line_size = size; + ret = buf_get_line(buffer, data, &line_size); + + /* A complete line should always be smaller than the size of our + * buffer. */ + tor_assert(ret != -1); + + /* Remove \n from the end of the line. */ + if (line_size >= 1 && data[line_size - 1] == '\n') { + data[line_size - 1] = '\0'; + --line_size; + } + + /* Remove \r from the end of the line. */ + if (line_size >= 1 && data[line_size - 1] == '\r') { + data[line_size - 1] = '\0'; + --line_size; + } + + if (ret == 1) { + log_debug(LD_PROCESS, "Read line from process: \"%s\"", data); + + if (callback) + callback(process, data, line_size); + + /* We have read a whole line, let's see if there is more lines to read. + * */ + continue; + } + + /* No complete line for us to read. We are done for now. */ + tor_assert_nonfatal(ret == 0); + break; + } + + tor_free(data); +} diff --git a/src/lib/process/process.h b/src/lib/process/process.h new file mode 100644 index 0000000000..8879ec4f21 --- /dev/null +++ b/src/lib/process/process.h @@ -0,0 +1,148 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.h + * \brief Header for process.c + **/ + +#ifndef TOR_PROCESS_H +#define TOR_PROCESS_H + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +#include <stdbool.h> + +/** Maximum number of bytes to write to a process' stdin. */ +#define PROCESS_MAX_WRITE (1024) + +/** Maximum number of bytes to read from a process' stdout/stderr. */ +#define PROCESS_MAX_READ (1024) + +typedef enum { + /** The process is not running. */ + PROCESS_STATUS_NOT_RUNNING, + + /** The process is running. */ + PROCESS_STATUS_RUNNING, + + /** The process is in an erroneous state. */ + PROCESS_STATUS_ERROR +} process_status_t; + +const char *process_status_to_string(process_status_t status); + +typedef enum { + /** Pass complete newline-terminated lines to the + * callback (with the LF or CRLF removed). */ + PROCESS_PROTOCOL_LINE, + + /** Pass the raw response from read() to the callback. */ + PROCESS_PROTOCOL_RAW +} process_protocol_t; + +const char *process_protocol_to_string(process_protocol_t protocol); + +void tor_disable_spawning_background_processes(void); + +struct smartlist_t; + +struct process_t; +typedef struct process_t process_t; + +typedef uint64_t process_exit_code_t; +typedef uint64_t process_pid_t; + +typedef void (*process_read_callback_t)(process_t *, + const char *, + size_t); +typedef bool +(*process_exit_callback_t)(process_t *, process_exit_code_t); + +void process_init(void); +void process_free_all(void); +const struct smartlist_t *process_get_all_processes(void); + +process_t *process_new(const char *command); +void process_free_(process_t *process); +#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s)) + +process_status_t process_exec(process_t *process); +bool process_terminate(process_t *process); + +process_pid_t process_get_pid(process_t *process); + +void process_set_stdout_read_callback(process_t *, + process_read_callback_t); +void process_set_stderr_read_callback(process_t *, + process_read_callback_t); +void process_set_exit_callback(process_t *, + process_exit_callback_t); + +const char *process_get_command(const process_t *process); + +void process_append_argument(process_t *process, const char *argument); +const struct smartlist_t *process_get_arguments(const process_t *process); +char **process_get_argv(const process_t *process); + +void process_reset_environment(process_t *process, + const struct smartlist_t *env); +void process_set_environment(process_t *process, + const char *key, + const char *value); + +struct process_environment_t; +struct process_environment_t *process_get_environment(const process_t *); + +void process_set_protocol(process_t *process, process_protocol_t protocol); +process_protocol_t process_get_protocol(const process_t *process); + +void process_set_data(process_t *process, void *data); +void *process_get_data(const process_t *process); + +void process_set_status(process_t *process, process_status_t status); +process_status_t process_get_status(const process_t *process); + +#ifndef _WIN32 +struct process_unix_t; +struct process_unix_t *process_get_unix_process(const process_t *process); +#else +struct process_win32_t; +struct process_win32_t *process_get_win32_process(const process_t *process); +#endif /* !defined(_WIN32) */ + +void process_write(process_t *process, + const uint8_t *data, size_t size); +void process_vprintf(process_t *process, + const char *format, va_list args) CHECK_PRINTF(2, 0); +void process_printf(process_t *process, + const char *format, ...) CHECK_PRINTF(2, 3); + +void process_notify_event_stdout(process_t *process); +void process_notify_event_stderr(process_t *process); +void process_notify_event_stdin(process_t *process); +void process_notify_event_exit(process_t *process, + process_exit_code_t); + +#ifdef PROCESS_PRIVATE +struct buf_t; +MOCK_DECL(STATIC int, process_read_stdout, (process_t *, struct buf_t *)); +MOCK_DECL(STATIC int, process_read_stderr, (process_t *, struct buf_t *)); +MOCK_DECL(STATIC void, process_write_stdin, (process_t *, struct buf_t *)); + +STATIC void process_read_data(process_t *process, + struct buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_buffer(process_t *process, + struct buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_lines(process_t *process, + struct buf_t *buffer, + process_read_callback_t callback); +#endif /* defined(PROCESS_PRIVATE) */ + +#endif /* !defined(TOR_PROCESS_H) */ diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c new file mode 100644 index 0000000000..c8332ba91e --- /dev/null +++ b/src/lib/process/process_sys.c @@ -0,0 +1,34 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.c + * \brief Subsystem object for process setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/process/process_sys.h" +#include "lib/process/process.h" + +static int +subsys_process_initialize(void) +{ + process_init(); + return 0; +} + +static void +subsys_process_shutdown(void) +{ + process_free_all(); +} + +const subsys_fns_t sys_process = { + .name = "process", + SUBSYS_DECLARE_LOCATION(), + .level = -18, + .supported = true, + .initialize = subsys_process_initialize, + .shutdown = subsys_process_shutdown +}; diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h new file mode 100644 index 0000000000..97b3aaebd0 --- /dev/null +++ b/src/lib/process/process_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_sys.h + * \brief Declare subsystem object for the process module. + **/ + +#ifndef TOR_PROCESS_SYS_H +#define TOR_PROCESS_SYS_H + +extern const struct subsys_fns_t sys_process; + +#endif /* !defined(TOR_PROCESS_SYS_H) */ diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c new file mode 100644 index 0000000000..2b47e1874d --- /dev/null +++ b/src/lib/process/process_unix.c @@ -0,0 +1,698 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.c + * \brief Module for working with Unix processes. + **/ + +#define PROCESS_UNIX_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/waitpid.h" +#include "lib/process/env.h" + +#include <stdio.h> + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif + +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif + +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) +#include <sys/prctl.h> +#endif + +#if HAVE_SIGNAL_H +#include <signal.h> +#endif + +#ifndef _WIN32 + +/** Maximum number of file descriptors, if we cannot get it via sysconf() */ +#define DEFAULT_MAX_FD 256 + +/** Internal state for Unix handles. */ +struct process_unix_handle_t { + /** Unix File Descriptor. */ + int fd; + + /** Have we reached end of file? */ + bool reached_eof; + + /** Event structure for libevent. */ + struct event *event; + + /** Are we writing? */ + bool is_writing; +}; + +/** Internal state for our Unix process. */ +struct process_unix_t { + /** Standard in handle. */ + process_unix_handle_t stdin_handle; + + /** Standard out handle. */ + process_unix_handle_t stdout_handle; + + /** Standard error handle. */ + process_unix_handle_t stderr_handle; + + /** The process identifier of our process. */ + pid_t pid; + + /** Waitpid Callback structure. */ + waitpid_callback_t *waitpid; +}; + +/** Returns a newly allocated <b>process_unix_t</b>. */ +process_unix_t * +process_unix_new(void) +{ + process_unix_t *unix_process; + unix_process = tor_malloc_zero(sizeof(process_unix_t)); + + unix_process->stdin_handle.fd = -1; + unix_process->stderr_handle.fd = -1; + unix_process->stdout_handle.fd = -1; + + return unix_process; +} + +/** Deallocates the given <b>unix_process</b>. */ +void +process_unix_free_(process_unix_t *unix_process) +{ + if (! unix_process) + return; + + /* Clean up our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + + /* FIXME(ahf): Refactor waitpid code? */ + unix_process->waitpid = NULL; + + /* Close all our file descriptors. */ + process_unix_close_file_descriptors(unix_process); + + tor_event_free(unix_process->stdout_handle.event); + tor_event_free(unix_process->stderr_handle.event); + tor_event_free(unix_process->stdin_handle.event); + + tor_free(unix_process); +} + +/** Executes the given process as a child process of Tor. This function is + * responsible for setting up the child process and run it. This includes + * setting up pipes for interprocess communication, initialize the waitpid + * callbacks, and finally run fork() followed by execve(). Returns + * <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_unix_exec(process_t *process) +{ + static int max_fd = -1; + + process_unix_t *unix_process; + pid_t pid; + int stdin_pipe[2]; + int stdout_pipe[2]; + int stderr_pipe[2]; + int retval, fd; + + unix_process = process_get_unix_process(process); + + /* Create standard in pipe. */ + retval = pipe(stdin_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdin " + "communication with process: %s", + strerror(errno)); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard out pipe. */ + retval = pipe(stdout_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdout " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard error pipe. */ + retval = pipe(stderr_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stderr " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdout_pipe[0]); + close(stdout_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + +#ifdef _SC_OPEN_MAX + if (-1 == max_fd) { + max_fd = (int)sysconf(_SC_OPEN_MAX); + + if (max_fd == -1) { + max_fd = DEFAULT_MAX_FD; + log_warn(LD_PROCESS, + "Cannot find maximum file descriptor, assuming: %d", max_fd); + } + } +#else /* !defined(_SC_OPEN_MAX) */ + max_fd = DEFAULT_MAX_FD; +#endif /* defined(_SC_OPEN_MAX) */ + + pid = fork(); + + if (0 == pid) { + /* This code is running in the child process context. */ + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) + /* Attempt to have the kernel issue a SIGTERM if the parent + * goes away. Certain attributes of the binary being execve()ed + * will clear this during the execve() call, but it's better + * than nothing. + */ + prctl(PR_SET_PDEATHSIG, SIGTERM); +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ + + /* Link process stdout to the write end of the pipe. */ + retval = dup2(stdout_pipe[1], STDOUT_FILENO); + if (-1 == retval) + goto error; + + /* Link process stderr to the write end of the pipe. */ + retval = dup2(stderr_pipe[1], STDERR_FILENO); + if (-1 == retval) + goto error; + + /* Link process stdin to the read end of the pipe */ + retval = dup2(stdin_pipe[0], STDIN_FILENO); + if (-1 == retval) + goto error; + + /* Close our pipes now after they have been dup2()'ed. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + close(stdout_pipe[0]); + close(stdout_pipe[1]); + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /* Close all other fds, including the read end of the pipe. XXX: We should + * now be doing enough FD_CLOEXEC setting to make this needless. + */ + for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) + close(fd); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Call the requested program. */ + execve(argv[0], argv, env->unixoid_environment_block); + + /* If we made it here it is because execve failed :-( */ + tor_free(argv); + process_environment_free(env); + + error: + fprintf(stderr, "Error from child process: %s", strerror(errno)); + _exit(1); + } + + /* We are in the parent process. */ + if (-1 == pid) { + log_warn(LD_PROCESS, + "Failed to create child process: %s", strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdout_pipe[0]); + close(stdout_pipe[1]); + + /** Cleanup standard error pipe. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Register our PID. */ + unix_process->pid = pid; + + /* Setup waitpid callbacks. */ + unix_process->waitpid = set_waitpid_callback(pid, + process_unix_waitpid_callback, + process); + + /* Handle standard out. */ + unix_process->stdout_handle.fd = stdout_pipe[0]; + retval = close(stdout_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s", + strerror(errno)); + } + + /* Handle standard error. */ + unix_process->stderr_handle.fd = stderr_pipe[0]; + retval = close(stderr_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Failed to close write end of standard error pipe: %s", + strerror(errno)); + } + + /* Handle standard in. */ + unix_process->stdin_handle.fd = stdin_pipe[1]; + retval = close(stdin_pipe[0]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s", + strerror(errno)); + } + + /* Setup our handles. */ + process_unix_setup_handle(process, + &unix_process->stdout_handle, + EV_READ|EV_PERSIST, + stdout_read_callback); + + process_unix_setup_handle(process, + &unix_process->stderr_handle, + EV_READ|EV_PERSIST, + stderr_read_callback); + + process_unix_setup_handle(process, + &unix_process->stdin_handle, + EV_WRITE|EV_PERSIST, + stdin_write_callback); + + /* Start reading from standard out and standard error. */ + process_unix_start_reading(&unix_process->stdout_handle); + process_unix_start_reading(&unix_process->stderr_handle); + + return PROCESS_STATUS_RUNNING; +} + +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_unix_terminate(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + + /* All running processes should have a waitpid. */ + if (BUG(unix_process->waitpid == NULL)) + return false; + + bool success = true; + + /* Send a SIGTERM to our child process. */ + int ret; + + ret = kill(unix_process->pid, SIGTERM); + + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to terminate process: %s", + strerror(errno)); + success = false; + } + + /* Close all our FD's. */ + if (! process_unix_close_file_descriptors(unix_process)) + success = false; + + return success; +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_unix_get_pid(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + return (process_pid_t)unix_process->pid; +} + +/** Write the given <b>buffer</b> as input to the given <b>process</b>'s + * standard input. Returns the number of bytes written. */ +int +process_unix_write(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + size_t buffer_flush_len = buf_datalen(buffer); + const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len); + + /* If we have data to write (when buffer_flush_len > 0) and we are not + * currently getting file descriptor events from the kernel, we tell the + * kernel to start notifying us about when we can write to our file + * descriptor and return. */ + if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) { + process_unix_start_writing(&unix_process->stdin_handle); + return 0; + } + + /* We don't have any data to write, but the kernel is currently notifying us + * about whether we are able to write or not. Tell the kernel to stop + * notifying us until we have data to write. */ + if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) { + process_unix_stop_writing(&unix_process->stdin_handle); + return 0; + } + + /* We have data to write and the kernel have told us to write it. */ + return buf_flush_to_pipe(buffer, + process_get_unix_process(process)->stdin_handle.fd, + max_to_write, &buffer_flush_len); +} + +/** Read data from the given process's standard output and put it into + * <b>buffer</b>. Returns the number of bytes read. */ +int +process_unix_read_stdout(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stdout_handle, + buffer); +} + +/** Read data from the given process's standard error and put it into + * <b>buffer</b>. Returns the number of bytes read. */ +int +process_unix_read_stderr(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stderr_handle, + buffer); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard output. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stdout_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdout(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard error. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stderr_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stderr(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * written the child process's standard input. We notify the Process subsystem, + * which is then responsible for calling back to us for doing the actual write + * of the data. */ +STATIC void +stdin_write_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdin(process); +} + +/** This function tells libevent that we are interested in receiving read + * events from the given <b>handle</b>. */ +STATIC void +process_unix_start_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); +} + +/** This function tells libevent that we are no longer interested in receiving + * read events from the given <b>handle</b>. */ +STATIC void +process_unix_stop_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); +} + +/** This function tells libevent that we are interested in receiving write + * events from the given <b>handle</b>. */ +STATIC void +process_unix_start_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); + + handle->is_writing = true; +} + +/** This function tells libevent that we are no longer interested in receiving + * write events from the given <b>handle</b>. */ +STATIC void +process_unix_stop_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); + + handle->is_writing = false; +} + +/** This function is called when the waitpid system have detected that our + * process have terminated. We disable the waitpid system and notify the + * Process subsystem that we have terminated. */ +STATIC void +process_unix_waitpid_callback(int status, void *data) +{ + tor_assert(data); + + process_t *process = data; + process_unix_t *unix_process = process_get_unix_process(process); + + /* Remove our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + unix_process->waitpid = NULL; + + /* Notify our process. */ + process_notify_event_exit(process, status); + + /* Make sure you don't modify the process after we have called + * process_notify_event_exit() on it, to allow users to process_free() it in + * the exit callback. */ +} + +/** This function sets the file descriptor in the <b>handle</b> as non-blocking + * and configures the libevent event structure based on the given <b>flags</b> + * to ensure that <b>callback</b> is called whenever we have events on the + * given <b>handle</b>. */ +STATIC void +process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(callback); + + /* Put our file descriptor into non-blocking mode. */ + if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) { + log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s", + strerror(errno)); + } + + /* Setup libevent event. */ + handle->event = tor_event_new(tor_libevent_get_base(), + handle->fd, + flags, + callback, + process); +} + +/** This function reads data from the given <b>handle</b> and puts it into + * <b>buffer</b>. Returns the number of bytes read this way. */ +STATIC int +process_unix_read_handle(process_t *process, + process_unix_handle_t *handle, + buf_t *buffer) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(buffer); + + int ret = 0; + int eof = 0; + int error = 0; + + ret = buf_read_from_pipe(buffer, + handle->fd, + PROCESS_MAX_READ, + &eof, + &error); + + if (error) + log_warn(LD_PROCESS, + "Unable to read data: %s", strerror(error)); + + if (eof) { + handle->reached_eof = true; + process_unix_stop_reading(handle); + } + + return ret; +} + +/** Close the standard in, out, and error handles of the given + * <b>unix_process</b>. */ +STATIC bool +process_unix_close_file_descriptors(process_unix_t *unix_process) +{ + tor_assert(unix_process); + + int ret; + bool success = true; + + /* Stop reading and writing before we close() our + * file descriptors. */ + if (! unix_process->stdout_handle.reached_eof) + process_unix_stop_reading(&unix_process->stdout_handle); + + if (! unix_process->stderr_handle.reached_eof) + process_unix_stop_reading(&unix_process->stderr_handle); + + if (unix_process->stdin_handle.is_writing) + process_unix_stop_writing(&unix_process->stdin_handle); + + if (unix_process->stdin_handle.fd != -1) { + ret = close(unix_process->stdin_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard in"); + success = false; + } + + unix_process->stdin_handle.fd = -1; + } + + if (unix_process->stdout_handle.fd != -1) { + ret = close(unix_process->stdout_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard out"); + success = false; + } + + unix_process->stdout_handle.fd = -1; + } + + if (unix_process->stderr_handle.fd != -1) { + ret = close(unix_process->stderr_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard error"); + success = false; + } + + unix_process->stderr_handle.fd = -1; + } + + return success; +} + +#endif /* !defined(_WIN32) */ diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h new file mode 100644 index 0000000000..a6b8304d48 --- /dev/null +++ b/src/lib/process/process_unix.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.h + * \brief Header for process_unix.c + **/ + +#ifndef TOR_PROCESS_UNIX_H +#define TOR_PROCESS_UNIX_H + +#ifndef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" + +#include <event2/event.h> + +struct process_t; + +struct process_unix_t; +typedef struct process_unix_t process_unix_t; + +process_unix_t *process_unix_new(void); +void process_unix_free_(process_unix_t *unix_process); +#define process_unix_free(s) \ + FREE_AND_NULL(process_unix_t, process_unix_free_, (s)) + +process_status_t process_unix_exec(struct process_t *process); +bool process_unix_terminate(struct process_t *process); + +process_pid_t process_unix_get_pid(struct process_t *process); + +int process_unix_write(struct process_t *process, buf_t *buffer); +int process_unix_read_stdout(struct process_t *process, buf_t *buffer); +int process_unix_read_stderr(struct process_t *process, buf_t *buffer); + +#ifdef PROCESS_UNIX_PRIVATE +struct process_unix_handle_t; +typedef struct process_unix_handle_t process_unix_handle_t; + +STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data); + +STATIC void process_unix_start_reading(process_unix_handle_t *); +STATIC void process_unix_stop_reading(process_unix_handle_t *); + +STATIC void process_unix_start_writing(process_unix_handle_t *); +STATIC void process_unix_stop_writing(process_unix_handle_t *); + +STATIC void process_unix_waitpid_callback(int status, void *data); + +STATIC void process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback); +STATIC int process_unix_read_handle(process_t *, + process_unix_handle_t *, + buf_t *); +STATIC bool process_unix_close_file_descriptors(process_unix_t *); +#endif /* defined(PROCESS_UNIX_PRIVATE) */ + +#endif /* !defined(_WIN32) */ + +#endif /* !defined(TOR_PROCESS_UNIX_H) */ diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c new file mode 100644 index 0000000000..8683e17fc2 --- /dev/null +++ b/src/lib/process/process_win32.c @@ -0,0 +1,1105 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.c + * \brief Module for working with Windows processes. + **/ + +#define PROCESS_WIN32_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/buf/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/log/win32err.h" +#include "lib/process/process.h" +#include "lib/process/process_win32.h" +#include "lib/process/env.h" + +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif + +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#ifdef _WIN32 + +/** The size of our intermediate buffers. */ +#define BUFFER_SIZE (1024) + +/** Timer that ticks once a second and calls the process_win32_timer_callback() + * function. */ +static periodic_timer_t *periodic_timer; + +/** Structure to represent the state around the pipe HANDLE. + * + * This structure is used to store state about a given HANDLE, including + * whether we have reached end of file, its intermediate buffers, and how much + * data that is available in the intermediate buffer. */ +struct process_win32_handle_t { + /** Standard out pipe handle. */ + HANDLE pipe; + + /** True iff we have reached EOF from the pipe. */ + bool reached_eof; + + /** How much data is available in buffer. */ + size_t data_available; + + /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */ + char buffer[BUFFER_SIZE]; + + /** Overlapped structure for ReadFileEx() and WriteFileEx(). */ + OVERLAPPED overlapped; + + /** Are we waiting for another I/O operation to complete? */ + bool busy; +}; + +/** Structure to represent the Windows specific implementation details of this + * Process backend. + * + * This structure is attached to <b>process_t</b> (see process.h) and is + * reachable from <b>process_t</b> via the <b>process_get_win32_process()</b> + * method. */ +struct process_win32_t { + /** Standard in state. */ + process_win32_handle_t stdin_handle; + + /** Standard out state. */ + process_win32_handle_t stdout_handle; + + /** Standard error state. */ + process_win32_handle_t stderr_handle; + + /** Process Information. */ + PROCESS_INFORMATION process_information; +}; + +/** Create a new <b>process_win32_t</b>. + * + * This function constructs a new <b>process_win32_t</b> and initializes the + * default values. */ +process_win32_t * +process_win32_new(void) +{ + process_win32_t *win32_process; + win32_process = tor_malloc_zero(sizeof(process_win32_t)); + + win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE; + + return win32_process; +} + +/** Free a given <b>process_win32_t</b>. + * + * This function deinitializes and frees up the resources allocated for the + * given <b>process_win32_t</b>. */ +void +process_win32_free_(process_win32_t *win32_process) +{ + if (! win32_process) + return; + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + tor_free(win32_process); +} + +/** Initialize the Windows backend of the Process subsystem. */ +void +process_win32_init(void) +{ + /* We don't start the periodic timer here because it makes no sense to have + * the timer running until we have some processes that benefits from the + * timer timer ticks. */ +} + +/** Deinitialize the Windows backend of the Process subsystem. */ +void +process_win32_deinit(void) +{ + /* Stop our timer, but only if it's running. */ + if (process_win32_timer_running()) + process_win32_timer_stop(); +} + +/** Execute the given process. This function is responsible for setting up + * named pipes for I/O between the child process and the Tor process. Returns + * <b>PROCESS_STATUS_RUNNING</b> upon success. */ +process_status_t +process_win32_exec(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + HANDLE stdout_pipe_read = NULL; + HANDLE stdout_pipe_write = NULL; + HANDLE stderr_pipe_read = NULL; + HANDLE stderr_pipe_write = NULL; + HANDLE stdin_pipe_read = NULL; + HANDLE stdin_pipe_write = NULL; + BOOL ret = FALSE; + + /* Setup our security attributes. */ + SECURITY_ATTRIBUTES security_attributes; + security_attributes.nLength = sizeof(security_attributes); + security_attributes.bInheritHandle = TRUE; + /* FIXME: should we set explicit security attributes? + * (See Ticket #2046, comment 5) */ + security_attributes.lpSecurityDescriptor = NULL; + + /* Create our standard out pipe. */ + if (! process_win32_create_pipe(&stdout_pipe_read, + &stdout_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create our standard error pipe. */ + if (! process_win32_create_pipe(&stderr_pipe_read, + &stderr_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create out standard in pipe. */ + if (! process_win32_create_pipe(&stdin_pipe_read, + &stdin_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_WRITER)) { + return PROCESS_STATUS_ERROR; + } + + /* Configure startup info for our child process. */ + STARTUPINFOA startup_info; + + memset(&startup_info, 0, sizeof(startup_info)); + startup_info.cb = sizeof(startup_info); + startup_info.hStdError = stderr_pipe_write; + startup_info.hStdOutput = stdout_pipe_write; + startup_info.hStdInput = stdin_pipe_read; + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Windows expects argv to be a whitespace delimited string, so join argv up + */ + char *joined_argv = tor_join_win_cmdline((const char **)argv); + + /* Create the child process */ + ret = CreateProcessA(NULL, + joined_argv, + NULL, + NULL, + TRUE, + CREATE_NO_WINDOW, + env->windows_environment_block[0] == '\0' ? + NULL : env->windows_environment_block, + NULL, + &startup_info, + &win32_process->process_information); + + tor_free(argv); + tor_free(joined_argv); + process_environment_free(env); + + if (! ret) { + log_warn(LD_PROCESS, "CreateProcessA() failed: %s", + format_win32_error(GetLastError())); + + /* Cleanup our handles. */ + CloseHandle(stdout_pipe_read); + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_read); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + CloseHandle(stdin_pipe_write); + + /* In the Unix backend, we do not get an error in the Tor process when a + * child process fails to spawn its target executable since we need to + * first do the fork() call in the Tor process and then the child process + * is responsible for doing the call to execve(). + * + * This means that the user of the process_exec() API must check for + * whether it returns PROCESS_STATUS_ERROR, which will rarely happen on + * Unix, but will happen for error cases on Windows where it does not + * happen on Unix. For example: when the target executable does not exist + * on the file system. + * + * To have somewhat feature compatibility between the Unix and the Windows + * backend, we here notify the process_t owner that the process have exited + * (even though it never managed to run) to ensure that the exit callback + * is executed. + */ + process_notify_event_exit(process, 0); + + return PROCESS_STATUS_ERROR; + } + + /* TODO: Should we close hProcess and hThread in + * process_handle->process_information? */ + win32_process->stdout_handle.pipe = stdout_pipe_read; + win32_process->stderr_handle.pipe = stderr_pipe_read; + win32_process->stdin_handle.pipe = stdin_pipe_write; + + /* Close our ends of the pipes that is now owned by the child process. */ + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + + /* Used by the callback functions from ReadFileEx() and WriteFileEx() such + * that we can figure out which process_t that was responsible for the event. + * + * Warning, here be dragons: + * + * MSDN says that the hEvent member of the overlapped structure is unused + * for ReadFileEx() and WriteFileEx, which allows us to store a pointer to + * our process state there. + */ + win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process; + + /* Start our timer if it is not already running. */ + if (! process_win32_timer_running()) + process_win32_timer_start(); + + /* We use Windows Extended I/O functions, so our completion callbacks are + * called automatically for us when there is data to read. Because of this + * we start the read of standard out and error right away. */ + process_notify_event_stdout(process); + process_notify_event_stderr(process); + + return PROCESS_STATUS_RUNNING; +} + +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_win32_terminate(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + /* Terminate our process. */ + BOOL ret; + + ret = TerminateProcess(win32_process->process_information.hProcess, 0); + + if (! ret) { + log_warn(LD_PROCESS, "TerminateProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + return true; +} + +/** Returns the unique process identifier for the given <b>process</b>. */ +process_pid_t +process_win32_get_pid(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + return (process_pid_t)win32_process->process_information.dwProcessId; +} + +/** Schedule an async write of the data found in <b>buffer</b> for the given + * process. This function runs an async write operation of the content of + * buffer, if we are not already waiting for a pending I/O request. Returns the + * number of bytes that Windows will hopefully write for us in the background. + * */ +int +process_win32_write(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD error_code = 0; + const size_t buffer_size = buf_datalen(buffer); + + /* Windows is still writing our buffer. */ + if (win32_process->stdin_handle.busy) + return 0; + + /* Nothing for us to do right now. */ + if (buffer_size == 0) + return 0; + + /* We have reached end of file already? */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return 0; + + /* Figure out how much data we should read. */ + const size_t write_size = MIN(buffer_size, + sizeof(win32_process->stdin_handle.buffer)); + + /* Read data from the process_t buffer into our intermediate buffer. */ + buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size); + + /* Because of the slightly weird API for WriteFileEx() we must set this to 0 + * before we call WriteFileEx() because WriteFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + + /* Schedule our write. */ + ret = WriteFileEx(win32_process->stdin_handle.pipe, + win32_process->stdin_handle.buffer, + write_size, + &win32_process->stdin_handle.overlapped, + process_win32_stdin_write_done); + + if (! ret) { + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "WriteFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "WriteFileEx() failed: %s", + format_win32_error(error_code)); + } + + win32_process->stdin_handle.reached_eof = true; + return 0; + } + + /* Here be dragons: According to MSDN's documentation for WriteFileEx() we + * should check GetLastError() after a call to WriteFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "WriteFileEx() failed after returning success: %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + return 0; + /* LCOV_EXCL_STOP */ + } + + /* This cast should be safe since our buffer can maximum be BUFFER_SIZE + * large. */ + return (int)write_size; +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard output of the child process + * and asks Windows to call process_win32_stdout_read_done() when that + * operation is finished. Returns the number of bytes moved into <b>buffer</b>. + * */ +int +process_win32_read_stdout(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stdout_handle, + buffer, + process_win32_stdout_read_done); +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard error of the child process + * and asks Windows to call process_win32_stderr_read_done() when that + * operation is finished. Returns the number of bytes moved into <b>buffer</b>. + * */ +int +process_win32_read_stderr(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stderr_handle, + buffer, + process_win32_stderr_read_done); +} + +/** This function is responsible for moving the Tor process into what Microsoft + * calls an "alertable" state. Once the process is in an alertable state the + * Windows kernel will notify us when our background I/O requests have finished + * and the callbacks will be executed. */ +void +process_win32_trigger_completion_callbacks(void) +{ + DWORD ret; + + /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep + * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause + * the process to enter alertable state, where the Windows kernel will notify + * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which + * will cause our completion callbacks to be executed. + * + * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION + * if one or more I/O callbacks were executed. */ + ret = SleepEx(0, TRUE); + + /* Warn us if the function returned something we did not anticipate. */ + if (ret != 0 && ret != WAIT_IO_COMPLETION) { + log_warn(LD_PROCESS, "SleepEx() returned %lu", ret); + } +} + +/** Start the periodic timer which is reponsible for checking whether processes + * are still alive and to make sure that the Tor process is periodically being + * moved into an alertable state. */ +void +process_win32_timer_start(void) +{ + /* Make sure we never start our timer if it's already running. */ + if (BUG(process_win32_timer_running())) + return; + + /* Wake up once a second. */ + static const struct timeval interval = {1, 0}; + + log_info(LD_PROCESS, "Starting Windows Process I/O timer"); + periodic_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + process_win32_timer_callback, + NULL); +} + +/** Stops the periodic timer. */ +void +process_win32_timer_stop(void) +{ + if (BUG(periodic_timer == NULL)) + return; + + log_info(LD_PROCESS, "Stopping Windows Process I/O timer"); + periodic_timer_free(periodic_timer); +} + +/** Returns true iff the periodic timer is running. */ +bool +process_win32_timer_running(void) +{ + return periodic_timer != NULL; +} + +/** This function is called whenever the periodic_timer ticks. The function is + * responsible for moving the Tor process into an alertable state once a second + * and checking for whether our child processes have terminated since the last + * tick. */ +STATIC void +process_win32_timer_callback(periodic_timer_t *timer, void *data) +{ + tor_assert(timer == periodic_timer); + tor_assert(data == NULL); + + /* Move the process into an alertable state. */ + process_win32_trigger_completion_callbacks(); + + /* Check if our processes are still alive. */ + + /* Since the call to process_win32_timer_test_process() might call + * process_notify_event_exit() which again might call process_free() which + * updates the list of processes returned by process_get_all_processes() it + * is important here that we make sure to not touch the list of processes if + * the call to process_win32_timer_test_process() returns true. */ + bool done; + + do { + const smartlist_t *processes = process_get_all_processes(); + done = true; + + SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) { + /* If process_win32_timer_test_process() returns true, it means that + * smartlist_remove() might have been called on the list returned by + * process_get_all_processes(). We start the loop over again until we + * have a succesful run over the entire list where the list was not + * modified. */ + if (process_win32_timer_test_process(process)) { + done = false; + break; + } + } SMARTLIST_FOREACH_END(process); + } while (! done); +} + +/** Test whether a given process is still alive. Notify the Process subsystem + * if our process have died. Returns true iff the given process have + * terminated. */ +STATIC bool +process_win32_timer_test_process(process_t *process) +{ + tor_assert(process); + + /* No need to look at processes that don't claim they are running. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD exit_code = 0; + + /* Sometimes the Windows kernel wont give us the EOF/Broken Pipe error + * message until some time after the process have actually terminated. We + * make sure that our ReadFileEx() calls for the process have *all* returned + * and both standard out and error have been marked as EOF before we try to + * see if the process terminated. + * + * This ensures that we *never* call the exit callback of the `process_t`, + * which potentially ends up calling `process_free()` on our `process_t`, + * before all data have been received from the process. + * + * We do NOT have a check here for whether standard in reached EOF since + * standard in's WriteFileEx() function is only called on-demand when we have + * something to write and is thus usually not awaiting to finish any + * operations. If we WriteFileEx() to a file that has terminated we'll simply + * get an error from ReadFileEx() or its completion routine and move on with + * life. */ + if (! win32_process->stdout_handle.reached_eof) + return false; + + if (! win32_process->stderr_handle.reached_eof) + return false; + + /* We start by testing whether our process is still running. */ + ret = GetExitCodeProcess(win32_process->process_information.hProcess, + &exit_code); + + if (! ret) { + log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Notify our process_t that our process have terminated. Since our + * exit_callback might decide to process_free() our process handle it is very + * important that we do not touch the process_t after the call to + * process_notify_event_exit(). */ + if (exit_code != STILL_ACTIVE) { + process_notify_event_exit(process, exit_code); + return true; + } + + return false; +} + +/** Create a new overlapped named pipe. This function creates a new connected, + * named, pipe in <b>*read_pipe</b> and <b>*write_pipe</b> if the function is + * succesful. Returns true on sucess, false on failure. */ +STATIC bool +process_win32_create_pipe(HANDLE *read_pipe, + HANDLE *write_pipe, + SECURITY_ATTRIBUTES *attributes, + process_win32_pipe_type_t pipe_type) +{ + tor_assert(read_pipe); + tor_assert(write_pipe); + tor_assert(attributes); + + BOOL ret = FALSE; + + /* Buffer size. */ + const size_t size = 4096; + + /* Our additional read/write modes that depends on which pipe type we are + * creating. */ + DWORD read_mode = 0; + DWORD write_mode = 0; + + /* Generate the unique pipe name. */ + char pipe_name[MAX_PATH]; + static DWORD process_id = 0; + static DWORD counter = 0; + + if (process_id == 0) + process_id = GetCurrentProcessId(); + + tor_snprintf(pipe_name, sizeof(pipe_name), + "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu", + process_id, counter++); + + /* Only one of our handles can be overlapped. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + read_mode = FILE_FLAG_OVERLAPPED; + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + write_mode = FILE_FLAG_OVERLAPPED; + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + /* Setup our read and write handles. */ + HANDLE read_handle; + HANDLE write_handle; + + /* Create our named pipe. */ + read_handle = CreateNamedPipeA(pipe_name, + (PIPE_ACCESS_INBOUND|read_mode), + (PIPE_TYPE_BYTE|PIPE_WAIT), + 1, + size, + size, + 1000, + attributes); + + if (read_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Create our file in the pipe namespace. */ + write_handle = CreateFileA(pipe_name, + GENERIC_WRITE, + 0, + attributes, + OPEN_EXISTING, + (FILE_ATTRIBUTE_NORMAL|write_mode), + NULL); + + if (write_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateFileA() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + + return false; + } + + /* Set the inherit flag for our pipe. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0); + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + if (! ret) { + log_warn(LD_PROCESS, "SetHandleInformation() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + CloseHandle(write_handle); + + return false; + } + + /* Everything is good. */ + *read_pipe = read_handle; + *write_pipe = write_handle; + + return true; +} + +/** Cleanup a given <b>handle</b>. */ +STATIC void +process_win32_cleanup_handle(process_win32_handle_t *handle) +{ + tor_assert(handle); + +#if 0 + BOOL ret; + DWORD error_code; + + /* Cancel any pending I/O requests: This means that instead of getting + * ERROR_BROKEN_PIPE we get ERROR_OPERATION_ABORTED, but it doesn't seem + * like this is needed. */ + ret = CancelIo(handle->pipe); + + if (! ret) { + error_code = GetLastError(); + + /* There was no pending I/O requests for our handle. */ + if (error_code != ERROR_NOT_FOUND) { + log_warn(LD_PROCESS, "CancelIo() failed: %s", + format_win32_error(error_code)); + } + } +#endif /* 0 */ + + /* Close our handle. */ + if (handle->pipe != INVALID_HANDLE_VALUE) { + CloseHandle(handle->pipe); + handle->pipe = INVALID_HANDLE_VALUE; + handle->reached_eof = true; + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard output. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stdout_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stdout_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stdout(process); + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard error. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stderr_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stderr_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stderr(process); + } +} + +/** This function is called when WriteFileEx() completes its background write + * to the child process's standard input. We notify the Process subsystem that + * it can write data to us again. */ +STATIC VOID WINAPI +process_win32_stdin_write_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + (void)byte_count; + + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + /* Mark our handle as not having any outstanding I/O requests. */ + win32_process->stdin_handle.busy = false; + + /* Check if we have been asked to write to the handle that have been marked + * as having reached EOF. */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return; + + if (error_code == 0) { + /** Our data have been succesfully written. Clear our state and schedule + * the next write. */ + win32_process->stdin_handle.data_available = 0; + memset(win32_process->stdin_handle.buffer, 0, + sizeof(win32_process->stdin_handle.buffer)); + + /* Schedule the next write. */ + process_notify_event_stdin(process); + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our WriteFileEx() call was succesful, but we reached the end of our + * file. We mark our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + win32_process->stdin_handle.reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from WriteFileEx(): %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + } +} + +/** This function reads data from the given <b>handle</b>'s internal buffer and + * moves it into the given <b>buffer</b>. Additionally, we start the next + * ReadFileEx() background operation with the given <b>callback</b> as + * completion callback. Returns the number of bytes written to the buffer. */ +STATIC int +process_win32_read_from_handle(process_win32_handle_t *handle, + buf_t *buffer, + LPOVERLAPPED_COMPLETION_ROUTINE callback) +{ + tor_assert(handle); + tor_assert(buffer); + tor_assert(callback); + + BOOL ret = FALSE; + int bytes_available = 0; + DWORD error_code = 0; + + /* We already have a request to read data that isn't complete yet. */ + if (BUG(handle->busy)) + return 0; + + /* Check if we have been asked to read from a handle that have already told + * us that we have reached the end of the file. */ + if (BUG(handle->reached_eof)) + return 0; + + /* This cast should be safe since our buffer can be at maximum up to + * BUFFER_SIZE in size. */ + bytes_available = (int)handle->data_available; + + if (handle->data_available > 0) { + /* Read data from our intermediate buffer into the process_t buffer. */ + buf_add(buffer, handle->buffer, handle->data_available); + + /* Reset our read state. */ + handle->data_available = 0; + memset(handle->buffer, 0, sizeof(handle->buffer)); + } + + /* Because of the slightly weird API for ReadFileEx() we must set this to 0 + * before we call ReadFileEx() because ReadFileEx() does not reset the last + * error itself when it's succesful. See comment below after the call to + * GetLastError(). */ + SetLastError(0); + + /* Ask the Windows kernel to read data from our pipe into our buffer and call + * the callback function when it is done. */ + ret = ReadFileEx(handle->pipe, + handle->buffer, + sizeof(handle->buffer), + &handle->overlapped, + callback); + + if (! ret) { + error_code = GetLastError(); + + /* No need to log at warning level for these two. */ + if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "ReadFileEx() returned EOF from pipe: %s", + format_win32_error(error_code)); + } else { + log_warn(LD_PROCESS, "ReadFileEx() failed: %s", + format_win32_error(error_code)); + } + + handle->reached_eof = true; + return bytes_available; + } + + /* Here be dragons: According to MSDN's documentation for ReadFileEx() we + * should check GetLastError() after a call to ReadFileEx() even though the + * `ret` return value was successful. If everything is good, GetLastError() + * returns `ERROR_SUCCESS` and nothing happens. + * + * XXX(ahf): I have not managed to trigger this code while stress-testing + * this code. */ + error_code = GetLastError(); + + if (error_code != ERROR_SUCCESS) { + /* LCOV_EXCL_START */ + log_warn(LD_PROCESS, "ReadFileEx() failed after returning success: %s", + format_win32_error(error_code)); + handle->reached_eof = true; + return bytes_available; + /* LCOV_EXCL_STOP */ + } + + /* We mark our handle as having a pending I/O request. */ + handle->busy = true; + + return bytes_available; +} + +/** This function checks the callback values from ReadFileEx() in + * <b>error_code</b> and <b>byte_count</b> if we have read data. Returns true + * iff our caller should request more data from ReadFileEx(). */ +STATIC bool +process_win32_handle_read_completion(process_win32_handle_t *handle, + DWORD error_code, + DWORD byte_count) +{ + tor_assert(handle); + + /* Mark our handle as not having any outstanding I/O requests. */ + handle->busy = false; + + if (error_code == 0) { + /* Our ReadFileEx() call was succesful and there is data for us. */ + + /* This cast should be safe since byte_count should never be larger than + * BUFFER_SIZE. */ + tor_assert(byte_count <= BUFFER_SIZE); + handle->data_available = (size_t)byte_count; + + /* Tell our caller to schedule the next read. */ + return true; + } else if (error_code == ERROR_HANDLE_EOF || + error_code == ERROR_BROKEN_PIPE) { + /* Our ReadFileEx() finished, but we reached the end of our file. We mark + * our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + handle->reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from ReadFileEx(): %s", + format_win32_error(error_code)); + + handle->reached_eof = true; + } + + /* Our caller should NOT schedule the next read. */ + return false; +} + +/** Format a single argument for being put on a Windows command line. + * Returns a newly allocated string */ +STATIC char * +format_win_cmdline_argument(const char *arg) +{ + char *formatted_arg; + char need_quotes; + const char *c; + int i; + int bs_counter = 0; + /* Backslash we can point to when one is inserted into the string */ + const char backslash = '\\'; + + /* Smartlist of *char */ + smartlist_t *arg_chars; + arg_chars = smartlist_new(); + + /* Quote string if it contains whitespace or is empty */ + need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); + + /* Build up smartlist of *chars */ + for (c=arg; *c != '\0'; c++) { + if ('"' == *c) { + /* Double up backslashes preceding a quote */ + for (i=0; i<(bs_counter*2); i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + /* Escape the quote */ + smartlist_add(arg_chars, (void*)&backslash); + smartlist_add(arg_chars, (void*)c); + } else if ('\\' == *c) { + /* Count backslashes until we know whether to double up */ + bs_counter++; + } else { + /* Don't double up slashes preceding a non-quote */ + for (i=0; i<bs_counter; i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + smartlist_add(arg_chars, (void*)c); + } + } + /* Don't double up trailing backslashes */ + for (i=0; i<bs_counter; i++) + smartlist_add(arg_chars, (void*)&backslash); + + /* Allocate space for argument, quotes (if needed), and terminator */ + const size_t formatted_arg_len = smartlist_len(arg_chars) + + (need_quotes ? 2 : 0) + 1; + formatted_arg = tor_malloc_zero(formatted_arg_len); + + /* Add leading quote */ + i=0; + if (need_quotes) + formatted_arg[i++] = '"'; + + /* Add characters */ + SMARTLIST_FOREACH(arg_chars, char*, ch, + { + formatted_arg[i++] = *ch; + }); + + /* Add trailing quote */ + if (need_quotes) + formatted_arg[i++] = '"'; + formatted_arg[i] = '\0'; + + smartlist_free(arg_chars); + return formatted_arg; +} + +/** Format a command line for use on Windows, which takes the command as a + * string rather than string array. Follows the rules from "Parsing C++ + * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the + * Python subprocess module. Returns a newly allocated string */ +STATIC char * +tor_join_win_cmdline(const char *argv[]) +{ + smartlist_t *argv_list; + char *joined_argv; + int i; + + /* Format each argument and put the result in a smartlist */ + argv_list = smartlist_new(); + for (i=0; argv[i] != NULL; i++) { + smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i])); + } + + /* Join the arguments with whitespace */ + joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL); + + /* Free the newly allocated arguments, and the smartlist */ + SMARTLIST_FOREACH(argv_list, char *, arg, + { + tor_free(arg); + }); + smartlist_free(argv_list); + + return joined_argv; +} + +#endif /* defined(_WIN32) */ diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h new file mode 100644 index 0000000000..0f264c8710 --- /dev/null +++ b/src/lib/process/process_win32.h @@ -0,0 +1,97 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.h + * \brief Header for process_win32.c + **/ + +#ifndef TOR_PROCESS_WIN32_H +#define TOR_PROCESS_WIN32_H + +#ifdef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/evloop/compat_libevent.h" + +#include <windows.h> + +struct process_t; + +struct process_win32_t; +typedef struct process_win32_t process_win32_t; + +process_win32_t *process_win32_new(void); +void process_win32_free_(process_win32_t *win32_process); +#define process_win32_free(s) \ + FREE_AND_NULL(process_win32_t, process_win32_free_, (s)) + +void process_win32_init(void); +void process_win32_deinit(void); + +process_status_t process_win32_exec(struct process_t *process); +bool process_win32_terminate(struct process_t *process); + +process_pid_t process_win32_get_pid(struct process_t *process); + +int process_win32_write(struct process_t *process, buf_t *buffer); +int process_win32_read_stdout(struct process_t *process, buf_t *buffer); +int process_win32_read_stderr(struct process_t *process, buf_t *buffer); + +void process_win32_trigger_completion_callbacks(void); + +/* Timer handling. */ +void process_win32_timer_start(void); +void process_win32_timer_stop(void); +bool process_win32_timer_running(void); + +#ifdef PROCESS_WIN32_PRIVATE +STATIC void process_win32_timer_callback(periodic_timer_t *, void *); +STATIC bool process_win32_timer_test_process(process_t *); + +/* I/O pipe handling. */ +struct process_win32_handle_t; +typedef struct process_win32_handle_t process_win32_handle_t; + +typedef enum process_win32_pipe_type_t { + /** This pipe is used for reading. */ + PROCESS_WIN32_PIPE_TYPE_READER, + + /** This pipe is used for writing. */ + PROCESS_WIN32_PIPE_TYPE_WRITER +} process_win32_pipe_type_t; + +STATIC bool process_win32_create_pipe(HANDLE *, + HANDLE *, + SECURITY_ATTRIBUTES *, + process_win32_pipe_type_t); + +STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle); + +STATIC VOID WINAPI process_win32_stdout_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stderr_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stdin_write_done(DWORD, + DWORD, + LPOVERLAPPED); + +STATIC int process_win32_read_from_handle(process_win32_handle_t *, + buf_t *, + LPOVERLAPPED_COMPLETION_ROUTINE); +STATIC bool process_win32_handle_read_completion(process_win32_handle_t *, + DWORD, + DWORD); + +STATIC char *format_win_cmdline_argument(const char *arg); +STATIC char *tor_join_win_cmdline(const char *argv[]); +#endif /* defined(PROCESS_WIN32_PRIVATE) */ + +#endif /* defined(_WIN32) */ + +#endif /* !defined(TOR_PROCESS_WIN32_H) */ diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c index 534b39d101..cd2a1c57b5 100644 --- a/src/lib/process/restrict.c +++ b/src/lib/process/restrict.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -85,7 +85,7 @@ tor_set_max_memlock(void) { /* Future consideration for Windows is probably SetProcessWorkingSetSize * This is similar to setting the memory rlimit of RLIMIT_MEMLOCK - * http://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx + * https://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx */ struct rlimit limit; @@ -128,7 +128,7 @@ tor_mlockall(void) * Future consideration for Windows may be VirtualLock * VirtualLock appears to implement mlock() but not mlockall() * - * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx + * https://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx */ #ifdef HAVE_UNIX_MLOCKALL @@ -152,7 +152,7 @@ tor_mlockall(void) "pages: %s", strerror(errno)); return -1; } -#else /* !(defined(HAVE_UNIX_MLOCKALL)) */ +#else /* !defined(HAVE_UNIX_MLOCKALL) */ log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?"); return -1; #endif /* defined(HAVE_UNIX_MLOCKALL) */ @@ -190,7 +190,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) /* Define some maximum connections values for systems where we cannot * automatically determine a limit. Re Cygwin, see - * http://archives.seul.org/or/talk/Aug-2006/msg00210.html + * https://archives.seul.org/or/talk/Aug-2006/msg00210.html * For an iPhone, 9999 should work. For Windows and all other unknown * systems we use 15000 as the default. */ #ifndef HAVE_GETRLIMIT @@ -214,7 +214,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out) return -1; } limit = MAX_CONNECTIONS; -#else /* !(!defined(HAVE_GETRLIMIT)) */ +#else /* defined(HAVE_GETRLIMIT) */ struct rlimit rlim; if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) { diff --git a/src/lib/process/restrict.h b/src/lib/process/restrict.h index 8491c99044..6173d5a438 100644 --- a/src/lib/process/restrict.h +++ b/src/lib/process/restrict.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c index 6e8258f279..3cfd520a4f 100644 --- a/src/lib/process/setuid.c +++ b/src/lib/process/setuid.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -64,7 +64,7 @@ log_credential_status(void) /* log UIDs */ #ifdef HAVE_GETRESUID - if (getresuid(&ruid, &euid, &suid) != 0 ) { + if (getresuid(&ruid, &euid, &suid) != 0) { log_warn(LD_GENERAL, "Error getting changed UIDs: %s", strerror(errno)); return -1; } else { @@ -72,7 +72,7 @@ log_credential_status(void) "UID is %u (real), %u (effective), %u (saved)", (unsigned)ruid, (unsigned)euid, (unsigned)suid); } -#else /* !(defined(HAVE_GETRESUID)) */ +#else /* !defined(HAVE_GETRESUID) */ /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */ ruid = getuid(); euid = geteuid(); @@ -85,7 +85,7 @@ log_credential_status(void) /* log GIDs */ #ifdef HAVE_GETRESGID - if (getresgid(&rgid, &egid, &sgid) != 0 ) { + if (getresgid(&rgid, &egid, &sgid) != 0) { log_warn(LD_GENERAL, "Error getting changed GIDs: %s", strerror(errno)); return -1; } else { @@ -93,7 +93,7 @@ log_credential_status(void) "GID is %u (real), %u (effective), %u (saved)", (unsigned)rgid, (unsigned)egid, (unsigned)sgid); } -#else /* !(defined(HAVE_GETRESGID)) */ +#else /* !defined(HAVE_GETRESGID) */ /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */ rgid = getgid(); egid = getegid(); @@ -154,7 +154,7 @@ have_capability_support(void) return 0; cap_free(caps); return 1; -#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ +#else /* !defined(HAVE_LINUX_CAPABILITIES) */ return 0; #endif /* defined(HAVE_LINUX_CAPABILITIES) */ } @@ -265,7 +265,7 @@ switch_id(const char *user, const unsigned flags) if (drop_capabilities(1)) return -1; } -#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */ +#else /* !defined(HAVE_LINUX_CAPABILITIES) */ (void) keep_bindlow; if (warn_if_no_caps) { log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support " @@ -376,7 +376,7 @@ switch_id(const char *user, const unsigned flags) #endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */ return 0; -#else /* !(!defined(_WIN32)) */ +#else /* defined(_WIN32) */ (void)user; (void)flags; diff --git a/src/lib/process/setuid.h b/src/lib/process/setuid.h index 7d03e1f025..fec35a1216 100644 --- a/src/lib/process/setuid.h +++ b/src/lib/process/setuid.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,4 +19,4 @@ int have_capability_support(void); #define SWITCH_ID_WARN_IF_NO_CAPS (1<<1) int switch_id(const char *user, unsigned flags); -#endif +#endif /* !defined(TOR_SETUID_H) */ diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c deleted file mode 100644 index f4429d7f76..0000000000 --- a/src/lib/process/subprocess.c +++ /dev/null @@ -1,1236 +0,0 @@ -/* Copyright (c) 2003, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file subprocess.c - * \brief Launch and monitor other processes. - **/ - -#define SUBPROCESS_PRIVATE -#include "lib/process/subprocess.h" - -#include "lib/container/smartlist.h" -#include "lib/err/torerr.h" -#include "lib/log/log.h" -#include "lib/log/util_bug.h" -#include "lib/log/win32err.h" -#include "lib/malloc/malloc.h" -#include "lib/process/env.h" -#include "lib/process/waitpid.h" -#include "lib/string/compat_ctype.h" - -#ifdef HAVE_SYS_TYPES_H -#include <sys/types.h> -#endif -#ifdef HAVE_SYS_PRCTL_H -#include <sys/prctl.h> -#endif -#ifdef HAVE_UNISTD_H -#include <unistd.h> -#endif -#ifdef HAVE_SIGNAL_H -#include <signal.h> -#endif -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif -#ifdef HAVE_SYS_WAIT_H -#include <sys/wait.h> -#endif -#include <errno.h> -#include <string.h> - -/** Format a single argument for being put on a Windows command line. - * Returns a newly allocated string */ -static char * -format_win_cmdline_argument(const char *arg) -{ - char *formatted_arg; - char need_quotes; - const char *c; - int i; - int bs_counter = 0; - /* Backslash we can point to when one is inserted into the string */ - const char backslash = '\\'; - - /* Smartlist of *char */ - smartlist_t *arg_chars; - arg_chars = smartlist_new(); - - /* Quote string if it contains whitespace or is empty */ - need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); - - /* Build up smartlist of *chars */ - for (c=arg; *c != '\0'; c++) { - if ('"' == *c) { - /* Double up backslashes preceding a quote */ - for (i=0; i<(bs_counter*2); i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - /* Escape the quote */ - smartlist_add(arg_chars, (void*)&backslash); - smartlist_add(arg_chars, (void*)c); - } else if ('\\' == *c) { - /* Count backslashes until we know whether to double up */ - bs_counter++; - } else { - /* Don't double up slashes preceding a non-quote */ - for (i=0; i<bs_counter; i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - smartlist_add(arg_chars, (void*)c); - } - } - /* Don't double up trailing backslashes */ - for (i=0; i<bs_counter; i++) - smartlist_add(arg_chars, (void*)&backslash); - - /* Allocate space for argument, quotes (if needed), and terminator */ - const size_t formatted_arg_len = smartlist_len(arg_chars) + - (need_quotes ? 2 : 0) + 1; - formatted_arg = tor_malloc_zero(formatted_arg_len); - - /* Add leading quote */ - i=0; - if (need_quotes) - formatted_arg[i++] = '"'; - - /* Add characters */ - SMARTLIST_FOREACH(arg_chars, char*, ch, - { - formatted_arg[i++] = *ch; - }); - - /* Add trailing quote */ - if (need_quotes) - formatted_arg[i++] = '"'; - formatted_arg[i] = '\0'; - - smartlist_free(arg_chars); - return formatted_arg; -} - -/** Format a command line for use on Windows, which takes the command as a - * string rather than string array. Follows the rules from "Parsing C++ - * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the - * Python subprocess module. Returns a newly allocated string */ -char * -tor_join_win_cmdline(const char *argv[]) -{ - smartlist_t *argv_list; - char *joined_argv; - int i; - - /* Format each argument and put the result in a smartlist */ - argv_list = smartlist_new(); - for (i=0; argv[i] != NULL; i++) { - smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i])); - } - - /* Join the arguments with whitespace */ - joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL); - - /* Free the newly allocated arguments, and the smartlist */ - SMARTLIST_FOREACH(argv_list, char *, arg, - { - tor_free(arg); - }); - smartlist_free(argv_list); - - return joined_argv; -} - -#ifndef _WIN32 -/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in - * <b>hex_errno</b>. Called between fork and _exit, so must be signal-handler - * safe. - * - * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available. - * - * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded - * with spaces. CHILD_STATE indicates where - * in the process of starting the child process did the failure occur (see - * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of - * errno when the failure occurred. - * - * On success return the number of characters added to hex_errno, not counting - * the terminating NUL; return -1 on error. - */ -STATIC int -format_helper_exit_status(unsigned char child_state, int saved_errno, - char *hex_errno) -{ - unsigned int unsigned_errno; - int written, left; - char *cur; - size_t i; - int res = -1; - - /* Fill hex_errno with spaces, and a trailing newline (memset may - not be signal handler safe, so we can't use it) */ - for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++) - hex_errno[i] = ' '; - hex_errno[HEX_ERRNO_SIZE - 1] = '\n'; - - /* Convert errno to be unsigned for hex conversion */ - if (saved_errno < 0) { - // Avoid overflow on the cast to unsigned int when result is INT_MIN - // by adding 1 to the signed int negative value, - // then, after it has been negated and cast to unsigned, - // adding the original 1 back (the double-addition is intentional). - // Otherwise, the cast to signed could cause a temporary int - // to equal INT_MAX + 1, which is undefined. - unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1; - } else { - unsigned_errno = (unsigned int) saved_errno; - } - - /* - * Count how many chars of space we have left, and keep a pointer into the - * current point in the buffer. - */ - left = HEX_ERRNO_SIZE+1; - cur = hex_errno; - - /* Emit child_state */ - written = format_hex_number_sigsafe(child_state, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - if (left <= 0) - goto err; - - /* Now the '/' */ - *cur = '/'; - - /* Adjust left and cur */ - ++cur; - --left; - if (left <= 0) - goto err; - - /* Need minus? */ - if (saved_errno < 0) { - *cur = '-'; - ++cur; - --left; - if (left <= 0) - goto err; - } - - /* Emit unsigned_errno */ - written = format_hex_number_sigsafe(unsigned_errno, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - - /* Check that we have enough space left for a newline and a NUL */ - if (left <= 1) - goto err; - - /* Emit the newline and NUL */ - *cur++ = '\n'; - *cur++ = '\0'; - - res = (int)(cur - hex_errno - 1); - - goto done; - - err: - /* - * In error exit, just write a '\0' in the first char so whatever called - * this at least won't fall off the end. - */ - *hex_errno = '\0'; - - done: - return res; -} -#endif /* !defined(_WIN32) */ - -/* Maximum number of file descriptors, if we cannot get it via sysconf() */ -#define DEFAULT_MAX_FD 256 - -/** Terminate the process of <b>process_handle</b>, if that process has not - * already exited. - * - * Return 0 if we succeeded in terminating the process (or if the process - * already exited), and -1 if we tried to kill the process but failed. - * - * Based on code originally borrowed from Python's os.kill. */ -int -tor_terminate_process(process_handle_t *process_handle) -{ -#ifdef _WIN32 - if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) { - HANDLE handle = process_handle->pid.hProcess; - - if (!TerminateProcess(handle, 0)) - return -1; - else - return 0; - } -#else /* !(defined(_WIN32)) */ - if (process_handle->waitpid_cb) { - /* We haven't got a waitpid yet, so we can just kill off the process. */ - return kill(process_handle->pid, SIGTERM); - } -#endif /* defined(_WIN32) */ - - return 0; /* We didn't need to kill the process, so report success */ -} - -/** Return the Process ID of <b>process_handle</b>. */ -int -tor_process_get_pid(process_handle_t *process_handle) -{ -#ifdef _WIN32 - return (int) process_handle->pid.dwProcessId; -#else - return (int) process_handle->pid; -#endif -} - -#ifdef _WIN32 -HANDLE -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#else /* !(defined(_WIN32)) */ -/* DOCDOC tor_process_get_stdout_pipe */ -int -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#endif /* defined(_WIN32) */ - -/* DOCDOC process_handle_new */ -static process_handle_t * -process_handle_new(void) -{ - process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t)); - -#ifdef _WIN32 - out->stdin_pipe = INVALID_HANDLE_VALUE; - out->stdout_pipe = INVALID_HANDLE_VALUE; - out->stderr_pipe = INVALID_HANDLE_VALUE; -#else - out->stdin_pipe = -1; - out->stdout_pipe = -1; - out->stderr_pipe = -1; -#endif /* defined(_WIN32) */ - - return out; -} - -#ifndef _WIN32 -/** Invoked when a process that we've launched via tor_spawn_background() has - * been found to have terminated. - */ -static void -process_handle_waitpid_cb(int status, void *arg) -{ - process_handle_t *process_handle = arg; - - process_handle->waitpid_exit_status = status; - clear_waitpid_callback(process_handle->waitpid_cb); - if (process_handle->status == PROCESS_STATUS_RUNNING) - process_handle->status = PROCESS_STATUS_NOTRUNNING; - process_handle->waitpid_cb = 0; -} -#endif /* !defined(_WIN32) */ - -/** - * @name child-process states - * - * Each of these values represents a possible state that a child process can - * be in. They're used to determine what to say when telling the parent how - * far along we were before failure. - * - * @{ - */ -#define CHILD_STATE_INIT 0 -#define CHILD_STATE_PIPE 1 -#define CHILD_STATE_MAXFD 2 -#define CHILD_STATE_FORK 3 -#define CHILD_STATE_DUPOUT 4 -#define CHILD_STATE_DUPERR 5 -#define CHILD_STATE_DUPIN 6 -#define CHILD_STATE_CLOSEFD 7 -#define CHILD_STATE_EXEC 8 -#define CHILD_STATE_FAILEXEC 9 -/** @} */ -/** - * Boolean. If true, then Tor may call execve or CreateProcess via - * tor_spawn_background. - **/ -static int may_spawn_background_process = 1; -/** - * Turn off may_spawn_background_process, so that all future calls to - * tor_spawn_background are guaranteed to fail. - **/ -void -tor_disable_spawning_background_processes(void) -{ - may_spawn_background_process = 0; -} -/** Start a program in the background. If <b>filename</b> contains a '/', then - * it will be treated as an absolute or relative path. Otherwise, on - * non-Windows systems, the system path will be searched for <b>filename</b>. - * On Windows, only the current directory will be searched. Here, to search the - * system path (as well as the application directory, current working - * directory, and system directories), set filename to NULL. - * - * The strings in <b>argv</b> will be passed as the command line arguments of - * the child program (following convention, argv[0] should normally be the - * filename of the executable, and this must be the case if <b>filename</b> is - * NULL). The last element of argv must be NULL. A handle to the child process - * will be returned in process_handle (which must be non-NULL). Read - * process_handle.status to find out if the process was successfully launched. - * For convenience, process_handle.status is returned by this function. - * - * Some parts of this code are based on the POSIX subprocess module from - * Python, and example code from - * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx. - */ -int -tor_spawn_background(const char *const filename, const char **argv, - process_environment_t *env, - process_handle_t **process_handle_out) -{ - if (BUG(may_spawn_background_process == 0)) { - /* We should never reach this point if we're forbidden to spawn - * processes. Instead we should have caught the attempt earlier. */ - return PROCESS_STATUS_ERROR; - } - -#ifdef _WIN32 - HANDLE stdout_pipe_read = NULL; - HANDLE stdout_pipe_write = NULL; - HANDLE stderr_pipe_read = NULL; - HANDLE stderr_pipe_write = NULL; - HANDLE stdin_pipe_read = NULL; - HANDLE stdin_pipe_write = NULL; - process_handle_t *process_handle; - int status; - - STARTUPINFOA siStartInfo; - BOOL retval = FALSE; - - SECURITY_ATTRIBUTES saAttr; - char *joined_argv; - - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - saAttr.bInheritHandle = TRUE; - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - saAttr.lpSecurityDescriptor = NULL; - - /* Assume failure to start process */ - status = PROCESS_STATUS_ERROR; - - /* Set up pipe for stdout */ - if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdout communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdout communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stderr */ - if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stderr communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stderr communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stdin */ - if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdin communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdin communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Create the child process */ - - /* Windows expects argv to be a whitespace delimited string, so join argv up - */ - joined_argv = tor_join_win_cmdline(argv); - - process_handle = process_handle_new(); - process_handle->status = status; - - ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION)); - ZeroMemory(&siStartInfo, sizeof(STARTUPINFO)); - siStartInfo.cb = sizeof(STARTUPINFO); - siStartInfo.hStdError = stderr_pipe_write; - siStartInfo.hStdOutput = stdout_pipe_write; - siStartInfo.hStdInput = stdin_pipe_read; - siStartInfo.dwFlags |= STARTF_USESTDHANDLES; - - /* Create the child process */ - - retval = CreateProcessA(filename, // module name - joined_argv, // command line - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - NULL, // process security attributes - NULL, // primary thread security attributes - TRUE, // handles are inherited - /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess() - * work?) */ - CREATE_NO_WINDOW, // creation flags - (env==NULL) ? NULL : env->windows_environment_block, - NULL, // use parent's current directory - &siStartInfo, // STARTUPINFO pointer - &(process_handle->pid)); // receives PROCESS_INFORMATION - - tor_free(joined_argv); - - if (!retval) { - log_warn(LD_GENERAL, - "Failed to create child process %s: %s", filename?filename:argv[0], - format_win32_error(GetLastError())); - tor_free(process_handle); - } else { - /* TODO: Close hProcess and hThread in process_handle->pid? */ - process_handle->stdout_pipe = stdout_pipe_read; - process_handle->stderr_pipe = stderr_pipe_read; - process_handle->stdin_pipe = stdin_pipe_write; - status = process_handle->status = PROCESS_STATUS_RUNNING; - } - - /* TODO: Close pipes on exit */ - *process_handle_out = process_handle; - return status; -#else /* !(defined(_WIN32)) */ - pid_t pid; - int stdout_pipe[2]; - int stderr_pipe[2]; - int stdin_pipe[2]; - int fd, retval; - process_handle_t *process_handle; - int status; - - const char *error_message = SPAWN_ERROR_MESSAGE; - size_t error_message_length; - - /* Represents where in the process of spawning the program is; - this is used for printing out the error message */ - unsigned char child_state = CHILD_STATE_INIT; - - char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */ - - static int max_fd = -1; - - status = PROCESS_STATUS_ERROR; - - /* We do the strlen here because strlen() is not signal handler safe, - and we are not allowed to use unsafe functions between fork and exec */ - error_message_length = strlen(error_message); - - // child_state = CHILD_STATE_PIPE; - - /* Set up pipe for redirecting stdout, stderr, and stdin of child */ - retval = pipe(stdout_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdout communication with child process: %s", - strerror(errno)); - return status; - } - - retval = pipe(stderr_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stderr communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - - return status; - } - - retval = pipe(stdin_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdin communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - - return status; - } - - // child_state = CHILD_STATE_MAXFD; - -#ifdef _SC_OPEN_MAX - if (-1 == max_fd) { - max_fd = (int) sysconf(_SC_OPEN_MAX); - if (max_fd == -1) { - max_fd = DEFAULT_MAX_FD; - log_warn(LD_GENERAL, - "Cannot find maximum file descriptor, assuming %d", max_fd); - } - } -#else /* !(defined(_SC_OPEN_MAX)) */ - max_fd = DEFAULT_MAX_FD; -#endif /* defined(_SC_OPEN_MAX) */ - - // child_state = CHILD_STATE_FORK; - - pid = fork(); - if (0 == pid) { - /* In child */ - -#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) - /* Attempt to have the kernel issue a SIGTERM if the parent - * goes away. Certain attributes of the binary being execve()ed - * will clear this during the execve() call, but it's better - * than nothing. - */ - prctl(PR_SET_PDEATHSIG, SIGTERM); -#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ - - child_state = CHILD_STATE_DUPOUT; - - /* Link child stdout to the write end of the pipe */ - retval = dup2(stdout_pipe[1], STDOUT_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPERR; - - /* Link child stderr to the write end of the pipe */ - retval = dup2(stderr_pipe[1], STDERR_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPIN; - - /* Link child stdin to the read end of the pipe */ - retval = dup2(stdin_pipe[0], STDIN_FILENO); - if (-1 == retval) - goto error; - - // child_state = CHILD_STATE_CLOSEFD; - - close(stderr_pipe[0]); - close(stderr_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - - /* Close all other fds, including the read end of the pipe */ - /* XXX: We should now be doing enough FD_CLOEXEC setting to make - * this needless. */ - for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) { - close(fd); - } - - // child_state = CHILD_STATE_EXEC; - - /* Call the requested program. We need the cast because - execvp doesn't define argv as const, even though it - does not modify the arguments */ - if (env) - execve(filename, (char *const *) argv, env->unixoid_environment_block); - else { - static char *new_env[] = { NULL }; - execve(filename, (char *const *) argv, new_env); - } - - /* If we got here, the exec or open(/dev/null) failed */ - - child_state = CHILD_STATE_FAILEXEC; - - error: - { - /* XXX: are we leaking fds from the pipe? */ - int n, err=0; - ssize_t nbytes; - - n = format_helper_exit_status(child_state, errno, hex_errno); - - if (n >= 0) { - /* Write the error message. GCC requires that we check the return - value, but there is nothing we can do if it fails */ - /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */ - nbytes = write(STDOUT_FILENO, error_message, error_message_length); - err = (nbytes < 0); - nbytes = write(STDOUT_FILENO, hex_errno, n); - err += (nbytes < 0); - } - - _exit(err?254:255); // exit ok: in child. - } - - /* Never reached, but avoids compiler warning */ - return status; // LCOV_EXCL_LINE - } - - /* In parent */ - - if (-1 == pid) { - log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno)); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - return status; - } - - process_handle = process_handle_new(); - process_handle->status = status; - process_handle->pid = pid; - - /* TODO: If the child process forked but failed to exec, waitpid it */ - - /* Return read end of the pipes to caller, and close write end */ - process_handle->stdout_pipe = stdout_pipe[0]; - retval = close(stdout_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stdout pipe in parent process: %s", - strerror(errno)); - } - - process_handle->waitpid_cb = set_waitpid_callback(pid, - process_handle_waitpid_cb, - process_handle); - - process_handle->stderr_pipe = stderr_pipe[0]; - retval = close(stderr_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stderr pipe in parent process: %s", - strerror(errno)); - } - - /* Return write end of the stdin pipe to caller, and close the read end */ - process_handle->stdin_pipe = stdin_pipe[1]; - retval = close(stdin_pipe[0]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close read end of stdin pipe in parent process: %s", - strerror(errno)); - } - - status = process_handle->status = PROCESS_STATUS_RUNNING; - /* Set stdin/stdout/stderr pipes to be non-blocking */ - if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) { - log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " - "nonblocking in parent process: %s", strerror(errno)); - } - - *process_handle_out = process_handle; - return status; -#endif /* defined(_WIN32) */ -} - -/** Destroy all resources allocated by the process handle in - * <b>process_handle</b>. - * If <b>also_terminate_process</b> is true, also terminate the - * process of the process handle. */ -MOCK_IMPL(void, -tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)) -{ - if (!process_handle) - return; - - if (also_terminate_process) { - if (tor_terminate_process(process_handle) < 0) { - const char *errstr = -#ifdef _WIN32 - format_win32_error(GetLastError()); -#else - strerror(errno); -#endif - log_notice(LD_GENERAL, "Failed to terminate process with " - "PID '%d' ('%s').", tor_process_get_pid(process_handle), - errstr); - } else { - log_info(LD_GENERAL, "Terminated process with PID '%d'.", - tor_process_get_pid(process_handle)); - } - } - - process_handle->status = PROCESS_STATUS_NOTRUNNING; - -#ifdef _WIN32 - if (process_handle->stdout_pipe) - CloseHandle(process_handle->stdout_pipe); - - if (process_handle->stderr_pipe) - CloseHandle(process_handle->stderr_pipe); - - if (process_handle->stdin_pipe) - CloseHandle(process_handle->stdin_pipe); -#else /* !(defined(_WIN32)) */ - close(process_handle->stdout_pipe); - close(process_handle->stderr_pipe); - close(process_handle->stdin_pipe); - - clear_waitpid_callback(process_handle->waitpid_cb); -#endif /* defined(_WIN32) */ - - memset(process_handle, 0x0f, sizeof(process_handle_t)); - tor_free(process_handle); -} - -/** Get the exit code of a process specified by <b>process_handle</b> and store - * it in <b>exit_code</b>, if set to a non-NULL value. If <b>block</b> is set - * to true, the call will block until the process has exited. Otherwise if - * the process is still running, the function will return - * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns - * PROCESS_EXIT_EXITED if the process did exit. If there is a failure, - * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if - * non-NULL) will be undefined. N.B. Under *nix operating systems, this will - * probably not work in Tor, because waitpid() is called in main.c to reap any - * terminated child processes.*/ -int -tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code) -{ -#ifdef _WIN32 - DWORD retval; - BOOL success; - - if (block) { - /* Wait for the process to exit */ - retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE); - if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } else { - retval = WaitForSingleObject(process_handle->pid.hProcess, 0); - if (WAIT_TIMEOUT == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } - - if (exit_code != NULL) { - success = GetExitCodeProcess(process_handle->pid.hProcess, - (PDWORD)exit_code); - if (!success) { - log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s", - format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } -#else /* !(defined(_WIN32)) */ - int stat_loc; - int retval; - - if (process_handle->waitpid_cb) { - /* We haven't processed a SIGCHLD yet. */ - retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG); - if (retval == process_handle->pid) { - clear_waitpid_callback(process_handle->waitpid_cb); - process_handle->waitpid_cb = NULL; - process_handle->waitpid_exit_status = stat_loc; - } - } else { - /* We already got a SIGCHLD for this process, and handled it. */ - retval = process_handle->pid; - stat_loc = process_handle->waitpid_exit_status; - } - - if (!block && 0 == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != process_handle->pid) { - log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s", - (int)process_handle->pid, strerror(errno)); - return PROCESS_EXIT_ERROR; - } - - if (!WIFEXITED(stat_loc)) { - log_warn(LD_GENERAL, "Process %d did not exit normally", - (int)process_handle->pid); - return PROCESS_EXIT_ERROR; - } - - if (exit_code != NULL) - *exit_code = WEXITSTATUS(stat_loc); -#endif /* defined(_WIN32) */ - - return PROCESS_EXIT_EXITED; -} - -#ifdef _WIN32 -/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes. If - * <b>hProcess</b> is NULL, the function will return immediately if there is - * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle - * to the process owning the <b>h</b>. In this case, the function will exit - * only once the process has exited, or <b>count</b> bytes are read. Returns - * the number of bytes read, or -1 on error. */ -ssize_t -tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process) -{ - size_t numread = 0; - BOOL retval; - DWORD byte_count; - BOOL process_exited = FALSE; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - /* Check if there is anything to read */ - retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); - if (!retval) { - log_warn(LD_GENERAL, - "Failed to peek from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* Nothing available: process exited or it is busy */ - - /* Exit if we don't know whether the process is running */ - if (NULL == process) - break; - - /* The process exited and there's nothing left to read from it */ - if (process_exited) - break; - - /* If process is not running, check for output one more time in case - it wrote something after the peek was performed. Otherwise keep on - waiting for output */ - tor_assert(process != NULL); - byte_count = WaitForSingleObject(process->pid.hProcess, 0); - if (WAIT_TIMEOUT != byte_count) - process_exited = TRUE; - - continue; - } - - /* There is data to read; read it */ - retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL); - tor_assert(byte_count + numread <= count); - if (!retval) { - log_warn(LD_GENERAL, "Failed to read from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* End of file */ - break; - } - numread += byte_count; - } - return (ssize_t)numread; -} -#else /* !(defined(_WIN32)) */ -/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes. If - * <b>process</b> is NULL, the function will return immediately if there is - * nothing more to read. Otherwise data will be read until end of file, or - * <b>count</b> bytes are read. Returns the number of bytes read, or -1 on - * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the - * file has been reached. */ -ssize_t -tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof) -{ - size_t numread = 0; - ssize_t result; - - if (eof) - *eof = 0; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - result = read(fd, buf+numread, count-numread); - - if (result == 0) { - log_debug(LD_GENERAL, "read() reached end of file"); - if (eof) - *eof = 1; - break; - } else if (result < 0 && errno == EAGAIN) { - if (process) - continue; - else - break; - } else if (result < 0) { - log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); - return -1; - } - - numread += result; - } - - log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); - return (ssize_t)numread; -} -#endif /* defined(_WIN32) */ - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stdout(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stderr(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Return a string corresponding to <b>stream_status</b>. */ -const char * -stream_status_to_string(enum stream_status stream_status) -{ - switch (stream_status) { - case IO_STREAM_OKAY: - return "okay"; - case IO_STREAM_EAGAIN: - return "temporarily unavailable"; - case IO_STREAM_TERM: - return "terminated"; - case IO_STREAM_CLOSED: - return "closed"; - default: - tor_fragile_assert(); - return "unknown"; - } -} - -/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be - * modified. The resulting smartlist will consist of pointers to buf, so there - * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated - * string. <b>len</b> should be set to the length of the buffer excluding the - * NUL. Non-printable characters (including NUL) will be replaced with "." */ -int -tor_split_lines(smartlist_t *sl, char *buf, int len) -{ - /* Index in buf of the start of the current line */ - int start = 0; - /* Index in buf of the current character being processed */ - int cur = 0; - /* Are we currently in a line */ - char in_line = 0; - - /* Loop over string */ - while (cur < len) { - /* Loop until end of line or end of string */ - for (; cur < len; cur++) { - if (in_line) { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* End of line */ - buf[cur] = '\0'; - /* Point cur to the next line */ - cur++; - /* Line starts at start and ends with a nul */ - break; - } else { - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } else { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* Skip leading vertical space */ - ; - } else { - in_line = 1; - start = cur; - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } - } - /* We are at the end of the line or end of string. If in_line is true there - * is a line which starts at buf+start and ends at a NUL. cur points to - * the character after the NUL. */ - if (in_line) - smartlist_add(sl, (void *)(buf+start)); - in_line = 0; - } - return smartlist_len(sl); -} - -#ifdef _WIN32 - -/** Return a smartlist containing lines outputted from - * <b>handle</b>. Return NULL on error, and set - * <b>stream_status_out</b> appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (HANDLE *handle, - enum stream_status *stream_status_out)) -{ - int pos; - char stdout_buf[600] = {0}; - smartlist_t *lines = NULL; - - tor_assert(stream_status_out); - - *stream_status_out = IO_STREAM_TERM; - - pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL); - if (pos < 0) { - *stream_status_out = IO_STREAM_TERM; - return NULL; - } - if (pos == 0) { - *stream_status_out = IO_STREAM_EAGAIN; - return NULL; - } - - /* End with a null even if there isn't a \r\n at the end */ - /* TODO: What if this is a partial line? */ - stdout_buf[pos] = '\0'; - - /* Split up the buffer */ - lines = smartlist_new(); - tor_split_lines(lines, stdout_buf, pos); - - /* Currently 'lines' is populated with strings residing on the - stack. Replace them with their exact copies on the heap: */ - SMARTLIST_FOREACH(lines, char *, line, - SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line))); - - *stream_status_out = IO_STREAM_OKAY; - - return lines; -} - -#else /* !(defined(_WIN32)) */ - -/** Return a smartlist containing lines outputted from - * <b>fd</b>. Return NULL on error, and set - * <b>stream_status_out</b> appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) -{ - enum stream_status stream_status; - char stdout_buf[400]; - smartlist_t *lines = NULL; - - while (1) { - memset(stdout_buf, 0, sizeof(stdout_buf)); - - stream_status = get_string_from_pipe(fd, - stdout_buf, sizeof(stdout_buf) - 1); - if (stream_status != IO_STREAM_OKAY) - goto done; - - if (!lines) lines = smartlist_new(); - smartlist_split_string(lines, stdout_buf, "\n", 0, 0); - } - - done: - *stream_status_out = stream_status; - return lines; -} - -#endif /* defined(_WIN32) */ - -/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making - * sure it's below <b>count</b> bytes. - * If the string has a trailing newline, we strip it off. - * - * This function is specifically created to handle input from managed - * proxies, according to the pluggable transports spec. Make sure it - * fits your needs before using it. - * - * Returns: - * IO_STREAM_CLOSED: If the stream is closed. - * IO_STREAM_EAGAIN: If there is nothing to read and we should check back - * later. - * IO_STREAM_TERM: If something is wrong with the stream. - * IO_STREAM_OKAY: If everything went okay and we got a string - * in <b>buf_out</b>. */ -enum stream_status -get_string_from_pipe(int fd, char *buf_out, size_t count) -{ - ssize_t ret; - - tor_assert(count <= INT_MAX); - - ret = read(fd, buf_out, count); - - if (ret == 0) - return IO_STREAM_CLOSED; - else if (ret < 0 && errno == EAGAIN) - return IO_STREAM_EAGAIN; - else if (ret < 0) - return IO_STREAM_TERM; - - if (buf_out[ret - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[ret - 1] = '\0'; - } else - buf_out[ret] = '\0'; - - return IO_STREAM_OKAY; -} diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h deleted file mode 100644 index aa3127d62d..0000000000 --- a/src/lib/process/subprocess.h +++ /dev/null @@ -1,134 +0,0 @@ -/* Copyright (c) 2003-2004, Roger Dingledine - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -/** - * \file subprocess.h - * \brief Header for subprocess.c - **/ - -#ifndef TOR_SUBPROCESS_H -#define TOR_SUBPROCESS_H - -#include "lib/cc/torint.h" -#include "lib/testsupport/testsupport.h" -#include <stddef.h> -#ifdef _WIN32 -#include <windows.h> -#endif - -struct smartlist_t; - -void tor_disable_spawning_background_processes(void); - -typedef struct process_handle_t process_handle_t; -struct process_environment_t; -int tor_spawn_background(const char *const filename, const char **argv, - struct process_environment_t *env, - process_handle_t **process_handle_out); - -#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code " - -/** Status of an I/O stream. */ -enum stream_status { - IO_STREAM_OKAY, - IO_STREAM_EAGAIN, - IO_STREAM_TERM, - IO_STREAM_CLOSED -}; - -const char *stream_status_to_string(enum stream_status stream_status); - -enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); - -/* Values of process_handle_t.status. */ -#define PROCESS_STATUS_NOTRUNNING 0 -#define PROCESS_STATUS_RUNNING 1 -#define PROCESS_STATUS_ERROR -1 - -#ifdef SUBPROCESS_PRIVATE -struct waitpid_callback_t; - -/** Structure to represent the state of a process with which Tor is - * communicating. The contents of this structure are private to util.c */ -struct process_handle_t { - /** One of the PROCESS_STATUS_* values */ - int status; -#ifdef _WIN32 - HANDLE stdin_pipe; - HANDLE stdout_pipe; - HANDLE stderr_pipe; - PROCESS_INFORMATION pid; -#else /* !(defined(_WIN32)) */ - int stdin_pipe; - int stdout_pipe; - int stderr_pipe; - pid_t pid; - /** If the process has not given us a SIGCHLD yet, this has the - * waitpid_callback_t that gets invoked once it has. Otherwise this - * contains NULL. */ - struct waitpid_callback_t *waitpid_cb; - /** The exit status reported by waitpid. */ - int waitpid_exit_status; -#endif /* defined(_WIN32) */ -}; -#endif /* defined(SUBPROCESS_PRIVATE) */ - -/* Return values of tor_get_exit_code() */ -#define PROCESS_EXIT_RUNNING 1 -#define PROCESS_EXIT_EXITED 0 -#define PROCESS_EXIT_ERROR -1 -int tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code); -int tor_split_lines(struct smartlist_t *sl, char *buf, int len); -#ifdef _WIN32 -ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process); -#else -ssize_t tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof); -#endif /* defined(_WIN32) */ -ssize_t tor_read_all_from_process_stdout( - const process_handle_t *process_handle, char *buf, size_t count); -ssize_t tor_read_all_from_process_stderr( - const process_handle_t *process_handle, char *buf, size_t count); -char *tor_join_win_cmdline(const char *argv[]); - -int tor_process_get_pid(process_handle_t *process_handle); -#ifdef _WIN32 -HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); -#else -int tor_process_get_stdout_pipe(process_handle_t *process_handle); -#endif - -#ifdef _WIN32 -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle, - enum stream_status *stream_status)); -#else -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd, - enum stream_status *stream_status)); -#endif /* defined(_WIN32) */ - -int tor_terminate_process(process_handle_t *process_handle); - -MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)); - -#ifdef SUBPROCESS_PRIVATE -/* Prototypes for private functions only used by util.c (and unit tests) */ - -#ifndef _WIN32 -STATIC int format_helper_exit_status(unsigned char child_state, - int saved_errno, char *hex_errno); - -/* Space for hex values of child state, a slash, saved_errno (with - leading minus) and newline (no null) */ -#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \ - 1 + sizeof(int) * 2 + 1) -#endif /* !defined(_WIN32) */ - -#endif /* defined(SUBPROCESS_PRIVATE) */ - -#endif diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c index 9b626394d2..33798f65f0 100644 --- a/src/lib/process/waitpid.c +++ b/src/lib/process/waitpid.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -16,7 +16,7 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/malloc/malloc.h" -#include "ht.h" +#include "ext/ht.h" #ifdef HAVE_SYS_WAIT_H #include <sys/wait.h> @@ -58,9 +58,9 @@ process_map_entries_eq_(const waitpid_callback_t *a, static HT_HEAD(process_map, waitpid_callback_t) process_map = HT_INITIALIZER(); HT_PROTOTYPE(process_map, waitpid_callback_t, node, process_map_entry_hash_, - process_map_entries_eq_) + process_map_entries_eq_); HT_GENERATE2(process_map, waitpid_callback_t, node, process_map_entry_hash_, - process_map_entries_eq_, 0.6, tor_reallocarray_, tor_free_) + process_map_entries_eq_, 0.6, tor_reallocarray_, tor_free_); /** * Begin monitoring the child pid <b>pid</b> to see if we get a SIGCHLD for diff --git a/src/lib/process/waitpid.h b/src/lib/process/waitpid.h index 5faef468c1..171cf206fb 100644 --- a/src/lib/process/waitpid.h +++ b/src/lib/process/waitpid.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2011-2019, The Tor Project, Inc. */ +/* Copyright (c) 2011-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/pubsub/.may_include b/src/lib/pubsub/.may_include new file mode 100644 index 0000000000..5623492f00 --- /dev/null +++ b/src/lib/pubsub/.may_include @@ -0,0 +1,10 @@ +orconfig.h + +lib/cc/*.h +lib/container/*.h +lib/dispatch/*.h +lib/intmath/*.h +lib/log/*.h +lib/malloc/*.h +lib/pubsub/*.h +lib/string/*.h diff --git a/src/lib/pubsub/include.am b/src/lib/pubsub/include.am new file mode 100644 index 0000000000..e2abebcd40 --- /dev/null +++ b/src/lib/pubsub/include.am @@ -0,0 +1,28 @@ + +noinst_LIBRARIES += src/lib/libtor-pubsub.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-pubsub-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_pubsub_a_SOURCES = \ + src/lib/pubsub/pubsub_build.c \ + src/lib/pubsub/pubsub_check.c \ + src/lib/pubsub/pubsub_publish.c + +src_lib_libtor_pubsub_testing_a_SOURCES = \ + $(src_lib_libtor_pubsub_a_SOURCES) +src_lib_libtor_pubsub_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_pubsub_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/pubsub/pub_binding_st.h \ + src/lib/pubsub/pubsub.h \ + src/lib/pubsub/pubsub_build.h \ + src/lib/pubsub/pubsub_builder_st.h \ + src/lib/pubsub/pubsub_connect.h \ + src/lib/pubsub/pubsub_flags.h \ + src/lib/pubsub/pubsub_macros.h \ + src/lib/pubsub/pubsub_publish.h diff --git a/src/lib/pubsub/lib_pubsub.md b/src/lib/pubsub/lib_pubsub.md new file mode 100644 index 0000000000..3f4c473436 --- /dev/null +++ b/src/lib/pubsub/lib_pubsub.md @@ -0,0 +1,14 @@ +@dir /lib/pubsub +@brief lib/pubsub: Publish-subscribe message passing. + +This module wraps the \refdir{lib/dispatch} module, to provide a more +ergonomic and type-safe approach to message passing. + +In general, we favor this mechanism for cases where higher-level modules +need to be notified when something happens in lower-level modules. (The +alternative would be calling up from the lower-level modules, which +would be error-prone; or maintaining lists of function-pointers, which +would be clumsy and tend to complicate the call graph.) + +See pubsub.c for more information. + diff --git a/src/lib/pubsub/pub_binding_st.h b/src/lib/pubsub/pub_binding_st.h new file mode 100644 index 0000000000..d7c562fc35 --- /dev/null +++ b/src/lib/pubsub/pub_binding_st.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pub_binding_st.h + * @brief Declaration of pub_binding_t. + * + * This is an internal type for the pubsub implementation. + */ + +#ifndef TOR_PUB_BINDING_ST_H +#define TOR_PUB_BINDING_ST_H + +#include "lib/dispatch/msgtypes.h" +struct dispatch_t; + +/** + * A pub_binding_t is an opaque object that subsystems use to publish + * messages. The DISPATCH_ADD_PUB*() macros set it up. + **/ +typedef struct pub_binding_t { + /** + * A pointer to a configured dispatch_t object. This is filled in + * when the dispatch_t is finally constructed. + **/ + struct dispatch_t *dispatch_ptr; + /** + * A template for the msg_t fields that are filled in for this message. + * This is copied into outgoing messages, ensuring that their fields are set + * corretly. + **/ + msg_t msg_template; +} pub_binding_t; + +#endif /* !defined(TOR_PUB_BINDING_ST_H) */ diff --git a/src/lib/pubsub/publish_subscribe.md b/src/lib/pubsub/publish_subscribe.md new file mode 100644 index 0000000000..bb05b100b1 --- /dev/null +++ b/src/lib/pubsub/publish_subscribe.md @@ -0,0 +1,144 @@ + +@page publish_subscribe Publish-subscribe message passing in Tor + +@tableofcontents + +## Introduction + +Tor has introduced a generic publish-subscribe mechanism for delivering +messages internally. It is meant to help us improve the modularity of +our code, by avoiding direct coupling between modules that don't +actually need to invoke one another. + +This publish-subscribe mechanism is *not* meant for handing +multithreading or multiprocess issues, thought we hope that eventually +it might be extended and adapted for that purpose. Instead, we use +publish-subscribe today to decouple modules that shouldn't be calling +each other directly. + +For example, there are numerous parts of our code that might need to +take action when a circuit is completed: a controller might need to be +informed, an onion service negotiation might need to be attached, a +guard might need to be marked as working, or a client connection might +need to be attached. But many of those actions occur at a higher layer +than circuit completion: calling them directly is a layering violation, +and makes our code harder to understand and analyze. + +But with message-passing, we can invert this layering violation: circuit +completion can become a "message" that the circuit code publishes, and +to which higher-level layers subscribe. This means that circuit +handling can be decoupled from higher-level modules, and stay nice and +simple. (@ref pubsub_notyet "1") + +> @anchor pubsub_notyet 1. Unfortunately, like most of our code, circuit +> handling is _not_ yet refactored to use publish-subscribe throughout. +> Instead, layer violations of the type described here are pretty common +> in Tor today. To see a small part of what happens when a circuit is +> completed today, have a look at circuit_build_no_more_hops() and its +> associated code. + +## Channels and delivery policies + +To work with messages, especially when refactoring existing code, you'll +need to understand "channels" and "delivery policies". + +Every message is delivered on a "message channel". Each channel +(conceptually) a queue-like structure that can support an arbitrarily +number of message types. Where channels vary is their delivery +mechanisms, and their guarantees about when messages are processed. + +Currently, three delivery policies are possible: + + - `DELIV_PROMPT` -- causes messages to be processed via a callback in + Tor's event loop. This is generally the best choice, since it + avoids unexpected growth of the stack. + + - `DELIV_IMMEDIATE` -- causes messages to be processed immediately + on the call stack when they are published. This choice grows the + stack, and can lead to unexpected complexity in the call graph. + We should only use it when necessary. + + - `DELIV_NEVER` -- causes messages not to be delivered by the message + dispatch system at all. Instead, some other part of the code must + call dispatch_flush() to get the messages delivered. + +See mainloop_pubsub.c and mainloop_pubsub.h for more information and +implementation details. + +## Layers: Dispatch vs publish-subsubscribe vs mainloop. + +At the lowest level, messages are sent via the "dispatcher" module in +@refdir{lib/dispatch}. For performance, this dispatcher works with a +untyped messages. Publishers, subscribers, channels, and messages are +distinguished by short integers. Associated data is handled as +dynamically-typed data pointers, and its types are also stored as short +integers. + +Naturally, this results in a type-unsafe C API, so most other modules +shouldn't invoke @refdir{lib/dispatch} directly. At a higher level, +@refdir{lib/pubsub} defines a set of functions and macros that make +messages named and type-safe. This is the one that other modules should +use when they want to send or receive a message. + +The two modules above do not handle message delivery. Instead, the +dispatch module takes a callback that it can invoke when a channel +becomes nonempty, and defines a dispatch_flush() function to deliver all +the messages queued in a channel. The work of actually making sure that +dispatch_flush() is called when appropriate falls to the main loop, +which needs to integrate the message dispatcher with the rest of our +events and callbacks. This work happens in mainloop_pubsub.c. + + +## How to publish and subscribe + +This section gives an overview of how to make new messages and how to +use them. For full details, see pubsub_macros.h. + +Before anybody can publish or subscribe to a message, the message must +be declared, typically in a header. This uses DECLARE_MESSAGE() or +DECLARE_MESSAGE_INT(). + +Only subsystems can publish or subscribe messages. For more information +about the subsystems architecture, see @ref initialization. + +To publish a message, you must: + - Include the header that declares the message. + - Declare a set of helper functions via DECLARE_PUBLISH(). These + must be visible wherever you call PUBLISH(). + - Call PUBLISH() to actually send a message. + - Connect your subsystem to the dispatcher by calling + DISPATCH_ADD_PUB() from your subsystem's subsys_fns_t.add_pubsub + callback. + +To subscribe to a message, you must: + - Include the header that declares the message. + - Declare a callback function to be invoked when the message is delivered. + - Use DISPATCH_SUBSCRIBE at file scope to define a set of wrapper + functions to call your callback function with the appropriate type. + - Connect your subsystem to the dispatcher by calling + DISPATCH_ADD_SUB() from your subsystem's subsys_fns_t.add_pubsub + callback. + +Again, the file-level documentation for pubsub_macros.h describes how to +declare a message, how to publish it, and how to subscribe to it. + +## Designing good messages + +**Frequency**: +The publish-subscribe system uses a few function calls +and allocations for each message sent. This makes it unsuitable for +very-high-bandwidth events, like "receiving a single data cell" or "a +socket has become writable." It's fine, however, for events that +ordinarily happen a bit less frequently than that, like a circuit +getting finished, a new connection getting opened, or so on. + +**Semantics**: +A message should declare that something has happened or is happening, +not that something in particular should be done. + +For example, suppose you want to set up a message so that onion services +clean up their replay caches whenever we're low on memory. The event +should be something like `memory_low`, not `clean_up_replay_caches`. +The latter name would imply that the publisher knew who was subscribing +to the message and what they intended to do about it, which would be a +layering violation. diff --git a/src/lib/pubsub/pubsub.h b/src/lib/pubsub/pubsub.h new file mode 100644 index 0000000000..d0a4d317f3 --- /dev/null +++ b/src/lib/pubsub/pubsub.h @@ -0,0 +1,89 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub.h + * @brief Header for OO publish-subscribe functionality. + * + * This module provides a wrapper around the "dispatch" module, + * ensuring type-safety and allowing us to do static analysis on + * publication and subscriptions. + * + * With this module, we enforce: + * <ul> + * <li>that every message has (potential) publishers and subscribers; + * <li>that every message is published and subscribed from the correct + * channels, with the correct type ID, every time it is published. + * <li>that type IDs correspond to a single C type, and that the C types are + * used correctly. + * <li>that when a message is published or subscribed, it is done with + * a correct subsystem identifier + * </ul> + * + * We do this by making "publication requests" and "subscription requests" + * into objects, and doing some computation on them before we create + * a dispatch_t with them. + * + * Rather than using the dispatch module directly, a publishing module + * receives a "binding" object that it uses to send messages with the right + * settings. + * + * Most users of this module will want to use this header, and the + * pubsub_macros.h header for convenience. + */ + +/* + * + * Overview: Messages are sent over channels. Before sending a message on a + * channel, or receiving a message on a channel, a subsystem needs to register + * that it publishes, or subscribes, to that message, on that channel. + * + * Messages, channels, and subsystems are represented internally as short + * integers, though they are associated with human-readable strings for + * initialization and debugging. + * + * When registering for a message, a subsystem must say whether it is an + * exclusive publisher/subscriber to that message type, or whether other + * subsystems may also publish/subscribe to it. + * + * All messages and their publishers/subscribers must be registered early in + * the initialization process. + * + * By default, it is an error for a message type to have publishers and no + * subscribers on a channel, or subscribers and no publishers on a channel. + * + * A subsystem may register for a message with a note that delivery or + * production is disabled -- for example, because the subsystem is + * disabled at compile-time. It is not an error for a message type to + * have all of its publishers or subscribers disabled. + * + * After a message is sent, it is delivered to every recipient. This + * delivery happens from the top level of the event loop; it may be + * interleaved with network events, timers, etc. + * + * Messages may have associated data. This data is typed, and is owned + * by the message. Strings, byte-arrays, and integers have built-in + * support. Other types may be added. If objects are to be sent, + * they should be identified by handle. If an object requires cleanup, + * it should be declared with an associated free function. + * + * Semantically, if two subsystems communicate only by this kind of + * message passing, neither is considered to depend on the other, though + * both are considered to have a dependency on the message and on any + * types it contains. + * + * (Or generational index?) + */ +#ifndef TOR_PUBSUB_PUBSUB_H +#define TOR_PUBSUB_PUBSUB_H + +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_connect.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pubsub_macros.h" +#include "lib/pubsub/pubsub_publish.h" + +#endif /* !defined(TOR_PUBSUB_PUBSUB_H) */ diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c new file mode 100644 index 0000000000..3c134f015c --- /dev/null +++ b/src/lib/pubsub/pubsub_build.c @@ -0,0 +1,307 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_build.c + * @brief Construct a dispatch_t in safer, more OO way. + **/ + +#define PUBSUB_PRIVATE + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_cfg.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/msgtypes.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_build.h" +#include "lib/pubsub/pubsub_builder_st.h" +#include "lib/pubsub/pubsub_connect.h" + +#include "lib/container/smartlist.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" + + #include <string.h> + +/** Construct and return a new empty pubsub_items_t. */ +static pubsub_items_t * +pubsub_items_new(void) +{ + pubsub_items_t *cfg = tor_malloc_zero(sizeof(*cfg)); + cfg->items = smartlist_new(); + cfg->type_items = smartlist_new(); + return cfg; +} + +/** Release all storage held in a pubsub_items_t. */ +void +pubsub_items_free_(pubsub_items_t *cfg) +{ + if (! cfg) + return; + SMARTLIST_FOREACH(cfg->items, pubsub_cfg_t *, item, tor_free(item)); + SMARTLIST_FOREACH(cfg->type_items, + pubsub_type_cfg_t *, item, tor_free(item)); + smartlist_free(cfg->items); + smartlist_free(cfg->type_items); + tor_free(cfg); +} + +/** Construct and return a new pubsub_builder_t. */ +pubsub_builder_t * +pubsub_builder_new(void) +{ + dispatch_naming_init(); + + pubsub_builder_t *pb = tor_malloc_zero(sizeof(*pb)); + pb->cfg = dcfg_new(); + pb->items = pubsub_items_new(); + return pb; +} + +/** + * Release all storage held by a pubsub_builder_t. + * + * You'll (mostly) only want to call this function on an error case: if you're + * constructing a dispatch_t instead, you should call + * pubsub_builder_finalize() to consume the pubsub_builder_t. + */ +void +pubsub_builder_free_(pubsub_builder_t *pb) +{ + if (pb == NULL) + return; + pubsub_items_free(pb->items); + dcfg_free(pb->cfg); + tor_free(pb); +} + +/** + * Create and return a pubsub_connector_t for the subsystem with ID + * <b>subsys</b> to use in adding publications, subscriptions, and types to + * <b>builder</b>. + **/ +pubsub_connector_t * +pubsub_connector_for_subsystem(pubsub_builder_t *builder, + subsys_id_t subsys) +{ + tor_assert(builder); + ++builder->n_connectors; + + pubsub_connector_t *con = tor_malloc_zero(sizeof(*con)); + + con->builder = builder; + con->subsys_id = subsys; + + return con; +} + +/** + * Release all storage held by a pubsub_connector_t. + **/ +void +pubsub_connector_free_(pubsub_connector_t *con) +{ + if (!con) + return; + + if (con->builder) { + --con->builder->n_connectors; + tor_assert(con->builder->n_connectors >= 0); + } + tor_free(con); +} + +/** + * Use <b>con</b> to add a request for being able to publish messages of type + * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>. + **/ +int +pubsub_add_pub_(pubsub_connector_t *con, + pub_binding_t *out, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line) +{ + pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + + memset(out, 0, sizeof(*out)); + cfg->is_publish = true; + + out->msg_template.sender = cfg->subsys = con->subsys_id; + out->msg_template.channel = cfg->channel = channel; + out->msg_template.msg = cfg->msg = msg; + out->msg_template.type = cfg->type = type; + + cfg->flags = flags; + cfg->added_by_file = file; + cfg->added_by_line = line; + + /* We're grabbing a pointer to the pub_binding_t so we can tell it about + * the dispatcher later on. + */ + cfg->pub_binding = out; + + smartlist_add(con->builder->items->items, cfg); + + if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0) + goto err; + if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0) + goto err; + + return 0; + err: + ++con->builder->n_errors; + return -1; +} + +/** + * Use <b>con</b> to add a request for being able to publish messages of type + * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>, + * passing them to the callback in <b>recv_fn</b>. + **/ +int +pubsub_add_sub_(pubsub_connector_t *con, + recv_fn_t recv_fn, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line) +{ + pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + + cfg->is_publish = false; + cfg->subsys = con->subsys_id; + cfg->channel = channel; + cfg->msg = msg; + cfg->type = type; + cfg->flags = flags; + cfg->added_by_file = file; + cfg->added_by_line = line; + + cfg->recv_fn = recv_fn; + + smartlist_add(con->builder->items->items, cfg); + + if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0) + goto err; + if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0) + goto err; + if (! (flags & DISP_FLAG_STUB)) { + if (dcfg_add_recv(con->builder->cfg, msg, cfg->subsys, recv_fn) < 0) + goto err; + } + + return 0; + err: + ++con->builder->n_errors; + return -1; +} + +/** + * Use <b>con</b> to define the functions to use for manipulating the type + * <b>type</b>. Any function pointers left as NULL will be implemented as + * no-ops. + **/ +int +pubsub_connector_register_type_(pubsub_connector_t *con, + msg_type_id_t type, + dispatch_typefns_t *fns, + const char *file, + unsigned line) +{ + pubsub_type_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg)); + cfg->type = type; + memcpy(&cfg->fns, fns, sizeof(*fns)); + cfg->subsys = con->subsys_id; + cfg->added_by_file = file; + cfg->added_by_line = line; + + smartlist_add(con->builder->items->type_items, cfg); + + if (dcfg_type_set_fns(con->builder->cfg, type, fns) < 0) + goto err; + + return 0; + err: + ++con->builder->n_errors; + return -1; +} + +/** + * Initialize the dispatch_ptr field in every relevant publish binding + * for <b>d</b>. + */ +static void +pubsub_items_install_bindings(pubsub_items_t *items, + dispatch_t *d) +{ + SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) { + if (cfg->pub_binding) { + // XXXX we could skip this for STUB publishers, and for any publishers + // XXXX where all subscribers are STUB. + cfg->pub_binding->dispatch_ptr = d; + } + } SMARTLIST_FOREACH_END(cfg); +} + +/** + * Remove the dispatch_ptr fields for all the relevant publish bindings + * in <b>items</b>. The prevents subsequent dispatch_pub_() calls from + * sending messages to a dispatcher that has been freed. + **/ +void +pubsub_items_clear_bindings(pubsub_items_t *items) +{ + SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) { + if (cfg->pub_binding) { + cfg->pub_binding->dispatch_ptr = NULL; + } + } SMARTLIST_FOREACH_END(cfg); +} + +/** + * Create a new dispatcher as configured in a pubsub_builder_t. + * + * Consumes and frees its input. + **/ +dispatch_t * +pubsub_builder_finalize(pubsub_builder_t *builder, + pubsub_items_t **items_out) +{ + dispatch_t *dispatcher = NULL; + tor_assert_nonfatal(builder->n_connectors == 0); + + if (pubsub_builder_check(builder) < 0) + goto err; + + if (builder->n_errors) { + log_warn(LD_GENERAL, "At least one error occurred previously when " + "configuring the dispatcher."); + goto err; + } + + dispatcher = dispatch_new(builder->cfg); + + if (!dispatcher) + goto err; + + pubsub_items_install_bindings(builder->items, dispatcher); + if (items_out) { + *items_out = builder->items; + builder->items = NULL; /* Prevent free */ + } + + err: + pubsub_builder_free(builder); + return dispatcher; +} diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h new file mode 100644 index 0000000000..2781b8251a --- /dev/null +++ b/src/lib/pubsub/pubsub_build.h @@ -0,0 +1,97 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_build.h + * @brief Header used for constructing the OO publish-subscribe facility. + * + * (See pubsub.h for more general information on this API.) + **/ + +#ifndef TOR_PUBSUB_BUILD_H +#define TOR_PUBSUB_BUILD_H + +#include "lib/dispatch/msgtypes.h" + +struct dispatch_t; +struct pubsub_connector_t; + +/** + * A "dispatch builder" is an incomplete dispatcher, used when + * registering messages. It does not have the same integrity guarantees + * as a dispatcher. It cannot actually handle messages itself: once all + * subsystems have registered, it is converted into a dispatch_t. + **/ +typedef struct pubsub_builder_t pubsub_builder_t; + +/** + * A "pubsub items" holds the configuration items used to configure a + * pubsub_builder. After the builder is finalized, this field is extracted, + * and used later to tear down pointers that enable publishing. + **/ +typedef struct pubsub_items_t pubsub_items_t; + +/** + * Create a new pubsub_builder. This should only happen in the + * main-init code. + */ +pubsub_builder_t *pubsub_builder_new(void); + +/** DOCDOC */ +int pubsub_builder_check(pubsub_builder_t *); + +/** + * Free a pubsub builder. This should only happen on error paths, where + * we have decided not to construct a dispatcher for some reason. + */ +#define pubsub_builder_free(db) \ + FREE_AND_NULL(pubsub_builder_t, pubsub_builder_free_, (db)) + +/** Internal implementation of pubsub_builder_free(). */ +void pubsub_builder_free_(pubsub_builder_t *); + +/** + * Create a pubsub connector that a single subsystem will use to + * register its messages. The main-init code does this during susbsystem + * initialization. + */ +struct pubsub_connector_t *pubsub_connector_for_subsystem(pubsub_builder_t *, + subsys_id_t); + +/** + * The main-init code does this after subsystem initialization. + */ +#define pubsub_connector_free(c) \ + FREE_AND_NULL(struct pubsub_connector_t, pubsub_connector_free_, (c)) + +void pubsub_connector_free_(struct pubsub_connector_t *); + +/** + * Constructs a dispatcher from a dispatch_builder, after checking that the + * invariances on the messages, channels, and connections have been + * respected. + * + * This should happen after every subsystem has initialized, and before + * entering the mainloop. + */ +struct dispatch_t *pubsub_builder_finalize(pubsub_builder_t *, + pubsub_items_t **items_out); + +/** + * Clear all pub_binding_t backpointers in <b>items</b>. + **/ +void pubsub_items_clear_bindings(pubsub_items_t *items); + +/** + * @copydoc pubsub_items_free_ + * + * Additionally, set the pointer <b>cfg</b> to NULL. + **/ +#define pubsub_items_free(cfg) \ + FREE_AND_NULL(pubsub_items_t, pubsub_items_free_, (cfg)) +void pubsub_items_free_(pubsub_items_t *cfg); + +#endif /* !defined(TOR_PUBSUB_BUILD_H) */ diff --git a/src/lib/pubsub/pubsub_builder_st.h b/src/lib/pubsub/pubsub_builder_st.h new file mode 100644 index 0000000000..57de1240ee --- /dev/null +++ b/src/lib/pubsub/pubsub_builder_st.h @@ -0,0 +1,161 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_builder_st.h + * + * @brief private structures used for configuring dispatchers and messages. + */ + +#ifndef TOR_PUBSUB_BUILDER_ST_H +#define TOR_PUBSUB_BUILDER_ST_H + +#ifdef PUBSUB_PRIVATE + +#include <stdbool.h> +#include <stddef.h> + +struct dispatch_cfg_t; +struct smartlist_t; +struct pub_binding_t; + +/** + * Configuration for a single publication or subscription request. + * + * These can be stored while the dispatcher is in use, but are only used for + * setup, teardown, and debugging. + * + * There are various fields in this request describing the message; all of + * them must match other descriptions of the message, or a bug has occurred. + **/ +typedef struct pubsub_cfg_t { + /** True if this is a publishing request; false for a subscribing request. */ + bool is_publish; + /** The system making this request. */ + subsys_id_t subsys; + /** The channel on which the message is to be sent. */ + channel_id_t channel; + /** The message ID to be sent or received. */ + message_id_t msg; + /** The C type associated with the message. */ + msg_type_id_t type; + /** One or more DISP_FLAGS_* items, combined with bitwise OR. */ + unsigned flags; + + /** + * Publishing only: a pub_binding object that will receive the binding for + * this request. We will finish filling this in when the dispatcher is + * constructed, so that the subsystem can publish then and not before. + */ + struct pub_binding_t *pub_binding; + + /** + * Subscribing only: a function to receive message objects for this request. + */ + recv_fn_t recv_fn; + + /** The file from which this message was configured */ + const char *added_by_file; + /** The line at which this message was configured */ + unsigned added_by_line; +} pubsub_cfg_t; + +/** + * Configuration request for a single C type. + * + * These are stored while the dispatcher is in use, but are only used for + * setup, teardown, and debugging. + **/ +typedef struct pubsub_type_cfg_t { + /** + * The identifier for this type. + */ + msg_type_id_t type; + /** + * Functions to use when manipulating the type. + */ + dispatch_typefns_t fns; + + /** The subsystem that configured this type. */ + subsys_id_t subsys; + /** The file from which this type was configured */ + const char *added_by_file; + /** The line at which this type was configured */ + unsigned added_by_line; +} pubsub_type_cfg_t; + +/** + * The set of configuration requests for a dispatcher, as made by various + * subsystems. + **/ +struct pubsub_items_t { + /** List of pubsub_cfg_t. */ + struct smartlist_t *items; + /** List of pubsub_type_cfg_t. */ + struct smartlist_t *type_items; +}; + +/** + * Type used to construct a dispatcher. We use this type to build up the + * configuration for a dispatcher, and then pass ownership of that + * configuration to the newly constructed dispatcher. + **/ +struct pubsub_builder_t { + /** Number of outstanding pubsub_connector_t objects pointing to this + * pubsub_builder_t. */ + int n_connectors; + /** Number of errors encountered while constructing this object so far. */ + int n_errors; + /** In-progress configuration that we're constructing, as a list of the + * requests that have been made. */ + struct pubsub_items_t *items; + /** In-progress configuration that we're constructing, in a form that can + * be converted to a dispatch_t. */ + struct dispatch_cfg_t *cfg; +}; + +/** + * Type given to a subsystem when adding connections to a pubsub_builder_t. + * We use this type to force each subsystem to get blamed for the + * publications, subscriptions, and types that it adds. + **/ +struct pubsub_connector_t { + /** The pubsub_builder that this connector refers to. */ + struct pubsub_builder_t *builder; + /** The subsystem that has been given this connector. */ + subsys_id_t subsys_id; +}; + +/** + * Helper structure used when constructing a dispatcher that sorts the + * pubsub_cfg_t objects in various ways. + **/ +typedef struct pubsub_adjmap_t { + /* XXXX The next three fields are currently constructed but not yet + * XXXX used. I believe we'll want them in the future, though. -nickm + */ + /** Number of subsystems; length of the *_by_subsys arrays. */ + size_t n_subsystems; + /** Array of lists of publisher pubsub_cfg_t objects, indexed by + * subsystem. */ + struct smartlist_t **pub_by_subsys; + /** Array of lists of subscriber pubsub_cfg_t objects, indexed by + * subsystem. */ + struct smartlist_t **sub_by_subsys; + + /** Number of message IDs; length of the *_by_msg arrays. */ + size_t n_msgs; + /** Array of lists of publisher pubsub_cfg_t objects, indexed by + * message ID. */ + struct smartlist_t **pub_by_msg; + /** Array of lists of subscriber pubsub_cfg_t objects, indexed by + * message ID. */ + struct smartlist_t **sub_by_msg; +} pubsub_adjmap_t; + +#endif /* defined(PUBSUB_PRIVATE) */ + +#endif /* !defined(TOR_PUBSUB_BUILDER_ST_H) */ diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c new file mode 100644 index 0000000000..dbcbb14746 --- /dev/null +++ b/src/lib/pubsub/pubsub_check.c @@ -0,0 +1,414 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_check.c + * @brief Enforce various requirements on a pubsub_builder. + **/ + +/** @{ */ +#define PUBSUB_PRIVATE +/** @} */ + +#include "lib/dispatch/dispatch_naming.h" +#include "lib/dispatch/msgtypes.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pubsub_builder_st.h" +#include "lib/pubsub/pubsub_build.h" + +#include "lib/container/bitarray.h" +#include "lib/container/smartlist.h" +#include "lib/log/util_bug.h" +#include "lib/malloc/malloc.h" +#include "lib/string/compat_string.h" + +#include <string.h> + +static void pubsub_adjmap_add(pubsub_adjmap_t *map, + const pubsub_cfg_t *item); + +/** + * Helper: contruct and return a new pubsub_adjacency_map from <b>cfg</b>. + * Return NULL on error. + **/ +static pubsub_adjmap_t * +pubsub_build_adjacency_map(const pubsub_items_t *cfg) +{ + pubsub_adjmap_t *map = tor_malloc_zero(sizeof(*map)); + const size_t n_subsystems = get_num_subsys_ids(); + const size_t n_msgs = get_num_message_ids(); + + map->n_subsystems = n_subsystems; + map->n_msgs = n_msgs; + + map->pub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*)); + map->sub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*)); + map->pub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*)); + map->sub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*)); + + SMARTLIST_FOREACH_BEGIN(cfg->items, const pubsub_cfg_t *, item) { + pubsub_adjmap_add(map, item); + } SMARTLIST_FOREACH_END(item); + + return map; +} + +/** + * Helper: add a single pubsub_cfg_t to an adjacency map. + **/ +static void +pubsub_adjmap_add(pubsub_adjmap_t *map, + const pubsub_cfg_t *item) +{ + smartlist_t **by_subsys; + smartlist_t **by_msg; + + tor_assert(item->subsys < map->n_subsystems); + tor_assert(item->msg < map->n_msgs); + + if (item->is_publish) { + by_subsys = &map->pub_by_subsys[item->subsys]; + by_msg = &map->pub_by_msg[item->msg]; + } else { + by_subsys = &map->sub_by_subsys[item->subsys]; + by_msg = &map->sub_by_msg[item->msg]; + } + + if (! *by_subsys) + *by_subsys = smartlist_new(); + if (! *by_msg) + *by_msg = smartlist_new(); + smartlist_add(*by_subsys, (void*) item); + smartlist_add(*by_msg, (void *) item); +} + +/** + * Release all storage held by m and set m to NULL. + **/ +#define pubsub_adjmap_free(m) \ + FREE_AND_NULL(pubsub_adjmap_t, pubsub_adjmap_free_, m) + +/** + * Free every element of an <b>n</b>-element array of smartlists, then + * free the array itself. + **/ +static void +pubsub_adjmap_free_helper(smartlist_t **lsts, size_t n) +{ + if (!lsts) + return; + + for (unsigned i = 0; i < n; ++i) { + smartlist_free(lsts[i]); + } + tor_free(lsts); +} + +/** + * Release all storage held by <b>map</b>. + **/ +static void +pubsub_adjmap_free_(pubsub_adjmap_t *map) +{ + if (!map) + return; + pubsub_adjmap_free_helper(map->pub_by_subsys, map->n_subsystems); + pubsub_adjmap_free_helper(map->sub_by_subsys, map->n_subsystems); + pubsub_adjmap_free_helper(map->pub_by_msg, map->n_msgs); + pubsub_adjmap_free_helper(map->sub_by_msg, map->n_msgs); + tor_free(map); +} + +/** + * Helper: return the length of <b>sl</b>, or 0 if sl is NULL. + **/ +static int +smartlist_len_opt(const smartlist_t *sl) +{ + if (sl) + return smartlist_len(sl); + else + return 0; +} + +/** Return a pointer to a statically allocated string encoding the + * dispatcher flags in <b>flags</b>. */ +static const char * +format_flags(unsigned flags) +{ + static char buf[32]; + buf[0] = 0; + if (flags & DISP_FLAG_EXCL) { + strlcat(buf, " EXCL", sizeof(buf)); + } + if (flags & DISP_FLAG_STUB) { + strlcat(buf, " STUB", sizeof(buf)); + } + return buf[0] ? buf+1 : buf; +} + +/** + * Log a message containing a description of <b>cfg</b> at severity, prefixed + * by the string <b>prefix</b>. + */ +static void +pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix) +{ + tor_assert(prefix); + + tor_log(severity, LD_MESG, + "%s%s %s: %s{%s} on %s (%s) <%u %u %u %u %x> [%s:%d]", + prefix, + get_subsys_id_name(cfg->subsys), + cfg->is_publish ? "PUB" : "SUB", + get_message_id_name(cfg->msg), + get_msg_type_id_name(cfg->type), + get_channel_id_name(cfg->channel), + format_flags(cfg->flags), + cfg->subsys, cfg->msg, cfg->type, cfg->channel, cfg->flags, + cfg->added_by_file, cfg->added_by_line); +} + +/** + * Helper: fill a bitarray <b>out</b> with entries corresponding to the + * subsystems listed in <b>items</b>. + **/ +static void +get_message_bitarray(const pubsub_adjmap_t *map, + const smartlist_t *items, + bitarray_t **out) +{ + *out = bitarray_init_zero((unsigned)map->n_subsystems); + if (! items) + return; + + SMARTLIST_FOREACH_BEGIN(items, const pubsub_cfg_t *, cfg) { + bitarray_set(*out, cfg->subsys); + } SMARTLIST_FOREACH_END(cfg); +} + +/** + * Helper for lint_message: check that all the pubsub_cfg_t items in the two + * respective smartlists obey our local graph topology rules. + * + * (Right now this is just a matter of "each subsystem only + * publishes/subscribes once; no subsystem is a publisher and subscriber for + * the same message.") + * + * Return 0 on success, -1 on failure. + **/ +static int +lint_message_graph(const pubsub_adjmap_t *map, + message_id_t msg, + const smartlist_t *pub, + const smartlist_t *sub) +{ + bitarray_t *published_by = NULL; + bitarray_t *subscribed_by = NULL; + bool ok = true; + + get_message_bitarray(map, pub, &published_by); + get_message_bitarray(map, sub, &subscribed_by); + + /* Check whether any subsystem is publishing and subscribing the same + * message. [??] + */ + for (unsigned i = 0; i < map->n_subsystems; ++i) { + if (bitarray_is_set(published_by, i) && + bitarray_is_set(subscribed_by, i)) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" is published and subscribed by the same " + "subsystem \"%s\".", + get_message_id_name(msg), + get_subsys_id_name(i)); + ok = false; + } + } + + bitarray_free(published_by); + bitarray_free(subscribed_by); + + return ok ? 0 : -1; +} + +/** + * Helper for lint_message: check that all the pubsub_cfg_t items in the two + * respective smartlists have compatible flags, channels, and types. + **/ +static int +lint_message_consistency(message_id_t msg, + const smartlist_t *pub, + const smartlist_t *sub) +{ + if (!smartlist_len_opt(pub) && !smartlist_len_opt(sub)) + return 0; // LCOV_EXCL_LINE -- this was already checked. + + /* The 'all' list has the publishers and the subscribers. */ + smartlist_t *all = smartlist_new(); + if (pub) + smartlist_add_all(all, pub); + if (sub) + smartlist_add_all(all, sub); + + const pubsub_cfg_t *item0 = smartlist_get(all, 0); + + /* Indicates which subsystems we've found publishing/subscribing here. */ + bool pub_excl = false, sub_excl = false, chan_same = true, type_same = true; + + /* Simple message consistency properties across messages. + */ + SMARTLIST_FOREACH_BEGIN(all, const pubsub_cfg_t *, cfg) { + chan_same &= (cfg->channel == item0->channel); + type_same &= (cfg->type == item0->type); + if (cfg->is_publish) + pub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0; + else + sub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0; + } SMARTLIST_FOREACH_END(cfg); + + bool ok = true; + + if (! chan_same) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" is associated with multiple inconsistent " + "channels.", + get_message_id_name(msg)); + ok = false; + } + if (! type_same) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" is associated with multiple inconsistent " + "message types.", + get_message_id_name(msg)); + ok = false; + } + + /* Enforce exclusive-ness for publishers and subscribers that have asked for + * it. + */ + if (pub_excl && smartlist_len_opt(pub) > 1) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" has multiple publishers, but at least one is " + "marked as exclusive.", + get_message_id_name(msg)); + ok = false; + } + if (sub_excl && smartlist_len_opt(sub) > 1) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" has multiple subscribers, but at least one is " + "marked as exclusive.", + get_message_id_name(msg)); + ok = false; + } + + smartlist_free(all); + + return ok ? 0 : -1; +} + +/** + * Check whether there are any errors or inconsistencies for the message + * described by <b>msg</b> in <b>map</b>. If there are problems, log about + * them, and return -1. Otherwise return 0. + **/ +static int +lint_message(const pubsub_adjmap_t *map, message_id_t msg) +{ + /* NOTE: Some of the checks in this function are maybe over-zealous, and we + * might not want to have them forever. I've marked them with [?] below. + */ + if (BUG(msg >= map->n_msgs)) + return 0; // LCOV_EXCL_LINE + + const smartlist_t *pub = map->pub_by_msg[msg]; + const smartlist_t *sub = map->sub_by_msg[msg]; + + const size_t n_pub = smartlist_len_opt(pub); + const size_t n_sub = smartlist_len_opt(sub); + + if (n_pub == 0 && n_sub == 0) { + log_info(LD_MESG, "Nobody is publishing or subscribing to message " + "\"%s\".", + get_message_id_name(msg)); + return 0; // No publishers or subscribers: nothing to do. + } + /* We'll set this to false if there are any problems. */ + bool ok = true; + + /* First make sure that if there are publishers, there are subscribers. */ + if (n_pub == 0) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" has subscribers, but no publishers.", + get_message_id_name(msg)); + ok = false; + } else if (n_sub == 0) { + log_warn(LD_MESG|LD_BUG, + "Message \"%s\" has publishers, but no subscribers.", + get_message_id_name(msg)); + ok = false; + } + + /* Check the message graph topology. */ + if (lint_message_graph(map, msg, pub, sub) < 0) + ok = false; + + /* Check whether the messages have the same fields set on them. */ + if (lint_message_consistency(msg, pub, sub) < 0) + ok = false; + + if (!ok) { + /* There was a problem -- let's log all the publishers and subscribers on + * this message */ + if (pub) { + SMARTLIST_FOREACH(pub, pubsub_cfg_t *, cfg, + pubsub_cfg_dump(cfg, LOG_WARN, " ")); + } + if (sub) { + SMARTLIST_FOREACH(sub, pubsub_cfg_t *, cfg, + pubsub_cfg_dump(cfg, LOG_WARN, " ")); + } + } + + return ok ? 0 : -1; +} + +/** + * Check all the messages in <b>map</b> for consistency. Return 0 on success, + * -1 on problems. + **/ +static int +pubsub_adjmap_check(const pubsub_adjmap_t *map) +{ + bool all_ok = true; + for (unsigned i = 0; i < map->n_msgs; ++i) { + if (lint_message(map, i) < 0) { + all_ok = false; + } + } + return all_ok ? 0 : -1; +} + +/** + * Check builder for consistency and various constraints. Return 0 on success, + * -1 on failure. + **/ +int +pubsub_builder_check(pubsub_builder_t *builder) +{ + pubsub_adjmap_t *map = pubsub_build_adjacency_map(builder->items); + int rv = -1; + + if (!map) + goto err; // should be impossible + + if (pubsub_adjmap_check(map) < 0) + goto err; + + rv = 0; + err: + pubsub_adjmap_free(map); + return rv; +} diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h new file mode 100644 index 0000000000..b0d6ae7e92 --- /dev/null +++ b/src/lib/pubsub/pubsub_connect.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_connect.h + * @brief Header for functions that add relationships to a pubsub builder. + * + * These functions are used by modules that need to add publication and + * subscription requests. Most users will want to call these functions + * indirectly, via the macros in pubsub_macros.h. + **/ + +#ifndef TOR_PUBSUB_CONNECT_H +#define TOR_PUBSUB_CONNECT_H + +#include "lib/dispatch/msgtypes.h" + +struct pub_binding_t; +/** + * A "dispatch connector" is a view of the dispatcher that a subsystem + * uses while initializing itself. It is specific to the subsystem, and + * ensures that each subsystem doesn't need to identify itself + * repeatedly while registering its messages. + **/ +typedef struct pubsub_connector_t pubsub_connector_t; + +int pubsub_add_pub_(struct pubsub_connector_t *con, + struct pub_binding_t *out, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line); + +int pubsub_add_sub_(struct pubsub_connector_t *con, + recv_fn_t recv_fn, + channel_id_t channel, + message_id_t msg, + msg_type_id_t type, + unsigned flags, + const char *file, + unsigned line); + +int pubsub_connector_register_type_(struct pubsub_connector_t *, + msg_type_id_t, + dispatch_typefns_t *, + const char *file, + unsigned line); + +#endif /* !defined(TOR_PUBSUB_CONNECT_H) */ diff --git a/src/lib/pubsub/pubsub_flags.h b/src/lib/pubsub/pubsub_flags.h new file mode 100644 index 0000000000..9912c1ae89 --- /dev/null +++ b/src/lib/pubsub/pubsub_flags.h @@ -0,0 +1,32 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_flags.h + * @brief Flags that can be set on publish/subscribe messages. + **/ + +#ifndef TOR_PUBSUB_FLAGS_H +#define TOR_PUBSUB_FLAGS_H + +/** + * Flag for registering a message: declare that no other module is allowed to + * publish this message if we are publishing it, or subscribe to it if we are + * subscribing to it. + */ +#define DISP_FLAG_EXCL (1u<<0) + +/** + * Flag for registering a message: declare that this message is a stub, and we + * will not actually publish/subscribe it, but that the dispatcher should + * treat us as if we did when typechecking. + * + * We use this so that messages aren't treated as "dangling" if they are + * potentially used by some other build of Tor. + */ +#define DISP_FLAG_STUB (1u<<1) + +#endif /* !defined(TOR_PUBSUB_FLAGS_H) */ diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h new file mode 100644 index 0000000000..e5ffbe501a --- /dev/null +++ b/src/lib/pubsub/pubsub_macros.h @@ -0,0 +1,373 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file pubsub_macros.h + * \brief Macros to help with the publish/subscribe dispatch API. + * + * The dispatch API allows different subsystems of Tor to communicate with + * another asynchronously via a shared "message" system. Some subsystems + * declare that they publish a given message, and others declare that they + * subscribe to it. Both subsystems depend on the message, but not upon one + * another. + * + * To declare a message, use DECLARE_MESSAGE() (for messages that take their + * data as a pointer) or DECLARE_MESSAGE_INT() (for messages that take their + * data as an integer. For example, you might say + * + * DECLARE_MESSAGE(new_circuit, circ, circuit_handle_t *); + * or + * DECLARE_MESSAGE_INT(shutdown_requested, boolean, bool); + * + * Every message has a unique name, a "type name" that the dispatch system + * uses to manage associated data, and a C type name. You'll want to put + * these declarations in a header, to be included by all publishers and all + * subscribers. + * + * When a subsystem wants to publish a message, it uses DECLARE_PUBLISH() at + * file scope to create necessary static functions. Then, in its subsystem + * initialization (in the "bind to dispatcher" callback) (TODO: name this + * properly!), it calls DISPATCH_ADD_PUB() to tell the dispatcher about its + * intent to publish. When it actually wants to publish, it uses the + * PUBLISH() macro. For example: + * + * // At file scope + * DECLARE_PUBLISH(shutdown_requested); + * + * static void bind_to_dispatcher(pubsub_connector_t *con) + * { + * DISPATCH_ADD_PUB(con, mainchannel, shutdown_requested); + * } + * + * // somewhere in a function + * { + * PUBLISH(shutdown_requested, true); + * } + * + * When a subsystem wants to subscribe to a message, it uses + * DECLARE_SUBSCRIBE() at file scope to declare static functions. It must + * declare a hook function that receives the message type. Then, in its "bind + * to dispatcher" function, it calls DISPATCHER_ADD_SUB() to tell the + * dispatcher about its intent to subscribe. When another module publishes + * the message, the dispatcher will call the provided hook function. + * + * // At file scope. The first argument is the message that you're + * // subscribing to; the second argument is the hook function to declare. + * DECLARE_SUBSCRIBE(shutdown_requested, on_shutdown_req_cb); + * + * // You need to declare this function. + * static void on_shutdown_req_cb(const msg_t *msg, + * bool value) + * { + * // (do something here.) + * } + * + * static void bind_to_dispatcher(pubsub_connector_t *con) + * { + * DISPATCH_ADD_SUB(con, mainchannel, shutdown_requested); + * } + * + * Where did these types come from? Somewhere in the code, you need to call + * DISPATCH_REGISTER_TYPE() to make sure that the dispatcher can manage the + * message auxiliary data. It associates a vtbl-like structure with the + * type name, so that the dispatcher knows how to manipulate the type you're + * giving it. + * + * For example, the "boolean" type we're using above could be defined as: + * + * static char *boolean_fmt(msg_aux_data_t d) + * { + * // This is used for debugging and dumping messages. + * if (d.u64) + * return tor_strdup("true"); + * else + * return tor_strdup("false"); + * } + * + * static void boolean_free(msg_aux_data_t d) + * { + * // We don't actually need to do anything to free a boolean. + * // We could use "NULL" instead of this function, but I'm including + * // it as an example. + * } + * + * static void bind_to_dispatcher(pubsub_connector_t *con) + * { + * dispatch_typefns_t boolean_fns = { + * .fmt_fn = boolean_fmt, + * .free_fn = boolean_free, + * }; + * DISPATCH_REGISTER_TYPE(con, boolean, &boolean_fns); + * } + * + * + * + * So, how does this all work? (You can stop reading here, unless you're + * debugging something.) + * + * When you declare a message in a header with DECLARE_MESSAGE() or + * DECLARE_MESSAGE_INT(), it creates five things: + * + * * two typedefs for the message argument (constant and non-constant + * variants). + * * a constant string to hold the declared message type name + * * two inline functions, to coerce the message argument type to and from + * a "msg_aux_data_t" union. + * + * All of these declarations have names based on the message name. + * + * Later, when you say DECLARE_PUBLISH() or DECLARE_SUBSCRIBE(), we use the + * elements defined by DECLARE_MESSAGE() to make sure that the publish + * function takes the correct argument type, and that the subscription hook is + * declared with the right argument type. + **/ + +#ifndef TOR_DISPATCH_MSG_H +#define TOR_DISPATCH_MSG_H + +#include "lib/cc/compat_compiler.h" +#include "lib/dispatch/dispatch_naming.h" +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_connect.h" +#include "lib/pubsub/pubsub_flags.h" +#include "lib/pubsub/pubsub_publish.h" + +/* Implemenation notes: + * + * For a messagename "foo", the DECLARE_MESSAGE*() macros must declare: + * + * msg_arg_type__foo -- a typedef for the argument type of the foo message. + * msg_arg_consttype__foo -- a typedef for the const argument type of the + * foo message. + * msg_arg_name__foo[] -- a static string constant holding the unique + * identifier for the type of the foo message. + * msg_arg_get__foo() -- an inline function taking a msg_aux_data_t and + * returning the C data type. + * msg_arg_set__foo() -- an inline function taking a msg_aux_data_t and + * the C type, setting the msg_aux_data_t to hold the C type. + * + * For a messagename "foo", the DECLARE_PUBLISH() macro must declare: + * + * pub_binding__foo -- A static pub_binding_t object used to send messages + * from this module. + * publish_fn__foo -- A function taking an argument of the appropriate + * C type, to be invoked by PUBLISH(). + * + * For a messagename "foo", the DECLARE_SUBSCRIBE() macro must declare: + * + * hookfn -- A user-provided function name, with the correct signature. + * recv_fn__foo -- A wrapper callback that takes a msg_t *, and calls + * hookfn with the appropriate arguments. + */ + +/** Macro to declare common elements shared by DECLARE_MESSAGE and + * DECLARE_MESSAGE_INT. Don't call this directly. + * + * Note that the "msg_arg_name" string constant is defined in each + * translation unit. This might be undesirable; we can tweak it in the + * future if need be. + */ +#define DECLARE_MESSAGE_COMMON__(messagename, typename, c_type) \ + typedef c_type msg_arg_type__ ##messagename; \ + typedef const c_type msg_arg_consttype__ ##messagename; \ + ATTR_UNUSED static const char msg_arg_name__ ##messagename[] = # typename; + +/** + * Use this macro in a header to declare the existence of a given message, + * taking a pointer as auxiliary data. + * + * "messagename" is a unique identifier for the message; it must be a valid + * C identifier. + * + * "typename" is a unique identifier for the type of the auxiliary data. + * It needs to be defined somewhere in Tor, using + * "DISPATCH_REGISTER_TYPE." + * + * "c_ptr_type" is a C pointer type (like "char *" or "struct foo *"). + * The "*" needs to be included. + */ +#define DECLARE_MESSAGE(messagename, typename, c_ptr_type) \ + DECLARE_MESSAGE_COMMON__(messagename, typename, c_ptr_type) \ + ATTR_UNUSED static inline c_ptr_type \ + msg_arg_get__ ##messagename(msg_aux_data_t m) \ + { \ + return m.ptr; \ + } \ + ATTR_UNUSED static inline void \ + msg_arg_set__ ##messagename(msg_aux_data_t *m, c_ptr_type v) \ + { \ + m->ptr = v; \ + } \ + EAT_SEMICOLON + +/** + * Use this macro in a header to declare the existence of a given message, + * taking an integer as auxiliary data. + * + * "messagename" is a unique identifier for the message; it must be a valid + * C identifier. + * + * "typename" is a unique identifier for the type of the auxiliary data. It + * needs to be defined somewhere in Tor, using "DISPATCH_REGISTER_TYPE." + * + * "c_type" is a C integer type, like "int" or "bool". It needs to fit inside + * a uint64_t. + */ +#define DECLARE_MESSAGE_INT(messagename, typename, c_type) \ + DECLARE_MESSAGE_COMMON__(messagename, typename, c_type) \ + ATTR_UNUSED static inline c_type \ + msg_arg_get__ ##messagename(msg_aux_data_t m) \ + { \ + return (c_type)m.u64; \ + } \ + ATTR_UNUSED static inline void \ + msg_arg_set__ ##messagename(msg_aux_data_t *m, c_type v) \ + { \ + m->u64 = (uint64_t)v; \ + } \ + EAT_SEMICOLON + +/** + * Use this macro inside a C module declare that we'll be publishing a given + * message type from within this module. + * + * It creates necessary functions and wrappers to publish a message whose + * unique identifier is "messagename". + * + * Before you use this, you need to include the header where DECLARE_MESSAGE*() + * was used for this message. + * + * You can only use this once per message in each subsystem. + */ +#define DECLARE_PUBLISH(messagename) \ + static pub_binding_t pub_binding__ ##messagename; \ + static void \ + publish_fn__ ##messagename(msg_arg_type__ ##messagename arg) \ + { \ + msg_aux_data_t data; \ + msg_arg_set__ ##messagename(&data, arg); \ + pubsub_pub_(&pub_binding__ ##messagename, data); \ + } \ + EAT_SEMICOLON + +/** + * Use this macro inside a C file to declare that we're subscribing to a + * given message and associating it with a given "hook function". It + * declares the hook function static, and helps with strong typing. + * + * Before you use this, you need to include the header where + * DECLARE_MESSAGE*() was used for the message whose unique identifier is + * "messagename". + * + * You will need to define a function with the name that you provide for + * "hookfn". The type of this function will be: + * static void hookfn(const msg_t *, const c_type) + * where c_type is the c type that you declared in the header. + * + * You can only use this once per message in each subsystem. + */ +#define DECLARE_SUBSCRIBE(messagename, hookfn) \ + static void hookfn(const msg_t *, \ + const msg_arg_consttype__ ##messagename); \ + static void recv_fn__ ## messagename(const msg_t *m) \ + { \ + msg_arg_type__ ## messagename arg; \ + arg = msg_arg_get__ ##messagename(m->aux_data__); \ + hookfn(m, arg); \ + } \ + EAT_SEMICOLON + +/** + * Add a fake use of the publish function for 'messagename', so that + * the compiler does not call it unused. + */ +#define DISPATCH__FAKE_USE_OF_PUBFN_(messagename) \ + ( 0 ? (publish_fn__ ##messagename((msg_arg_type__##messagename)0), 1) \ + : 1) + +/** + * This macro is for internal use. It backs DISPATCH_ADD_PUB*() + */ +#define DISPATCH_ADD_PUB_(connector, channel, messagename, flags) \ + ( \ + DISPATCH__FAKE_USE_OF_PUBFN_(messagename), \ + pubsub_add_pub_((connector), \ + &pub_binding__ ##messagename, \ + get_channel_id(# channel), \ + get_message_id(# messagename), \ + get_msg_type_id(msg_arg_name__ ## messagename), \ + (flags), \ + __FILE__, \ + __LINE__) \ + ) + +/** + * Use a given connector and channel name to declare that this subsystem will + * publish a given message type. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_PUB(connector, channel, messagename) \ + DISPATCH_ADD_PUB_(connector, channel, messagename, 0) + +/** + * Use a given connector and channel name to declare that this subsystem will + * publish a given message type, and that no other subsystem is allowed to. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_PUB_EXCL(connector, channel, messagename) \ + DISPATCH_ADD_PUB_(connector, channel, messagename, DISP_FLAG_EXCL) + +/** + * This macro is for internal use. It backs DISPATCH_ADD_SUB*() + */ +#define DISPATCH_ADD_SUB_(connector, channel, messagename, flags) \ + pubsub_add_sub_((connector), \ + recv_fn__ ##messagename, \ + get_channel_id(#channel), \ + get_message_id(# messagename), \ + get_msg_type_id(msg_arg_name__ ##messagename), \ + (flags), \ + __FILE__, \ + __LINE__) +/** + * Use a given connector and channel name to declare that this subsystem will + * receive a given message type. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_SUB(connector, channel, messagename) \ + DISPATCH_ADD_SUB_(connector, channel, messagename, 0) +/** + * Use a given connector and channel name to declare that this subsystem will + * receive a given message type, and that no other subsystem is allowed to do + * so. + * + * Call this macro from within the add_subscriptions() function of a module. + */ +#define DISPATCH_ADD_SUB_EXCL(connector, channel, messagename) \ + DISPATCH_ADD_SUB_(connector, channel, messagename, DISP_FLAG_EXCL) + +/** + * Publish a given message with a given argument. (Takes ownership of the + * argument if it is a pointer.) + */ +#define PUBLISH(messagename, arg) \ + publish_fn__ ##messagename(arg) + +/** + * Use a given connector to declare that the functions to be used to manipuate + * a certain C type. + **/ +#define DISPATCH_REGISTER_TYPE(con, type, fns) \ + pubsub_connector_register_type_((con), \ + get_msg_type_id(#type), \ + (fns), \ + __FILE__, \ + __LINE__) + +#endif /* !defined(TOR_DISPATCH_MSG_H) */ diff --git a/src/lib/pubsub/pubsub_publish.c b/src/lib/pubsub/pubsub_publish.c new file mode 100644 index 0000000000..84c7dae02c --- /dev/null +++ b/src/lib/pubsub/pubsub_publish.c @@ -0,0 +1,72 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_publish.c + * @brief Header for functions to publish using a pub_binding_t. + **/ + +#define PUBSUB_PRIVATE +#define DISPATCH_PRIVATE +#include "orconfig.h" + +#include "lib/dispatch/dispatch.h" +#include "lib/dispatch/dispatch_st.h" + +#include "lib/pubsub/pub_binding_st.h" +#include "lib/pubsub/pubsub_publish.h" + +#include "lib/malloc/malloc.h" +#include "lib/log/util_bug.h" + +#include <string.h> + +/** + * Publish a message from the publication binding <b>pub</b> using the + * auxiliary data <b>auxdata</b>. + * + * Return 0 on success, -1 on failure. + **/ +int +pubsub_pub_(const pub_binding_t *pub, msg_aux_data_t auxdata) +{ + dispatch_t *d = pub->dispatch_ptr; + if (BUG(! d)) { + /* Tried to publish a message before the dispatcher was configured. */ + /* (Without a dispatcher, we don't know how to free auxdata.) */ + return -1; + } + + if (BUG(pub->msg_template.type >= d->n_types)) { + /* The type associated with this message is not known to the dispatcher. */ + /* (Without a correct type, we don't know how to free auxdata.) */ + return -1; + } + + if (BUG(pub->msg_template.msg >= d->n_msgs) || + BUG(pub->msg_template.channel >= d->n_queues)) { + /* The message ID or channel ID was out of bounds. */ + // LCOV_EXCL_START + d->typefns[pub->msg_template.type].free_fn(auxdata); + return -1; + // LCOV_EXCL_STOP + } + + if (! d->table[pub->msg_template.msg]) { + /* Fast path: nobody wants this data. */ + + // XXXX Faster path: we could store this in the pub_binding_t. + d->typefns[pub->msg_template.type].free_fn(auxdata); + return 0; + } + + /* Construct the message object */ + msg_t *m = tor_malloc(sizeof(msg_t)); + memcpy(m, &pub->msg_template, sizeof(msg_t)); + m->aux_data__ = auxdata; + + return dispatch_send_msg_unchecked(d, m); +} diff --git a/src/lib/pubsub/pubsub_publish.h b/src/lib/pubsub/pubsub_publish.h new file mode 100644 index 0000000000..d9d6fa9ba5 --- /dev/null +++ b/src/lib/pubsub/pubsub_publish.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file pubsub_publish.h + * @brief Header for pubsub_publish.c + **/ + +#ifndef TOR_PUBSUB_PUBLISH_H +#define TOR_PUBSUB_PUBLISH_H + +#include "lib/dispatch/msgtypes.h" +struct pub_binding_t; + +int pubsub_pub_(const struct pub_binding_t *pub, msg_aux_data_t auxdata); + +#endif /* !defined(TOR_PUBSUB_PUBLISH_H) */ diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include index 84906dfb3d..853dae7880 100644 --- a/src/lib/sandbox/.may_include +++ b/src/lib/sandbox/.may_include @@ -5,11 +5,10 @@ lib/container/*.h lib/err/*.h lib/log/*.h lib/malloc/*.h -lib/net/*.h lib/sandbox/*.h lib/sandbox/*.inc lib/string/*.h -ht.h -siphash.h -tor_queue.h +ext/ht.h +ext/siphash.h +ext/tor_queue.h diff --git a/src/lib/sandbox/include.am b/src/lib/sandbox/include.am index adfda6bde5..e81f14b55f 100644 --- a/src/lib/sandbox/include.am +++ b/src/lib/sandbox/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-sandbox-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_sandbox_a_SOURCES = \ src/lib/sandbox/sandbox.c @@ -13,6 +14,7 @@ src_lib_libtor_sandbox_testing_a_SOURCES = \ src_lib_libtor_sandbox_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_sandbox_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/sandbox/linux_syscalls.inc \ src/lib/sandbox/sandbox.h diff --git a/src/lib/sandbox/lib_sandbox.md b/src/lib/sandbox/lib_sandbox.md new file mode 100644 index 0000000000..dd168c9b13 --- /dev/null +++ b/src/lib/sandbox/lib_sandbox.md @@ -0,0 +1,15 @@ +@dir /lib/sandbox +@brief lib/sandbox: Linux seccomp2-based sandbox. + +This module uses Linux's seccomp2 facility via the +[`libseccomp` library](https://github.com/seccomp/libseccomp), to restrict +the set of system calls that Tor is allowed to invoke while it is running. + +Because there are many libc versions that invoke different system calls, and +because handling strings is quite complex, this module is more complex and +less portable than it needs to be. + +A better architecture would put the responsibility for invoking tricky system +calls (like open()) in another, less restricted process, and give that +process responsibility for enforcing our sandbox rules. + diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c index 8f577b0660..d4f0da8397 100644 --- a/src/lib/sandbox/sandbox.c +++ b/src/lib/sandbox/sandbox.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -38,13 +38,12 @@ #include "lib/err/torerr.h" #include "lib/log/log.h" #include "lib/cc/torint.h" -#include "lib/net/resolve.h" #include "lib/malloc/malloc.h" #include "lib/string/scanf.h" -#include "tor_queue.h" -#include "ht.h" -#include "siphash.h" +#include "ext/tor_queue.h" +#include "ext/ht.h" +#include "ext/siphash.h" #define DEBUGGING_CLOSE @@ -83,7 +82,7 @@ #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) #define USE_BACKTRACE -#define EXPOSE_CLEAN_BACKTRACE +#define BACKTRACE_PRIVATE #include "lib/err/backtrace.h" #endif /* defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && ... */ @@ -118,6 +117,10 @@ #endif /* defined(__i386__) || ... */ +#ifdef M_SYSCALL +#define SYSCALL_NAME_DEBUGGING +#endif + /**Determines if at least one sandbox is active.*/ static int sandbox_active = 0; /** Holds the parameter list configuration for the sandbox.*/ @@ -134,6 +137,10 @@ static sandbox_cfg_t *filter_dynamic = NULL; * the high bits of the value might get masked out improperly. */ #define SCMP_CMP_MASKED(a,b,c) \ SCMP_CMP4((a), SCMP_CMP_MASKED_EQ, ~(scmp_datum_t)(b), (c)) +/* For negative constants, the rule to add depends on the glibc version. */ +#define SCMP_CMP_NEG(a,op,b) (libc_negative_constant_needs_cast() ? \ + (SCMP_CMP((a), (op), (unsigned int)(b))) : \ + (SCMP_CMP_STR((a), (op), (b)))) /** Variable used for storing all syscall numbers that will be allowed with the * stage 1 general Tor sandbox. @@ -144,6 +151,7 @@ static int filter_nopar_gen[] = { SCMP_SYS(clock_gettime), SCMP_SYS(close), SCMP_SYS(clone), + SCMP_SYS(dup), SCMP_SYS(epoll_create), SCMP_SYS(epoll_wait), #ifdef __NR_epoll_pwait @@ -166,6 +174,7 @@ static int filter_nopar_gen[] = { #ifdef __NR_fstat64 SCMP_SYS(fstat64), #endif + SCMP_SYS(fsync), SCMP_SYS(futex), SCMP_SYS(getdents), SCMP_SYS(getdents64), @@ -265,13 +274,27 @@ static int filter_nopar_gen[] = { SCMP_SYS(listen), SCMP_SYS(connect), SCMP_SYS(getsockname), +#ifdef ENABLE_NSS +#ifdef __NR_getpeername + SCMP_SYS(getpeername), +#endif +#endif SCMP_SYS(recvmsg), SCMP_SYS(recvfrom), SCMP_SYS(sendto), SCMP_SYS(unlink), +#ifdef __NR_unlinkat + SCMP_SYS(unlinkat), +#endif SCMP_SYS(poll) }; +/* opendir is not a syscall but it will use either open or openat. We do not + * want the decision to allow open/openat to be the callers reponsability, so + * we create a phony syscall number for opendir and sb_opendir will choose the + * correct syscall. */ +#define PHONY_OPENDIR_SYSCALL -2 + /* These macros help avoid the error where the number of filters we add on a * single rule don't match the arg_cnt param. */ #define seccomp_rule_add_0(ctx,act,call) \ @@ -295,6 +318,7 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter) unsigned i; int rc; int param[] = { SIGINT, SIGTERM, SIGPIPE, SIGUSR1, SIGUSR2, SIGHUP, SIGCHLD, + SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, SIGIO, #ifdef SIGXFSZ SIGXFSZ #endif @@ -424,31 +448,59 @@ sb_mmap2(scmp_filter_ctx ctx, sandbox_cfg_t *filter) #endif #endif -/* Return true if we think we're running with a libc that always uses - * openat on linux. */ +/* Return true the libc version is greater or equal than + * <b>major</b>.<b>minor</b>. Returns false otherwise. */ static int -libc_uses_openat_for_everything(void) +is_libc_at_least(int major, int minor) { #ifdef CHECK_LIBC_VERSION const char *version = gnu_get_libc_version(); if (version == NULL) return 0; - int major = -1; - int minor = -1; + int libc_major = -1; + int libc_minor = -1; - tor_sscanf(version, "%d.%d", &major, &minor); - if (major >= 3) + tor_sscanf(version, "%d.%d", &libc_major, &libc_minor); + if (libc_major > major) return 1; - else if (major == 2 && minor >= 26) + else if (libc_major == major && libc_minor >= minor) return 1; else return 0; -#else /* !(defined(CHECK_LIBC_VERSION)) */ +#else /* !defined(CHECK_LIBC_VERSION) */ + (void)major; + (void)minor; return 0; #endif /* defined(CHECK_LIBC_VERSION) */ } +/* Return true if we think we're running with a libc that uses openat for the + * open function on linux. */ +static int +libc_uses_openat_for_open(void) +{ + return is_libc_at_least(2, 26); +} + +/* Return true if we think we're running with a libc that uses openat for the + * opendir function on linux. */ +static int +libc_uses_openat_for_opendir(void) +{ + // libc 2.27 and above or between 2.15 (inclusive) and 2.22 (exclusive) + return is_libc_at_least(2, 27) || + (is_libc_at_least(2, 15) && !is_libc_at_least(2, 22)); +} + +/* Return true if we think we're running with a libc that needs to cast + * negative arguments like AT_FDCWD for seccomp rules. */ +static int +libc_negative_constant_needs_cast(void) +{ + return is_libc_at_least(2, 27); +} + /** Allow a single file to be opened. If <b>use_openat</b> is true, * we're using a libc that remaps all the opens into openats. */ static int @@ -456,7 +508,7 @@ allow_file_open(scmp_filter_ctx ctx, int use_openat, const char *file) { if (use_openat) { return seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(openat), - SCMP_CMP(0, SCMP_CMP_EQ, (unsigned int)AT_FDCWD), + SCMP_CMP_NEG(0, SCMP_CMP_EQ, AT_FDCWD), SCMP_CMP_STR(1, SCMP_CMP_EQ, file)); } else { return seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(open), @@ -474,7 +526,7 @@ sb_open(scmp_filter_ctx ctx, sandbox_cfg_t *filter) int rc; sandbox_cfg_t *elem = NULL; - int use_openat = libc_uses_openat_for_everything(); + int use_openat = libc_uses_openat_for_open(); // for each dynamic parameter filters for (elem = filter; elem != NULL; elem = elem->next) { @@ -592,7 +644,7 @@ sb_openat(scmp_filter_ctx ctx, sandbox_cfg_t *filter) if (param != NULL && param->prot == 1 && param->syscall == SCMP_SYS(openat)) { rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(openat), - SCMP_CMP(0, SCMP_CMP_EQ, AT_FDCWD), + SCMP_CMP_NEG(0, SCMP_CMP_EQ, AT_FDCWD), SCMP_CMP_STR(1, SCMP_CMP_EQ, param->value), SCMP_CMP(2, SCMP_CMP_EQ, O_RDONLY|O_NONBLOCK|O_LARGEFILE|O_DIRECTORY| O_CLOEXEC)); @@ -607,6 +659,30 @@ sb_openat(scmp_filter_ctx ctx, sandbox_cfg_t *filter) return 0; } +static int +sb_opendir(scmp_filter_ctx ctx, sandbox_cfg_t *filter) +{ + int rc; + sandbox_cfg_t *elem = NULL; + + // for each dynamic parameter filters + for (elem = filter; elem != NULL; elem = elem->next) { + smp_param_t *param = elem->param; + + if (param != NULL && param->prot == 1 && param->syscall + == PHONY_OPENDIR_SYSCALL) { + rc = allow_file_open(ctx, libc_uses_openat_for_opendir(), param->value); + if (rc != 0) { + log_err(LD_BUG,"(Sandbox) failed to add openat syscall, received " + "libseccomp error %d", rc); + return rc; + } + } + } + + return 0; +} + /** * Function responsible for setting up the socket syscall for * the seccomp filter sandbox. @@ -647,6 +723,15 @@ sb_socket(scmp_filter_ctx ctx, sandbox_cfg_t *filter) } } +#ifdef ENABLE_NSS + rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket), + SCMP_CMP(0, SCMP_CMP_EQ, PF_INET), + SCMP_CMP(1, SCMP_CMP_EQ, SOCK_STREAM), + SCMP_CMP(2, SCMP_CMP_EQ, IPPROTO_IP)); + if (rc) + return rc; +#endif + rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket), SCMP_CMP(0, SCMP_CMP_EQ, PF_UNIX), SCMP_CMP_MASKED(1, SOCK_CLOEXEC|SOCK_NONBLOCK, SOCK_STREAM), @@ -798,6 +883,12 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter) if (rc) return rc; + rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), + SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), + SCMP_CMP(2, SCMP_CMP_EQ, SO_ACCEPTCONN)); + if (rc) + return rc; + #ifdef HAVE_SYSTEMD rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt), SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET), @@ -1106,6 +1197,7 @@ static sandbox_filter_func_t filter_func[] = { sb_chmod, sb_open, sb_openat, + sb_opendir, sb_rename, #ifdef __NR_fcntl64 sb_fcntl64, @@ -1425,6 +1517,19 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) return 0; } +int +sandbox_cfg_allow_opendir_dirname(sandbox_cfg_t **cfg, char *dir) +{ + sandbox_cfg_t *elem = NULL; + + elem = new_element(PHONY_OPENDIR_SYSCALL, dir); + + elem->next = *cfg; + *cfg = elem; + + return 0; +} + /** * Function responsible for going through the parameter syscall filters and * call each function pointer in the list. @@ -1517,15 +1622,16 @@ install_syscall_filter(sandbox_cfg_t* cfg) // marking the sandbox as active sandbox_active = 1; - tor_make_getaddrinfo_cache_active(); end: seccomp_release(ctx); return (rc < 0 ? -rc : rc); } +#ifdef SYSCALL_NAME_DEBUGGING #include "lib/sandbox/linux_syscalls.inc" +/** Return a string containing the name of a given syscall (if we know it) */ static const char * get_syscall_name(int syscall_num) { @@ -1543,6 +1649,28 @@ get_syscall_name(int syscall_num) } } +/** Return the syscall number from a ucontext_t that we got in a signal + * handler (if we know how to do that). */ +static int +get_syscall_from_ucontext(const ucontext_t *ctx) +{ + return (int) ctx->uc_mcontext.M_SYSCALL; +} +#else +static const char * +get_syscall_name(int syscall_num) +{ + (void) syscall_num; + return "unknown"; +} +static int +get_syscall_from_ucontext(const ucontext_t *ctx) +{ + (void) ctx; + return -1; +} +#endif + #ifdef USE_BACKTRACE #define MAX_DEPTH 256 static void *syscall_cb_buf[MAX_DEPTH]; @@ -1558,7 +1686,6 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context) { ucontext_t *ctx = (ucontext_t *) (void_context); const char *syscall_name; - int syscall; #ifdef USE_BACKTRACE size_t depth; int n_fds, i; @@ -1573,7 +1700,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context) if (!ctx) return; - syscall = (int) ctx->uc_mcontext.M_SYSCALL; + int syscall = get_syscall_from_ucontext(ctx); #ifdef USE_BACKTRACE depth = backtrace(syscall_cb_buf, MAX_DEPTH); @@ -1731,6 +1858,13 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file) } int +sandbox_cfg_allow_opendir_dirname(sandbox_cfg_t **cfg, char *dir) +{ + (void)cfg; (void)dir; + return 0; +} + +int sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file) { (void)cfg; (void)file; @@ -1764,9 +1898,4 @@ sandbox_is_active(void) return 0; } -void -sandbox_disable_getaddrinfo_cache(void) -{ -} - #endif /* !defined(USE_LIBSECCOMP) */ diff --git a/src/lib/sandbox/sandbox.h b/src/lib/sandbox/sandbox.h index 5bec09a36a..a2b3227b90 100644 --- a/src/lib/sandbox/sandbox.h +++ b/src/lib/sandbox/sandbox.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,10 +29,10 @@ #define USE_LIBSECCOMP #endif -struct sandbox_cfg_elem; +struct sandbox_cfg_elem_t; /** Typedef to structure used to manage a sandbox configuration. */ -typedef struct sandbox_cfg_elem sandbox_cfg_t; +typedef struct sandbox_cfg_elem_t sandbox_cfg_t; /** * Linux definitions @@ -58,7 +58,7 @@ typedef enum { * Configuration parameter structure associated with the LIBSECCOMP2 * implementation. */ -typedef struct smp_param { +typedef struct smp_param_t { /** syscall associated with parameter. */ int syscall; @@ -77,7 +77,7 @@ typedef struct smp_param { * It is implemented as a linked list of parameters. Currently only controls * parameters for open, openat, execve, stat64. */ -struct sandbox_cfg_elem { +struct sandbox_cfg_elem_t { /** Sandbox implementation which dictates the parameter type. */ SB_IMPL implem; @@ -85,7 +85,7 @@ struct sandbox_cfg_elem { smp_param_t *param; /** Next element of the configuration*/ - struct sandbox_cfg_elem *next; + struct sandbox_cfg_elem_t *next; }; /** Function pointer defining the prototype of a filter function.*/ @@ -108,7 +108,7 @@ typedef struct { * it matches the parameter. */ const char* sandbox_intern_string(const char *param); -#else /* !(defined(USE_LIBSECCOMP)) */ +#else /* !defined(USE_LIBSECCOMP) */ #define sandbox_intern_string(s) (s) #endif /* defined(USE_LIBSECCOMP) */ @@ -136,6 +136,13 @@ int sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2); int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file); /** + * Function used to add a opendir allowed filename to a supplied configuration. + * The (char*) specifies the path to the allowed dir; we steal the pointer to + * that dir. + */ +int sandbox_cfg_allow_opendir_dirname(sandbox_cfg_t **cfg, char *dir); + +/** * Function used to add a stat/stat64 allowed filename to a configuration. * The (char*) specifies the path to the allowed file; that pointer is stolen. */ diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include index a8507761a4..2f0c8d341e 100644 --- a/src/lib/smartlist_core/.may_include +++ b/src/lib/smartlist_core/.may_include @@ -4,4 +4,4 @@ lib/malloc/*.h lib/err/*.h lib/string/*.h lib/smartlist_core/*.h -lib/testsupport/testsupport.h +lib/testsupport/*.h diff --git a/src/lib/smartlist_core/include.am b/src/lib/smartlist_core/include.am index 99d65f0b23..548179bc4f 100644 --- a/src/lib/smartlist_core/include.am +++ b/src/lib/smartlist_core/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-smartlist-core-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_smartlist_core_a_SOURCES = \ src/lib/smartlist_core/smartlist_core.c \ src/lib/smartlist_core/smartlist_split.c @@ -15,6 +16,7 @@ src_lib_libtor_smartlist_core_testing_a_CPPFLAGS = \ $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_smartlist_core_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/smartlist_core/smartlist_core.h \ src/lib/smartlist_core/smartlist_foreach.h \ diff --git a/src/lib/smartlist_core/lib_smartlist_core.md b/src/lib/smartlist_core/lib_smartlist_core.md new file mode 100644 index 0000000000..c031dd6f24 --- /dev/null +++ b/src/lib/smartlist_core/lib_smartlist_core.md @@ -0,0 +1,10 @@ +@dir /lib/smartlist_core +@brief lib/smartlist_core: Minimal dynamic array implementation + +A `smartlist_t` is a dynamic array type for holding `void *`. We use it +throughout the rest of the codebase. + +There are higher-level pieces in \refdir{lib/container} but +the ones in lib/smartlist_core are used by the logging code, and therefore +cannot use the logging code. + diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c index ac85a6cc84..571d17aa5d 100644 --- a/src/lib/smartlist_core/smartlist_core.c +++ b/src/lib/smartlist_core/smartlist_core.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -88,6 +88,30 @@ smartlist_ensure_capacity(smartlist_t *sl, size_t size) #undef MAX_CAPACITY } +/** Expand <b>sl</b> so that its length is at least <b>new_size</b>, + * filling in previously unused entries with NULL> + * + * Do nothing if <b>sl</b> already had at least <b>new_size</b> elements. + */ +void +smartlist_grow(smartlist_t *sl, size_t new_size) +{ + smartlist_ensure_capacity(sl, new_size); + + if (new_size > (size_t)sl->num_used) { + /* This memset() should be a no-op: everything else in the smartlist code + * tries to make sure that unused entries are always NULL. Still, that is + * meant as a safety mechanism, so let's clear the memory here. + */ + memset(sl->list + sl->num_used, 0, + sizeof(void *) * (new_size - sl->num_used)); + + /* This cast is safe, since we already asserted that we were below + * MAX_CAPACITY in smartlist_ensure_capacity(). */ + sl->num_used = (int)new_size; + } +} + /** Append element to the end of the list. */ void smartlist_add(smartlist_t *sl, void *element) @@ -153,6 +177,8 @@ smartlist_remove_keeporder(smartlist_t *sl, const void *element) sl->list[i++] = sl->list[j]; } } + memset(sl->list + sl->num_used, 0, + sizeof(void *) * (num_used_orig - sl->num_used)); } /** If <b>sl</b> is nonempty, remove and return the final element. Otherwise, diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h index a7fbaa099b..de6fe69d3a 100644 --- a/src/lib/smartlist_core/smartlist_core.h +++ b/src/lib/smartlist_core/smartlist_core.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -43,6 +43,7 @@ void smartlist_clear(smartlist_t *sl); void smartlist_add(smartlist_t *sl, void *element); void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2); void smartlist_add_strdup(struct smartlist_t *sl, const char *string); +void smartlist_grow(smartlist_t *sl, size_t new_size); void smartlist_remove(smartlist_t *sl, const void *element); void smartlist_remove_keeporder(smartlist_t *sl, const void *element); @@ -76,7 +77,7 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) { raw_assert(sl->num_used > idx); sl->list[idx] = val; } -#else /* !(defined(DEBUG_SMARTLIST)) */ +#else /* !defined(DEBUG_SMARTLIST) */ #define smartlist_len(sl) ((sl)->num_used) #define smartlist_get(sl, idx) ((sl)->list[idx]) #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val)) @@ -97,4 +98,4 @@ void smartlist_del(smartlist_t *sl, int idx); void smartlist_del_keeporder(smartlist_t *sl, int idx); void smartlist_insert(smartlist_t *sl, int idx, void *val); -#endif /* !defined(TOR_CONTAINER_H) */ +#endif /* !defined(TOR_SMARTLIST_CORE_H) */ diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h index 0f6fe30074..03edb80f05 100644 --- a/src/lib/smartlist_core/smartlist_foreach.h +++ b/src/lib/smartlist_core/smartlist_foreach.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -83,6 +83,19 @@ ++var ## _sl_idx) { \ var = (sl)->list[var ## _sl_idx]; +/** Iterates over the items in smartlist <b>sl</b> in reverse order, similar to + * SMARTLIST_FOREACH_BEGIN + * + * NOTE: This macro is incompatible with SMARTLIST_DEL_CURRENT. + */ +#define SMARTLIST_FOREACH_REVERSE_BEGIN(sl, type, var) \ + STMT_BEGIN \ + int var ## _sl_idx, var ## _sl_len=(sl)->num_used; \ + type var; \ + for (var ## _sl_idx = var ## _sl_len-1; var ## _sl_idx >= 0; \ + --var ## _sl_idx) { \ + var = (sl)->list[var ## _sl_idx]; + #define SMARTLIST_FOREACH_END(var) \ var = NULL; \ (void) var ## _sl_idx; \ diff --git a/src/lib/smartlist_core/smartlist_split.c b/src/lib/smartlist_core/smartlist_split.c index c9cf59851f..b76b87406d 100644 --- a/src/lib/smartlist_core/smartlist_split.c +++ b/src/lib/smartlist_core/smartlist_split.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/smartlist_core/smartlist_split.h b/src/lib/smartlist_core/smartlist_split.h index 4f72376125..fc964201e9 100644 --- a/src/lib/smartlist_core/smartlist_split.h +++ b/src/lib/smartlist_core/smartlist_split.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,4 +17,4 @@ int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max); -#endif +#endif /* !defined(TOR_SMARTLIST_SPLIT_H) */ diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include index ec5c769831..1fb9127f19 100644 --- a/src/lib/string/.may_include +++ b/src/lib/string/.may_include @@ -6,5 +6,5 @@ lib/malloc/*.h lib/ctime/*.h lib/string/*.h -strlcat.c -strlcpy.c +ext/strlcat.c +ext/strlcpy.c diff --git a/src/lib/string/compat_ctype.c b/src/lib/string/compat_ctype.c index f5d82be3ae..a7668bfbfb 100644 --- a/src/lib/string/compat_ctype.c +++ b/src/lib/string/compat_ctype.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -29,6 +29,7 @@ const uint32_t TOR_ISPRINT_TABLE[8] = { 0, 0xffffffff, 0xffffffff, 0x7fffffff, 0, 0, 0, 0x0 }; const uint32_t TOR_ISUPPER_TABLE[8] = { 0, 0, 0x7fffffe, 0, 0, 0, 0, 0 }; const uint32_t TOR_ISLOWER_TABLE[8] = { 0, 0, 0, 0x7fffffe, 0, 0, 0, 0 }; +/**@}*/ /** Upper-casing and lowercasing tables to map characters to upper/lowercase * equivalents. Used by tor_toupper() and tor_tolower(). */ diff --git a/src/lib/string/compat_ctype.h b/src/lib/string/compat_ctype.h index dbddd356c1..53ee6066f8 100644 --- a/src/lib/string/compat_ctype.h +++ b/src/lib/string/compat_ctype.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c index e125c921a4..2bd3c2f2b4 100644 --- a/src/lib/string/compat_string.c +++ b/src/lib/string/compat_string.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,10 +14,10 @@ /* Inline the strl functions if the platform doesn't have them. */ #ifndef HAVE_STRLCPY -#include "strlcpy.c" +#include "ext/strlcpy.c" #endif #ifndef HAVE_STRLCAT -#include "strlcat.c" +#include "ext/strlcat.c" #endif #include <stdlib.h> diff --git a/src/lib/string/compat_string.h b/src/lib/string/compat_string.h index a0e37bb6dc..f05265bdcc 100644 --- a/src/lib/string/compat_string.h +++ b/src/lib/string/compat_string.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -25,20 +25,23 @@ static inline int strncasecmp(const char *a, const char *b, size_t n); static inline int strncasecmp(const char *a, const char *b, size_t n) { return _strnicmp(a,b,n); } -#endif +#endif /* !defined(HAVE_STRNCASECMP) */ #ifndef HAVE_STRCASECMP static inline int strcasecmp(const char *a, const char *b); static inline int strcasecmp(const char *a, const char *b) { return _stricmp(a,b); } -#endif -#endif +#endif /* !defined(HAVE_STRCASECMP) */ +#endif /* defined(_WIN32) */ #if defined __APPLE__ /* On OSX 10.9 and later, the overlap-checking code for strlcat would * appear to have a severe bug that can sometimes cause aborts in Tor. * Instead, use the non-checking variants. This is sad. * + * (If --enable-fragile-hardening is passed to configure, we use the hardened + * variants, which do not suffer from this issue.) + * * See https://trac.torproject.org/projects/tor/ticket/15205 */ #undef strlcat @@ -59,4 +62,4 @@ char *tor_strtok_r_impl(char *str, const char *sep, char **lasts); #define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts) #endif -#endif +#endif /* !defined(TOR_COMPAT_STRING_H) */ diff --git a/src/lib/string/include.am b/src/lib/string/include.am index edd74b8a3e..82d35cc5af 100644 --- a/src/lib/string/include.am +++ b/src/lib/string/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-string-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_string_a_SOURCES = \ src/lib/string/compat_ctype.c \ src/lib/string/compat_string.c \ @@ -18,6 +19,7 @@ src_lib_libtor_string_testing_a_SOURCES = \ src_lib_libtor_string_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_string_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/string/compat_ctype.h \ src/lib/string/compat_string.h \ diff --git a/src/lib/string/lib_string.md b/src/lib/string/lib_string.md new file mode 100644 index 0000000000..98e3e652ed --- /dev/null +++ b/src/lib/string/lib_string.md @@ -0,0 +1,13 @@ +@dir /lib/string +@brief lib/string: Low-level string manipulation. + +We have a number of compatibility functions here: some are for handling +functionality that is not implemented (or not implemented the same) on every +platform; some are for providing locale-independent versions of libc +functions that would otherwise be defined differently for different users. + +Other functions here are for common string-manipulation operations that we do +in the rest of the codebase. + +Any string function high-level enough to need logging belongs in a +higher-level module. diff --git a/src/lib/string/parse_int.c b/src/lib/string/parse_int.c index fbdd554a47..11ce0fa415 100644 --- a/src/lib/string/parse_int.c +++ b/src/lib/string/parse_int.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -9,6 +9,7 @@ **/ #include "lib/string/parse_int.h" +#include "lib/cc/compat_compiler.h" #include <errno.h> #include <stdlib.h> @@ -17,6 +18,7 @@ /* Helper: common code to check whether the result of a strtol or strtoul or * strtoll is correct. */ #define CHECK_STRTOX_RESULT() \ + STMT_BEGIN \ /* Did an overflow occur? */ \ if (errno == ERANGE) \ goto err; \ @@ -38,7 +40,8 @@ err: \ if (ok) *ok = 0; \ if (next) *next = endptr; \ - return 0 + return 0; \ + STMT_END /** Extract a long from the start of <b>s</b>, in the given numeric * <b>base</b>. If <b>base</b> is 0, <b>s</b> is parsed as a decimal, diff --git a/src/lib/string/parse_int.h b/src/lib/string/parse_int.h index 925547942e..27939ade61 100644 --- a/src/lib/string/parse_int.h +++ b/src/lib/string/parse_int.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,4 +22,4 @@ double tor_parse_double(const char *s, double min, double max, int *ok, uint64_t tor_parse_uint64(const char *s, int base, uint64_t min, uint64_t max, int *ok, char **next); -#endif +#endif /* !defined(TOR_PARSE_INT_H) */ diff --git a/src/lib/string/printf.c b/src/lib/string/printf.c index a5cb71ce09..86d860935e 100644 --- a/src/lib/string/printf.c +++ b/src/lib/string/printf.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -117,8 +117,8 @@ tor_vasprintf(char **strp, const char *fmt, va_list args) *strp = NULL; return -1; } - strp_tmp = tor_malloc(len + 1); - r = _vsnprintf(strp_tmp, len+1, fmt, args); + strp_tmp = tor_malloc((size_t)len + 1); + r = _vsnprintf(strp_tmp, (size_t)len+1, fmt, args); if (r != len) { tor_free(strp_tmp); *strp = NULL; @@ -153,9 +153,9 @@ tor_vasprintf(char **strp, const char *fmt, va_list args) *strp = tor_strdup(buf); return len; } - strp_tmp = tor_malloc(len+1); + strp_tmp = tor_malloc((size_t)len+1); /* use of tor_vsnprintf() will ensure string is null terminated */ - r = tor_vsnprintf(strp_tmp, len+1, fmt, args); + r = tor_vsnprintf(strp_tmp, (size_t)len+1, fmt, args); if (r != len) { tor_free(strp_tmp); *strp = NULL; diff --git a/src/lib/string/printf.h b/src/lib/string/printf.h index 2cc13d6bee..5ab751b338 100644 --- a/src/lib/string/printf.h +++ b/src/lib/string/printf.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -27,4 +27,4 @@ int tor_asprintf(char **strp, const char *fmt, ...) int tor_vasprintf(char **strp, const char *fmt, va_list args) CHECK_PRINTF(2,0); -#endif /* !defined(TOR_UTIL_STRING_H) */ +#endif /* !defined(TOR_UTIL_PRINTF_H) */ diff --git a/src/lib/string/scanf.c b/src/lib/string/scanf.c index 1bc39b5182..89d1683204 100644 --- a/src/lib/string/scanf.c +++ b/src/lib/string/scanf.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/string/scanf.h b/src/lib/string/scanf.h index 6673173de5..67e9c5eb78 100644 --- a/src/lib/string/scanf.h +++ b/src/lib/string/scanf.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -21,4 +21,4 @@ int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \ int tor_sscanf(const char *buf, const char *pattern, ...) CHECK_SCANF(2, 3); -#endif /* !defined(TOR_UTIL_STRING_H) */ +#endif /* !defined(TOR_UTIL_SCANF_H) */ diff --git a/src/lib/string/strings.md b/src/lib/string/strings.md new file mode 100644 index 0000000000..b22574a05a --- /dev/null +++ b/src/lib/string/strings.md @@ -0,0 +1,102 @@ + +@page strings String processing in Tor + +Since you're reading about a C program, you probably expected this +section: it's full of functions for manipulating the (notoriously +dubious) C string abstraction. I'll describe some often-missed +highlights here. + +### Comparing strings and memory chunks ### + +We provide strcmpstart() and strcmpend() to perform a strcmp with the start +or end of a string. + + tor_assert(!strcmpstart("Hello world","Hello")); + tor_assert(!strcmpend("Hello world","world")); + + tor_assert(!strcasecmpstart("HELLO WORLD","Hello")); + tor_assert(!strcasecmpend("HELLO WORLD","world")); + +To compare two string pointers, either of which might be NULL, use +strcmp_opt(). + +To search for a string or a chunk of memory within a non-null +terminated memory block, use tor_memstr or tor_memmem respectively. + +We avoid using memcmp() directly, since it tends to be used in cases +when having a constant-time operation would be better. Instead, we +recommend tor_memeq() and tor_memneq() for when you need a +constant-time operation. In cases when you need a fast comparison, +and timing leaks are not a danger, you can use fast_memeq() and +fast_memneq(). + +It's a common pattern to take a string representing one or more lines +of text, and search within it for some other string, at the start of a +line. You could search for "\\ntarget", but that would miss the first +line. Instead, use find_str_at_start_of_line. + +### Parsing text ### + +Over the years, we have accumulated lots of ways to parse text -- +probably too many. Refactoring them to be safer and saner could be a +good project! The one that seems most error-resistant is tokenizing +text with smartlist_split_strings(). This function takes a smartlist, +a string, and a separator, and splits the string along occurrences of +the separator, adding new strings for the sub-elements to the given +smartlist. + +To handle time, you can use one of the functions mentioned above in +"Parsing and encoding time values". + +For numbers in general, use the tor_parse_{long,ulong,double,uint64} +family of functions. Each of these can be called in a few ways. The +most general is as follows: + + const int BASE = 10; + const int MINVAL = 10, MAXVAL = 10000; + const char *next; + int ok; + long lng = tor_parse_long("100", BASE, MINVAL, MAXVAL, &ok, &next); + +The return value should be ignored if "ok" is set to false. The input +string needs to contain an entire number, or it's considered +invalid... unless the "next" pointer is available, in which case extra +characters at the end are allowed, and "next" is set to point to the +first such character. + +### Generating blocks of text ### + +For not-too-large blocks of text, we provide tor_asprintf(), which +behaves like other members of the sprintf() family, except that it +always allocates enough memory on the heap for its output. + +For larger blocks: Rather than using strlcat and strlcpy to build +text, or keeping pointers to the interior of a memory block, we +recommend that you use the smartlist_* functions to build a smartlist +full of substrings in order. Then you can concatenate them into a +single string with smartlist_join_strings(), which also takes optional +separator and terminator arguments. + +Alternatively, you might find it more convenient (and more +allocation-efficient) to use the buffer API in buffers.c: Construct a buf_t +object, add your data to it with buf_add_string(), buf_add_printf(), and so +on, then call buf_extract() to get the resulting output. + +As a convenience, we provide smartlist_add_asprintf(), which combines +the two methods above together. Many of the cryptographic digest +functions also accept a not-yet-concatenated smartlist of strings. + +### Logging helpers ### + +Often we'd like to log a value that comes from an untrusted source. +To do this, use escaped() to escape the nonprintable characters and +other confusing elements in a string, and surround it by quotes. (Use +esc_for_log() if you need to allocate a new string.) + +It's also handy to put memory chunks into hexadecimal before logging; +you can use hex_str(memory, length) for that. + +The escaped() and hex_str() functions both provide outputs that are +only valid till they are next invoked; they are not threadsafe. + +*/ diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c index f934f66f02..c8f12d780e 100644 --- a/src/lib/string/util_string.c +++ b/src/lib/string/util_string.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -71,7 +71,7 @@ tor_memstr(const void *haystack, size_t hlen, const char *needle) /** Return true iff the 'len' bytes at 'mem' are all zero. */ int -tor_mem_is_zero(const char *mem, size_t len) +fast_mem_is_zero(const char *mem, size_t len) { static const char ZERO[] = { 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, @@ -95,17 +95,14 @@ tor_mem_is_zero(const char *mem, size_t len) int tor_digest_is_zero(const char *digest) { - static const uint8_t ZERO_DIGEST[] = { - 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 - }; - return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN); + return safe_mem_is_zero(digest, DIGEST_LEN); } /** Return true iff the DIGEST256_LEN bytes in digest are all zero. */ int tor_digest256_is_zero(const char *digest) { - return tor_mem_is_zero(digest, DIGEST256_LEN); + return safe_mem_is_zero(digest, DIGEST256_LEN); } /** Remove from the string <b>s</b> every character which appears in @@ -212,21 +209,6 @@ strcmpstart(const char *s1, const char *s2) return strncmp(s1, s2, n); } -/** Compare the s1_len-byte string <b>s1</b> with <b>s2</b>, - * without depending on a terminating nul in s1. Sorting order is first by - * length, then lexically; return values are as for strcmp. - */ -int -strcmp_len(const char *s1, const char *s2, size_t s1_len) -{ - size_t s2_len = strlen(s2); - if (s1_len < s2_len) - return -1; - if (s1_len > s2_len) - return 1; - return fast_memcmp(s1, s2, s2_len); -} - /** Compares the first strlen(s2) characters of s1 with s2. Returns as for * strcasecmp. */ @@ -524,6 +506,23 @@ validate_char(const uint8_t *c, uint8_t len) int string_is_utf8(const char *str, size_t len) { + // If str is NULL, don't try to read it + if (!str) { + // We could test for this case, but the low-level logs would produce + // confusing test output. + // LCOV_EXCL_START + if (len) { + // Use the low-level logging function, so that the log module can + // validate UTF-8 (if needed in future code) + tor_log_err_sigsafe( + "BUG: string_is_utf8() called with NULL str but non-zero len."); + // Since it's a bug, we should probably reject this string + return false; + } + // LCOV_EXCL_STOP + return true; + } + for (size_t i = 0; i < len;) { uint8_t num_bytes = bytes_in_char(str[i]); if (num_bytes == 0) // Invalid leading byte found. @@ -541,3 +540,16 @@ string_is_utf8(const char *str, size_t len) } return true; } + +/** As string_is_utf8(), but returns false if the string begins with a UTF-8 + * byte order mark (BOM). + */ +int +string_is_utf8_no_bom(const char *str, size_t len) +{ + if (str && len >= 3 && (!strcmpstart(str, "\uFEFF") || + !strcmpstart(str, "\uFFFE"))) { + return false; + } + return string_is_utf8(str, len); +} diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h index d9fbf8c61e..e89233df88 100644 --- a/src/lib/string/util_string.h +++ b/src/lib/string/util_string.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,7 +20,10 @@ const void *tor_memmem(const void *haystack, size_t hlen, const void *needle, size_t nlen); const void *tor_memstr(const void *haystack, size_t hlen, const char *needle); -int tor_mem_is_zero(const char *mem, size_t len); +int fast_mem_is_zero(const char *mem, size_t len); +#define fast_digest_is_zero(d) fast_mem_is_zero((d), DIGEST_LEN) +#define fast_digetst256_is_zero(d) fast_mem_is_zero((d), DIGEST256_LEN) + int tor_digest_is_zero(const char *digest); int tor_digest256_is_zero(const char *digest); @@ -33,7 +36,6 @@ int tor_strisnonupper(const char *s); int tor_strisspace(const char *s); int strcmp_opt(const char *s1, const char *s2); int strcmpstart(const char *s1, const char *s2); -int strcmp_len(const char *s1, const char *s2, size_t len); int strcasecmpstart(const char *s1, const char *s2); int strcmpend(const char *s1, const char *s2); int strcasecmpend(const char *s1, const char *s2); @@ -53,5 +55,6 @@ const char *find_str_at_start_of_line(const char *haystack, int string_is_C_identifier(const char *string); int string_is_utf8(const char *str, size_t len); +int string_is_utf8_no_bom(const char *str, size_t len); #endif /* !defined(TOR_UTIL_STRING_H) */ diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include new file mode 100644 index 0000000000..2b06e8519c --- /dev/null +++ b/src/lib/subsys/.may_include @@ -0,0 +1 @@ +orconfig.h diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am new file mode 100644 index 0000000000..c9ab54ca73 --- /dev/null +++ b/src/lib/subsys/include.am @@ -0,0 +1,4 @@ + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/subsys/subsys.h diff --git a/src/lib/subsys/initialization.md b/src/lib/subsys/initialization.md new file mode 100644 index 0000000000..012ab7000d --- /dev/null +++ b/src/lib/subsys/initialization.md @@ -0,0 +1,75 @@ + +@page initialization Initialization and shutdown + +@tableofcontents + +@section overview Overview + +Tor has a single entry point: tor_run_main() in main.c. All the ways of +starting a Tor process (ntmain.c, tor_main.c, and tor_api.c) work by invoking tor_run_main(). + +The tor_run_main() function normally exits (@ref init_exceptwhen "1") by +returning: not by calling abort() or exit(). Before it returns, it calls +tor_cleanup() in shutdown.c. + +Conceptually, there are several stages in running Tor. + +1. First, we initialize those modules that do not depend on the + configuration. This happens in the first half of tor_run_main(), and the + first half of tor_init(). (@ref init_pending_refactor "2") + +2. Second, we parse the command line and our configuration, and configure + systems that depend on our configuration or state. This configuration + happens midway through tor_init(), which invokes + options_init_from_torrc(). We then initialize more systems from the + second half of tor_init(). + +3. At this point we may exit early if we have been asked to do something + requiring no further initialization, like printing our version number or + creating a new signing key. Otherwise, we proceed to run_tor_main_loop(), + which initializes some network-specific parts of Tor, grabs some + daemon-only resources (like the data directory lock) and starts Tor itself + running. + + +> @anchor init_exceptwhen 1. tor_run_main() _can_ terminate with a call to +> abort() or exit(), but only when crashing due to a bug, or when forking to +> run as a daemon. + +> @anchor init_pending_refactor 2. The pieces of code that I'm describing as +> "the first part of tor_init()" and so on deserve to be functions with their +> own name. I'd like to refactor them, but before I do so, there is some +> slight reorganization that needs to happen. Notably, the +> nt_service_parse_options() call ought logically to be later in our +> initialization sequence. See @ticket{32447} for our refactoring progress. + + +@section subsys Subsystems and initialization + +Our current convention is to use the subsystem mechanism to initialize and +clean up pieces of Tor. The more recently updated pieces of Tor will use +this mechanism. For examples, see e.g. time_sys.c or log_sys.c. + +In simplest terms, a **subsytem** is a logically separate part of Tor that +can be initialized, shut down, managed, and configured somewhat independently +of the rest of the program. + +The subsys_fns_t type describes a subsystem and a set of functions that +initialize it, desconstruct it, and so on. To define a subsystem, we declare +a `const` instance of subsys_fns_t. See the documentation for subsys_fns_t +for a full list of these functions. + +After defining a subsytem, it must be inserted in subsystem_list.c. At that +point, table-driven mechanisms in subsysmgr.c will invoke its functions when +appropriate. + +@subsection vsconfig Initialization versus configuration + +We note that the initialization phase of Tor occurs before any configuration +is read from disk -- and therefore before any other files are read from +disk. Therefore, any behavior that depends on Tor's configuration or state +must occur _after_ the initialization process, during configuration. + + + + diff --git a/src/lib/subsys/lib_subsys.md b/src/lib/subsys/lib_subsys.md new file mode 100644 index 0000000000..764d25d1b6 --- /dev/null +++ b/src/lib/subsys/lib_subsys.md @@ -0,0 +1,32 @@ +@dir /lib/subsys +@brief lib/subsys: Types for declaring a "subsystem". + +## Subsystems in Tor + +A subsystem is a module with support for initialization, shutdown, +configuration, and so on. + +Many parts of Tor can be initialized, cleaned up, and configured somewhat +independently through a table-driven mechanism. Each such part is called a +"subsystem". + +To declare a subsystem, make a global `const` instance of the `subsys_fns_t` +type, filling in the function pointer fields that you require with ones +corresponding to your subsystem. Any function pointers left as "NULL" will +be a no-op. Each system must have a name and a "level", which corresponds to +the order in which it is initialized. (See `app/main/subsystem_list.c` for a +list of current subsystems and their levels.) + +Then, insert your subsystem in the list in `app/main/subsystem_list.c`. It +will need to occupy a position corresponding to its level. + +At this point, your subsystem will be handled like the others: it will get +initialized at startup, torn down at exit, and so on. + +Historical note: Not all of Tor's code is currently handled as +subsystems. As you work with older code, you may see some parts of the code +that are initialized from `tor_init()` or `run_tor_main_loop()` or +`tor_run_main()`; and torn down from `tor_cleanup()`. We aim to migrate +these to subsystems over time; please don't add any new code that follows +this pattern. + diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h new file mode 100644 index 0000000000..62c0de026d --- /dev/null +++ b/src/lib/subsys/subsys.h @@ -0,0 +1,218 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * @file subsys.h + * @brief Types used to declare a subsystem. + **/ + +#ifndef TOR_SUBSYS_T +#define TOR_SUBSYS_T + +#include <stdbool.h> + +struct pubsub_connector_t; +struct config_format_t; + +/** + * A subsystem is a part of Tor that is initialized, shut down, configured, + * and connected to other parts of Tor. + * + * All callbacks are optional -- if a callback is set to NULL, the subsystem + * manager will treat it as a no-op. + * + * You should use c99 named-field initializers with this structure, for + * readability and safety. (There are a lot of functions here, all of them + * optional, and many of them with similar signatures.) + * + * See @ref initialization for more information about initialization and + * shutdown in Tor. + * + * To make a new subsystem, you declare a const instance of this type, and + * include it on the list in subsystem_list.c. The code that manages these + * subsystems is in subsysmgr.c. + **/ +typedef struct subsys_fns_t { + /** + * The name of this subsystem. It should be a programmer-readable + * identifier. + **/ + const char *name; + + /** + * The file in which the subsystem object is declared. Used for debugging. + **/ + const char *location; + + /** + * Whether this subsystem is supported -- that is, whether it is compiled + * into Tor. For most subsystems, this should be true. + **/ + bool supported; + + /** + * The 'initialization level' for the subsystem. It should run from -100 + * through +100. The subsystems are initialized from lowest level to + * highest, and shut down from highest level to lowest. + **/ + int level; + + /** + * Initialize any global components of this subsystem. + * + * This function MAY rely on any lower-level subsystem being initialized. + * + * This function MUST NOT rely on any runtime configuration information; + * it is only for global state or pre-configuration state. + * + * (If you need to do any setup that depends on configuration, you'll need + * to declare a configuration callback instead. (Not yet designed)) + * + * This function MUST NOT have any parts that can fail. + **/ + int (*initialize)(void); + + /** + * Connect a subsystem to the message dispatch system. + * + * This function should use the macros in @refdir{lib/pubsub} to register a + * set of messages that this subsystem may publish, and may subscribe to. + * + * See pubsub_macros.h for more information, and for examples. + **/ + int (*add_pubsub)(struct pubsub_connector_t *); + + /** + * Perform any necessary pre-fork cleanup. This function may not fail. + * + * On Windows (and any other platforms without fork()), this function will + * never be invoked. Otherwise it is used when we are about to start + * running as a background daemon, or when we are about to run a unit test + * in a subprocess. Unlike the subsys_fns_t.postfork callback, it is run + * from the parent process. + * + * Note that we do not invoke this function when the child process's only + * purpose is to call exec() and run another program. + */ + void (*prefork)(void); + + /** + * Perform any necessary post-fork setup. This function may not fail. + * + * On Windows (and any other platforms without fork()), this function will + * never be invoked. Otherwise it is used when we are about to start + * running as a background daemon, or when we are about to run a unit test + * in a subprocess. Unlike the subsys_fns_t.prefork callback, it is run + * from the child process. + * + * Note that we do not invoke this function when the child process's only + * purpose is to call exec() and run another program. + */ + void (*postfork)(void); + + /** + * Free any thread-local resources held by this subsystem. Called before + * the thread exits. + * + * This function is not allowed to fail. + * + * \bug Note that this callback is currently buggy: See \ticket{32103}. + */ + void (*thread_cleanup)(void); + + /** + * Free all resources held by this subsystem. + * + * This function is not allowed to fail. + * + * Subsystems are shut down when Tor is about to exit or return control to + * an embedding program. This callback must return the process to a state + * such that subsys_fns_t.init will succeed if invoked again. + **/ + void (*shutdown)(void); + + /** + * A config_format_t describing all of the torrc fields owned by this + * subsystem. + * + * This object, if present, is registered in a confmgr_t for Tor's options, + * and used to parse option fields from the command line and torrc file. + **/ + const struct config_format_t *options_format; + + /** + * A config_format_t describing all of the DataDir/state fields owned by + * this subsystem. + * + * This object, if present, is registered in a confmgr_t for Tor's state, + * and used to parse state fields from the DataDir/state file. + **/ + const struct config_format_t *state_format; + + /** + * Receive an options object as defined by options_format. Return 0 + * on success, -1 on failure. + * + * It is safe to store the pointer to the object until set_options() + * is called again. + * + * This function is only called after all the validation code defined + * by subsys_fns_t.options_format has passed. + **/ + int (*set_options)(void *); + + /* XXXX Add an implementation for options_act_reversible() later in this + * branch. */ + + /** + * Receive a state object as defined by state_format. Return 0 on success, + * -1 on failure. + * + * It is safe to store the pointer to the object; set_state() is only + * called on startup. + * + * This function is only called after all the validation code defined + * by subsys_fns_t.state_format has passed. + * + * This function will only be called once per invocation of Tor, since + * Tor does not reload its state while it is running. + **/ + int (*set_state)(void *); + + /** + * Update any information that needs to be stored in the provided state + * object (as defined by state_format). Return 0 on success, -1 on failure. + * + * The object provided here will be the same one as provided earlier to + * set_state(). This method is called when we are about to save the state + * to disk. + **/ + int (*flush_state)(void *); +} subsys_fns_t; + +#ifndef COCCI +/** + * Macro to declare a subsystem's location. + **/ +#define SUBSYS_DECLARE_LOCATION() \ + .location = __FILE__ +#endif /* !defined(COCCI) */ + +/** + * Lowest allowed subsystem level. + **/ +#define MIN_SUBSYS_LEVEL -100 +/** + * Highest allowed subsystem level. + **/ +#define MAX_SUBSYS_LEVEL 100 + +/** + * All tor "libraries" (in src/libs) should have a subsystem level equal to or + * less than this value. + */ +#define SUBSYS_LEVEL_LIBS -10 + +#endif /* !defined(TOR_SUBSYS_T) */ diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include index c93a06e59e..306fa57b7a 100644 --- a/src/lib/term/.may_include +++ b/src/lib/term/.may_include @@ -5,5 +5,4 @@ lib/log/*.h lib/term/*.h lib/malloc/*.h -# From src/ext -tor_readpassphrase.h +ext/tor_readpassphrase.h diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c index c6459f250f..d2d6cb2b7b 100644 --- a/src/lib/term/getpass.c +++ b/src/lib/term/getpass.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #elif defined(HAVE_READPASSPHRASE_H) #include <readpassphrase.h> #else -#include "tor_readpassphrase.h" +#include "ext/tor_readpassphrase.h" #endif /* defined(_WIN32) || ... */ #include <stdlib.h> diff --git a/src/lib/term/getpass.h b/src/lib/term/getpass.h index a9c146ea8f..b080ad2473 100644 --- a/src/lib/term/getpass.h +++ b/src/lib/term/getpass.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,4 +15,4 @@ ssize_t tor_getpass(const char *prompt, char *output, size_t buflen); -#endif +#endif /* !defined(TOR_GETPASS_H) */ diff --git a/src/lib/term/include.am b/src/lib/term/include.am index 55fe548ebc..a120bba0cb 100644 --- a/src/lib/term/include.am +++ b/src/lib/term/include.am @@ -11,6 +11,7 @@ else readpassphrase_source= endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_term_a_SOURCES = \ src/lib/term/getpass.c \ $(readpassphrase_source) @@ -20,5 +21,6 @@ src_lib_libtor_term_testing_a_SOURCES = \ src_lib_libtor_term_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_term_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/term/getpass.h diff --git a/src/lib/term/lib_term.md b/src/lib/term/lib_term.md new file mode 100644 index 0000000000..f96d25ffe8 --- /dev/null +++ b/src/lib/term/lib_term.md @@ -0,0 +1,2 @@ +@dir /lib/term +@brief lib/term: Terminal operations (password input). diff --git a/src/lib/testsupport/include.am b/src/lib/testsupport/include.am index b2aa620985..a5ed46eb67 100644 --- a/src/lib/testsupport/include.am +++ b/src/lib/testsupport/include.am @@ -1,3 +1,4 @@ +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/testsupport/testsupport.h diff --git a/src/lib/testsupport/lib_testsupport.md b/src/lib/testsupport/lib_testsupport.md new file mode 100644 index 0000000000..7358e6a80f --- /dev/null +++ b/src/lib/testsupport/lib_testsupport.md @@ -0,0 +1,2 @@ +@dir /lib/testsupport +@brief lib/testsupport: Helpers for test-only code and for function mocking. diff --git a/src/lib/testsupport/testsupport.h b/src/lib/testsupport/testsupport.h index 9363a9ba66..165c497f71 100644 --- a/src/lib/testsupport/testsupport.h +++ b/src/lib/testsupport/testsupport.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2019, The Tor Project, Inc. */ +/* Copyright (c) 2013-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,16 +15,41 @@ #ifndef TOR_TESTSUPPORT_H #define TOR_TESTSUPPORT_H -#ifdef TOR_UNIT_TESTS /** The "STATIC" macro marks a function or variable that is static when * building Tor for production, but non-static when building the unit - * tests. */ + * tests. + * + * For example, a function declared as: + * + * STATIC int internal_function(void); + * + * should be only visible for the file on which it is declared, and in the + * unit tests. + */ +#ifdef TOR_UNIT_TESTS #define STATIC +#else /* !defined(TOR_UNIT_TESTS) */ +#define STATIC static +#endif /* defined(TOR_UNIT_TESTS) */ + +/** The "EXTERN" macro is used along with "STATIC" for variables declarations: + * it expands to an extern declaration when Tor building unit tests, and to + * nothing otherwise. + * + * For example, to declare a variable as visible only visible in one + * file and in the unit tests, you would put this in the header: + * + * EXTERN(int, local_variable) + * + * and this in the source: + * + * STATIC int local_variable; + */ +#ifdef TOR_UNIT_TESTS #define EXTERN(type, name) extern type name; #else -#define STATIC static #define EXTERN(type, name) -#endif /* defined(TOR_UNIT_TESTS) */ +#endif /** Quick and dirty macros to implement test mocking. * @@ -70,32 +95,42 @@ * * @{ */ #ifdef TOR_UNIT_TESTS +/** Declare a mocked function. For use in headers. */ #define MOCK_DECL(rv, funcname, arglist) \ rv funcname ##__real arglist; \ extern rv(*funcname) arglist +/** Define the implementation of a mocked function. */ #define MOCK_IMPL(rv, funcname, arglist) \ rv(*funcname) arglist = funcname ##__real; \ rv funcname ##__real arglist +/** As MOCK_DECL(), but allow attributes. */ #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \ rv funcname ##__real arglist attr; \ extern rv(*funcname) arglist -#define MOCK_IMPL(rv, funcname, arglist) \ - rv(*funcname) arglist = funcname ##__real; \ - rv funcname ##__real arglist +/** + * Replace <b>func</b> (a mockable function) with a replacement function. + * + * Only usable when Tor has been built for unit tests. */ #define MOCK(func, replacement) \ do { \ (func) = (replacement); \ } while (0) +/** Replace <b>func</b> (a mockable function) with its original value. + * + * Only usable when Tor has been built for unit tests. */ #define UNMOCK(func) \ do { \ func = func ##__real; \ } while (0) -#else /* !(defined(TOR_UNIT_TESTS)) */ +#else /* !defined(TOR_UNIT_TESTS) */ +/** Declare a mocked function. For use in headers. */ #define MOCK_DECL(rv, funcname, arglist) \ rv funcname arglist -#define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \ +/** As MOCK_DECL(), but allow */ +#define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \ rv funcname arglist attr -#define MOCK_IMPL(rv, funcname, arglist) \ +/** Define the implementation of a mocked function. */ +#define MOCK_IMPL(rv, funcname, arglist) \ rv funcname arglist #endif /* defined(TOR_UNIT_TESTS) */ /** @} */ diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include index fc56f46836..02711348c5 100644 --- a/src/lib/thread/.may_include +++ b/src/lib/thread/.may_include @@ -2,6 +2,7 @@ orconfig.h lib/cc/*.h lib/lock/*.h lib/log/*.h +lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h lib/wallclock/*.h diff --git a/src/lib/thread/compat_pthreads.c b/src/lib/thread/compat_pthreads.c index 6f7ecd17da..d143b80252 100644 --- a/src/lib/thread/compat_pthreads.c +++ b/src/lib/thread/compat_pthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c index 94ab021c52..75ade9c9f2 100644 --- a/src/lib/thread/compat_threads.c +++ b/src/lib/thread/compat_threads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -14,9 +14,11 @@ #include "orconfig.h" #include <stdlib.h> #include "lib/thread/threads.h" +#include "lib/thread/thread_sys.h" #include "lib/log/log.h" #include "lib/log/util_bug.h" +#include "lib/subsys/subsys.h" #include <string.h> @@ -65,7 +67,15 @@ atomic_counter_init(atomic_counter_t *counter) memset(counter, 0, sizeof(*counter)); tor_mutex_init_nonrecursive(&counter->mutex); } -/** Clean up all resources held by an atomic counter. */ +/** Clean up all resources held by an atomic counter. + * + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ void atomic_counter_destroy(atomic_counter_t *counter) { @@ -109,3 +119,18 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return oldval; } #endif /* !defined(HAVE_WORKING_STDATOMIC) */ + +static int +subsys_threads_initialize(void) +{ + tor_threads_init(); + return 0; +} + +const subsys_fns_t sys_threads = { + .name = "threads", + SUBSYS_DECLARE_LOCATION(), + .supported = true, + .level = -89, + .initialize = subsys_threads_initialize, +}; diff --git a/src/lib/thread/compat_winthreads.c b/src/lib/thread/compat_winthreads.c index f0b1430e84..2ca5620d23 100644 --- a/src/lib/thread/compat_winthreads.c +++ b/src/lib/thread/compat_winthreads.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am index 9ec23d166e..cd8016b5df 100644 --- a/src/lib/thread/include.am +++ b/src/lib/thread/include.am @@ -12,6 +12,7 @@ if THREADS_WIN32 threads_impl_source=src/lib/thread/compat_winthreads.c endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_thread_a_SOURCES = \ src/lib/thread/compat_threads.c \ src/lib/thread/numcpus.c \ @@ -22,6 +23,8 @@ src_lib_libtor_thread_testing_a_SOURCES = \ src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ - src/lib/thread/threads.h \ - src/lib/thread/numcpus.h + src/lib/thread/numcpus.h \ + src/lib/thread/thread_sys.h \ + src/lib/thread/threads.h diff --git a/src/lib/thread/lib_thread.md b/src/lib/thread/lib_thread.md new file mode 100644 index 0000000000..5870ad790f --- /dev/null +++ b/src/lib/thread/lib_thread.md @@ -0,0 +1,7 @@ +@dir /lib/thread +@brief lib/thread: Mid-level threading. + +This module contains compatibility and convenience code for multithreading, +except for low-level locks (which are in \refdir{lib/lock} and +workqueue/threadpool code (which belongs in \refdir{lib/evloop}.) + diff --git a/src/lib/thread/numcpus.c b/src/lib/thread/numcpus.c index b293d965d2..18454ce3ad 100644 --- a/src/lib/thread/numcpus.c +++ b/src/lib/thread/numcpus.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/thread/numcpus.h b/src/lib/thread/numcpus.h index 3f0a29ce7c..65e6c430cf 100644 --- a/src/lib/thread/numcpus.h +++ b/src/lib/thread/numcpus.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -13,4 +13,4 @@ int compute_num_cpus(void); -#endif +#endif /* !defined(TOR_NUMCPUS_H) */ diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h new file mode 100644 index 0000000000..6206fac9d6 --- /dev/null +++ b/src/lib/thread/thread_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file thread_sys.h + * \brief Declare subsystem object for threads library + **/ + +#ifndef TOR_THREADS_SYS_H +#define TOR_THREADS_SYS_H + +extern const struct subsys_fns_t sys_threads; + +#endif /* !defined(TOR_THREADS_SYS_H) */ diff --git a/src/lib/thread/threading.md b/src/lib/thread/threading.md new file mode 100644 index 0000000000..a1058c97de --- /dev/null +++ b/src/lib/thread/threading.md @@ -0,0 +1,26 @@ + +@page threading Threading in Tor + +Tor is based around a single main thread and one or more worker +threads. We aim (with middling success) to use worker threads for +CPU-intensive activities and the main thread for our networking. +Fortunately (?) we have enough cryptography that moving what we can +of the cryptographic processes to the workers should achieve good +parallelism under most loads. Unfortunately, we only have a small +fraction of our cryptography done in our worker threads right now. + +Our threads-and-workers abstraction is defined in workqueue.c, which +combines a work queue with a thread pool, and integrates the +signalling with libevent. Tor's main instance of a work queue is +instantiated in cpuworker.c. It will probably need some refactoring +as more types of work are added. + +On a lower level, we provide locks with tor_mutex_t in \refdir{lib/lock}, and +higher-level locking/threading tools in \refdir{lib/thread}, including +conditions (tor_cond_t), thread-local storage (tor_threadlocal_t), and more. + + +Try to minimize sharing between threads: it is usually best to simply +make the worker "own" all the data it needs while the work is in +progress, and to give up ownership when it's complete. + diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h index ecf60641b5..fcc0c23a87 100644 --- a/src/lib/thread/threads.h +++ b/src/lib/thread/threads.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -63,7 +63,7 @@ int tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex, void tor_cond_signal_one(tor_cond_t *cond); void tor_cond_signal_all(tor_cond_t *cond); -typedef struct tor_threadlocal_s { +typedef struct tor_threadlocal_t { #ifdef _WIN32 DWORD index; #else @@ -106,8 +106,10 @@ void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value); typedef struct atomic_counter_t { atomic_size_t val; } atomic_counter_t; +#ifndef COCCI #define ATOMIC_LINKAGE static -#else /* !(defined(HAVE_WORKING_STDATOMIC)) */ +#endif +#else /* !defined(HAVE_WORKING_STDATOMIC) */ typedef struct atomic_counter_t { tor_mutex_t mutex; size_t val; @@ -131,7 +133,17 @@ atomic_counter_init(atomic_counter_t *counter) { atomic_init(&counter->val, 0); } -/** Clean up all resources held by an atomic counter. */ +/** Clean up all resources held by an atomic counter. + * + * This usage note applies to the compat_threads implementation of + * atomic_counter_destroy(): + * Destroying a locked mutex is undefined behaviour. Global mutexes may be + * locked when they are passed to this function, because multiple threads can + * still access them. So we can either: + * - destroy on shutdown, and re-initialise when tor re-initialises, or + * - skip destroying and re-initialisation, using a sentinel variable. + * See #31735 for details. + */ static inline void atomic_counter_destroy(atomic_counter_t *counter) { @@ -162,7 +174,7 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval) return atomic_exchange(&counter->val, newval); } -#else /* !(defined(HAVE_WORKING_STDATOMIC)) */ +#else /* !defined(HAVE_WORKING_STDATOMIC) */ #endif /* defined(HAVE_WORKING_STDATOMIC) */ #endif /* !defined(TOR_COMPAT_THREADS_H) */ diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include index 2c7e37a836..ae01431b60 100644 --- a/src/lib/time/.may_include +++ b/src/lib/time/.may_include @@ -4,8 +4,10 @@ lib/cc/*.h lib/err/*.h lib/intmath/*.h lib/log/*.h +lib/subsys/*.h lib/time/*.h lib/wallclock/*.h +lib/defs/time.h # For load_windows_system_lib. lib/fs/winlib.h
\ No newline at end of file diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c index 98854bad2c..6bbad4f98a 100644 --- a/src/lib/time/compat_time.c +++ b/src/lib/time/compat_time.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -164,6 +164,8 @@ static int64_t last_tick_count = 0; * to be monotonic; increments them as appropriate so that they actually * _are_ monotonic. * + * The returned time may be the same as the previous returned time. + * * Caller must hold lock. */ STATIC int64_t ratchet_performance_counter(int64_t count_raw) @@ -202,6 +204,8 @@ static struct timeval timeofday_offset = { 0, 0 }; * supposed to be monotonic; increments them as appropriate so that they * actually _are_ monotonic. * + * The returned time may be the same as the previous returned time. + * * Caller must hold lock. */ STATIC void ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out) @@ -270,7 +274,9 @@ monotime_init_internal(void) } /** - * Set "out" to the most recent monotonic time value + * Set "out" to the most recent monotonic time value. + * + * The returned time may be the same as the previous returned time. */ void monotime_get(monotime_t *out) @@ -298,10 +304,12 @@ monotime_coarse_get(monotime_coarse_t *out) #endif /* defined(TOR_UNIT_TESTS) */ out->abstime_ = mach_approximate_time(); } -#endif +#endif /* defined(HAVE_MACH_APPROXIMATE_TIME) */ /** * Return the number of nanoseconds between <b>start</b> and <b>end</b>. + * + * The returned value may be equal to zero. */ int64_t monotime_diff_nsec(const monotime_t *start, @@ -522,7 +530,9 @@ monotime_init_internal(void) GetTickCount64_fn = (GetTickCount64_fn_t) (void(*)(void)) GetProcAddress(h, "GetTickCount64"); } - // FreeLibrary(h) ? + // We can't call FreeLibrary(h) here, because freeing the handle may + // unload the library, and cause future calls to GetTickCount64_fn() + // to fail. See 29642 for details. } void @@ -757,7 +767,7 @@ monotime_coarse_zero(monotime_coarse_t *out) { memset(out, 0, sizeof(*out)); } -#endif +#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ int64_t monotime_diff_usec(const monotime_t *start, @@ -787,8 +797,8 @@ monotime_absolute_nsec(void) return monotime_diff_nsec(&initialized_at, &now); } -uint64_t -monotime_absolute_usec(void) +MOCK_IMPL(uint64_t, +monotime_absolute_usec,(void)) { return monotime_absolute_nsec() / 1000; } @@ -823,7 +833,7 @@ monotime_coarse_absolute_msec(void) { return monotime_coarse_absolute_nsec() / ONE_MILLION; } -#else +#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ #define initialized_at_coarse initialized_at #endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ @@ -855,7 +865,7 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) mach_time_info.numer; return abstime_val >> monotime_shift; } -#else +#else /* !defined(__APPLE__) */ uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units) { @@ -866,4 +876,4 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) { return (msec * STAMP_TICKS_PER_SECOND) / 1000; } -#endif +#endif /* defined(__APPLE__) */ diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h index 480d426ac7..5089e16ca5 100644 --- a/src/lib/time/compat_time.h +++ b/src/lib/time/compat_time.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -15,6 +15,120 @@ * of tens of milliseconds. */ +/* Q: When should I use monotonic time? + * + * A: If you need a time that never decreases, use monotonic time. If you need + * to send a time to a user or another process, or store a time, use the + * wall-clock time. + * + * Q: Should you use monotime or monotime_coarse as your source? + * + * A: Generally, you get better precision with monotime, but better + * performance with monotime_coarse. + * + * Q: What is a "monotonic" time, exactly? + * + * A: Monotonic times are strictly non-decreasing. The difference between any + * previous monotonic time, and the current monotonic time, is always greater + * than *or equal to* zero. + * Zero deltas happen more often: + * - on Windows (due to an OS bug), + * - when using monotime_coarse, or on systems with low-resolution timers, + * - on platforms where we emulate monotonic time using wall-clock time, and + * - when using time units that are larger than nanoseconds (due to + * truncation on division). + * + * Q: Should you use monotime_t or monotime_coarse_t directly? Should you use + * usec? msec? "stamp units?" + * + * A: Using monotime_t and monotime_coarse_t directly is most time-efficient, + * since no conversion needs to happen. But they can potentially use more + * memory than you would need for a usec/msec/"stamp unit" count. + * + * Converting to usec or msec on some platforms, and working with them in + * general, creates a risk of doing a 64-bit division. 64-bit division is + * expensive on 32-bit platforms, which still do exist. + * + * The "stamp unit" type is designed to give a type that is cheap to convert + * from monotime_coarse, has resolution of about 1-2ms, and fits nicely in a + * 32-bit integer. Its downside is that it does not correspond directly + * to a natural unit of time. + * + * There is not much point in using "coarse usec" or "coarse nsec", since the + * current coarse monotime implementations give you on the order of + * milliseconds of precision. + * + * Q: So, what backends is monotime_coarse using? + * + * A: Generally speaking, it uses "whatever monotonic-ish time implemenation + * does not require a context switch." The various implementations provide + * this by having a view of the current time in a read-only memory page that + * is updated with a frequency corresponding to the kernel's tick count. + * + * On Windows, monotime_coarse uses GetCount64() [or GetTickCount() on + * obsolete systems]. MSDN claims that the resolution is "typically in the + * range of 10-16 msec", but it has said that for years. Storing + * monotime_coarse_t uses 8 bytes. + * + * On OSX/iOS, monotime_coarse uses uses mach_approximate_time() where + * available, and falls back to regular monotime. The precision is not + * documented, but the implementation is open-source: it reads from a page + * that the kernel updates. Storing monotime_coarse_t uses 8 bytes. + * + * On unixy systems, monotime_coarse uses clock_gettime() with + * CLOCK_MONOTONIC_COARSE where available, and falls back to CLOCK_MONOTONIC. + * It typically uses vdso tricks to read from a page that the kernel updates. + * Its precision fixed, but you can get it with clock_getres(): on my Linux + * desktop, it claims to be 1 msec, but it will depend on the system HZ + * setting. Storing monotime_coarse_t uses 16 bytes. + * + * [TODO: Try CLOCK_MONOTONIC_FAST on foobsd.] + * + * Q: What backends is regular monotonic time using? + * + * A: In general, regular monotime uses something that requires a system call. + * On platforms where system calls are cheap, you win! Otherwise, you lose. + * + * On Windows, monotonic time uses QuereyPerformanceCounter. Storing + * monotime_t costs 8 bytes. + * + * On OSX/Apple, monotonic time uses mach_absolute_time. Storing + * monotime_t costs 8 bytes. + * + * On unixy systems, monotonic time uses CLOCK_MONOTONIC. Storing + * monotime_t costs 16 bytes. + * + * Q: Tell me about the costs of converting to a 64-bit nsec, usec, or msec + * count. + * + * A: Windows, coarse: Cheap, since it's all multiplication. + * + * Windows, precise: Expensive on 32-bit: it needs 64-bit division. + * + * Apple, all: Expensive on 32-bit: it needs 64-bit division. + * + * Unixy, all: Fairly cheap, since the only division required is dividing + * tv_nsec 1000, and nanoseconds-per-second fits in a 32-bit value. + * + * All, "timestamp units": Cheap everywhere: it never divides. + * + * Q: This is only somewhat related, but how much precision could I hope for + * from a libevent time? + * + * A: Actually, it's _very_ related if you're timing in order to have a + * timeout happen. + * + * On Windows, it uses select: you could in theory have a microsecond + * resolution, but it usually isn't that accurate. + * + * On OSX, iOS, and BSD, you have kqueue: You could in theory have a nanosecond + * resolution, but it usually isn't that accurate. + * + * On Linux, you have epoll: It has a millisecond resolution. Some recent + * Libevents can also use timerfd for higher resolution if + * EVENT_BASE_FLAG_PRECISE_TIMER is set: Tor doesn't set that flag. + */ + #ifndef TOR_COMPAT_TIME_H #define TOR_COMPAT_TIME_H @@ -86,26 +200,36 @@ void monotime_init(void); void monotime_get(monotime_t *out); /** * Return the number of nanoseconds between <b>start</b> and <b>end</b>. + * The returned value may be equal to zero. */ int64_t monotime_diff_nsec(const monotime_t *start, const monotime_t *end); /** * Return the number of microseconds between <b>start</b> and <b>end</b>. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ int64_t monotime_diff_usec(const monotime_t *start, const monotime_t *end); /** * Return the number of milliseconds between <b>start</b> and <b>end</b>. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ int64_t monotime_diff_msec(const monotime_t *start, const monotime_t *end); /** * Return the number of nanoseconds since the timer system was initialized. + * The returned value may be equal to zero. */ uint64_t monotime_absolute_nsec(void); /** * Return the number of microseconds since the timer system was initialized. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ -uint64_t monotime_absolute_usec(void); +MOCK_DECL(uint64_t, monotime_absolute_usec,(void)); /** * Return the number of milliseconds since the timer system was initialized. + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ uint64_t monotime_absolute_msec(void); @@ -129,10 +253,13 @@ void monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec); * Set <b>out</b> to the current coarse time. */ void monotime_coarse_get(monotime_coarse_t *out); +/** + * Like monotime_absolute_*(), but faster on some platforms. + */ uint64_t monotime_coarse_absolute_nsec(void); uint64_t monotime_coarse_absolute_usec(void); uint64_t monotime_coarse_absolute_msec(void); -#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */ +#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */ #define monotime_coarse_get monotime_get #define monotime_coarse_absolute_nsec monotime_absolute_nsec #define monotime_coarse_absolute_usec monotime_absolute_usec @@ -152,23 +279,32 @@ uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t); /** * Convert a difference, expressed in the units of monotime_coarse_to_stamp, * into an approximate number of milliseconds. + * + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units); uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec); uint32_t monotime_coarse_get_stamp(void); #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) +/** + * Like monotime_diff_*(), but faster on some platforms. + */ int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start, const monotime_coarse_t *end); int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start, const monotime_coarse_t *end); int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start, const monotime_coarse_t *end); +/** + * Like monotime_*(), but faster on some platforms. + */ void monotime_coarse_zero(monotime_coarse_t *out); int monotime_coarse_is_zero(const monotime_coarse_t *val); void monotime_coarse_add_msec(monotime_coarse_t *out, const monotime_coarse_t *val, uint32_t msec); -#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */ +#else /* !defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ #define monotime_coarse_diff_nsec monotime_diff_nsec #define monotime_coarse_diff_usec monotime_diff_usec #define monotime_coarse_diff_msec monotime_diff_msec @@ -182,6 +318,9 @@ void monotime_coarse_add_msec(monotime_coarse_t *out, * * Requires that the difference fit into an int32_t; not for use with * large time differences. + * + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start, const monotime_coarse_t *end); @@ -191,6 +330,9 @@ int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start, * * Requires that the difference fit into an int32_t; not for use with * large time differences. + * + * The returned value may be equal to zero. + * Fractional units are truncated, not rounded. */ static inline int32_t monotime_coarse_diff_msec32(const monotime_coarse_t *start, @@ -202,7 +344,7 @@ monotime_coarse_diff_msec32(const monotime_coarse_t *start, #else #define USING_32BIT_MSEC_HACK return monotime_coarse_diff_msec32_(start, end); -#endif +#endif /* SIZEOF_VOID_P == 8 */ } #ifdef TOR_UNIT_TESTS diff --git a/src/lib/time/include.am b/src/lib/time/include.am index a3f93a3744..dcb199b142 100644 --- a/src/lib/time/include.am +++ b/src/lib/time/include.am @@ -5,8 +5,10 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-time-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_time_a_SOURCES = \ src/lib/time/compat_time.c \ + src/lib/time/time_sys.c \ src/lib/time/tvdiff.c src_lib_libtor_time_testing_a_SOURCES = \ @@ -14,6 +16,8 @@ src_lib_libtor_time_testing_a_SOURCES = \ src_lib_libtor_time_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/time/compat_time.h \ + src/lib/time/time_sys.h \ src/lib/time/tvdiff.h diff --git a/src/lib/time/lib_time.md b/src/lib/time/lib_time.md new file mode 100644 index 0000000000..8e58aafcd8 --- /dev/null +++ b/src/lib/time/lib_time.md @@ -0,0 +1,9 @@ +@dir /lib/time +@brief lib/time: Higher-level time functions + +This includes both fine-grained timers and monotonic timers, along with +wrappers for them to try to improve efficiency. + +For "what time is it" in UTC, see \refdir{lib/wallclock}. For parsing and +encoding times and dates, see \refdir{lib/encoding}. + diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c new file mode 100644 index 0000000000..1c1bc4cd18 --- /dev/null +++ b/src/lib/time/time_sys.c @@ -0,0 +1,29 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.c + * \brief Subsystem object for monotime setup. + **/ + +#include "orconfig.h" +#include "lib/subsys/subsys.h" +#include "lib/time/time_sys.h" +#include "lib/time/compat_time.h" + +static int +subsys_time_initialize(void) +{ + monotime_init(); + return 0; +} + +const subsys_fns_t sys_time = { + .name = "time", + SUBSYS_DECLARE_LOCATION(), + /* Monotonic time depends on logging, and a lot of other modules depend on + * monotonic time. */ + .level = -80, + .supported = true, + .initialize = subsys_time_initialize, +}; diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h new file mode 100644 index 0000000000..f2401e1911 --- /dev/null +++ b/src/lib/time/time_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file time_sys.h + * \brief Declare subsystem object for the time module. + **/ + +#ifndef TOR_TIME_SYS_H +#define TOR_TIME_SYS_H + +extern const struct subsys_fns_t sys_time; + +#endif /* !defined(TOR_TIME_SYS_H) */ diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c index a87d0d96dc..cbad5a48b8 100644 --- a/src/lib/time/tvdiff.c +++ b/src/lib/time/tvdiff.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,6 +11,7 @@ #include "lib/time/tvdiff.h" #include "lib/cc/compat_compiler.h" +#include "lib/defs/time.h" #include "lib/log/log.h" #ifdef _WIN32 @@ -20,8 +21,6 @@ #include <sys/time.h> #endif -#define TOR_USEC_PER_SEC 1000000 - /** Return the difference between start->tv_sec and end->tv_sec. * Returns INT64_MAX on overflow and underflow. */ diff --git a/src/lib/time/tvdiff.h b/src/lib/time/tvdiff.h index 724af1528a..e779e758f1 100644 --- a/src/lib/time/tvdiff.h +++ b/src/lib/time/tvdiff.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,4 +20,4 @@ int64_t tv_to_msec(const struct timeval *tv); time_t time_diff(const time_t from, const time_t to); -#endif +#endif /* !defined(TOR_TVDIFF_H) */ diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include index 2840e590b8..c550bde024 100644 --- a/src/lib/tls/.may_include +++ b/src/lib/tls/.may_include @@ -1,6 +1,7 @@ orconfig.h lib/arch/*.h +lib/buf/*.h lib/cc/*.h lib/container/*.h lib/crypt_ops/*.h @@ -11,7 +12,7 @@ lib/log/*.h lib/malloc/*.h lib/net/*.h lib/string/*.h -lib/testsupport/testsupport.h +lib/subsys/*.h +lib/testsupport/*.h lib/tls/*.h - -ciphers.inc +lib/tls/*.inc diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c index b570216df0..b92a14d6a1 100644 --- a/src/lib/tls/buffers_tls.c +++ b/src/lib/tls/buffers_tls.c @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -12,7 +12,7 @@ #define BUFFERS_PRIVATE #include "orconfig.h" #include <stddef.h> -#include "lib/container/buffers.h" +#include "lib/buf/buffers.h" #include "lib/tls/buffers_tls.h" #include "lib/cc/torint.h" #include "lib/log/log.h" @@ -68,10 +68,10 @@ buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most) check_no_tls_errors(); - IF_BUG_ONCE(buf->datalen >= INT_MAX) - return -1; - IF_BUG_ONCE(buf->datalen >= INT_MAX - at_most) - return -1; + IF_BUG_ONCE(buf->datalen > BUF_MAX_LEN) + return TOR_TLS_ERROR_MISC; + IF_BUG_ONCE(buf->datalen > BUF_MAX_LEN - at_most) + return TOR_TLS_ERROR_MISC; while (at_most > total_read) { size_t readlen = at_most - total_read; @@ -90,7 +90,7 @@ buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most) r = read_to_chunk_tls(buf, chunk, tls, readlen); if (r < 0) return r; /* Error */ - tor_assert(total_read+r < INT_MAX); + tor_assert(total_read+r <= BUF_MAX_LEN); total_read += r; if ((size_t)r < readlen) /* eof, block, or no more to read. */ break; @@ -177,6 +177,6 @@ buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen, if (r == 0) /* Can't flush any more now. */ break; } while (sz > 0); - tor_assert(flushed < INT_MAX); + tor_assert(flushed <= BUF_MAX_LEN); return (int)flushed; } diff --git a/src/lib/tls/buffers_tls.h b/src/lib/tls/buffers_tls.h index 65788c3f34..587426801d 100644 --- a/src/lib/tls/buffers_tls.h +++ b/src/lib/tls/buffers_tls.h @@ -1,7 +1,7 @@ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am index a664b29fb2..7e05ef4f8c 100644 --- a/src/lib/tls/include.am +++ b/src/lib/tls/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-tls-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_tls_a_SOURCES = \ src/lib/tls/buffers_tls.c \ src/lib/tls/tortls.c \ @@ -29,6 +30,7 @@ src_lib_libtor_tls_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_tls_testing_a_CFLAGS = \ $(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/tls/ciphers.inc \ src/lib/tls/buffers_tls.h \ @@ -36,5 +38,6 @@ noinst_HEADERS += \ src/lib/tls/tortls.h \ src/lib/tls/tortls_internal.h \ src/lib/tls/tortls_st.h \ + src/lib/tls/tortls_sys.h \ src/lib/tls/x509.h \ src/lib/tls/x509_internal.h diff --git a/src/lib/tls/lib_tls.md b/src/lib/tls/lib_tls.md new file mode 100644 index 0000000000..26fea723f9 --- /dev/null +++ b/src/lib/tls/lib_tls.md @@ -0,0 +1,11 @@ +@dir /lib/tls +@brief lib/tls: TLS library wrappers + +This module has compatibility wrappers around the library (NSS or OpenSSL, +depending on configuration) that Tor uses to implement the TLS link security +protocol. + +It also implements the logic for some legacy TLS protocol usage we used to +support in old versions of Tor, involving conditional delivery of certificate +chains (v1 link protocol) and conditional renegotiation (v2 link protocol). + diff --git a/src/lib/tls/nss_countbytes.c b/src/lib/tls/nss_countbytes.c index 7761727acd..4b98df80ec 100644 --- a/src/lib/tls/nss_countbytes.c +++ b/src/lib/tls/nss_countbytes.c @@ -1,4 +1,4 @@ -/* Copyright 2018-2019, The Tor Project Inc. */ +/* Copyright 2018-2020, The Tor Project Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/nss_countbytes.h b/src/lib/tls/nss_countbytes.h index 8b31603923..36ed55e10d 100644 --- a/src/lib/tls/nss_countbytes.h +++ b/src/lib/tls/nss_countbytes.h @@ -1,4 +1,4 @@ -/* Copyright 2018-2019, The Tor Project, Inc. */ +/* Copyright 2018-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,4 +22,4 @@ int tor_get_prfiledesc_byte_counts(struct PRFileDesc *fd, uint64_t *n_read_out, uint64_t *n_written_out); -#endif +#endif /* !defined(TOR_NSS_COUNTBYTES_H) */ diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c index 4ca7c7d9d3..9e70e54725 100644 --- a/src/lib/tls/tortls.c +++ b/src/lib/tls/tortls.c @@ -1,12 +1,18 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * @file tortls.c + * @brief Shared functionality for our TLS backends. + **/ + #define TORTLS_PRIVATE #define TOR_X509_PRIVATE #include "lib/tls/x509.h" #include "lib/tls/x509_internal.h" +#include "lib/tls/tortls_sys.h" #include "lib/tls/tortls.h" #include "lib/tls/tortls_st.h" #include "lib/tls/tortls_internal.h" @@ -15,6 +21,7 @@ #include "lib/crypt_ops/crypto_rsa.h" #include "lib/crypt_ops/crypto_rand.h" #include "lib/net/socket.h" +#include "lib/subsys/subsys.h" #ifdef _WIN32 #include <winsock2.h> @@ -440,3 +447,16 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity) return rv; } + +static void +subsys_tortls_shutdown(void) +{ + tor_tls_free_all(); +} + +const subsys_fns_t sys_tortls = { + .name = "tortls", + SUBSYS_DECLARE_LOCATION(), + .level = -50, + .shutdown = subsys_tortls_shutdown +}; diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h index 9398318035..3703ecad56 100644 --- a/src/lib/tls/tortls.h +++ b/src/lib/tls/tortls.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TORTLS_H @@ -25,12 +25,12 @@ struct ssl_ctx_st; struct ssl_session_st; typedef struct ssl_ctx_st tor_tls_context_impl_t; typedef struct ssl_st tor_tls_impl_t; -#else +#else /* !defined(ENABLE_OPENSSL) */ struct PRFileDesc; typedef struct PRFileDesc tor_tls_context_impl_t; typedef struct PRFileDesc tor_tls_impl_t; -#endif -#endif +#endif /* defined(ENABLE_OPENSSL) */ +#endif /* defined(TORTLS_PRIVATE) */ struct tor_x509_cert_t; @@ -81,6 +81,7 @@ void tor_tls_free_all(void); void tor_tls_init(void); void tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing); +const char *tor_tls_get_last_error_msg(const tor_tls_t *tls); int tor_tls_context_init(unsigned flags, crypto_pk_t *client_identity, crypto_pk_t *server_identity, @@ -143,9 +144,9 @@ void check_no_tls_errors_(const char *fname, int line); void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err, int severity, int domain, const char *doing); -#else +#else /* !defined(ENABLE_OPENSSL) */ #define check_no_tls_errors() STMT_NIL -#endif +#endif /* defined(ENABLE_OPENSSL) */ int tor_tls_get_my_certs(int server, const struct tor_x509_cert_t **link_cert_out, diff --git a/src/lib/tls/tortls_internal.h b/src/lib/tls/tortls_internal.h index 071c506561..3f56f181ee 100644 --- a/src/lib/tls/tortls_internal.h +++ b/src/lib/tls/tortls_internal.h @@ -1,11 +1,18 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +/** + * @file tortls_internal.h + * @brief Declare internal functions for lib/tls + **/ + #ifndef TORTLS_INTERNAL_H #define TORTLS_INTERNAL_H +#include "lib/tls/x509.h" + int tor_errno_to_tls_error(int e); #ifdef ENABLE_OPENSSL int tor_tls_get_error(tor_tls_t *tls, int r, int extra, @@ -61,8 +68,8 @@ STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret, void *arg); STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher); -#endif -#endif +#endif /* defined(TORTLS_OPENSSL_PRIVATE) */ +#endif /* defined(ENABLE_OPENSSL) */ #ifdef TOR_UNIT_TESTS extern int tor_tls_object_ex_data_index; @@ -73,4 +80,4 @@ extern uint64_t total_bytes_written_over_tls; extern uint64_t total_bytes_written_by_tls; #endif /* defined(TOR_UNIT_TESTS) */ -#endif /* defined(TORTLS_INTERNAL_H) */ +#endif /* !defined(TORTLS_INTERNAL_H) */ diff --git a/src/lib/tls/tortls_nss.c b/src/lib/tls/tortls_nss.c index b7f3513150..25446a81af 100644 --- a/src/lib/tls/tortls_nss.c +++ b/src/lib/tls/tortls_nss.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -34,7 +34,7 @@ #include "lib/tls/nss_countbytes.h" #include "lib/log/util_bug.h" -DISABLE_GCC_WARNING(strict-prototypes) +DISABLE_GCC_WARNING("-Wstrict-prototypes") #include <prio.h> // For access to rar sockets. #include <private/pprio.h> @@ -42,7 +42,7 @@ DISABLE_GCC_WARNING(strict-prototypes) #include <sslt.h> #include <sslproto.h> #include <certt.h> -ENABLE_GCC_WARNING(strict-prototypes) +ENABLE_GCC_WARNING("-Wstrict-prototypes") static SECStatus always_accept_cert_cb(void *, PRFileDesc *, PRBool, PRBool); @@ -369,6 +369,8 @@ tls_log_errors(tor_tls_t *tls, int severity, int domain, (void)tls; PRErrorCode code = PORT_GetError(); + if (tls) + tls->last_error = code; const char *addr = tls ? tls->address : NULL; const char *string = PORT_ErrorToString(code); @@ -391,6 +393,17 @@ tls_log_errors(tor_tls_t *tls, int severity, int domain, with, addr); } } +const char * +tor_tls_get_last_error_msg(const tor_tls_t *tls) +{ + IF_BUG_ONCE(!tls) { + return NULL; + } + if (tls->last_error == 0) { + return NULL; + } + return PORT_ErrorToString((PRErrorCode)tls->last_error); +} tor_tls_t * tor_tls_new(tor_socket_t sock, int is_server) diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c index c5031a00aa..5c92df1259 100644 --- a/src/lib/tls/tortls_openssl.c +++ b/src/lib/tls/tortls_openssl.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -25,7 +25,7 @@ * <winsock.h> and mess things up, in at least some openssl versions. */ #include <winsock2.h> #include <ws2tcpip.h> -#endif +#endif /* defined(_WIN32) */ #include "lib/crypt_ops/crypto_cipher.h" #include "lib/crypt_ops/crypto_rand.h" @@ -37,7 +37,7 @@ /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */ -DISABLE_GCC_WARNING(redundant-decls) +DISABLE_GCC_WARNING("-Wredundant-decls") #include <openssl/opensslv.h> @@ -54,7 +54,7 @@ DISABLE_GCC_WARNING(redundant-decls) #include <openssl/bn.h> #include <openssl/rsa.h> -ENABLE_GCC_WARNING(redundant-decls) +ENABLE_GCC_WARNING("-Wredundant-decls") #include "lib/tls/tortls.h" #include "lib/tls/tortls_st.h" @@ -245,10 +245,30 @@ tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing) unsigned long err; while ((err = ERR_get_error()) != 0) { + if (tls) + tls->last_error = err; tor_tls_log_one_error(tls, err, severity, domain, doing); } } +/** + * Return a string representing more detail about the last error received + * on TLS. + * + * May return null if no error was found. + **/ +const char * +tor_tls_get_last_error_msg(const tor_tls_t *tls) +{ + IF_BUG_ONCE(!tls) { + return NULL; + } + if (tls->last_error == 0) { + return NULL; + } + return (const char*)ERR_reason_error_string(tls->last_error); +} + #define CATCH_SYSCALL 1 #define CATCH_ZERO 2 @@ -318,7 +338,7 @@ tor_tls_init(void) #else SSL_library_init(); SSL_load_error_strings(); -#endif +#endif /* defined(OPENSSL_1_1_API) */ #if (SIZEOF_VOID_P >= 8 && \ OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1)) @@ -383,7 +403,7 @@ static const char SERVER_CIPHER_LIST[] = * conclude that it has no valid ciphers if it's running with TLS1.3. */ TLS1_3_TXT_AES_128_GCM_SHA256 ":" -#endif +#endif /* defined(TLS1_3_TXT_AES_128_GCM_SHA256) */ TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":" TLS1_TXT_DHE_RSA_WITH_AES_128_SHA; @@ -464,7 +484,9 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /** List of ciphers that clients should advertise, omitting items that * our OpenSSL doesn't know about. */ static const char CLIENT_CIPHER_LIST[] = -#include "ciphers.inc" +#ifndef COCCI +#include "lib/tls/ciphers.inc" +#endif /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version * of any cipher we say. */ "!SSLv2" @@ -657,7 +679,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, if (r < 0) goto error; } -#else +#else /* !(defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SE...)) */ if (! is_client) { int nid; EC_KEY *ec_key; @@ -673,7 +695,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime, SSL_CTX_set_tmp_ecdh(result->ctx, ec_key); EC_KEY_free(ec_key); } -#endif +#endif /* defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1...) */ SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER, always_accept_verify_cb); /* let us realloc bufs that we're writing from */ @@ -764,7 +786,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher) tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher); return c != NULL; } -#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */ +#else /* !defined(HAVE_SSL_CIPHER_FIND) */ # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR) if (m && m->get_cipher_by_char) { @@ -1062,7 +1084,7 @@ tor_tls_new(tor_socket_t sock, int isServer) /* We can't actually use TLS 1.3 until this bug is fixed. */ SSL_set_max_proto_version(result->ssl, TLS1_2_VERSION); } -#endif +#endif /* defined(SSL_CTRL_SET_MAX_PROTO_VERSION) */ if (!SSL_set_cipher_list(result->ssl, isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) { @@ -1715,7 +1737,7 @@ tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out, else return -1; } -#endif +#endif /* defined(TLS1_3_VERSION) */ return (r == 1) ? 0 : -1; } diff --git a/src/lib/tls/tortls_st.h b/src/lib/tls/tortls_st.h index 3f7ea8ac6a..34abe52ee3 100644 --- a/src/lib/tls/tortls_st.h +++ b/src/lib/tls/tortls_st.h @@ -1,11 +1,19 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_TORTLS_ST_H #define TOR_TORTLS_ST_H +/** + * @file tortls_st.h + * @brief Structure declarations for internal TLS types. + * + * These should generally be treated as opaque outside of the + * lib/tls module. + **/ + #include "lib/net/socket.h" #define TOR_TLS_MAGIC 0x71571571 @@ -59,17 +67,20 @@ struct tor_tls_t { */ unsigned long last_write_count; unsigned long last_read_count; + /** Most recent error value from ERR_get_error(). */ + unsigned long last_error; /** If set, a callback to invoke whenever the client tries to renegotiate * the handshake. */ void (*negotiated_callback)(tor_tls_t *tls, void *arg); /** Argument to pass to negotiated_callback. */ void *callback_arg; -#endif +#endif /* defined(ENABLE_OPENSSL) */ #ifdef ENABLE_NSS /** Last values retried from tor_get_prfiledesc_byte_counts(). */ uint64_t last_write_count; uint64_t last_read_count; + long last_error; #endif }; -#endif +#endif /* !defined(TOR_TORTLS_ST_H) */ diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h new file mode 100644 index 0000000000..177c198f71 --- /dev/null +++ b/src/lib/tls/tortls_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file tortls_sys.h + * \brief Declare subsystem object for the tortls module + **/ + +#ifndef TOR_TORTLS_SYS_H +#define TOR_TORTLS_SYS_H + +extern const struct subsys_fns_t sys_tortls; + +#endif /* !defined(TOR_TORTLS_SYS_H) */ diff --git a/src/lib/tls/x509.c b/src/lib/tls/x509.c index b4a0f8dabf..2515499298 100644 --- a/src/lib/tls/x509.c +++ b/src/lib/tls/x509.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/tls/x509.h b/src/lib/tls/x509.h index 5e6660de5c..5919b9089d 100644 --- a/src/lib/tls/x509.h +++ b/src/lib/tls/x509.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_X509_H @@ -35,7 +35,7 @@ struct tor_x509_cert_t { common_digests_t cert_digests; common_digests_t pkey_digests; }; -#endif +#endif /* defined(TOR_X509_PRIVATE) */ void tor_tls_pick_certificate_lifetime(time_t now, unsigned cert_lifetime, @@ -47,7 +47,7 @@ tor_x509_cert_t *tor_x509_cert_replace_expiration( const tor_x509_cert_t *inp, time_t new_expiration_time, crypto_pk_t *signing_key); -#endif +#endif /* defined(TOR_UNIT_TESTS) */ tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert); @@ -72,4 +72,4 @@ int tor_tls_cert_is_valid(int severity, time_t now, int check_rsa_1024); -#endif +#endif /* !defined(TOR_X509_H) */ diff --git a/src/lib/tls/x509_internal.h b/src/lib/tls/x509_internal.h index bf2bec9689..145be7e71c 100644 --- a/src/lib/tls/x509_internal.h +++ b/src/lib/tls/x509_internal.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_X509_INTERNAL_H @@ -50,4 +50,4 @@ int tor_x509_cert_set_cached_der_encoding(tor_x509_cert_t *cert); #define tor_x509_cert_set_cached_der_encoding(cert) (0) #endif -#endif +#endif /* !defined(TOR_X509_INTERNAL_H) */ diff --git a/src/lib/tls/x509_nss.c b/src/lib/tls/x509_nss.c index fb4af54c52..341bb57104 100644 --- a/src/lib/tls/x509_nss.c +++ b/src/lib/tls/x509_nss.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -120,13 +120,13 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa, der.data, der.len, (SECKEYPrivateKey *)signing_key,//const &cert->signature); -#else +#else /* !(0) */ s = SEC_DerSignData(cert->arena, &signed_der, der.data, der.len, (SECKEYPrivateKey *)signing_key,//const SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION); -#endif +#endif /* 0 */ if (s != SECSuccess) goto err; @@ -145,7 +145,7 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa, &result_cert->signatureWrap, issuer_pk, NULL); tor_assert(cert_ok == SECSuccess); } -#endif +#endif /* 1 */ err: if (subject_spki) @@ -455,4 +455,4 @@ tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp, return newcert ? tor_x509_cert_new(newcert) : NULL; } -#endif +#endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c index a344279c22..2abf02851d 100644 --- a/src/lib/tls/x509_openssl.c +++ b/src/lib/tls/x509_openssl.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,7 +19,7 @@ /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */ -DISABLE_GCC_WARNING(redundant-decls) +DISABLE_GCC_WARNING("-Wredundant-decls") #include <openssl/opensslv.h> @@ -36,7 +36,7 @@ DISABLE_GCC_WARNING(redundant-decls) #include <openssl/rsa.h> #include <openssl/x509.h> -ENABLE_GCC_WARNING(redundant-decls) +ENABLE_GCC_WARNING("-Wredundant-decls") #include "lib/log/log.h" #include "lib/log/util_bug.h" @@ -59,12 +59,12 @@ ENABLE_GCC_WARNING(redundant-decls) #define X509_get_notAfter(cert) \ X509_getm_notAfter(cert) #endif -#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */ +#else /* !defined(OPENSSL_1_1_API) */ #define X509_get_notBefore_const(cert) \ ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert)) #define X509_get_notAfter_const(cert) \ ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert)) -#endif +#endif /* defined(OPENSSL_1_1_API) */ /** Return a newly allocated X509 name with commonName <b>cname</b>. */ static X509_NAME * diff --git a/src/lib/trace/debug.h b/src/lib/trace/debug.h index e35616cf50..87b3074e0b 100644 --- a/src/lib/trace/debug.h +++ b/src/lib/trace/debug.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* Copyright (c) 2017-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -27,4 +27,4 @@ "\"" XSTR(subsystem) "\" hit. " \ "(line "XSTR(__LINE__) ")") -#endif /* TOR_TRACE_LOG_DEBUG_H */ +#endif /* !defined(TOR_TRACE_LOG_DEBUG_H) */ diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h index 1e1e7b9d16..368f85dd02 100644 --- a/src/lib/trace/events.h +++ b/src/lib/trace/events.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* Copyright (c) 2017-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -34,12 +34,12 @@ #include "lib/trace/debug.h" #endif -#else /* TOR_EVENT_TRACING_ENABLED */ +#else /* !defined(TOR_EVENT_TRACING_ENABLED) */ /* Reaching this point, we NOP every event declaration because event tracing * is not been enabled at compile time. */ #define tor_trace(subsystem, name, args...) -#endif /* TOR_EVENT_TRACING_ENABLED */ +#endif /* defined(TOR_EVENT_TRACING_ENABLED) */ -#endif /* TOR_TRACE_EVENTS_H */ +#endif /* !defined(TOR_TRACE_EVENTS_H) */ diff --git a/src/lib/trace/include.am b/src/lib/trace/include.am index 6f10c98744..98098c87f4 100644 --- a/src/lib/trace/include.am +++ b/src/lib/trace/include.am @@ -2,6 +2,7 @@ noinst_LIBRARIES += \ src/lib/libtor-trace.a +# ADD_C_FILE: INSERT HEADERS HERE. TRACEHEADERS = \ src/lib/trace/trace.h \ src/lib/trace/events.h @@ -11,7 +12,7 @@ TRACEHEADERS += \ src/lib/trace/debug.h endif -# Library source files. +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_trace_a_SOURCES = \ src/lib/trace/trace.c diff --git a/src/lib/trace/lib_trace.md b/src/lib/trace/lib_trace.md new file mode 100644 index 0000000000..a7a32529b0 --- /dev/null +++ b/src/lib/trace/lib_trace.md @@ -0,0 +1,6 @@ +@dir /lib/trace +@brief lib/trace: Function-tracing functionality API. + +This module is used for adding "trace" support (low-granularity function +logging) to Tor. Right now it doesn't have many users. + diff --git a/src/lib/trace/trace.c b/src/lib/trace/trace.c index 18be63c5a8..4e5c66b4c6 100644 --- a/src/lib/trace/trace.c +++ b/src/lib/trace/trace.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* Copyright (c) 2017-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/trace/trace.h b/src/lib/trace/trace.h index 606d435568..5e24678c3c 100644 --- a/src/lib/trace/trace.h +++ b/src/lib/trace/trace.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017-2019, The Tor Project, Inc. */ +/* Copyright (c) 2017-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -11,4 +11,4 @@ void tor_trace_init(void); -#endif // TOR_TRACE_TRACE_H +#endif /* !defined(TOR_TRACE_TRACE_H) */ diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include new file mode 100644 index 0000000000..d159ceb41f --- /dev/null +++ b/src/lib/version/.may_include @@ -0,0 +1,3 @@ +orconfig.h +micro-revision.i +lib/version/*.h
\ No newline at end of file diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c index 7d27549cad..09f11aa316 100644 --- a/src/lib/log/git_revision.c +++ b/src/lib/version/git_revision.c @@ -1,24 +1,39 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #include "orconfig.h" -#include "lib/log/git_revision.h" +#include "lib/version/git_revision.h" + +/** + * @file git_revision.c + * @brief Strings to describe the current Git commit. + **/ /** String describing which Tor Git repository version the source was * built from. This string is generated by a bit of shell kludging in * src/core/include.am, and is usually right. */ const char tor_git_revision[] = +#ifndef COCCI #ifndef _MSC_VER #include "micro-revision.i" #endif +#endif ""; +/** + * String appended to Tor bug messages describing the Tor version. + * + * It has the form "(on Tor 0.4.3.1-alpha)" or + * "(on Tor 0.4.3.1-alpha git-b994397f1af193f8)" + **/ const char tor_bug_suffix[] = " (on Tor " VERSION +#ifndef COCCI #ifndef _MSC_VER " " #include "micro-revision.i" #endif +#endif /* !defined(COCCI) */ ")"; diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h index 79e3c6684b..80b6c4734e 100644 --- a/src/lib/log/git_revision.h +++ b/src/lib/version/git_revision.h @@ -1,11 +1,16 @@ /* Copyright 2001-2004 Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ #ifndef TOR_GIT_REVISION_H #define TOR_GIT_REVISION_H +/** + * @file git_revision.h + * @brief Header for git_revision.c + **/ + extern const char tor_git_revision[]; extern const char tor_bug_suffix[]; diff --git a/src/lib/version/include.am b/src/lib/version/include.am new file mode 100644 index 0000000000..0ae31be1b2 --- /dev/null +++ b/src/lib/version/include.am @@ -0,0 +1,27 @@ + +noinst_LIBRARIES += src/lib/libtor-version.a + +if UNITTESTS_ENABLED +noinst_LIBRARIES += src/lib/libtor-version-testing.a +endif + +# ADD_C_FILE: INSERT SOURCES HERE. +src_lib_libtor_version_a_SOURCES = \ + src/lib/version/git_revision.c \ + src/lib/version/version.c + +src_lib_libtor_version_testing_a_SOURCES = \ + $(src_lib_libtor_version_a_SOURCES) +src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) +src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) + +# Declare that these object files depend on micro-revision.i. Without this +# rule, we could try to build them before micro-revision.i was created. +src/lib/version/git_revision.$(OBJEXT) \ + src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \ + micro-revision.i + +# ADD_C_FILE: INSERT HEADERS HERE. +noinst_HEADERS += \ + src/lib/version/git_revision.h \ + src/lib/version/torversion.h diff --git a/src/lib/version/lib_version.md b/src/lib/version/lib_version.md new file mode 100644 index 0000000000..ccc45920f9 --- /dev/null +++ b/src/lib/version/lib_version.md @@ -0,0 +1,2 @@ +@dir /lib/version +@brief lib/version: holds the current version of Tor. diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h new file mode 100644 index 0000000000..679df74381 --- /dev/null +++ b/src/lib/version/torversion.h @@ -0,0 +1,17 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_VERSION_H +#define TOR_VERSION_H + +/** + * @file torversion.h + * @brief Header for version.c. + **/ + +const char *get_version(void); +const char *get_short_version(void); + +#endif /* !defined(TOR_VERSION_H) */ diff --git a/src/lib/version/version.c b/src/lib/version/version.c new file mode 100644 index 0000000000..ec1d0bea2f --- /dev/null +++ b/src/lib/version/version.c @@ -0,0 +1,59 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "lib/version/torversion.h" +#include "lib/version/git_revision.h" + +#include <stdio.h> +#include <string.h> + +/** + * @file version.c + * @brief Functions to get the version of Tor. + **/ + +/** A shorter version of this Tor process's version, for export in our router + * descriptor. (Does not include the git version, if any.) */ +static const char the_short_tor_version[] = + VERSION +#ifdef TOR_BUILD_TAG + " ("TOR_BUILD_TAG")" +#endif + ""; + +/** + * Longest possible version length. We make this a constant so that we + * can statically allocate the_tor_version. + **/ +#define MAX_VERSION_LEN 128 + +/** The version of this Tor process, possibly including git version */ +static char the_tor_version[MAX_VERSION_LEN] = ""; + +/** Return the current Tor version. */ +const char * +get_version(void) +{ + if (the_tor_version[0] == 0) { + if (strlen(tor_git_revision)) { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s (git-%s)", the_short_tor_version, tor_git_revision); + } else { + snprintf(the_tor_version, sizeof(the_tor_version), + "%s", the_short_tor_version); + } + the_tor_version[sizeof(the_tor_version)-1] = 0; + } + + return the_tor_version; +} + +/** Return the current Tor version, without any git tag. */ +const char * +get_short_version(void) +{ + return the_short_tor_version; +} diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include index dc010da063..ce7a26472b 100644 --- a/src/lib/wallclock/.may_include +++ b/src/lib/wallclock/.may_include @@ -3,4 +3,5 @@ lib/cc/*.h lib/err/*.h lib/wallclock/*.h lib/string/*.h +lib/subsys/*.h lib/testsupport/*.h diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c index ee498702d5..c815f20e51 100644 --- a/src/lib/wallclock/approx_time.c +++ b/src/lib/wallclock/approx_time.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -9,7 +9,9 @@ **/ #include "orconfig.h" +#include "lib/subsys/subsys.h" #include "lib/wallclock/approx_time.h" +#include "lib/wallclock/wallclock_sys.h" /* ===== * Cached time @@ -41,3 +43,26 @@ update_approx_time(time_t now) cached_approx_time = now; } #endif /* !defined(TIME_IS_FAST) */ + +/** + * Initialize the "wallclock" subsystem by setting the current cached time. + **/ +static int +subsys_wallclock_initialize(void) +{ + update_approx_time(time(NULL)); + return 0; +} + +/** + * Subsystem function table describing the "wallclock" subsystem. + **/ +const subsys_fns_t sys_wallclock = { + .name = "wallclock", + SUBSYS_DECLARE_LOCATION(), + .supported = true, + /* Approximate time is a diagnostic feature, we want it to init right after + * low-level error handling. */ + .level = -98, + .initialize = subsys_wallclock_initialize, +}; diff --git a/src/lib/wallclock/approx_time.h b/src/lib/wallclock/approx_time.h index e6b53f2c27..42040a1f52 100644 --- a/src/lib/wallclock/approx_time.h +++ b/src/lib/wallclock/approx_time.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -22,4 +22,4 @@ time_t approx_time(void); void update_approx_time(time_t now); #endif /* defined(TIME_IS_FAST) */ -#endif +#endif /* !defined(TOR_APPROX_TIME_H) */ diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am index 1961639bd7..2b50d6ccbb 100644 --- a/src/lib/wallclock/include.am +++ b/src/lib/wallclock/include.am @@ -5,6 +5,7 @@ if UNITTESTS_ENABLED noinst_LIBRARIES += src/lib/libtor-wallclock-testing.a endif +# ADD_C_FILE: INSERT SOURCES HERE. src_lib_libtor_wallclock_a_SOURCES = \ src/lib/wallclock/approx_time.c \ src/lib/wallclock/time_to_tm.c \ @@ -15,8 +16,10 @@ src_lib_libtor_wallclock_testing_a_SOURCES = \ src_lib_libtor_wallclock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_lib_libtor_wallclock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) +# ADD_C_FILE: INSERT HEADERS HERE. noinst_HEADERS += \ src/lib/wallclock/approx_time.h \ src/lib/wallclock/timeval.h \ src/lib/wallclock/time_to_tm.h \ - src/lib/wallclock/tor_gettimeofday.h + src/lib/wallclock/tor_gettimeofday.h \ + src/lib/wallclock/wallclock_sys.h diff --git a/src/lib/wallclock/lib_wallclock.md b/src/lib/wallclock/lib_wallclock.md new file mode 100644 index 0000000000..f21721f6f6 --- /dev/null +++ b/src/lib/wallclock/lib_wallclock.md @@ -0,0 +1,11 @@ +@dir /lib/wallclock +@brief lib/wallclock: Inspect and manipulate the current time. + +This module handles our concept of "what time is it" or "what time does the +world agree it is?" Generally, if you want something derived from UTC, this +is the module for you. + +For versions of the time that are more local, more monotonic, or more +accurate, see \refdir{lib/time}. For parsing and encoding times and dates, +see \refdir{lib/encoding}. + diff --git a/src/lib/wallclock/time_to_tm.c b/src/lib/wallclock/time_to_tm.c index f7cb21827b..8c747b4c7b 100644 --- a/src/lib/wallclock/time_to_tm.c +++ b/src/lib/wallclock/time_to_tm.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -198,3 +198,4 @@ tor_gmtime_r_msg(const time_t *timep, struct tm *result, char **err_out) return correct_tm(0, timep, result, r, err_out); } #endif /* defined(HAVE_GMTIME_R) || ... */ +/**@}*/ diff --git a/src/lib/wallclock/time_to_tm.h b/src/lib/wallclock/time_to_tm.h index abe78c0efe..bfa8fa3689 100644 --- a/src/lib/wallclock/time_to_tm.h +++ b/src/lib/wallclock/time_to_tm.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -19,4 +19,4 @@ struct tm *tor_localtime_r_msg(const time_t *timep, struct tm *result, struct tm *tor_gmtime_r_msg(const time_t *timep, struct tm *result, char **err_out); -#endif +#endif /* !defined(TOR_WALLCLOCK_TIME_TO_TM_H) */ diff --git a/src/lib/wallclock/timeval.h b/src/lib/wallclock/timeval.h index 4967e939bf..d7d5bda99f 100644 --- a/src/lib/wallclock/timeval.h +++ b/src/lib/wallclock/timeval.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -20,6 +20,27 @@ #include <sys/time.h> #endif +#ifdef TOR_COVERAGE +/* For coverage builds, we use a slower definition of these macros without + * branches, to make coverage consistent. */ +#undef timeradd +#undef timersub +#define timeradd(tv1,tv2,tvout) \ + do { \ + (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec; \ + (tvout)->tv_usec = (tv1)->tv_usec + (tv2)->tv_usec; \ + (tvout)->tv_sec += (tvout)->tv_usec / 1000000; \ + (tvout)->tv_usec %= 1000000; \ + } while (0) +#define timersub(tv1,tv2,tvout) \ + do { \ + (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec - 1; \ + (tvout)->tv_usec = (tv1)->tv_usec - (tv2)->tv_usec + 1000000; \ + (tvout)->tv_sec += (tvout)->tv_usec / 1000000; \ + (tvout)->tv_usec %= 1000000; \ + } while (0) +#endif /* defined(TOR_COVERAGE) */ + #ifndef timeradd /** Replacement for timeradd on platforms that do not have it: sets tvout to * the sum of tv1 and tv2. */ @@ -48,6 +69,7 @@ } while (0) #endif /* !defined(timersub) */ +#ifndef COCCI #ifndef timercmp /** Replacement for timercmp on platforms that do not have it: returns true * iff the relational operator "op" makes the expression tv1 op tv2 true. @@ -61,5 +83,6 @@ ((tv1)->tv_usec op (tv2)->tv_usec) : \ ((tv1)->tv_sec op (tv2)->tv_sec)) #endif /* !defined(timercmp) */ +#endif /* !defined(COCCI) */ -#endif +#endif /* !defined(TOR_TIMEVAL_H) */ diff --git a/src/lib/wallclock/tor_gettimeofday.c b/src/lib/wallclock/tor_gettimeofday.c index 63538f3b81..a07f83220d 100644 --- a/src/lib/wallclock/tor_gettimeofday.c +++ b/src/lib/wallclock/tor_gettimeofday.c @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** diff --git a/src/lib/wallclock/tor_gettimeofday.h b/src/lib/wallclock/tor_gettimeofday.h index c7fff9747a..c1a8afca3a 100644 --- a/src/lib/wallclock/tor_gettimeofday.h +++ b/src/lib/wallclock/tor_gettimeofday.h @@ -1,6 +1,6 @@ /* Copyright (c) 2003-2004, Roger Dingledine * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2019, The Tor Project, Inc. */ + * Copyright (c) 2007-2020, The Tor Project, Inc. */ /* See LICENSE for licensing information */ /** @@ -17,4 +17,4 @@ struct timeval; MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval)); -#endif +#endif /* !defined(TOR_GETTIMEOFDAY_H) */ diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h new file mode 100644 index 0000000000..3997d11e7a --- /dev/null +++ b/src/lib/wallclock/wallclock_sys.h @@ -0,0 +1,14 @@ +/* Copyright (c) 2018-2020, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file wallclock_sys.h + * \brief Declare subsystem object for the wallclock module. + **/ + +#ifndef TOR_WALLCLOCK_SYS_H +#define TOR_WALLCLOCK_SYS_H + +extern const struct subsys_fns_t sys_wallclock; + +#endif /* !defined(TOR_WALLCLOCK_SYS_H) */ |