diff options
59 files changed, 1567 insertions, 243 deletions
@@ -1,3 +1,64 @@ +Changes in version 0.2.4.21 - 2014-02-28 + Tor 0.2.4.21 further improves security against potential adversaries who + find breaking 1024-bit crypto doable, and backports several stability + and robustness patches from the 0.2.5 branch. + + o Major features (client security): + - When we choose a path for a 3-hop circuit, make sure it contains + at least one relay that supports the NTor circuit extension + handshake. Otherwise, there is a chance that we're building + a circuit that's worth attacking by an adversary who finds + breaking 1024-bit crypto doable, and that chance changes the game + theory. Implements ticket 9777. + + o Major bugfixes: + - Do not treat streams that fail with reason + END_STREAM_REASON_INTERNAL as indicating a definite circuit failure, + since it could also indicate an ENETUNREACH connection error. Fixes + part of bug 10777; bugfix on 0.2.4.8-alpha. + + o Code simplification and refactoring: + - Remove data structures which were introduced to implement the + CellStatistics option: they are now redundant with the new timestamp + field in the regular packed_cell_t data structure, which we did + in 0.2.4.18-rc in order to resolve bug 9093. Resolves ticket 10870. + + o Minor features: + - Always clear OpenSSL bignums before freeing them -- even bignums + that don't contain secrets. Resolves ticket 10793. Patch by + Florent Daigniere. + - Build without warnings under clang 3.4. (We have some macros that + define static functions only some of which will get used later in + the module. Starting with clang 3.4, these give a warning unless the + unused attribute is set on them.) Resolves ticket 10904. + - Update geoip and geoip6 files to the February 7 2014 Maxmind + GeoLite2 Country database. + + o Minor bugfixes: + - Set the listen() backlog limit to the largest actually supported + on the system, not to the value in a header file. Fixes bug 9716; + bugfix on every released Tor. + - Treat ENETUNREACH, EACCES, and EPERM connection failures at an + exit node as a NOROUTE error, not an INTERNAL error, since they + can apparently happen when trying to connect to the wrong sort + of netblocks. Fixes part of bug 10777; bugfix on 0.1.0.1-rc. + - Fix build warnings about missing "a2x" comment when building the + manpages from scratch on OpenBSD; OpenBSD calls it "a2x.py". + Fixes bug 10929; bugfix on 0.2.2.9-alpha. Patch from Dana Koch. + - Avoid a segfault on SIGUSR1, where we had freed a connection but did + not entirely remove it from the connection lists. Fixes bug 9602; + bugfix on 0.2.4.4-alpha. + - Fix a segmentation fault in our benchmark code when running with + Fedora's OpenSSL package, or any other OpenSSL that provides + ECDH but not P224. Fixes bug 10835; bugfix on 0.2.4.8-alpha. + - Turn "circuit handshake stats since last time" log messages into a + heartbeat message. Fixes bug 10485; bugfix on 0.2.4.17-rc. + + o Documentation fixes: + - Document that all but one DirPort entry must have the NoAdvertise + flag set. Fixes bug 10470; bugfix on 0.2.3.3-alpha / 0.2.3.16-alpha. + + Changes in version 0.2.5.2-alpha - 2013-02-13 Tor 0.2.5.2-alpha includes all the fixes from 0.2.4.18-rc and 0.2.4.20, like the "poor random number generation" fix and the "building too many @@ -22,3 +22,6 @@ Frequently Asked Questions: https://www.torproject.org/faq.html https://wiki.torproject.org/noreply/TheOnionRouter/TorFAQ + +To get started working on Tor development: + See the doc/HACKING file. diff --git a/ReleaseNotes b/ReleaseNotes index 54195f82ae..8709d0d48f 100644 --- a/ReleaseNotes +++ b/ReleaseNotes @@ -3,6 +3,67 @@ This document summarizes new features and bugfixes in each stable release of Tor. If you want to see more detailed descriptions of the changes in each development snapshot, see the ChangeLog file. +Changes in version 0.2.4.21 - 2014-02-28 + Tor 0.2.4.21 further improves security against potential adversaries who + find breaking 1024-bit crypto doable, and backports several stability + and robustness patches from the 0.2.5 branch. + + o Major features (client security): + - When we choose a path for a 3-hop circuit, make sure it contains + at least one relay that supports the NTor circuit extension + handshake. Otherwise, there is a chance that we're building + a circuit that's worth attacking by an adversary who finds + breaking 1024-bit crypto doable, and that chance changes the game + theory. Implements ticket 9777. + + o Major bugfixes: + - Do not treat streams that fail with reason + END_STREAM_REASON_INTERNAL as indicating a definite circuit failure, + since it could also indicate an ENETUNREACH connection error. Fixes + part of bug 10777; bugfix on 0.2.4.8-alpha. + + o Code simplification and refactoring: + - Remove data structures which were introduced to implement the + CellStatistics option: they are now redundant with the new timestamp + field in the regular packed_cell_t data structure, which we did + in 0.2.4.18-rc in order to resolve bug 9093. Resolves ticket 10870. + + o Minor features: + - Always clear OpenSSL bignums before freeing them -- even bignums + that don't contain secrets. Resolves ticket 10793. Patch by + Florent Daigniere. + - Build without warnings under clang 3.4. (We have some macros that + define static functions only some of which will get used later in + the module. Starting with clang 3.4, these give a warning unless the + unused attribute is set on them.) Resolves ticket 10904. + - Update geoip and geoip6 files to the February 7 2014 Maxmind + GeoLite2 Country database. + + o Minor bugfixes: + - Set the listen() backlog limit to the largest actually supported + on the system, not to the value in a header file. Fixes bug 9716; + bugfix on every released Tor. + - Treat ENETUNREACH, EACCES, and EPERM connection failures at an + exit node as a NOROUTE error, not an INTERNAL error, since they + can apparently happen when trying to connect to the wrong sort + of netblocks. Fixes part of bug 10777; bugfix on 0.1.0.1-rc. + - Fix build warnings about missing "a2x" comment when building the + manpages from scratch on OpenBSD; OpenBSD calls it "a2x.py". + Fixes bug 10929; bugfix on 0.2.2.9-alpha. Patch from Dana Koch. + - Avoid a segfault on SIGUSR1, where we had freed a connection but did + not entirely remove it from the connection lists. Fixes bug 9602; + bugfix on 0.2.4.4-alpha. + - Fix a segmentation fault in our benchmark code when running with + Fedora's OpenSSL package, or any other OpenSSL that provides + ECDH but not P224. Fixes bug 10835; bugfix on 0.2.4.8-alpha. + - Turn "circuit handshake stats since last time" log messages into a + heartbeat message. Fixes bug 10485; bugfix on 0.2.4.17-rc. + + o Documentation fixes: + - Document that all but one DirPort entry must have the NoAdvertise + flag set. Fixes bug 10470; bugfix on 0.2.3.3-alpha / 0.2.3.16-alpha. + + Changes in version 0.2.4.20 - 2013-12-22 Tor 0.2.4.20 fixes potentially poor random number generation for users who 1) use OpenSSL 1.0.0 or later, 2) set "HardwareAccel 1" in their diff --git a/changes/bug10169 b/changes/bug10169 new file mode 100644 index 0000000000..979c4e4288 --- /dev/null +++ b/changes/bug10169 @@ -0,0 +1,4 @@ + o Major features: + - Also consider stream buffer sizes when calculating OOM + conditions. Rename MaxMemInCellQueues to MaxMemInQueues. Fixes + bug 10169. diff --git a/changes/bug11043 b/changes/bug11043 new file mode 100644 index 0000000000..cdf534da87 --- /dev/null +++ b/changes/bug11043 @@ -0,0 +1,5 @@ + o Minor features (log messages): + - Improve the message that gets displayed when Tor as a bridge is + using pluggable transports but doesn't have an Extended ORPort + listener. Furthermore, we now log the message in the log file + too. Resolves ticket 11043. diff --git a/changes/bug11108 b/changes/bug11108 new file mode 100644 index 0000000000..b2c1f50038 --- /dev/null +++ b/changes/bug11108 @@ -0,0 +1,8 @@ + o Minor features: + - Warn the user if they put any ports in the SocksPolicy, + DirPolicy, AuthDirReject, AuthDirInvalid, AuthDirBadDir, or + AuthDirBadExit options. Fixes ticket #11108. + + o Documentation: + - Explain that SocksPolicy, DirPolicy, and their allies don't take + port arguments. Fixes ticket #11108. diff --git a/changes/bug11118 b/changes/bug11118 new file mode 100644 index 0000000000..6354ab2db9 --- /dev/null +++ b/changes/bug11118 @@ -0,0 +1,4 @@ + o Documentation: + - Fix the max client name length in the manpage's description of + HiddenServiceAuthorizeClient description: it should have been + 16, not 19. Fixes bug 11118; bugfix on 0.2.1.6-alpha. diff --git a/changes/bug11156 b/changes/bug11156 new file mode 100644 index 0000000000..17ac23f445 --- /dev/null +++ b/changes/bug11156 @@ -0,0 +1,5 @@ + o Minor bugfixes (clients): + - Fix a bug where we would attempt to connect to bridges before + our pluggable transports were configured, which resulted in some + erroneous log messages. Fixes bug 11156; bugfix on + 0.2.3.2-alpha. diff --git a/changes/bug8788 b/changes/bug8788 new file mode 100644 index 0000000000..26fb5bc4f3 --- /dev/null +++ b/changes/bug8788 @@ -0,0 +1,3 @@ + o Minor bugfixes: + - Avoid crashing on a malformed resolv.conf file when running a + server using Libevent 1. Fixes bug 8788; bugfix on 0.1.1.23. diff --git a/changes/bug9004 b/changes/bug9004 new file mode 100644 index 0000000000..5bab7ea0b4 --- /dev/null +++ b/changes/bug9004 @@ -0,0 +1,6 @@ + o Minor bugfixes (testing): + - Improve the tinytest implementation of string operation tests + so that comparisons NULL strings no longer crash the tests; + they now just fail, normally. Fixes bug 9004; bugfix on + 0.2.2.4-alpha. + diff --git a/changes/bug9099 b/changes/bug9099 new file mode 100644 index 0000000000..5cd963ea8f --- /dev/null +++ b/changes/bug9099 @@ -0,0 +1,5 @@ + o Minor bugfixes: + - Fix a comment about the rend_server_descriptor_t.protocols field + to more accurately describe its range. Also, make that + field unsigned, to more accurately reflect its usage. + Fixes bug 9099; bugfix on 0.2.1.5-alpha. diff --git a/changes/bug9222 b/changes/bug9222 new file mode 100644 index 0000000000..bd66277118 --- /dev/null +++ b/changes/bug9222 @@ -0,0 +1,5 @@ + o Documentation: + + - Document in the manpage that "KBytes" may also be written as + "kilobytes" or "KB", that "Kbits" may also be written as + "kilobits", and so forth. Closes ticket #9222. diff --git a/changes/bug9393 b/changes/bug9393 new file mode 100644 index 0000000000..9aedd1260b --- /dev/null +++ b/changes/bug9393 @@ -0,0 +1,4 @@ + o Minor bugfixes: + - Give the correct URL in the warning message that we present + when the user is trying to run a Tor relay on an ancient version + of Windows. Fixes bug 9393. diff --git a/changes/bug9686 b/changes/bug9686 new file mode 100644 index 0000000000..82a8f44471 --- /dev/null +++ b/changes/bug9686 @@ -0,0 +1,3 @@ + o Minor changes: + - Decrease the lower limit of MaxMemInQueues to 256 MBytes, to + appease raspberry pi users. Fixes bug 9686.
\ No newline at end of file diff --git a/changes/ticket5528 b/changes/ticket5528 new file mode 100644 index 0000000000..69b2c1d168 --- /dev/null +++ b/changes/ticket5528 @@ -0,0 +1,4 @@ + o Code simplifications and refactoring: + - Get rid of router->address, since in all cases it was just the + string representation of router->addr. Resolves ticket 5528. + diff --git a/changes/ticket9176 b/changes/ticket9176 new file mode 100644 index 0000000000..53c30e330e --- /dev/null +++ b/changes/ticket9176 @@ -0,0 +1,5 @@ + o Minor features: + + - Made PREDICTED_CIRCS_RELEVANCE_TIME configurable from config + file with a new option, PredictedPortsRelevanceTime. Implements + ticket #9176. Patch by unixninja92. diff --git a/changes/tinytest_upgrade b/changes/tinytest_upgrade new file mode 100644 index 0000000000..ccac4020b2 --- /dev/null +++ b/changes/tinytest_upgrade @@ -0,0 +1,2 @@ + o Testing: + - Update to the latest version of tinytest diff --git a/doc/HACKING b/doc/HACKING index 39eafec204..3c1d0d9577 100644 --- a/doc/HACKING +++ b/doc/HACKING @@ -93,7 +93,7 @@ down bugs. Jenkins ~~~~~~~ -http://jenkins.torproject.org +https://jenkins.torproject.org Dmalloc ~~~~~~~ diff --git a/doc/tor.1.txt b/doc/tor.1.txt index dc67889353..c49692c69d 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -143,7 +143,17 @@ GENERAL OPTIONS the specified number of bytes per second, and the average outgoing bandwidth usage to that same value. If you want to run a relay in the public network, this needs to be _at the very least_ 30 KBytes (that is, - 30720 bytes). (Default: 1 GByte) + 30720 bytes). (Default: 1 GByte) + + + + With this option, and in other options that take arguments in bytes, + KBytes, and so on, other formats are also supported. Notably, "KBytes" can + also be written as "kilobytes" or "kb"; "MBytes" can be written as + "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth. + Tor also accepts "byte" and "bit" in the singular. + The prefixes "tera" and "T" are also recognized. + If no units are given, we default to bytes. + To avoid confusion, we recommend writing "bytes" or "bits" explicitly, + since it's easy to forget that "B" means bytes, not bits. [[BandwidthBurst]] **BandwidthBurst** __N__ **bytes**|**KBytes**|**MBytes**|**GBytes**|**KBits**|**MBits**|**GBits**:: Limit the maximum token bucket size (also known as the burst) to the given @@ -534,6 +544,12 @@ GENERAL OPTIONS following the Tor specification. Otherwise, they are logged with severity \'info'. (Default: 0) +[[PredictedPortsRelevanceTime]] **PredictedPortsRelevanceTime** __NUM__:: + Set how long, after the client has mad an anonymized connection to a + given port, we will try to make sure that we build circuits to + exits that support that port. The maximum value for this option is 1 + hour. (Default: 1 hour) + [[RunAsDaemon]] **RunAsDaemon** **0**|**1**:: If 1, Tor forks and daemonizes to the background. This option has no effect on Windows; instead you should use the --service command-line option. @@ -1025,7 +1041,8 @@ The following options are useful only for clients (that is, if [[SocksPolicy]] **SocksPolicy** __policy__,__policy__,__...__:: Set an entrance policy for this server, to limit who can connect to the SocksPort and DNSPort ports. The policies have the same form as exit - policies below. + policies below, except that port specifiers are ignored. Any address + not matched by some entry in the policy is accepted. [[SocksTimeout]] **SocksTimeout** __NUM__:: Let a socks connection wait NUM seconds handshaking, and NUM seconds @@ -1727,13 +1744,13 @@ is non-zero): localhost, RFC1918 addresses, and so on. This can create security issues; you should probably leave it off. (Default: 0) -[[MaxMemInCellQueues]] **MaxMemInCellQueues** __N__ **bytes**|**KB**|**MB**|**GB**:: +[[MaxMemInQueues]] **MaxMemInQueues** __N__ **bytes**|**KB**|**MB**|**GB**:: This option configures a threshold above which Tor will assume that it - needs to stop queueing cells because it's about to run out of memory. - If it hits this threshold, it will begin killing circuits until it - has recovered at least 10% of this memory. Do not set this option too + needs to stop queueing or buffering data because it's about to run out of + memory. If it hits this threshold, it will begin killing circuits until + it has recovered at least 10% of this memory. Do not set this option too low, or your relay may be unreliable under load. This option only - affects circuit queues, so the actual process size will be larger than + affects some queues, so the actual process size will be larger than this. (Default: 8GB) DIRECTORY SERVER OPTIONS @@ -1822,7 +1839,9 @@ if DirPort is non-zero): [[DirPolicy]] **DirPolicy** __policy__,__policy__,__...__:: Set an entrance policy for this server, to limit who can connect to the - directory ports. The policies have the same form as exit policies above. + directory ports. The policies have the same form as exit policies above, + except that port specifiers are ignored. Any address not matched by + some entry in the policy is accepted. [[FetchV2Networkstatus]] **FetchV2Networkstatus** **0**|**1**:: If set, we try to fetch the (obsolete, unused) version 2 network status @@ -1866,7 +1885,11 @@ DIRECTORY AUTHORITY SERVER OPTIONS [[AuthDirBadDir]] **AuthDirBadDir** __AddressPattern...__:: Authoritative directories only. A set of address patterns for servers that will be listed as bad directories in any network status document this - authority publishes, if **AuthDirListBadDirs** is set. + authority publishes, if **AuthDirListBadDirs** is set. + + + + (The address pattern syntax here and in the options below + is the same as for exit policies, except that you don't need to say + "accept" or "reject", and ports are not needed.) [[AuthDirBadExit]] **AuthDirBadExit** __AddressPattern...__:: Authoritative directories only. A set of address patterns for servers that @@ -2032,7 +2055,7 @@ The following options are used to configure a hidden service. authorization protocol or \'stealth' for a less scalable protocol that also hides service activity from unauthorized clients. Only clients that are listed here are authorized to access the hidden service. Valid client names - are 1 to 19 characters long and only use characters in A-Za-z0-9+-_ (no + are 1 to 16 characters long and only use characters in A-Za-z0-9+-_ (no spaces). If this option is set, the hidden service is not accessible for clients without authorization any more. Generated authorization data can be found in the hostname file. Clients need to put this authorization data in diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 4d0fff833b..8525b4a721 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -626,7 +626,8 @@ tor_add_bufferevent_to_rate_limit_group(struct bufferevent *bev, } #endif -#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,1,1) +#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= V(2,1,1) \ + && !defined(TOR_UNIT_TESTS) void tor_gettimeofday_cached(struct timeval *tv) { @@ -659,5 +660,45 @@ tor_gettimeofday_cache_clear(void) { cached_time_hires.tv_sec = 0; } + +#ifdef TOR_UNIT_TESTS +/** For testing: force-update the cached time to a given value. */ +void +tor_gettimeofday_cache_set(const struct timeval *tv) +{ + tor_assert(tv); + memcpy(&cached_time_hires, tv, sizeof(*tv)); +} #endif +#endif + +/** + * As tor_gettimeofday_cached, but can never move backwards in time. + * + * The returned value may diverge from wall-clock time, since wall-clock time + * can trivially be adjusted backwards, and this can't. Don't mix wall-clock + * time with these values in the same calculation. + * + * Depending on implementation, this function may or may not "smooth out" huge + * jumps forward in wall-clock time. It may or may not keep its results + * advancing forward (as opposed to stalling) if the wall-clock time goes + * backwards. The current implementation does neither of of these. + * + * This function is not thread-safe; do not call it outside the main thread. + * + * In future versions of Tor, this may return a time does not have its + * origin at the Unix epoch. + */ +void +tor_gettimeofday_cached_monotonic(struct timeval *tv) +{ + struct timeval last_tv = { 0, 0 }; + + tor_gettimeofday_cached(tv); + if (timercmp(tv, &last_tv, <)) { + memcpy(tv, &last_tv, sizeof(struct timeval)); + } else { + memcpy(&last_tv, tv, sizeof(struct timeval)); + } +} diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index fda8733592..f0d1828b7b 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -91,6 +91,10 @@ int tor_add_bufferevent_to_rate_limit_group(struct bufferevent *bev, void tor_gettimeofday_cached(struct timeval *tv); void tor_gettimeofday_cache_clear(void); +#ifdef TOR_UNIT_TESTS +void tor_gettimeofday_cache_set(const struct timeval *tv); +#endif +void tor_gettimeofday_cached_monotonic(struct timeval *tv); #endif diff --git a/src/ext/eventdns.c b/src/ext/eventdns.c index 8b934c4430..5ac9c1230c 100644 --- a/src/ext/eventdns.c +++ b/src/ext/eventdns.c @@ -3014,7 +3014,8 @@ resolv_conf_parse_line(char *const start, int flags) { if (!strcmp(first_token, "nameserver") && (flags & DNS_OPTION_NAMESERVERS)) { const char *const nameserver = NEXT_TOKEN; - evdns_nameserver_ip_add(nameserver); + if (nameserver) + evdns_nameserver_ip_add(nameserver); } else if (!strcmp(first_token, "domain") && (flags & DNS_OPTION_SEARCH)) { const char *const domain = NEXT_TOKEN; if (domain) { diff --git a/src/ext/tinytest.c b/src/ext/tinytest.c index 4d9afacce4..3a8e331055 100644 --- a/src/ext/tinytest.c +++ b/src/ext/tinytest.c @@ -31,6 +31,8 @@ #include <string.h> #include <assert.h> +#ifndef NO_FORKING + #ifdef _WIN32 #include <windows.h> #else @@ -39,6 +41,17 @@ #include <unistd.h> #endif +#if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) +#if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ + __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) +/* Workaround for a stupid bug in OSX 10.6 */ +#define FORK_BREAKS_GCOV +#include <vproc.h> +#endif +#endif + +#endif /* !NO_FORKING */ + #ifndef __GNUC__ #define __attribute__(x) #endif @@ -58,6 +71,8 @@ static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ const char *verbosity_flag = ""; +const struct testlist_alias_t *cfg_aliases=NULL; + enum outcome { SKIP=2, OK=1, FAIL=0 }; static enum outcome cur_test_outcome = 0; const char *cur_test_prefix = NULL; /**< prefix of the current test group */ @@ -71,6 +86,7 @@ static char commandname[MAX_PATH+1]; static void usage(struct testgroup_t *groups, int list_groups) __attribute__((noreturn)); +static int process_test_option(struct testgroup_t *groups, const char *test); static enum outcome testcase_run_bare_(const struct testcase_t *testcase) @@ -99,6 +115,8 @@ testcase_run_bare_(const struct testcase_t *testcase) #define MAGIC_EXITCODE 42 +#ifndef NO_FORKING + static enum outcome testcase_run_forked_(const struct testgroup_t *group, const struct testcase_t *testcase) @@ -160,6 +178,9 @@ testcase_run_forked_(const struct testgroup_t *group, if (opt_verbosity>0) printf("[forking] "); pid = fork(); +#ifdef FORK_BREAKS_GCOV + vproc_transaction_begin(0); +#endif if (!pid) { /* child. */ int test_r, write_r; @@ -196,16 +217,19 @@ testcase_run_forked_(const struct testgroup_t *group, #endif } +#endif /* !NO_FORKING */ + int testcase_run_one(const struct testgroup_t *group, const struct testcase_t *testcase) { enum outcome outcome; - if (testcase->flags & TT_SKIP) { + if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) { if (opt_verbosity>0) - printf("%s%s: SKIPPED\n", - group->prefix, testcase->name); + printf("%s%s: %s\n", + group->prefix, testcase->name, + (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED"); ++n_skipped; return SKIP; } @@ -218,9 +242,13 @@ testcase_run_one(const struct testgroup_t *group, cur_test_name = testcase->name; } +#ifndef NO_FORKING if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { outcome = testcase_run_forked_(group, testcase); } else { +#else + { +#endif outcome = testcase_run_bare_(testcase); } @@ -247,7 +275,7 @@ testcase_run_one(const struct testgroup_t *group, } int -tinytest_set_flag_(struct testgroup_t *groups, const char *arg, unsigned long flag) +tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag) { int i, j; size_t length = LONGEST_TEST_NAME; @@ -257,12 +285,23 @@ tinytest_set_flag_(struct testgroup_t *groups, const char *arg, unsigned long fl length = strstr(arg,"..")-arg; for (i=0; groups[i].prefix; ++i) { for (j=0; groups[i].cases[j].name; ++j) { + struct testcase_t *testcase = &groups[i].cases[j]; snprintf(fullname, sizeof(fullname), "%s%s", - groups[i].prefix, groups[i].cases[j].name); - if (!flag) /* Hack! */ - printf(" %s\n", fullname); + groups[i].prefix, testcase->name); + if (!flag) { /* Hack! */ + printf(" %s", fullname); + if (testcase->flags & TT_OFF_BY_DEFAULT) + puts(" (Off by default)"); + else if (testcase->flags & TT_SKIP) + puts(" (DISABLED)"); + else + puts(""); + } if (!strncmp(fullname, arg, length)) { - groups[i].cases[j].flags |= flag; + if (set) + testcase->flags |= flag; + else + testcase->flags &= ~flag; ++found; } } @@ -275,15 +314,69 @@ usage(struct testgroup_t *groups, int list_groups) { puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); puts(" Specify tests by name, or using a prefix ending with '..'"); - puts(" To skip a test, list give its name prefixed with a colon."); + puts(" To skip a test, prefix its name with a colon."); + puts(" To enable a disabled test, prefix its name with a plus."); puts(" Use --list-tests for a list of tests."); if (list_groups) { puts("Known tests are:"); - tinytest_set_flag_(groups, "..", 0); + tinytest_set_flag_(groups, "..", 1, 0); } exit(0); } +static int +process_test_alias(struct testgroup_t *groups, const char *test) +{ + int i, j, n, r; + for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) { + if (!strcmp(cfg_aliases[i].name, test)) { + n = 0; + for (j = 0; cfg_aliases[i].tests[j]; ++j) { + r = process_test_option(groups, cfg_aliases[i].tests[j]); + if (r<0) + return -1; + n += r; + } + return n; + } + } + printf("No such test alias as @%s!",test); + return -1; +} + +static int +process_test_option(struct testgroup_t *groups, const char *test) +{ + int flag = TT_ENABLED_; + int n = 0; + if (test[0] == '@') { + return process_test_alias(groups, test + 1); + } else if (test[0] == ':') { + ++test; + flag = TT_SKIP; + } else if (test[0] == '+') { + ++test; + ++n; + if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) { + printf("No such test as %s!\n", test); + return -1; + } + } else { + ++n; + } + if (!tinytest_set_flag_(groups, test, 1, flag)) { + printf("No such test as %s!\n", test); + return -1; + } + return n; +} + +void +tinytest_set_aliases(const struct testlist_alias_t *aliases) +{ + cfg_aliases = aliases; +} + int tinytest_main(int c, const char **v, struct testgroup_t *groups) { @@ -321,24 +414,18 @@ tinytest_main(int c, const char **v, struct testgroup_t *groups) return -1; } } else { - const char *test = v[i]; - int flag = TT_ENABLED_; - if (test[0] == ':') { - ++test; - flag = TT_SKIP; - } else { - ++n; - } - if (!tinytest_set_flag_(groups, test, flag)) { - printf("No such test as %s!\n", v[i]); + int r = process_test_option(groups, v[i]); + if (r<0) return -1; - } + n += r; } } if (!n) - tinytest_set_flag_(groups, "..", TT_ENABLED_); + tinytest_set_flag_(groups, "..", 1, TT_ENABLED_); +#ifdef _IONBF setvbuf(stdout, NULL, _IONBF, 0); +#endif ++in_tinytest_main; for (i=0; groups[i].prefix; ++i) @@ -385,3 +472,22 @@ tinytest_set_test_skipped_(void) cur_test_outcome = SKIP; } +char * +tinytest_format_hex_(const void *val_, unsigned long len) +{ + const unsigned char *val = val_; + char *result, *cp; + size_t i; + + if (!val) + return strdup("null"); + if (!(result = malloc(len*2+1))) + return strdup("<allocation failure>"); + cp = result; + for (i=0;i<len;++i) { + *cp++ = "0123456789ABCDEF"[val[i] >> 4]; + *cp++ = "0123456789ABCDEF"[val[i] & 0x0f]; + } + *cp = 0; + return result; +} diff --git a/src/ext/tinytest.h b/src/ext/tinytest.h index bcac9f079c..ed07b26bc0 100644 --- a/src/ext/tinytest.h +++ b/src/ext/tinytest.h @@ -32,8 +32,10 @@ #define TT_SKIP (1<<1) /** Internal runtime flag for a test we've decided to run. */ #define TT_ENABLED_ (1<<2) +/** Flag for a test that's off by default. */ +#define TT_OFF_BY_DEFAULT (1<<3) /** If you add your own flags, make them start at this point. */ -#define TT_FIRST_USER_FLAG (1<<3) +#define TT_FIRST_USER_FLAG (1<<4) typedef void (*testcase_fn)(void *); @@ -64,6 +66,12 @@ struct testgroup_t { }; #define END_OF_GROUPS { NULL, NULL} +struct testlist_alias_t { + const char *name; + const char **tests; +}; +#define END_OF_ALIASES { NULL, NULL } + /** Implementation: called from a test to indicate failure, before logging. */ void tinytest_set_test_failed_(void); /** Implementation: called from a test to indicate that we're skipping. */ @@ -72,14 +80,19 @@ void tinytest_set_test_skipped_(void); int tinytest_get_verbosity_(void); /** Implementation: Set a flag on tests matching a name; returns number * of tests that matched. */ -int tinytest_set_flag_(struct testgroup_t *, const char *, unsigned long); +int tinytest_set_flag_(struct testgroup_t *, const char *, int set, unsigned long); +/** Implementation: Put a chunk of memory into hex. */ +char *tinytest_format_hex_(const void *, unsigned long); /** Set all tests in 'groups' matching the name 'named' to be skipped. */ #define tinytest_skip(groups, named) \ - tinytest_set_flag_(groups, named, TT_SKIP) + tinytest_set_flag_(groups, named, 1, TT_SKIP) /** Run a single testcase in a single group. */ int testcase_run_one(const struct testgroup_t *,const struct testcase_t *); + +void tinytest_set_aliases(const struct testlist_alias_t *aliases); + /** Run a set of testcases from an END_OF_GROUPS-terminated array of groups, as selected from the command line. */ int tinytest_main(int argc, const char **argv, struct testgroup_t *groups); diff --git a/src/ext/tinytest_demo.c b/src/ext/tinytest_demo.c index be95ce4c1d..bdd0e60089 100644 --- a/src/ext/tinytest_demo.c +++ b/src/ext/tinytest_demo.c @@ -35,6 +35,10 @@ #include <stdlib.h> #include <string.h> #include <errno.h> +#include <time.h> +#ifndef _WIN32 +#include <unistd.h> +#endif /* ============================================================ */ @@ -148,6 +152,9 @@ test_memcpy(void *ptr) memcpy(db->buffer2, db->buffer1, sizeof(db->buffer1)); tt_str_op(db->buffer1, ==, db->buffer2); + /* This one works if there's an internal NUL. */ + tt_mem_op(db->buffer1, <, db->buffer2, sizeof(db->buffer1)); + /* Now we've allocated memory that's referenced by a local variable. The end block of the function will clean it up. */ mem = strdup("Hello world."); @@ -162,6 +169,27 @@ test_memcpy(void *ptr) free(mem); } +void +test_timeout(void *ptr) +{ + time_t t1, t2; + (void)ptr; + t1 = time(NULL); +#ifdef _WIN32 + Sleep(5000); +#else + sleep(5); +#endif + t2 = time(NULL); + + tt_int_op(t2-t1, >=, 4); + + tt_int_op(t2-t1, <=, 6); + + end: + ; +} + /* ============================================================ */ /* Now we need to make sure that our tests get invoked. First, you take @@ -178,6 +206,10 @@ struct testcase_t demo_tests[] = { its environment. */ { "memcpy", test_memcpy, TT_FORK, &data_buffer_setup }, + /* This flag is off-by-default, since it takes a while to run. You + * can enable it manually by passing +demo/timeout at the command line.*/ + { "timeout", test_timeout, TT_OFF_BY_DEFAULT }, + /* The array has to end with END_OF_TESTCASES. */ END_OF_TESTCASES }; @@ -192,6 +224,18 @@ struct testgroup_t groups[] = { END_OF_GROUPS }; +/* We can also define test aliases. These can be used for types of tests that + * cut across groups. */ +const char *alltests[] = { "+..", NULL }; +const char *slowtests[] = { "+demo/timeout", NULL }; +struct testlist_alias_t aliases[] = { + + { "ALL", alltests }, + { "SLOW", slowtests }, + + END_OF_ALIASES +}; + int main(int c, const char **v) @@ -211,5 +255,6 @@ main(int c, const char **v) "tinytest-demo" and "tinytest-demo .." mean the same thing. */ + tinytest_set_aliases(aliases); return tinytest_main(c, v, groups); } diff --git a/src/ext/tinytest_macros.h b/src/ext/tinytest_macros.h index 9ff69b1d50..db2dfcbe6b 100644 --- a/src/ext/tinytest_macros.h +++ b/src/ext/tinytest_macros.h @@ -144,6 +144,10 @@ tt_assert_test_fmt_type(a,b,str_test,type,test,type,fmt, \ {print_=value_;},{},die_on_fail) +#define tt_assert_test_type_opt(a,b,str_test,type,test,fmt,die_on_fail) \ + tt_assert_test_fmt_type(a,b,str_test,type,test,type,fmt, \ + {print_=value_?value_:"<NULL>";},{},die_on_fail) + /* Helper: assert that a op b, when cast to type. Format the values with * printf format fmt on failure. */ #define tt_assert_op_type(a,op,b,type,fmt) \ @@ -163,8 +167,19 @@ (val1_ op val2_),"%p",TT_EXIT_TEST_FUNCTION) #define tt_str_op(a,op,b) \ - tt_assert_test_type(a,b,#a" "#op" "#b,const char *, \ - (strcmp(val1_,val2_) op 0),"<%s>",TT_EXIT_TEST_FUNCTION) + tt_assert_test_type_opt(a,b,#a" "#op" "#b,const char *, \ + (val1_ && val2_ && strcmp(val1_,val2_) op 0),"<%s>", \ + TT_EXIT_TEST_FUNCTION) + +#define tt_mem_op(expr1, op, expr2, len) \ + tt_assert_test_fmt_type(expr1,expr2,#expr1" "#op" "#expr2, \ + const char *, \ + (val1_ && val2_ && memcmp(val1_, val2_, len) op 0), \ + char *, "%s", \ + { print_ = tinytest_format_hex_(value_, (len)); }, \ + { if (print_) free(print_); }, \ + TT_EXIT_TEST_FUNCTION \ + ); #define tt_want_int_op(a,op,b) \ tt_assert_test_type(a,b,#a" "#op" "#b,long,(val1_ op val2_),"%ld",(void)0) diff --git a/src/or/buffers.c b/src/or/buffers.c index 50016d3a86..012ced6d32 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -62,6 +62,8 @@ static int parse_socks_client(const uint8_t *data, size_t datalen, int state, char **reason, ssize_t *drain_out); +#define DEBUG_CHUNK_ALLOC + /* Chunk manipulation functions */ /** A single chunk on a buffer or in a freelist. */ @@ -69,7 +71,12 @@ typedef struct chunk_t { struct chunk_t *next; /**< The next chunk on the buffer or freelist. */ size_t datalen; /**< The number of bytes stored in this chunk */ size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */ +#ifdef DEBUG_CHUNK_ALLOC + size_t DBG_alloc; +#endif char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */ + uint32_t inserted_time; /**< Timestamp in truncated ms since epoch + * when this chunk was inserted. */ char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in * this chunk. */ } chunk_t; @@ -141,6 +148,9 @@ static chunk_freelist_t freelists[] = { * could help with? */ static uint64_t n_freelist_miss = 0; +/** DOCDOC */ +static size_t total_bytes_allocated_in_chunks = 0; + static void assert_freelist_ok(chunk_freelist_t *fl); /** Return the freelist to hold chunks of size <b>alloc</b>, or NULL if @@ -174,6 +184,11 @@ chunk_free_unchecked(chunk_t *chunk) } else { if (freelist) ++freelist->n_free; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(alloc == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= alloc); + total_bytes_allocated_in_chunks -= alloc; tor_free(chunk); } } @@ -200,6 +215,10 @@ chunk_new_with_alloc_size(size_t alloc) else ++n_freelist_miss; ch = tor_malloc(alloc); +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif + total_bytes_allocated_in_chunks += alloc; } ch->next = NULL; ch->datalen = 0; @@ -211,6 +230,14 @@ chunk_new_with_alloc_size(size_t alloc) static void chunk_free_unchecked(chunk_t *chunk) { + if (!chunk) + return; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); } static INLINE chunk_t * @@ -220,7 +247,11 @@ chunk_new_with_alloc_size(size_t alloc) ch = tor_malloc(alloc); ch->next = NULL; ch->datalen = 0; +#ifdef DEBUG_CHUNK_ALLOC + ch->DBG_alloc = alloc; +#endif ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc); + total_bytes_allocated_in_chunks += alloc; ch->data = &ch->mem[0]; return ch; } @@ -232,11 +263,18 @@ static INLINE chunk_t * chunk_grow(chunk_t *chunk, size_t sz) { off_t offset; + size_t memlen_orig = chunk->memlen; tor_assert(sz > chunk->memlen); offset = chunk->data - chunk->mem; chunk = tor_realloc(chunk, CHUNK_ALLOC_SIZE(sz)); chunk->memlen = sz; chunk->data = chunk->mem + offset; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == CHUNK_ALLOC_SIZE(memlen_orig)); + chunk->DBG_alloc = CHUNK_ALLOC_SIZE(sz); +#endif + total_bytes_allocated_in_chunks += + CHUNK_ALLOC_SIZE(sz) - CHUNK_ALLOC_SIZE(memlen_orig); return chunk; } @@ -261,12 +299,14 @@ preferred_chunk_size(size_t target) } /** Remove from the freelists most chunks that have not been used since the - * last call to buf_shrink_freelists(). */ -void + * last call to buf_shrink_freelists(). Return the amount of memory + * freed. */ +size_t buf_shrink_freelists(int free_all) { #ifdef ENABLE_BUF_FREELISTS int i; + size_t total_freed = 0; disable_control_logging(); for (i = 0; freelists[i].alloc_size; ++i) { int slack = freelists[i].slack; @@ -298,6 +338,13 @@ buf_shrink_freelists(int free_all) *chp = NULL; while (chunk) { chunk_t *next = chunk->next; +#ifdef DEBUG_CHUNK_ALLOC + tor_assert(chunk->DBG_alloc == CHUNK_ALLOC_SIZE(chunk->memlen)); +#endif + tor_assert(total_bytes_allocated_in_chunks >= + CHUNK_ALLOC_SIZE(chunk->memlen)); + total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen); + total_freed += CHUNK_ALLOC_SIZE(chunk->memlen); tor_free(chunk); chunk = next; --n_to_free; @@ -315,18 +362,21 @@ buf_shrink_freelists(int free_all) } // tor_assert(!n_to_free); freelists[i].cur_length = new_length; + tor_assert(orig_n_to_skip == new_length); log_info(LD_MM, "Cleaned freelist for %d-byte chunks: original " - "length %d, kept %d, dropped %d.", + "length %d, kept %d, dropped %d. New length is %d", (int)freelists[i].alloc_size, orig_length, - orig_n_to_skip, orig_n_to_free); + orig_n_to_skip, orig_n_to_free, new_length); } freelists[i].lowest_length = freelists[i].cur_length; assert_freelist_ok(&freelists[i]); } done: enable_control_logging(); + return total_freed; #else (void) free_all; + return 0; #endif } @@ -376,9 +426,10 @@ struct buf_t { * * If <b>nulterminate</b> is true, ensure that there is a 0 byte in * buf->head->mem right after all the data. */ -static void +STATIC void buf_pullup(buf_t *buf, size_t bytes, int nulterminate) { + /* XXXX nothing uses nulterminate; remove it. */ chunk_t *dest, *src; size_t capacity; if (!buf->head) @@ -450,6 +501,20 @@ buf_pullup(buf_t *buf, size_t bytes, int nulterminate) check(); } +#ifdef TOR_UNIT_TESTS +void +buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz) +{ + if (!buf || !buf->head) { + *cp = NULL; + *sz = 0; + } else { + *cp = buf->head->data; + *sz = buf->head->datalen; + } +} +#endif + /** Resize buf so it won't hold extra memory that we haven't been * using lately. */ @@ -504,6 +569,12 @@ buf_new(void) return buf; } +size_t +buf_get_default_chunk_size(const buf_t *buf) +{ + return buf->default_chunk_size; +} + /** Remove all data from <b>buf</b>. */ void buf_clear(buf_t *buf) @@ -531,7 +602,7 @@ buf_allocation(const buf_t *buf) size_t total = 0; const chunk_t *chunk; for (chunk = buf->head; chunk; chunk = chunk->next) { - total += chunk->memlen; + total += CHUNK_ALLOC_SIZE(chunk->memlen); } return total; } @@ -564,6 +635,10 @@ static chunk_t * chunk_copy(const chunk_t *in_chunk) { chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen)); + total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen); +#ifdef DEBUG_CHUNK_ALLOC + newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen); +#endif newch->next = NULL; if (in_chunk->data) { off_t offset = in_chunk->data - in_chunk->mem; @@ -599,6 +674,7 @@ static chunk_t * buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) { chunk_t *chunk; + struct timeval now; if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) { chunk = chunk_new_with_alloc_size(buf->default_chunk_size); } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) { @@ -606,6 +682,10 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) } else { chunk = chunk_new_with_alloc_size(preferred_chunk_size(capacity)); } + + tor_gettimeofday_cached_monotonic(&now); + chunk->inserted_time = (uint32_t)tv_to_msec(&now); + if (buf->tail) { tor_assert(buf->head); buf->tail->next = chunk; @@ -618,6 +698,26 @@ buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped) return chunk; } +/** Return the age of the oldest chunk in the buffer <b>buf</b>, in + * milliseconds. Requires the current time, in truncated milliseconds since + * the epoch, as its input <b>now</b>. + */ +uint32_t +buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now) +{ + if (buf->head) { + return now - buf->head->inserted_time; + } else { + return 0; + } +} + +size_t +buf_get_total_allocation(void) +{ + return total_bytes_allocated_in_chunks; +} + /** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set * *<b>reached_eof</b> to 1. Return -1 on error, 0 on eof or blocking, diff --git a/src/or/buffers.h b/src/or/buffers.h index 48b1185204..a201282da6 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -16,17 +16,21 @@ buf_t *buf_new(void); buf_t *buf_new_with_capacity(size_t size); +size_t buf_get_default_chunk_size(const buf_t *buf); void buf_free(buf_t *buf); void buf_clear(buf_t *buf); buf_t *buf_copy(const buf_t *buf); void buf_shrink(buf_t *buf); -void buf_shrink_freelists(int free_all); +size_t buf_shrink_freelists(int free_all); void buf_dump_freelist_sizes(int severity); size_t buf_datalen(const buf_t *buf); size_t buf_allocation(const buf_t *buf); size_t buf_slack(const buf_t *buf); +uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now); +size_t buf_get_total_allocation(void); + int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error); int read_to_buf_tls(tor_tls_t *tls, size_t at_most, buf_t *buf); @@ -100,6 +104,8 @@ void assert_buf_ok(buf_t *buf); #ifdef BUFFERS_PRIVATE STATIC int buf_find_string_offset(const buf_t *buf, const char *s, size_t n); +STATIC void buf_pullup(buf_t *buf, size_t bytes, int nulterminate); +void buf_get_first_chunk_data(const buf_t *buf, const char **cp, size_t *sz); #endif #endif diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index ffcca4666c..98fef4c142 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -606,27 +606,30 @@ int inform_testing_reachability(void) { char dirbuf[128]; + char *address; const routerinfo_t *me = router_get_my_routerinfo(); if (!me) return 0; + address = tor_dup_ip(me->addr); control_event_server_status(LOG_NOTICE, "CHECKING_REACHABILITY ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); if (me->dir_port) { tor_snprintf(dirbuf, sizeof(dirbuf), " and DirPort %s:%d", - me->address, me->dir_port); + address, me->dir_port); control_event_server_status(LOG_NOTICE, "CHECKING_REACHABILITY DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); } log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... " "(this may take up to %d minutes -- look for log " "messages indicating success)", - me->address, me->or_port, + address, me->or_port, me->dir_port ? dirbuf : "", me->dir_port ? "are" : "is", TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT/60); + tor_free(address); return 1; } diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 9474896367..b2eb730c8c 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1435,9 +1435,9 @@ circuit_mark_all_dirty_circs_as_unusable(void) * - If circ->rend_splice is set (we are the midpoint of a joined * rendezvous stream), then mark the other circuit to close as well. */ -void -circuit_mark_for_close_(circuit_t *circ, int reason, int line, - const char *file) +MOCK_IMPL(void, +circuit_mark_for_close_, (circuit_t *circ, int reason, int line, + const char *file)) { int orig_reason = reason; /* Passed to the controller */ assert_circuit_ok(circ); @@ -1612,6 +1612,38 @@ marked_circuit_free_cells(circuit_t *circ) cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_chan_cells); } +/** Aggressively free buffer contents on all the buffers of all streams in the + * list starting at <b>stream</b>. Return the number of bytes recovered. */ +static size_t +marked_circuit_streams_free_bytes(edge_connection_t *stream) +{ + size_t result = 0; + for ( ; stream; stream = stream->next_stream) { + connection_t *conn = TO_CONN(stream); + if (conn->inbuf) { + result += buf_allocation(conn->inbuf); + buf_clear(conn->inbuf); + } + if (conn->outbuf) { + result += buf_allocation(conn->outbuf); + buf_clear(conn->outbuf); + } + } + return result; +} + +/** Aggressively free buffer contents on all the buffers of all streams on + * circuit <b>c</b>. Return the number of bytes recovered. */ +static size_t +marked_circuit_free_stream_bytes(circuit_t *c) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return marked_circuit_streams_free_bytes(TO_ORIGIN_CIRCUIT(c)->p_streams); + } else { + return marked_circuit_streams_free_bytes(TO_OR_CIRCUIT(c)->n_streams); + } +} + /** Return the number of cells used by the circuit <b>c</b>'s cell queues. */ STATIC size_t n_cells_in_circ_queues(const circuit_t *c) @@ -1632,7 +1664,7 @@ n_cells_in_circ_queues(const circuit_t *c) * This function will return incorrect results if the oldest cell queued on * the circuit is older than 2**32 msec (about 49 days) old. */ -static uint32_t +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) { uint32_t age = 0; @@ -1652,20 +1684,68 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) return age; } -/** Temporary variable for circuits_compare_by_oldest_queued_cell_ This is a - * kludge to work around the fact that qsort doesn't provide a way for - * comparison functions to take an extra argument. */ -static uint32_t circcomp_now_tmp; +/** Return the age in milliseconds of the oldest buffer chunk on any stream in + * the linked list <b>stream</b>, where age is taken in milliseconds before + * the time <b>now</b> (in truncated milliseconds since the epoch). */ +static uint32_t +circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) +{ + uint32_t age = 0, age2; + for (; stream; stream = stream->next_stream) { + const connection_t *conn = TO_CONN(stream); + if (conn->outbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); + if (age2 > age) + age = age2; + } + if (conn->inbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); + if (age2 > age) + age = age2; + } + } + + return age; +} + +/** Return the age in milliseconds of the oldest buffer chunk on any stream + * attached to the circuit <b>c</b>, where age is taken in milliseconds before + * the time <b>now</b> (in truncated milliseconds since the epoch). */ +STATIC uint32_t +circuit_max_queued_data_age(const circuit_t *c, uint32_t now) +{ + if (CIRCUIT_IS_ORIGIN(c)) { + return circuit_get_streams_max_data_age( + TO_ORIGIN_CIRCUIT((circuit_t*)c)->p_streams, now); + } else { + return circuit_get_streams_max_data_age( + TO_OR_CIRCUIT((circuit_t*)c)->n_streams, now); + } +} -/** Helper to sort a list of circuit_t by age of oldest cell, in descending - * order. Requires that circcomp_now_tmp is set correctly. */ +/** Return the age of the oldest cell or stream buffer chunk on the circuit + * <b>c</b>, where age is taken in milliseconds before the time <b>now</b> (in + * truncated milliseconds since the epoch). */ +STATIC uint32_t +circuit_max_queued_item_age(const circuit_t *c, uint32_t now) +{ + uint32_t cell_age = circuit_max_queued_cell_age(c, now); + uint32_t data_age = circuit_max_queued_data_age(c, now); + if (cell_age > data_age) + return cell_age; + else + return data_age; +} + +/** Helper to sort a list of circuit_t by age of oldest item, in descending + * order. */ static int -circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) +circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) { const circuit_t *a = *a_; const circuit_t *b = *b_; - uint32_t age_a = circuit_max_queued_cell_age(a, circcomp_now_tmp); - uint32_t age_b = circuit_max_queued_cell_age(b, circcomp_now_tmp); + uint32_t age_a = a->age_tmp; + uint32_t age_b = b->age_tmp; if (age_a < age_b) return 1; @@ -1675,67 +1755,88 @@ circuits_compare_by_oldest_queued_cell_(const void **a_, const void **b_) return -1; } -#define FRACTION_OF_CELLS_TO_RETAIN_ON_OOM 0.90 +#define FRACTION_OF_DATA_TO_RETAIN_ON_OOM 0.90 /** We're out of memory for cells, having allocated <b>current_allocation</b> * bytes' worth. Kill the 'worst' circuits until we're under - * FRACTION_OF_CIRCS_TO_RETAIN_ON_OOM of our maximum usage. */ + * FRACTION_OF_DATA_TO_RETAIN_ON_OOM of our maximum usage. */ void circuits_handle_oom(size_t current_allocation) { /* Let's hope there's enough slack space for this allocation here... */ smartlist_t *circlist = smartlist_new(); circuit_t *circ; - size_t n_cells_removed=0, n_cells_to_remove; + size_t mem_to_recover; + size_t mem_recovered=0; int n_circuits_killed=0; struct timeval now; + uint32_t now_ms; log_notice(LD_GENERAL, "We're low on memory. Killing circuits with " "over-long queues. (This behavior is controlled by " - "MaxMemInCellQueues.)"); + "MaxMemInQueues.)"); + + { + const size_t recovered = buf_shrink_freelists(1); + if (recovered >= current_allocation) { + log_warn(LD_BUG, "We somehow recovered more memory from freelists " + "than we thought we had allocated"); + current_allocation = 0; + } else { + current_allocation -= recovered; + } + } { - size_t mem_target = (size_t)(get_options()->MaxMemInCellQueues * - FRACTION_OF_CELLS_TO_RETAIN_ON_OOM); - size_t mem_to_recover; + size_t mem_target = (size_t)(get_options()->MaxMemInQueues * + FRACTION_OF_DATA_TO_RETAIN_ON_OOM); if (current_allocation <= mem_target) return; mem_to_recover = current_allocation - mem_target; - n_cells_to_remove = CEIL_DIV(mem_to_recover, packed_cell_mem_cost()); } + tor_gettimeofday_cached_monotonic(&now); + now_ms = (uint32_t)tv_to_msec(&now); + /* This algorithm itself assumes that you've got enough memory slack * to actually run it. */ - TOR_LIST_FOREACH(circ, &global_circuitlist, head) + TOR_LIST_FOREACH(circ, &global_circuitlist, head) { + circ->age_tmp = circuit_max_queued_item_age(circ, now_ms); smartlist_add(circlist, circ); - - /* Set circcomp_now_tmp so that the sort can work. */ - tor_gettimeofday_cached(&now); - circcomp_now_tmp = (uint32_t)tv_to_msec(&now); + } /* This is O(n log n); there are faster algorithms we could use instead. * Let's hope this doesn't happen enough to be in the critical path. */ - smartlist_sort(circlist, circuits_compare_by_oldest_queued_cell_); + smartlist_sort(circlist, circuits_compare_by_oldest_queued_item_); /* Okay, now the worst circuits are at the front of the list. Let's mark * them, and reclaim their storage aggressively. */ SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { size_t n = n_cells_in_circ_queues(circ); + size_t freed; if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } marked_circuit_free_cells(circ); + freed = marked_circuit_free_stream_bytes(circ); ++n_circuits_killed; - n_cells_removed += n; - if (n_cells_removed >= n_cells_to_remove) + + mem_recovered += n * packed_cell_mem_cost(); + mem_recovered += freed; + + if (mem_recovered >= mem_to_recover) break; } SMARTLIST_FOREACH_END(circ); clean_cell_pool(); /* In case this helps. */ - - log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits.", - U64_PRINTF_ARG(n_cells_removed * packed_cell_mem_cost()), - n_circuits_killed); + buf_shrink_freelists(1); /* This is necessary to actually release buffer + chunks. */ + + log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; " + "%d circuits remain alive.", + U64_PRINTF_ARG(mem_recovered), + n_circuits_killed, + smartlist_len(circlist) - n_circuits_killed); smartlist_free(circlist); } diff --git a/src/or/circuitlist.h b/src/or/circuitlist.h index 1c8cf7de24..a29c29a49a 100644 --- a/src/or/circuitlist.h +++ b/src/or/circuitlist.h @@ -53,8 +53,8 @@ origin_circuit_t *circuit_find_to_cannibalize(uint8_t purpose, extend_info_t *info, int flags); void circuit_mark_all_unused_circs(void); void circuit_mark_all_dirty_circs_as_unusable(void); -void circuit_mark_for_close_(circuit_t *circ, int reason, - int line, const char *file); +MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason, + int line, const char *file)); int circuit_get_cpath_len(origin_circuit_t *circ); void circuit_clear_cpath(origin_circuit_t *circ); crypt_path_t *circuit_get_cpath_hop(origin_circuit_t *circ, int hopnum); @@ -76,6 +76,9 @@ void channel_note_destroy_not_pending(channel_t *chan, circid_t id); #ifdef CIRCUITLIST_PRIVATE STATIC void circuit_free(circuit_t *circ); STATIC size_t n_cells_in_circ_queues(const circuit_t *c); +STATIC uint32_t circuit_max_queued_data_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_cell_age(const circuit_t *c, uint32_t now); +STATIC uint32_t circuit_max_queued_item_age(const circuit_t *c, uint32_t now); #endif #endif diff --git a/src/or/config.c b/src/or/config.c index 5633696d0c..0187caf104 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -85,6 +85,7 @@ static config_abbrev_t option_abbrevs_[] = { { "DirFetchPostPeriod", "StatusFetchPeriod", 0, 0}, { "DirServer", "DirAuthority", 0, 0}, /* XXXX024 later, make this warn? */ { "MaxConn", "ConnLimit", 0, 1}, + { "MaxMemInCellQueues", "MaxMemInQueues", 0, 0}, { "ORBindAddress", "ORListenAddress", 0, 0}, { "DirBindAddress", "DirListenAddress", 0, 0}, { "SocksBindAddress", "SocksListenAddress", 0, 0}, @@ -306,7 +307,7 @@ static config_var_t option_vars_[] = { V(MaxAdvertisedBandwidth, MEMUNIT, "1 GB"), V(MaxCircuitDirtiness, INTERVAL, "10 minutes"), V(MaxClientCircuitsPending, UINT, "32"), - V(MaxMemInCellQueues, MEMUNIT, "8 GB"), + V(MaxMemInQueues, MEMUNIT, "8 GB"), OBSOLETE("MaxOnionsPending"), V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), @@ -317,6 +318,7 @@ static config_var_t option_vars_[] = { V(NATDListenAddress, LINELIST, NULL), VPORT(NATDPort, LINELIST, NULL), V(Nickname, STRING, NULL), + V(PredictedPortsRelevanceTime, INTERVAL, "1 hour"), V(WarnUnsafeSocks, BOOL, "1"), OBSOLETE("NoPublish"), VAR("NodeFamily", LINELIST, NodeFamilies, NULL), @@ -1341,6 +1343,20 @@ options_act(const or_options_t *old_options) } #endif + /* If we are a bridge with a pluggable transport proxy but no + Extended ORPort, inform the user that she is missing out. */ + if (server_mode(options) && options->ServerTransportPlugin && + !options->ExtORPort_lines) { + log_notice(LD_CONFIG, "We use pluggable transports but the Extended " + "ORPort is disabled. Tor and your pluggable transports proxy " + "communicate with each other via the Extended ORPort so it " + "is suggested you enable it: it will also allow your Bridge " + "to collect statistics about its clients that use pluggable " + "transports. Please enable it using the ExtORPort torrc option " + "(e.g. set 'ExtORPort auto')."); + + } + if (options->SafeLogging_ != SAFELOG_SCRUB_ALL && (!old_options || old_options->SafeLogging_ != options->SafeLogging_)) { log_warn(LD_GENERAL, "Your log may contain sensitive information - you " @@ -2380,6 +2396,11 @@ compute_publishserverdescriptor(or_options_t *options) * services can overload the directory system. */ #define MIN_REND_POST_PERIOD (10*60) +/** Higest allowable value for PredictedPortsRelevanceTime; if this is + * too high, our selection of exits will decrease for an extended + * period of time to an uncomfortable level .*/ +#define MAX_PREDICTED_CIRCS_RELEVANCE (60*60) + /** Highest allowable value for RendPostPeriod. */ #define MAX_DIR_PERIOD (MIN_ONION_KEY_LIFETIME/2) @@ -2448,7 +2469,7 @@ options_validate(or_options_t *old_options, or_options_t *options, !strcmpstart(uname, "Windows Me"))) { log_warn(LD_CONFIG, "Tor is running as a server, but you are " "running %s; this probably won't work. See " - "https://wiki.torproject.org/TheOnionRouter/TorFAQ#ServerOS " + "https://www.torproject.org/docs/faq.html#BestOSForRelay " "for details.", uname); } @@ -2758,10 +2779,10 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("If EntryNodes is set, UseEntryGuards must be enabled."); } - if (options->MaxMemInCellQueues < (500 << 20)) { - log_warn(LD_CONFIG, "MaxMemInCellQueues must be at least 500 MB for now. " + if (options->MaxMemInQueues < (256 << 20)) { + log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. " "Ideally, have it as large as you can afford."); - options->MaxMemInCellQueues = (500 << 20); + options->MaxMemInQueues = (256 << 20); } options->AllowInvalid_ = 0; @@ -2840,6 +2861,13 @@ options_validate(or_options_t *old_options, or_options_t *options, options->RendPostPeriod = MAX_DIR_PERIOD; } + if (options->PredictedPortsRelevanceTime > + MAX_PREDICTED_CIRCS_RELEVANCE) { + log_warn(LD_CONFIG, "PredictedPortsRelevanceTime is too large; " + "clipping to %ds.", MAX_PREDICTED_CIRCS_RELEVANCE); + options->PredictedPortsRelevanceTime = MAX_PREDICTED_CIRCS_RELEVANCE; + } + if (options->Tor2webMode && options->LearnCircuitBuildTimeout) { /* LearnCircuitBuildTimeout and Tor2webMode are incompatible in * two ways: @@ -3258,17 +3286,6 @@ options_validate(or_options_t *old_options, or_options_t *options, smartlist_free(options_sl); } - /* If we are a bridge with a pluggable transport proxy but no - Extended ORPort, inform the user that she is missing out. */ - if (server_mode(options) && options->ServerTransportPlugin && - !options->ExtORPort_lines) { - log_notice(LD_CONFIG, "We are a bridge with a pluggable transport " - "proxy but the Extended ORPort is disabled. The " - "Extended ORPort helps Tor communicate with the pluggable " - "transport proxy. Please enable it using the ExtORPort " - "torrc option."); - } - if (options->ConstrainedSockets) { /* If the user wants to constrain socket buffer use, make sure the desired * limit is between MIN|MAX_TCPSOCK_BUFFER in k increments. */ diff --git a/src/or/connection.c b/src/or/connection.c index 653fa1cff7..19944161fb 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1008,9 +1008,9 @@ tor_listen(tor_socket_t fd) */ static connection_t * connection_listener_new(const struct sockaddr *listensockaddr, - socklen_t socklen, - int type, const char *address, - const port_cfg_t *port_cfg) + socklen_t socklen, + int type, const char *address, + const port_cfg_t *port_cfg) { listener_connection_t *lis_conn; connection_t *conn = NULL; @@ -2227,7 +2227,7 @@ retry_listener_ports(smartlist_t *old_conns, if (listensockaddr) { conn = connection_listener_new(listensockaddr, listensocklen, - port->type, address, port); + port->type, address, port); tor_free(listensockaddr); tor_free(address); } else { diff --git a/src/or/control.c b/src/or/control.c index 48f9b57d93..23e2054f9e 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -3176,6 +3176,10 @@ connection_control_reached_eof(control_connection_t *conn) return 0; } +static void lost_owning_controller(const char *owner_type, + const char *loss_manner) + ATTR_NORETURN; + /** Shut down this Tor instance in the same way that SIGINT would, but * with a log message appropriate for the loss of an owning controller. */ static void @@ -4670,6 +4674,8 @@ static char *owning_controller_process_spec = NULL; * if this Tor instance is not currently owned by a process. */ static tor_process_monitor_t *owning_controller_process_monitor = NULL; +static void owning_controller_procmon_cb(void *unused) ATTR_NORETURN; + /** Process-termination monitor callback for Tor's owning controller * process. */ static void diff --git a/src/or/directory.c b/src/or/directory.c index 739885c766..5eccb2cabd 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -82,8 +82,7 @@ static void dir_microdesc_download_failed(smartlist_t *failed, static void note_client_request(int purpose, int compressed, size_t bytes); static int client_likes_consensus(networkstatus_t *v, const char *want_url); -static void directory_initiate_command_rend(const char *address, - const tor_addr_t *addr, +static void directory_initiate_command_rend(const tor_addr_t *addr, uint16_t or_port, uint16_t dir_port, const char *digest, @@ -464,7 +463,7 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, tor_addr_t addr; routerinfo_t *ri = node->ri; node_get_addr(node, &addr); - directory_initiate_command(ri->address, &addr, + directory_initiate_command(&addr, ri->or_port, 0/*no dirport*/, ri->cache_info.identity_digest, dir_purpose, @@ -593,9 +592,6 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, { const or_options_t *options = get_options(); const node_t *node; - char address_buf[INET_NTOA_BUF_LEN+1]; - struct in_addr in; - const char *address; tor_addr_t addr; const int anonymized_connection = dirind_is_anon(indirection); node = node_get_by_id(status->identity_digest); @@ -605,13 +601,6 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, "don't have its router descriptor.", routerstatus_describe(status)); return; - } else if (node) { - node_get_address_string(node, address_buf, sizeof(address_buf)); - address = address_buf; - } else { - in.s_addr = htonl(status->addr); - tor_inet_ntoa(&in, address_buf, sizeof(address_buf)); - address = address_buf; } tor_addr_from_ipv4h(&addr, status->addr); @@ -625,7 +614,7 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, return; } - directory_initiate_command_rend(address, &addr, + directory_initiate_command_rend(&addr, status->or_port, status->dir_port, status->identity_digest, dir_purpose, router_purpose, @@ -867,7 +856,7 @@ directory_command_should_use_begindir(const or_options_t *options, * <b>supports_begindir</b>, and whose identity key digest is * <b>digest</b>. */ void -directory_initiate_command(const char *address, const tor_addr_t *_addr, +directory_initiate_command(const tor_addr_t *_addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, @@ -875,7 +864,7 @@ directory_initiate_command(const char *address, const tor_addr_t *_addr, const char *payload, size_t payload_len, time_t if_modified_since) { - directory_initiate_command_rend(address, _addr, or_port, dir_port, + directory_initiate_command_rend(_addr, or_port, dir_port, digest, dir_purpose, router_purpose, indirection, resource, payload, payload_len, @@ -897,7 +886,7 @@ is_sensitive_dir_purpose(uint8_t dir_purpose) /** Same as directory_initiate_command(), but accepts rendezvous data to * fetch a hidden service descriptor. */ static void -directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, +directory_initiate_command_rend(const tor_addr_t *_addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, @@ -915,7 +904,6 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, const int anonymized_connection = dirind_is_anon(indirection); tor_addr_t addr; - tor_assert(address); tor_assert(_addr); tor_assert(or_port || dir_port); tor_assert(digest); @@ -948,7 +936,7 @@ directory_initiate_command_rend(const char *address, const tor_addr_t *_addr, /* set up conn so it's got all the data we need to remember */ tor_addr_copy(&conn->base_.addr, &addr); conn->base_.port = use_begindir ? or_port : dir_port; - conn->base_.address = tor_strdup(address); + conn->base_.address = tor_dup_addr(&addr); memcpy(conn->identity_digest, digest, DIGEST_LEN); conn->base_.purpose = dir_purpose; diff --git a/src/or/directory.h b/src/or/directory.h index 3de69329a4..bc200797d4 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -63,7 +63,7 @@ int connection_dir_process_inbuf(dir_connection_t *conn); int connection_dir_finished_flushing(dir_connection_t *conn); int connection_dir_finished_connecting(dir_connection_t *conn); void connection_dir_about_to_close(dir_connection_t *dir_conn); -void directory_initiate_command(const char *address, const tor_addr_t *addr, +void directory_initiate_command(const tor_addr_t *addr, uint16_t or_port, uint16_t dir_port, const char *digest, uint8_t dir_purpose, uint8_t router_purpose, diff --git a/src/or/dirserv.c b/src/or/dirserv.c index cd0e23cc2c..984b47d2f5 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -79,7 +79,6 @@ static int add_fingerprint_to_dir(const char *nickname, const char *fp, struct authdir_config_t *list); static uint32_t dirserv_get_status_impl(const char *fp, const char *nickname, - const char *address, uint32_t addr, uint16_t or_port, const char *platform, const char *contact, const char **msg, int should_log); @@ -327,7 +326,6 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg) } return dirserv_get_status_impl(d, router->nickname, - router->address, router->addr, router->or_port, router->platform, router->contact_info, msg, 1); @@ -341,7 +339,6 @@ dirserv_would_reject_router(const routerstatus_t *rs) uint32_t res; res = dirserv_get_status_impl(rs->identity_digest, rs->nickname, - "", /* address is only used in logs */ rs->addr, rs->or_port, NULL, NULL, NULL, 0); @@ -380,7 +377,6 @@ dirserv_get_name_status(const char *id_digest, const char *nickname) */ static uint32_t dirserv_get_status_impl(const char *id_digest, const char *nickname, - const char *address, uint32_t addr, uint16_t or_port, const char *platform, const char *contact, const char **msg, int should_log) @@ -452,14 +448,14 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (should_log) log_info(LD_DIRSERV, "Marking '%s' as bad directory because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_BADDIR; } if (authdir_policy_badexit_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Marking '%s' as bad exit because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_BADEXIT; } @@ -467,7 +463,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (!authdir_policy_permits_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Rejecting '%s' because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); if (msg) *msg = "Authdir is rejecting routers in this range."; return FP_REJECT; @@ -475,7 +471,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (!authdir_policy_valid_address(addr, or_port)) { if (should_log) log_info(LD_DIRSERV, "Not marking '%s' valid because of address '%s'", - nickname, address); + nickname, fmt_addr32(addr)); result |= FP_INVALID; } if (reject_unlisted) { @@ -524,19 +520,12 @@ dirserv_free_fingerprint_list(void) static int dirserv_router_has_valid_address(routerinfo_t *ri) { - struct in_addr iaddr; if (get_options()->DirAllowPrivateAddresses) return 0; /* whatever it is, we're fine with it */ - if (!tor_inet_aton(ri->address, &iaddr)) { - log_info(LD_DIRSERV,"Router %s published non-IP address '%s'. Refusing.", - router_describe(ri), - ri->address); - return -1; - } - if (is_internal_IP(ntohl(iaddr.s_addr), 0)) { + if (is_internal_IP(ri->addr, 0)) { log_info(LD_DIRSERV, - "Router %s published internal IP address '%s'. Refusing.", - router_describe(ri), ri->address); + "Router %s published internal IP address. Refusing.", + router_describe(ri)); return -1; /* it's a private IP, we should reject it */ } return 0; @@ -588,12 +577,10 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg, } if (dirserv_router_has_valid_address(ri) < 0) { log_fn(severity, LD_DIRSERV, - "Router %s has invalid address '%s'. " - "Not adding (%s).", + "Router %s has invalid address. Not adding (%s).", router_describe(ri), - ri->address, esc_router_info(ri)); - *msg = "Rejected: Address is not an IP, or IP is a private address."; + *msg = "Rejected: Address is a private address."; return -1; } @@ -3279,7 +3266,7 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router) /* IPv4. */ log_debug(LD_OR,"Testing reachability of %s at %s:%u.", - router->nickname, router->address, router->or_port); + router->nickname, fmt_addr32(router->addr), router->or_port); tor_addr_from_ipv4h(&router_addr, router->addr); chan = channel_tls_connect(&router_addr, router->or_port, router->cache_info.identity_digest); diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 17c5b13e79..b374ac7a34 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -1967,7 +1967,6 @@ get_socks_args_by_bridge_addrport(const tor_addr_t *addr, uint16_t port) static void launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) { - char *address; const or_options_t *options = get_options(); if (connection_get_by_type_addr_port_purpose( @@ -1982,15 +1981,12 @@ launch_direct_bridge_descriptor_fetch(bridge_info_t *bridge) return; } - address = tor_dup_addr(&bridge->addr); - - directory_initiate_command(address, &bridge->addr, + directory_initiate_command(&bridge->addr, bridge->port, 0/*no dirport*/, bridge->identity, DIR_PURPOSE_FETCH_SERVERDESC, ROUTER_PURPOSE_BRIDGE, DIRIND_ONEHOP, "authority.z", NULL, 0, 0); - tor_free(address); } /** Fetching the bridge descriptor from the bridge authority returned a @@ -2108,13 +2104,11 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node) } else { if (tor_addr_family(&bridge->addr) == AF_INET) { ri->addr = tor_addr_to_ipv4h(&bridge->addr); - tor_free(ri->address); - ri->address = tor_dup_ip(ri->addr); ri->or_port = bridge->port; log_info(LD_DIR, "Adjusted bridge routerinfo for '%s' to match configured " "address %s:%d.", - ri->nickname, ri->address, ri->or_port); + ri->nickname, fmt_addr32(ri->addr), ri->or_port); } else if (tor_addr_family(&bridge->addr) == AF_INET6) { tor_addr_copy(&ri->ipv6_addr, &bridge->addr); ri->ipv6_orport = bridge->port; diff --git a/src/or/main.c b/src/or/main.c index b0529cde8a..7294c8955a 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1670,24 +1670,28 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) /* every 20 minutes, check and complain if necessary */ const routerinfo_t *me = router_get_my_routerinfo(); if (me && !check_whether_orport_reachable()) { + char *address = tor_dup_ip(me->addr); log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " "its ORPort is reachable. Please check your firewalls, ports, " "address, /etc/hosts file, etc.", - me->address, me->or_port); + address, me->or_port); control_event_server_status(LOG_WARN, "REACHABILITY_FAILED ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); + tor_free(address); } if (me && !check_whether_dirport_reachable()) { + char *address = tor_dup_ip(me->addr); log_warn(LD_CONFIG, "Your server (%s:%d) has not managed to confirm that its " "DirPort is reachable. Please check your firewalls, ports, " "address, /etc/hosts file, etc.", - me->address, me->dir_port); + address, me->dir_port); control_event_server_status(LOG_WARN, "REACHABILITY_FAILED DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); + tor_free(address); } } diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 8052ca998c..6419ea79f8 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -721,7 +721,7 @@ update_microdesc_downloads(time_t now) smartlist_t *missing; digestmap_t *pending; - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (directory_too_idle_to_fetch_descriptors(options, now)) return; diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 2b0242b569..49478a7341 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -31,6 +31,7 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "transports.h" /** Map from lowercase nickname to identity digest of named server, if any. */ static strmap_t *named_server_map = NULL; @@ -884,14 +885,37 @@ update_consensus_networkstatus_fetch_time(time_t now) /** Return 1 if there's a reason we shouldn't try any directory * fetches yet (e.g. we demand bridges and none are yet known). - * Else return 0. */ + * Else return 0. + + * If we return 1 and <b>msg_out</b> is provided, set <b>msg_out</b> + * to an explanation of why directory fetches are delayed. (If we + * return 0, we set msg_out to NULL.) + */ int -should_delay_dir_fetches(const or_options_t *options) +should_delay_dir_fetches(const or_options_t *options, const char **msg_out) { - if (options->UseBridges && !any_bridge_descriptors_known()) { - log_info(LD_DIR, "delaying dir fetches (no running bridges known)"); - return 1; + if (msg_out) { + *msg_out = NULL; } + + if (options->UseBridges) { + if (!any_bridge_descriptors_known()) { + if (msg_out) { + *msg_out = "No running bridges"; + } + log_info(LD_DIR, "Delaying dir fetches (no running bridges known)"); + return 1; + } + + if (pt_proxies_configuration_pending()) { + if (msg_out) { + *msg_out = "Pluggable transport proxies still configuring"; + } + log_info(LD_DIR, "Delaying dir fetches (pt proxies still configuring)"); + return 1; + } + } + return 0; } @@ -901,7 +925,7 @@ void update_networkstatus_downloads(time_t now) { const or_options_t *options = get_options(); - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; update_consensus_networkstatus_downloads(now); update_certificate_downloads(now); diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index df5895c4b8..1659818f0a 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -53,7 +53,7 @@ int networkstatus_nickname_is_unnamed(const char *nickname); void networkstatus_consensus_download_failed(int status_code, const char *flavname); void update_consensus_networkstatus_fetch_time(time_t now); -int should_delay_dir_fetches(const or_options_t *options); +int should_delay_dir_fetches(const or_options_t *options,const char **msg_out); void update_networkstatus_downloads(time_t now); void update_certificate_downloads(time_t now); int consensus_is_waiting_for_certs(void); diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 03fa836d4e..3704822c72 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -785,7 +785,7 @@ void node_get_address_string(const node_t *node, char *buf, size_t len) { if (node->ri) { - strlcpy(buf, node->ri->address, len); + strlcpy(buf, fmt_addr32(node->ri->addr), len); } else if (node->rs) { tor_addr_t addr; tor_addr_from_ipv4h(&addr, node->rs->addr); @@ -1477,6 +1477,7 @@ update_router_have_minimum_dir_info(void) const networkstatus_t *consensus = networkstatus_get_reasonably_live_consensus(now,usable_consensus_flavor()); int using_md; + const char *delay_fetches_msg = NULL; if (!consensus) { if (!networkstatus_get_latest_consensus()) @@ -1489,10 +1490,9 @@ update_router_have_minimum_dir_info(void) goto done; } - if (should_delay_dir_fetches(get_options())) { - log_notice(LD_DIR, "no known bridge descriptors running yet; stalling"); - strlcpy(dir_info_status, "No live bridge descriptors.", - sizeof(dir_info_status)); + if (should_delay_dir_fetches(get_options(), &delay_fetches_msg)) { + log_notice(LD_DIR, "Delaying dir fetches: %s", delay_fetches_msg); + strlcpy(dir_info_status, "%s", sizeof(dir_info_status)); res = 0; goto done; } diff --git a/src/or/or.h b/src/or/or.h index c47ae23ed1..546adaa3a2 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -2009,7 +2009,6 @@ typedef int16_t country_t; /** Information about another onion router in the network. */ typedef struct { signed_descriptor_t cache_info; - char *address; /**< Location of OR: either a hostname or an IP address. */ char *nickname; /**< Human-readable OR name. */ uint32_t addr; /**< IPv4 address of OR, in host order. */ @@ -2816,6 +2815,9 @@ typedef struct circuit_t { * more. */ int deliver_window; + /** Temporary field used during circuits_handle_oom. */ + uint32_t age_tmp; + /** For storage while n_chan is pending (state CIRCUIT_STATE_CHAN_WAIT). */ struct create_cell_t *n_chan_create_cell; @@ -3469,9 +3471,8 @@ typedef struct { config_line_t *DirPort_lines; config_line_t *DNSPort_lines; /**< Ports to listen on for DNS requests. */ - uint64_t MaxMemInCellQueues; /**< If we have more memory than this allocated - * for circuit cell queues, run the OOM handler - */ + uint64_t MaxMemInQueues; /**< If we have more memory than this allocated + * for queues and buffers, run the OOM handler */ /** @name port booleans * @@ -3637,6 +3638,10 @@ typedef struct { * a new one? */ int MaxCircuitDirtiness; /**< Never use circs that were first used more than this interval ago. */ + int PredictedPortsRelevanceTime; /** How long after we've requested a + * connection for a given port, do we want + * to continue to pick exits that support + * that port? */ uint64_t BandwidthRate; /**< How much bandwidth, on average, are we willing * to use in a second? */ uint64_t BandwidthBurst; /**< How much bandwidth, at maximum, are we willing @@ -4836,9 +4841,9 @@ typedef struct rend_service_descriptor_t { crypto_pk_t *pk; /**< This service's public key. */ int version; /**< Version of the descriptor format: 0 or 2. */ time_t timestamp; /**< Time when the descriptor was generated. */ - /** Bitmask: which rendezvous protocols are supported? - * (We allow bits '0', '1', and '2' to be set.) */ - int protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; + /** Bitmask: which introduce/rendezvous protocols are supported? + * (We allow bits '0', '1', '2' and '3' to be set.) */ + unsigned protocols : REND_PROTOCOL_VERSION_BITMASK_WIDTH; /** List of the service's introduction points. Elements are removed if * introduction attempts fail. */ smartlist_t *intro_nodes; diff --git a/src/or/policies.c b/src/or/policies.c index 05377ec205..42dc46b7fd 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -438,7 +438,7 @@ validate_addr_policies(const or_options_t *options, char **msg) if (policies_parse_exit_policy(options->ExitPolicy, &addr_policy, options->IPv6Exit, - options->ExitPolicyRejectPrivate, NULL, + options->ExitPolicyRejectPrivate, 0, !options->BridgeRelay)) REJECT("Error in ExitPolicy entry."); @@ -482,10 +482,12 @@ validate_addr_policies(const or_options_t *options, char **msg) * Ignore port specifiers. */ static int -load_policy_from_option(config_line_t *config, smartlist_t **policy, +load_policy_from_option(config_line_t *config, const char *option_name, + smartlist_t **policy, int assume_action) { int r; + int killed_any_ports = 0; addr_policy_list_free(*policy); *policy = NULL; r = parse_addr_policy(config, policy, assume_action); @@ -504,9 +506,13 @@ load_policy_from_option(config_line_t *config, smartlist_t **policy, c = addr_policy_get_canonical_entry(&newp); SMARTLIST_REPLACE_CURRENT(*policy, n, c); addr_policy_free(n); + killed_any_ports = 1; } } SMARTLIST_FOREACH_END(n); } + if (killed_any_ports) { + log_warn(LD_CONFIG, "Ignoring ports in %s option.", option_name); + } return 0; } @@ -516,20 +522,22 @@ int policies_parse_from_options(const or_options_t *options) { int ret = 0; - if (load_policy_from_option(options->SocksPolicy, &socks_policy, -1) < 0) + if (load_policy_from_option(options->SocksPolicy, "SocksPolicy", + &socks_policy, -1) < 0) ret = -1; - if (load_policy_from_option(options->DirPolicy, &dir_policy, -1) < 0) + if (load_policy_from_option(options->DirPolicy, "DirPolicy", + &dir_policy, -1) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirReject, + if (load_policy_from_option(options->AuthDirReject, "AuthDirReject", &authdir_reject_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirInvalid, + if (load_policy_from_option(options->AuthDirInvalid, "AuthDirInvalid", &authdir_invalid_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirBadDir, + if (load_policy_from_option(options->AuthDirBadDir, "AuthDirBadDir", &authdir_baddir_policy, ADDR_POLICY_REJECT) < 0) ret = -1; - if (load_policy_from_option(options->AuthDirBadExit, + if (load_policy_from_option(options->AuthDirBadExit, "AuthDirBadExit", &authdir_badexit_policy, ADDR_POLICY_REJECT) < 0) ret = -1; if (parse_reachable_addresses() < 0) @@ -962,7 +970,7 @@ exit_policy_remove_redundancies(smartlist_t *dest) int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, int ipv6_exit, - int rejectprivate, const char *local_address, + int rejectprivate, uint32_t local_address, int add_default_policy) { if (!ipv6_exit) { @@ -972,7 +980,7 @@ policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, append_exit_policy_string(dest, "reject private:*"); if (local_address) { char buf[POLICY_BUF_LEN]; - tor_snprintf(buf, sizeof(buf), "reject %s:*", local_address); + tor_snprintf(buf, sizeof(buf), "reject %s:*", fmt_addr32(local_address)); append_exit_policy_string(dest, buf); } } diff --git a/src/or/policies.h b/src/or/policies.h index facbbb6b5a..91ac427492 100644 --- a/src/or/policies.h +++ b/src/or/policies.h @@ -45,7 +45,7 @@ addr_policy_result_t compare_tor_addr_to_node_policy(const tor_addr_t *addr, int policies_parse_exit_policy(config_line_t *cfg, smartlist_t **dest, int ipv6exit, - int rejectprivate, const char *local_address, + int rejectprivate, uint32_t local_address, int add_default_policy); void policies_exit_policy_append_reject_star(smartlist_t **dest); void addr_policy_append_reject_addr(smartlist_t **dest, diff --git a/src/or/relay.c b/src/or/relay.c index dc234c1f2a..d6742d25e1 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -2153,7 +2153,8 @@ cell_queue_append_packed_copy(circuit_t *circ, cell_queue_t *queue, (void)circ; (void)exitward; (void)use_stats; - tor_gettimeofday_cached(&now); + tor_gettimeofday_cached_monotonic(&now); + copy->inserted_time = (uint32_t)tv_to_msec(&now); cell_queue_append(queue, copy); @@ -2201,13 +2202,21 @@ packed_cell_mem_cost(void) return sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD; } +/** DOCDOC */ +STATIC size_t +cell_queues_get_total_allocation(void) +{ + return total_cells_allocated * packed_cell_mem_cost(); +} + /** Check whether we've got too much space used for cells. If so, * call the OOM handler and return 1. Otherwise, return 0. */ -static int +STATIC int cell_queues_check_size(void) { - size_t alloc = total_cells_allocated * packed_cell_mem_cost(); - if (alloc >= get_options()->MaxMemInCellQueues) { + size_t alloc = cell_queues_get_total_allocation(); + alloc += buf_get_total_allocation(); + if (alloc >= get_options()->MaxMemInQueues) { circuits_handle_oom(alloc); return 1; } diff --git a/src/or/relay.h b/src/or/relay.h index 20eecfb400..2c7d0d8ae4 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -85,6 +85,8 @@ STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out); STATIC packed_cell_t *packed_cell_new(void); STATIC packed_cell_t *cell_queue_pop(cell_queue_t *queue); +STATIC size_t cell_queues_get_total_allocation(void); +STATIC int cell_queues_check_size(void); #endif #endif diff --git a/src/or/rephist.c b/src/or/rephist.c index 66dc5f611f..16e16ab651 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -1862,22 +1862,20 @@ rep_hist_note_used_port(time_t now, uint16_t port) add_predicted_port(now, port); } -/** For this long after we've seen a request for a given port, assume that - * we'll want to make connections to the same port in the future. */ -#define PREDICTED_CIRCS_RELEVANCE_TIME (60*60) - /** Return a newly allocated pointer to a list of uint16_t * for ports that * are likely to be asked for in the near future. */ smartlist_t * rep_hist_get_predicted_ports(time_t now) { + int predicted_circs_relevance_time; smartlist_t *out = smartlist_new(); tor_assert(predicted_ports_list); + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; /* clean out obsolete entries */ SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) { - if (pp->time + PREDICTED_CIRCS_RELEVANCE_TIME < now) { + if (pp->time + predicted_circs_relevance_time < now) { log_debug(LD_CIRC, "Expiring predicted port %d", pp->port); rephist_total_alloc -= sizeof(predicted_port_t); @@ -1944,14 +1942,17 @@ int rep_hist_get_predicted_internal(time_t now, int *need_uptime, int *need_capacity) { + int predicted_circs_relevance_time; + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + if (!predicted_internal_time) { /* initialize it */ predicted_internal_time = now; predicted_internal_uptime_time = now; predicted_internal_capacity_time = now; } - if (predicted_internal_time + PREDICTED_CIRCS_RELEVANCE_TIME < now) + if (predicted_internal_time + predicted_circs_relevance_time < now) return 0; /* too long ago */ - if (predicted_internal_uptime_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now) + if (predicted_internal_uptime_time + predicted_circs_relevance_time >= now) *need_uptime = 1; // Always predict that we need capacity. *need_capacity = 1; @@ -1963,8 +1964,11 @@ rep_hist_get_predicted_internal(time_t now, int *need_uptime, int any_predicted_circuits(time_t now) { + int predicted_circs_relevance_time; + predicted_circs_relevance_time = get_options()->PredictedPortsRelevanceTime; + return smartlist_len(predicted_ports_list) || - predicted_internal_time + PREDICTED_CIRCS_RELEVANCE_TIME >= now; + predicted_internal_time + predicted_circs_relevance_time >= now; } /** Return 1 if we have no need for circuits currently, else return 0. */ diff --git a/src/or/router.c b/src/or/router.c index 6fa9f65e10..4828a8df67 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1177,7 +1177,7 @@ consider_testing_reachability(int test_or, int test_dir) /* XXX IPv6 self testing */ log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", !orport_reachable ? "reachability" : "bandwidth", - me->address, me->or_port); + fmt_addr32(me->addr), me->or_port); circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei, CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); extend_info_free(ei); @@ -1189,7 +1189,7 @@ consider_testing_reachability(int test_or, int test_dir) CONN_TYPE_DIR, &addr, me->dir_port, DIR_PURPOSE_FETCH_SERVERDESC)) { /* ask myself, via tor, for my server descriptor. */ - directory_initiate_command(me->address, &addr, + directory_initiate_command(&addr, me->or_port, me->dir_port, me->cache_info.identity_digest, DIR_PURPOSE_FETCH_SERVERDESC, @@ -1204,6 +1204,7 @@ router_orport_found_reachable(void) { const routerinfo_t *me = router_get_my_routerinfo(); if (!can_reach_or_port && me) { + char *address = tor_dup_ip(me->addr); log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from " "the outside. Excellent.%s", get_options()->PublishServerDescriptor_ != NO_DIRINFO ? @@ -1212,7 +1213,8 @@ router_orport_found_reachable(void) mark_my_descriptor_dirty("ORPort found reachable"); control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d", - me->address, me->or_port); + address, me->or_port); + tor_free(address); } } @@ -1222,6 +1224,7 @@ router_dirport_found_reachable(void) { const routerinfo_t *me = router_get_my_routerinfo(); if (!can_reach_dir_port && me) { + char *address = tor_dup_ip(me->addr); log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable " "from the outside. Excellent."); can_reach_dir_port = 1; @@ -1229,7 +1232,8 @@ router_dirport_found_reachable(void) mark_my_descriptor_dirty("DirPort found reachable"); control_event_server_status(LOG_NOTICE, "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d", - me->address, me->dir_port); + address, me->dir_port); + tor_free(address); } } @@ -1800,7 +1804,6 @@ router_rebuild_descriptor(int force) ri = tor_malloc_zero(sizeof(routerinfo_t)); ri->cache_info.routerlist_index = -1; - ri->address = tor_dup_ip(addr); ri->nickname = tor_strdup(options->Nickname); ri->addr = addr; ri->or_port = router_get_advertised_or_port(options); @@ -1865,7 +1868,7 @@ router_rebuild_descriptor(int force) policies_parse_exit_policy(options->ExitPolicy, &ri->exit_policy, options->IPv6Exit, options->ExitPolicyRejectPrivate, - ri->address, !options->BridgeRelay); + ri->addr, !options->BridgeRelay); } ri->policy_is_reject_star = policy_is_reject_star(ri->exit_policy, AF_INET) && @@ -2271,8 +2274,7 @@ char * router_dump_router_to_string(routerinfo_t *router, crypto_pk_t *ident_key) { - /* XXXX025 Make this look entirely at its arguments, and not at globals. - */ + char *address = NULL; char *onion_pkey = NULL; /* Onion key, PEM-encoded. */ char *identity_pkey = NULL; /* Identity key, PEM-encoded. */ char digest[DIGEST_LEN]; @@ -2346,7 +2348,9 @@ router_dump_router_to_string(routerinfo_t *router, } } + address = tor_dup_ip(router->addr); chunks = smartlist_new(); + /* Generate the easy portion of the router descriptor. */ smartlist_add_asprintf(chunks, "router %s %s %d 0 %d\n" @@ -2362,7 +2366,7 @@ router_dump_router_to_string(routerinfo_t *router, "signing-key\n%s" "%s%s%s%s", router->nickname, - router->address, + address, router->or_port, decide_to_advertise_dirport(options, router->dir_port), extra_or_address ? extra_or_address : "", @@ -2476,6 +2480,7 @@ router_dump_router_to_string(routerinfo_t *router, SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); smartlist_free(chunks); } + tor_free(address); tor_free(family_line); tor_free(onion_pkey); tor_free(identity_pkey); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index d636a1e25c..8d29b89ea9 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -669,7 +669,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) char id_digest_str[2*DIGEST_LEN+1]; char sk_digest_str[2*DIGEST_LEN+1]; - if (should_delay_dir_fetches(get_options())) + if (should_delay_dir_fetches(get_options(), NULL)) return; pending_cert = fp_pair_map_new(); @@ -2709,7 +2709,6 @@ routerinfo_free(routerinfo_t *router) return; tor_free(router->cache_info.signed_descriptor_body); - tor_free(router->address); tor_free(router->nickname); tor_free(router->platform); tor_free(router->contact_info); @@ -4593,7 +4592,7 @@ void update_router_descriptor_downloads(time_t now) { const or_options_t *options = get_options(); - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (!we_fetch_router_descriptors(options)) return; @@ -4614,7 +4613,7 @@ update_extrainfo_downloads(time_t now) int n_no_ei = 0, n_pending = 0, n_have = 0, n_delay = 0; if (! options->DownloadExtraInfo) return; - if (should_delay_dir_fetches(options)) + if (should_delay_dir_fetches(options, NULL)) return; if (!router_have_minimum_dir_info()) return; @@ -4720,7 +4719,7 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2) } /* If any key fields differ, they're different. */ - if (strcasecmp(r1->address, r2->address) || + if (r1->addr != r2->addr || strcasecmp(r1->nickname, r2->nickname) || r1->or_port != r2->or_port || !tor_addr_eq(&r1->ipv6_addr, &r2->ipv6_addr) || diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 9414a533b9..ad3cf3b388 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -1195,8 +1195,7 @@ router_parse_entry_from_string(const char *s, const char *end, log_warn(LD_DIR,"Router nickname is invalid"); goto err; } - router->address = tor_strdup(tok->args[1]); - if (!tor_inet_aton(router->address, &in)) { + if (!tor_inet_aton(tok->args[1], &in)) { log_warn(LD_DIR,"Router address is not an IP address."); goto err; } diff --git a/src/test/include.am b/src/test/include.am index e7aebac389..c6743a19b0 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -32,6 +32,7 @@ src_test_test_SOURCES = \ src/test/test_introduce.c \ src/test/test_logging.c \ src/test/test_microdesc.c \ + src/test/test_oom.c \ src/test/test_options.c \ src/test/test_pt.c \ src/test/test_replay.c \ diff --git a/src/test/test.c b/src/test/test.c index 2529e2902a..456dde1412 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -638,7 +638,7 @@ test_policy_summary_helper(const char *policy_str, line.value = (char *)policy_str; line.next = NULL; - r = policies_parse_exit_policy(&line, &policy, 1, 0, NULL, 1); + r = policies_parse_exit_policy(&line, &policy, 1, 0, 0, 1); test_eq(r, 0); summary = policy_summarize(policy, AF_INET); @@ -695,7 +695,7 @@ test_policies(void) test_assert(ADDR_POLICY_REJECTED == compare_tor_addr_to_addr_policy(&tar, 2, policy)); - test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, 1, NULL, 1)); + test_assert(0 == policies_parse_exit_policy(NULL, &policy2, 1, 1, 0, 1)); test_assert(policy2); policy3 = smartlist_new(); @@ -782,7 +782,7 @@ test_policies(void) line.key = (char*)"foo"; line.value = (char*)"accept *:80,reject private:*,reject *:*"; line.next = NULL; - test_assert(0 == policies_parse_exit_policy(&line, &policy, 1, 0, NULL, 1)); + test_assert(0 == policies_parse_exit_policy(&line, &policy, 1, 0, 0, 1)); test_assert(policy); //test_streq(policy->string, "accept *:80"); //test_streq(policy->next->string, "reject *:*"); @@ -1630,6 +1630,7 @@ extern struct testcase_t backtrace_tests[]; extern struct testcase_t hs_tests[]; extern struct testcase_t nodelist_tests[]; extern struct testcase_t routerkeys_tests[]; +extern struct testcase_t oom_tests[]; static struct testgroup_t testgroups[] = { { "", test_array }, @@ -1656,6 +1657,7 @@ static struct testgroup_t testgroups[] = { { "hs/", hs_tests }, { "nodelist/", nodelist_tests }, { "routerkeys/", routerkeys_tests }, + { "oom/", oom_tests }, END_OF_GROUPS }; diff --git a/src/test/test.h b/src/test/test.h index a89b558e5a..ba82f52add 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -36,17 +36,7 @@ #define test_strneq(expr1, expr2) tt_str_op((expr1), !=, (expr2)) #define test_mem_op(expr1, op, expr2, len) \ - tt_assert_test_fmt_type(expr1,expr2,#expr1" "#op" "#expr2, \ - const char *, \ - (memcmp(val1_, val2_, len) op 0), \ - char *, "%s", \ - { size_t printlen = (len)*2+1; \ - print_ = tor_malloc(printlen); \ - base16_encode(print_, printlen, value_, \ - (len)); }, \ - { tor_free(print_); }, \ - TT_EXIT_TEST_FUNCTION \ - ); + tt_mem_op((expr1), op, (expr2), (len)) #define test_memeq(expr1, expr2, len) test_mem_op((expr1), ==, (expr2), len) #define test_memneq(expr1, expr2, len) test_mem_op((expr1), !=, (expr2), len) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index 79ddd95090..036380fe85 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -971,6 +971,32 @@ test_addr_is_loopback(void *data) ; } +static void +test_addr_make_null(void *data) +{ + tor_addr_t *addr = tor_malloc(sizeof(*addr)); + tor_addr_t *zeros = tor_malloc_zero(sizeof(*addr)); + char buf[TOR_ADDR_BUF_LEN]; + (void) data; + /* Ensure that before tor_addr_make_null, addr != 0's */ + memset(addr, 1, sizeof(*addr)); + tt_int_op(memcmp(addr, zeros, sizeof(*addr)), !=, 0); + /* Test with AF == AF_INET */ + zeros->family = AF_INET; + tor_addr_make_null(addr, AF_INET); + tt_int_op(memcmp(addr, zeros, sizeof(*addr)), ==, 0); + tt_str_op(tor_addr_to_str(buf, addr, sizeof(buf), 0), ==, "0.0.0.0"); + /* Test with AF == AF_INET6 */ + memset(addr, 1, sizeof(*addr)); + zeros->family = AF_INET6; + tor_addr_make_null(addr, AF_INET6); + tt_int_op(memcmp(addr, zeros, sizeof(*addr)), ==, 0); + tt_str_op(tor_addr_to_str(buf, addr, sizeof(buf), 0), ==, "::"); + done: + tor_free(addr); + tor_free(zeros); +} + #define ADDR_LEGACY(name) \ { #name, legacy_test_helper, 0, &legacy_setup, test_addr_ ## name } @@ -983,6 +1009,7 @@ struct testcase_t addr_tests[] = { { "dup_ip", test_addr_dup_ip, 0, NULL, NULL }, { "sockaddr_to_str", test_addr_sockaddr_to_str, 0, NULL, NULL }, { "is_loopback", test_addr_is_loopback, 0, NULL, NULL }, + { "make_null", test_addr_make_null, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index a009faa0be..6dd7715936 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -193,7 +193,120 @@ test_buffers_basic(void *arg) buf_free(buf); if (buf2) buf_free(buf2); + buf_shrink_freelists(1); } + +static void +test_buffer_pullup(void *arg) +{ + buf_t *buf; + char *stuff, *tmp; + const char *cp; + size_t sz; + (void)arg; + stuff = tor_malloc(16384); + tmp = tor_malloc(16384); + + /* Note: this test doesn't check the nulterminate argument to buf_pullup, + since nothing actually uses it. We should remove it some time. */ + + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + + tt_assert(buf); + tt_int_op(buf_get_default_chunk_size(buf), ==, 4096); + + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* There are a bunch of cases for pullup. One is the trivial case. Let's + mess around with an empty buffer. */ + buf_pullup(buf, 16, 1); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, ==, NULL); + tt_ptr_op(sz, ==, 0); + + /* Let's make sure nothing got allocated */ + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* Case 1: everything puts into the first chunk with some moving. */ + + /* Let's add some data. */ + crypto_rand(stuff, 16384); + write_to_buf(stuff, 3000, buf); + write_to_buf(stuff+3000, 3000, buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, <=, 4096); + + /* Make room for 3000 bytes in the first chunk, so that the pullup-move code + * can get tested. */ + tt_int_op(fetch_from_buf(tmp, 3000, buf), ==, 3000); + test_memeq(tmp, stuff, 3000); + buf_pullup(buf, 2048, 0); + assert_buf_ok(buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, >=, 2048); + test_memeq(cp, stuff+3000, 2048); + tt_int_op(3000, ==, buf_datalen(buf)); + tt_int_op(fetch_from_buf(tmp, 3000, buf), ==, 0); + test_memeq(tmp, stuff+3000, 2048); + + buf_free(buf); + + /* Now try the large-chunk case. */ + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + write_to_buf(stuff, 4000, buf); + write_to_buf(stuff+4000, 4000, buf); + write_to_buf(stuff+8000, 4000, buf); + write_to_buf(stuff+12000, 4000, buf); + tt_int_op(buf_datalen(buf), ==, 16000); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, <=, 4096); + + buf_pullup(buf, 12500, 0); + assert_buf_ok(buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, >=, 12500); + test_memeq(cp, stuff, 12500); + tt_int_op(buf_datalen(buf), ==, 16000); + + fetch_from_buf(tmp, 12400, buf); + test_memeq(tmp, stuff, 12400); + tt_int_op(buf_datalen(buf), ==, 3600); + fetch_from_buf(tmp, 3500, buf); + test_memeq(tmp, stuff+12400, 3500); + fetch_from_buf(tmp, 100, buf); + test_memeq(tmp, stuff+15900, 10); + + buf_free(buf); + + /* Make sure that the pull-up-whole-buffer case works */ + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + write_to_buf(stuff, 4000, buf); + write_to_buf(stuff+4000, 4000, buf); + fetch_from_buf(tmp, 100, buf); /* dump 100 bytes from first chunk */ + buf_pullup(buf, 16000, 0); /* Way too much. */ + assert_buf_ok(buf); + buf_get_first_chunk_data(buf, &cp, &sz); + tt_ptr_op(cp, !=, NULL); + tt_int_op(sz, ==, 7900); + test_memeq(cp, stuff+100, 7900); + + buf_free(buf); + buf = NULL; + + buf_shrink_freelists(1); + + tt_int_op(buf_get_total_allocation(), ==, 0); + done: + buf_free(buf); + buf_shrink_freelists(1); + tor_free(stuff); + tor_free(tmp); +} + static void test_buffer_copy(void *arg) { @@ -257,6 +370,7 @@ test_buffer_copy(void *arg) generic_buffer_free(buf); if (buf2) generic_buffer_free(buf2); + buf_shrink_freelists(1); } static void @@ -331,12 +445,156 @@ test_buffer_ext_or_cmd(void *arg) ext_or_cmd_free(cmd); generic_buffer_free(buf); tor_free(tmp); + buf_shrink_freelists(1); +} + +static void +test_buffer_allocation_tracking(void *arg) +{ + char *junk = tor_malloc(16384); + buf_t *buf1 = NULL, *buf2 = NULL; + int i; + + (void)arg; + + crypto_rand(junk, 16384); + tt_int_op(buf_get_total_allocation(), ==, 0); + + buf1 = buf_new(); + tt_assert(buf1); + buf2 = buf_new(); + tt_assert(buf2); + + tt_int_op(buf_allocation(buf1), ==, 0); + tt_int_op(buf_get_total_allocation(), ==, 0); + + write_to_buf(junk, 4000, buf1); + write_to_buf(junk, 4000, buf1); + write_to_buf(junk, 4000, buf1); + write_to_buf(junk, 4000, buf1); + tt_int_op(buf_allocation(buf1), ==, 16384); + fetch_from_buf(junk, 100, buf1); + tt_int_op(buf_allocation(buf1), ==, 16384); /* still 4 4k chunks */ + + tt_int_op(buf_get_total_allocation(), ==, 16384); + + fetch_from_buf(junk, 4096, buf1); /* drop a 1k chunk... */ + tt_int_op(buf_allocation(buf1), ==, 3*4096); /* now 3 4k chunks */ + + tt_int_op(buf_get_total_allocation(), ==, 16384); /* that chunk went onto + the freelist. */ + + write_to_buf(junk, 4000, buf2); + tt_int_op(buf_allocation(buf2), ==, 4096); /* another 4k chunk. */ + tt_int_op(buf_get_total_allocation(), ==, 16384); /* that chunk came from + the freelist. */ + write_to_buf(junk, 4000, buf2); + tt_int_op(buf_allocation(buf2), ==, 8192); /* another 4k chunk. */ + tt_int_op(buf_get_total_allocation(), ==, 5*4096); /* that chunk was new. */ + + /* Make a really huge buffer */ + for (i = 0; i < 1000; ++i) { + write_to_buf(junk, 4000, buf2); + } + tt_int_op(buf_allocation(buf2), >=, 4008000); + tt_int_op(buf_get_total_allocation(), >=, 4008000); + buf_free(buf2); + buf2 = NULL; + + tt_int_op(buf_get_total_allocation(), <, 4008000); + buf_shrink_freelists(1); + tt_int_op(buf_get_total_allocation(), ==, buf_allocation(buf1)); + buf_free(buf1); + buf1 = NULL; + buf_shrink_freelists(1); + tt_int_op(buf_get_total_allocation(), ==, 0); + + done: + buf_free(buf1); + buf_free(buf2); + buf_shrink_freelists(1); +} + +static void +test_buffer_time_tracking(void *arg) +{ + buf_t *buf=NULL, *buf2=NULL; + struct timeval tv0; + const time_t START = 1389288246; + const uint32_t START_MSEC = (uint32_t) ((uint64_t)START * 1000); + int i; + char tmp[4096]; + (void)arg; + + crypto_rand(tmp, sizeof(tmp)); + + tv0.tv_sec = START; + tv0.tv_usec = 0; + + buf = buf_new_with_capacity(3000); /* rounds up to next power of 2. */ + tt_assert(buf); + + /* Empty buffer means the timestamp is 0. */ + tt_int_op(0, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC)); + tt_int_op(0, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000)); + + tor_gettimeofday_cache_set(&tv0); + write_to_buf("ABCDEFG", 7, buf); + tt_int_op(1000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+1000)); + + buf2 = buf_copy(buf); + tt_assert(buf2); + tt_int_op(1234, ==, buf_get_oldest_chunk_timestamp(buf2, START_MSEC+1234)); + + /* Now add more bytes; enough to overflow the first chunk. */ + tv0.tv_usec += 123 * 1000; + tor_gettimeofday_cache_set(&tv0); + for (i = 0; i < 600; ++i) + write_to_buf("ABCDEFG", 7, buf); + tt_int_op(4207, ==, buf_datalen(buf)); + + /* The oldest bytes are still in the front. */ + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000)); + + /* Once those bytes are dropped, the chunk is still on the first + * timestamp. */ + fetch_from_buf(tmp, 100, buf); + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2000)); + + /* But once we discard the whole first chunk, we get the data in the second + * chunk. */ + fetch_from_buf(tmp, 4000, buf); + tt_int_op(107, ==, buf_datalen(buf)); + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123)); + + /* This time we'll be grabbing a chunk from the freelist, and making sure + its time gets updated */ + tv0.tv_sec += 5; + tv0.tv_usec = 617*1000; + tor_gettimeofday_cache_set(&tv0); + for (i = 0; i < 600; ++i) + write_to_buf("ABCDEFG", 7, buf); + tt_int_op(4307, ==, buf_datalen(buf)); + + tt_int_op(2000, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+2123)); + fetch_from_buf(tmp, 4000, buf); + fetch_from_buf(tmp, 306, buf); + tt_int_op(0, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+5617)); + tt_int_op(383, ==, buf_get_oldest_chunk_timestamp(buf, START_MSEC+6000)); + + done: + buf_free(buf); + buf_free(buf2); } struct testcase_t buffer_tests[] = { - { "basic", test_buffers_basic, 0, NULL, NULL }, - { "copy", test_buffer_copy, 0, NULL, NULL }, - { "ext_or_cmd", test_buffer_ext_or_cmd, 0, NULL, NULL }, + { "basic", test_buffers_basic, TT_FORK, NULL, NULL }, + { "copy", test_buffer_copy, TT_FORK, NULL, NULL }, + { "pullup", test_buffer_pullup, TT_FORK, NULL, NULL }, + { "ext_or_cmd", test_buffer_ext_or_cmd, TT_FORK, NULL, NULL }, + { "allocation_tracking", test_buffer_allocation_tracking, TT_FORK, + NULL, NULL }, + { "time_tracking", test_buffer_time_tracking, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_dir.c b/src/test/test_dir.c index dcfe98dd11..9e01bdbd48 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -97,7 +97,6 @@ test_dir_formats(void) get_platform_str(platform, sizeof(platform)); r1 = tor_malloc_zero(sizeof(routerinfo_t)); - r1->address = tor_strdup("18.244.0.1"); r1->addr = 0xc0a80001u; /* 192.168.0.1 */ r1->cache_info.published_on = 0; r1->or_port = 9000; @@ -124,7 +123,6 @@ test_dir_formats(void) ex2->maskbits = 8; ex2->prt_min = ex2->prt_max = 24; r2 = tor_malloc_zero(sizeof(routerinfo_t)); - r2->address = tor_strdup("1.1.1.1"); r2->addr = 0x0a030201u; /* 10.3.2.1 */ r2->platform = tor_strdup(platform); r2->cache_info.published_on = 5; @@ -153,7 +151,7 @@ test_dir_formats(void) tor_free(options->ContactInfo); test_assert(buf); - strlcpy(buf2, "router Magri 18.244.0.1 9000 0 9003\n" + strlcpy(buf2, "router Magri 192.168.0.1 9000 0 9003\n" "or-address [1:2:3:4::]:9999\n" "platform Tor "VERSION" on ", sizeof(buf2)); strlcat(buf2, get_uname(), sizeof(buf2)); @@ -187,7 +185,7 @@ test_dir_formats(void) cp = buf; rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); test_assert(rp1); - test_streq(rp1->address, r1->address); + test_eq(rp1->addr, r1->addr); test_eq(rp1->or_port, r1->or_port); //test_eq(rp1->dir_port, r1->dir_port); test_eq(rp1->bandwidthrate, r1->bandwidthrate); @@ -198,7 +196,7 @@ test_dir_formats(void) //test_assert(rp1->exit_policy == NULL); strlcpy(buf2, - "router Fred 1.1.1.1 9005 0 0\n" + "router Fred 10.3.2.1 9005 0 0\n" "platform Tor "VERSION" on ", sizeof(buf2)); strlcat(buf2, get_uname(), sizeof(buf2)); strlcat(buf2, "\n" @@ -231,7 +229,7 @@ test_dir_formats(void) cp = buf; rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL); test_assert(rp2); - test_streq(rp2->address, r2->address); + test_eq(rp2->addr, r2->addr); test_eq(rp2->or_port, r2->or_port); test_eq(rp2->dir_port, r2->dir_port); test_eq(rp2->bandwidthrate, r2->bandwidthrate); diff --git a/src/test/test_oom.c b/src/test/test_oom.c new file mode 100644 index 0000000000..cc6e532358 --- /dev/null +++ b/src/test/test_oom.c @@ -0,0 +1,348 @@ +/* Copyright (c) 2014, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* Unit tests for OOM handling logic */ + +#define RELAY_PRIVATE +#define BUFFERS_PRIVATE +#define CIRCUITLIST_PRIVATE +#include "or.h" +#include "buffers.h" +#include "circuitlist.h" +#include "compat_libevent.h" +#include "connection.h" +#include "config.h" +#include "mempool.h" +#include "relay.h" +#include "test.h" + +/* small replacement mock for circuit_mark_for_close_ to avoid doing all + * the other bookkeeping that comes with marking circuits. */ +static void +circuit_mark_for_close_dummy_(circuit_t *circ, int reason, int line, + const char *file) +{ + (void) reason; + if (circ->marked_for_close) { + TT_FAIL(("Circuit already marked for close at %s:%d, but we are marking " + "it again at %s:%d", + circ->marked_for_close_file, (int)circ->marked_for_close, + file, line)); + } + + circ->marked_for_close = line; + circ->marked_for_close_file = file; +} + +static circuit_t * +dummy_or_circuit_new(int n_p_cells, int n_n_cells) +{ + or_circuit_t *circ = or_circuit_new(0, NULL); + int i; + cell_t cell; + + for (i=0; i < n_p_cells; ++i) { + crypto_rand((void*)&cell, sizeof(cell)); + cell_queue_append_packed_copy(TO_CIRCUIT(circ), &circ->p_chan_cells, + 0, &cell, 1, 0); + } + + for (i=0; i < n_n_cells; ++i) { + crypto_rand((void*)&cell, sizeof(cell)); + cell_queue_append_packed_copy(TO_CIRCUIT(circ), + &TO_CIRCUIT(circ)->n_chan_cells, + 1, &cell, 1, 0); + } + + TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_OR; + return TO_CIRCUIT(circ); +} + +static circuit_t * +dummy_origin_circuit_new(int n_cells) +{ + origin_circuit_t *circ = origin_circuit_new(); + int i; + cell_t cell; + + for (i=0; i < n_cells; ++i) { + crypto_rand((void*)&cell, sizeof(cell)); + cell_queue_append_packed_copy(TO_CIRCUIT(circ), + &TO_CIRCUIT(circ)->n_chan_cells, + 1, &cell, 1, 0); + } + + TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_GENERAL; + return TO_CIRCUIT(circ); +} + +static void +add_bytes_to_buf(generic_buffer_t *buf, size_t n_bytes) +{ + char b[3000]; + + while (n_bytes) { + size_t this_add = n_bytes > sizeof(buf) ? sizeof(buf) : n_bytes; + crypto_rand(b, sizeof(b)); + generic_buffer_add(buf, b, this_add); + n_bytes -= this_add; + } +} + +static edge_connection_t * +dummy_edge_conn_new(circuit_t *circ, + int type, size_t in_bytes, size_t out_bytes) +{ + edge_connection_t *conn; + + if (type == CONN_TYPE_EXIT) + conn = edge_connection_new(type, AF_INET); + else + conn = ENTRY_TO_EDGE_CONN(entry_connection_new(type, AF_INET)); + + /* We add these bytes directly to the buffers, to avoid all the + * edge connection read/write machinery. */ + add_bytes_to_buf(TO_CONN(conn)->inbuf, in_bytes); + add_bytes_to_buf(TO_CONN(conn)->outbuf, out_bytes); + + conn->on_circuit = circ; + if (type == CONN_TYPE_EXIT) { + or_circuit_t *oc = TO_OR_CIRCUIT(circ); + conn->next_stream = oc->n_streams; + oc->n_streams = conn; + } else { + origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); + conn->next_stream = oc->p_streams; + oc->p_streams = conn; + } + + return conn; +} + +/** Run unit tests for buffers.c */ +static void +test_oom_circbuf(void *arg) +{ + or_options_t *options = get_options_mutable(); + circuit_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL; + struct timeval tv = { 1389631048, 0 }; + + (void) arg; + + MOCK(circuit_mark_for_close_, circuit_mark_for_close_dummy_); + init_cell_pool(); + + /* Far too low for real life. */ + options->MaxMemInQueues = 256*packed_cell_mem_cost(); + options->CellStatistics = 0; + + tt_int_op(cell_queues_check_size(), ==, 0); /* We don't start out OOM. */ + tt_int_op(cell_queues_get_total_allocation(), ==, 0); + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* Now we're going to fake up some circuits and get them added to the global + circuit list. */ + tv.tv_usec = 0; + tor_gettimeofday_cache_set(&tv); + c1 = dummy_origin_circuit_new(30); + tv.tv_usec = 10*1000; + tor_gettimeofday_cache_set(&tv); + c2 = dummy_or_circuit_new(20, 20); + + tt_int_op(packed_cell_mem_cost(), ==, + sizeof(packed_cell_t) + MP_POOL_ITEM_OVERHEAD); + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 70); + tt_int_op(cell_queues_check_size(), ==, 0); /* We are still not OOM */ + + tv.tv_usec = 20*1000; + tor_gettimeofday_cache_set(&tv); + c3 = dummy_or_circuit_new(100, 85); + tt_int_op(cell_queues_check_size(), ==, 0); /* We are still not OOM */ + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 255); + + tv.tv_usec = 30*1000; + tor_gettimeofday_cache_set(&tv); + /* Adding this cell will trigger our OOM handler. */ + c4 = dummy_or_circuit_new(2, 0); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 257); + + tt_int_op(cell_queues_check_size(), ==, 1); /* We are now OOM */ + + tt_assert(c1->marked_for_close); + tt_assert(! c2->marked_for_close); + tt_assert(! c3->marked_for_close); + tt_assert(! c4->marked_for_close); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * (257 - 30)); + + circuit_free(c1); + tv.tv_usec = 0; + tor_gettimeofday_cache_set(&tv); /* go back in time */ + c1 = dummy_or_circuit_new(90, 0); + + tv.tv_usec = 40*1000; /* go back to the future */ + tor_gettimeofday_cache_set(&tv); + + tt_int_op(cell_queues_check_size(), ==, 1); /* We are now OOM */ + + tt_assert(c1->marked_for_close); + tt_assert(! c2->marked_for_close); + tt_assert(! c3->marked_for_close); + tt_assert(! c4->marked_for_close); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * (257 - 30)); + + done: + circuit_free(c1); + circuit_free(c2); + circuit_free(c3); + circuit_free(c4); + + UNMOCK(circuit_mark_for_close_); +} + +/** Run unit tests for buffers.c */ +static void +test_oom_streambuf(void *arg) +{ + or_options_t *options = get_options_mutable(); + circuit_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *c4 = NULL, *c5 = NULL; + struct timeval tv = { 1389641159, 0 }; + uint32_t tvms; + int i; + + (void) arg; + + MOCK(circuit_mark_for_close_, circuit_mark_for_close_dummy_); + init_cell_pool(); + + /* Far too low for real life. */ + options->MaxMemInQueues = 81*packed_cell_mem_cost() + 4096 * 34; + options->CellStatistics = 0; + + tt_int_op(cell_queues_check_size(), ==, 0); /* We don't start out OOM. */ + tt_int_op(cell_queues_get_total_allocation(), ==, 0); + tt_int_op(buf_get_total_allocation(), ==, 0); + + /* Start all circuits with a bit of data queued in cells */ + tv.tv_usec = 500*1000; /* go halfway into the second. */ + tor_gettimeofday_cache_set(&tv); + c1 = dummy_or_circuit_new(10,10); + tv.tv_usec = 510*1000; + tor_gettimeofday_cache_set(&tv); + c2 = dummy_origin_circuit_new(20); + tv.tv_usec = 520*1000; + tor_gettimeofday_cache_set(&tv); + c3 = dummy_or_circuit_new(20,20); + tv.tv_usec = 530*1000; + tor_gettimeofday_cache_set(&tv); + c4 = dummy_or_circuit_new(0,0); + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 80); + + tv.tv_usec = 600*1000; + tor_gettimeofday_cache_set(&tv); + + /* Add some connections to c1...c4. */ + for (i = 0; i < 4; ++i) { + edge_connection_t *ec; + /* link it to a circuit */ + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c1, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c2, CONN_TYPE_AP, 1000, 1000); + tt_assert(ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); /* Yes, 4 twice*/ + tt_assert(ec); + tv.tv_usec += 10*1000; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + } + + tv.tv_sec += 1; + tv.tv_usec = 0; + tvms = (uint32_t) tv_to_msec(&tv); + + tt_int_op(circuit_max_queued_cell_age(c1, tvms), ==, 500); + tt_int_op(circuit_max_queued_cell_age(c2, tvms), ==, 490); + tt_int_op(circuit_max_queued_cell_age(c3, tvms), ==, 480); + tt_int_op(circuit_max_queued_cell_age(c4, tvms), ==, 0); + + tt_int_op(circuit_max_queued_data_age(c1, tvms), ==, 390); + tt_int_op(circuit_max_queued_data_age(c2, tvms), ==, 380); + tt_int_op(circuit_max_queued_data_age(c3, tvms), ==, 0); + tt_int_op(circuit_max_queued_data_age(c4, tvms), ==, 370); + + tt_int_op(circuit_max_queued_item_age(c1, tvms), ==, 500); + tt_int_op(circuit_max_queued_item_age(c2, tvms), ==, 490); + tt_int_op(circuit_max_queued_item_age(c3, tvms), ==, 480); + tt_int_op(circuit_max_queued_item_age(c4, tvms), ==, 370); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 80); + tt_int_op(buf_get_total_allocation(), ==, 4096*16*2); + + /* Now give c4 a very old buffer of modest size */ + { + edge_connection_t *ec; + tv.tv_sec -= 1; + tv.tv_usec = 0; + tor_gettimeofday_cache_set(&tv); + ec = dummy_edge_conn_new(c4, CONN_TYPE_EXIT, 1000, 1000); + tt_assert(ec); + } + tt_int_op(buf_get_total_allocation(), ==, 4096*17*2); + tt_int_op(circuit_max_queued_item_age(c4, tvms), ==, 1000); + + tt_int_op(cell_queues_check_size(), ==, 0); + + /* And run over the limit. */ + tv.tv_usec = 800*1000; + tor_gettimeofday_cache_set(&tv); + c5 = dummy_or_circuit_new(0,5); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 85); + tt_int_op(buf_get_total_allocation(), ==, 4096*17*2); + + tt_int_op(cell_queues_check_size(), ==, 1); /* We are now OOM */ + + /* C4 should have died. */ + tt_assert(! c1->marked_for_close); + tt_assert(! c2->marked_for_close); + tt_assert(! c3->marked_for_close); + tt_assert(c4->marked_for_close); + tt_assert(! c5->marked_for_close); + + tt_int_op(cell_queues_get_total_allocation(), ==, + packed_cell_mem_cost() * 85); + tt_int_op(buf_get_total_allocation(), ==, 4096*8*2); + + done: + circuit_free(c1); + circuit_free(c2); + circuit_free(c3); + circuit_free(c4); + circuit_free(c5); + + UNMOCK(circuit_mark_for_close_); +} + +struct testcase_t oom_tests[] = { + { "circbuf", test_oom_circbuf, TT_FORK, NULL, NULL }, + { "streambuf", test_oom_streambuf, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + |