From ec59167cae1f5b3057ed722857d78ec78239e991 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Mon, 18 Aug 2014 15:21:50 -0400 Subject: When counting memory from closing a connection, count the dir conn too Fix part of bug 11972 --- src/or/circuitlist.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'src/or') diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index f3a83503ef..9aeb3eb19a 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1799,6 +1799,21 @@ marked_circuit_free_cells(circuit_t *circ) cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_chan_cells); } +static size_t +marked_circuit_single_conn_free_bytes(connection_t *conn) +{ + size_t result = 0; + if (conn->inbuf) { + result += buf_allocation(conn->inbuf); + buf_clear(conn->inbuf); + } + if (conn->outbuf) { + result += buf_allocation(conn->outbuf); + buf_clear(conn->outbuf); + } + return result; +} + /** Aggressively free buffer contents on all the buffers of all streams in the * list starting at stream. Return the number of bytes recovered. */ static size_t @@ -1807,13 +1822,9 @@ marked_circuit_streams_free_bytes(edge_connection_t *stream) size_t result = 0; for ( ; stream; stream = stream->next_stream) { connection_t *conn = TO_CONN(stream); - if (conn->inbuf) { - result += buf_allocation(conn->inbuf); - buf_clear(conn->inbuf); - } - if (conn->outbuf) { - result += buf_allocation(conn->outbuf); - buf_clear(conn->outbuf); + result += marked_circuit_single_conn_free_bytes(conn); + if (conn->linked_conn) { + result += marked_circuit_single_conn_free_bytes(conn->linked_conn); } } return result; -- cgit v1.2.3-54-g00ecf From 8e55cafd672353979e7628d5dad9e12429b401dd Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 19 Aug 2014 10:59:15 -0400 Subject: Count zlib buffer memory towards OOM totals. Part of 11792. (Uses the zlib-endorsed formula for memory needs for inflate/deflate from "zconf.h".) --- changes/bug11792 | 8 ++++++- src/common/torgzip.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++-- src/common/torgzip.h | 3 +++ src/or/circuitlist.c | 8 +++++++ src/or/relay.c | 1 + 5 files changed, 82 insertions(+), 3 deletions(-) (limited to 'src/or') diff --git a/changes/bug11792 b/changes/bug11792 index 1d381891db..7395b4aba5 100644 --- a/changes/bug11792 +++ b/changes/bug11792 @@ -2,4 +2,10 @@ - When closing an edge connection because we've run out of memory, also count the amount of memory that any tunnelled directory connection attached to that connection had consumed. Part of - ticket 11792. \ No newline at end of file + ticket 11792. + + - When considering whether we're running low on memory, consider + memory that was allocated as part of zlib buffers as well. + Count that memory as reclaimed by our OOM handler. Part of + ticket 11792. + diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 15451ee30d..67fd742818 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -46,6 +46,12 @@ #include +static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static size_t total_zlib_allocation = 0; + /** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't; * set to -1 if we haven't checked yet. */ static int gzip_is_supported = -1; @@ -411,6 +417,9 @@ struct tor_zlib_state_t { size_t input_so_far; /** Number of bytes written so far. Used to detect zlib bombs. */ size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; }; /** Construct and return a tor_zlib_state_t object using method. If @@ -420,6 +429,7 @@ tor_zlib_state_t * tor_zlib_new(int compress, compress_method_t method) { tor_zlib_state_t *out; + int bits; if (method == GZIP_METHOD && !is_gzip_supported()) { /* Old zlib version don't support gzip in inflateInit2 */ @@ -432,14 +442,19 @@ tor_zlib_new(int compress, compress_method_t method) out->stream.zfree = Z_NULL; out->stream.opaque = NULL; out->compress = compress; + bits = method_bits(method); if (compress) { if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method), 8, Z_DEFAULT_STRATEGY) != Z_OK) + bits, 8, Z_DEFAULT_STRATEGY) != Z_OK) goto err; } else { - if (inflateInit2(&out->stream, method_bits(method)) != Z_OK) + if (inflateInit2(&out->stream, bits) != Z_OK) goto err; } + out->allocation = tor_zlib_state_size_precalc(!compress, bits, 8); + + total_zlib_allocation += out->allocation; + return out; err: @@ -517,6 +532,8 @@ tor_zlib_free(tor_zlib_state_t *state) if (!state) return; + total_zlib_allocation -= state->allocation; + if (state->compress) deflateEnd(&state->stream); else @@ -525,3 +542,47 @@ tor_zlib_free(tor_zlib_state_t *state) tor_free(state); } +/** Return an approximate number of bytes used in RAM to hold a state with + * window bits windowBits and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Return the approximate number of bytes allocated for state. */ +size_t +tor_zlib_state_size(const tor_zlib_state_t *state) +{ + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return total_zlib_allocation; +} diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 5db03fe6e0..d407bf48c6 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -55,5 +55,8 @@ tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, int finish); void tor_zlib_free(tor_zlib_state_t *state); +size_t tor_zlib_state_size(const tor_zlib_state_t *state); +size_t tor_zlib_get_total_allocation(void); + #endif diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 9aeb3eb19a..c738f9357f 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1811,6 +1811,14 @@ marked_circuit_single_conn_free_bytes(connection_t *conn) result += buf_allocation(conn->outbuf); buf_clear(conn->outbuf); } + if (conn->type == CONN_TYPE_DIR) { + dir_connection_t *dir_conn = TO_DIR_CONN(conn); + if (dir_conn->zlib_state) { + result += tor_zlib_state_size(dir_conn->zlib_state); + tor_zlib_free(dir_conn->zlib_state); + dir_conn->zlib_state = NULL; + } + } return result; } diff --git a/src/or/relay.c b/src/or/relay.c index f42602d412..7d7ef7a835 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -2433,6 +2433,7 @@ cell_queues_check_size(void) { size_t alloc = cell_queues_get_total_allocation(); alloc += buf_get_total_allocation(); + alloc += tor_zlib_get_total_allocation(); if (alloc >= get_options()->MaxMemInQueues) { circuits_handle_oom(alloc); return 1; -- cgit v1.2.3-54-g00ecf From 68e430a6fb3056c1265da04fbb08004ddd449525 Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 19 Aug 2014 11:25:48 -0400 Subject: Kill non-tunneled directory connections when handling OOM. Another part of 11792. --- changes/bug11792 | 4 ++ src/or/circuitlist.c | 118 ++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 101 insertions(+), 21 deletions(-) (limited to 'src/or') diff --git a/changes/bug11792 b/changes/bug11792 index 7395b4aba5..66f7df833c 100644 --- a/changes/bug11792 +++ b/changes/bug11792 @@ -9,3 +9,7 @@ Count that memory as reclaimed by our OOM handler. Part of ticket 11792. + - When handling out-of-memory conditions, also look at + non-tunnneled directory connections, and kill the ones that have + had data sitting on them for the longest. Part of ticket 11792. + diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index c738f9357f..bcdd858be0 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -21,6 +21,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "main.h" #include "networkstatus.h" #include "nodelist.h" #include "onion.h" @@ -1800,7 +1801,7 @@ marked_circuit_free_cells(circuit_t *circ) } static size_t -marked_circuit_single_conn_free_bytes(connection_t *conn) +single_conn_free_bytes(connection_t *conn) { size_t result = 0; if (conn->inbuf) { @@ -1830,9 +1831,9 @@ marked_circuit_streams_free_bytes(edge_connection_t *stream) size_t result = 0; for ( ; stream; stream = stream->next_stream) { connection_t *conn = TO_CONN(stream); - result += marked_circuit_single_conn_free_bytes(conn); + result += single_conn_free_bytes(conn); if (conn->linked_conn) { - result += marked_circuit_single_conn_free_bytes(conn->linked_conn); + result += single_conn_free_bytes(conn->linked_conn); } } return result; @@ -1890,6 +1891,28 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now) return age; } +/** Return the age in milliseconds of the oldest buffer chunk on conn, + * where age is taken in milliseconds before the time now (in truncated + * milliseconds since the epoch). If the connection has no data, treat + * it as having age zero. + **/ +static uint32_t +conn_get_buffer_age(const connection_t *conn, uint32_t now) +{ + uint32_t age = 0, age2; + if (conn->outbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); + if (age2 > age) + age = age2; + } + if (conn->inbuf) { + age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); + if (age2 > age) + age = age2; + } + return age; +} + /** Return the age in milliseconds of the oldest buffer chunk on any stream in * the linked list stream, where age is taken in milliseconds before * the time now (in truncated milliseconds since the epoch). */ @@ -1899,18 +1922,10 @@ circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) uint32_t age = 0, age2; for (; stream; stream = stream->next_stream) { const connection_t *conn = TO_CONN(stream); - if (conn->outbuf) { - age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now); - if (age2 > age) - age = age2; - } - if (conn->inbuf) { - age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now); - if (age2 > age) - age = age2; - } + age2 = conn_get_buffer_age(conn, now); + if (age2 > age) + age = age2; } - return age; } @@ -1961,6 +1976,26 @@ circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_) return -1; } +static uint32_t now_ms_for_buf_cmp; + +/** Helper to sort a list of circuit_t by age of oldest item, in descending + * order. */ +static int +conns_compare_by_buffer_age_(const void **a_, const void **b_) +{ + const connection_t *a = *a_; + const connection_t *b = *b_; + time_t age_a = conn_get_buffer_age(a, now_ms_for_buf_cmp); + time_t age_b = conn_get_buffer_age(b, now_ms_for_buf_cmp); + + if (age_a < age_b) + return 1; + else if (age_a == age_b) + return 0; + else + return -1; +} + #define FRACTION_OF_DATA_TO_RETAIN_ON_OOM 0.90 /** We're out of memory for cells, having allocated current_allocation @@ -1971,10 +2006,13 @@ circuits_handle_oom(size_t current_allocation) { /* Let's hope there's enough slack space for this allocation here... */ smartlist_t *circlist = smartlist_new(); + smartlist_t *connection_array = get_connection_array(); + int conn_idx; circuit_t *circ; size_t mem_to_recover; size_t mem_recovered=0; int n_circuits_killed=0; + int n_dirconns_killed=0; struct timeval now; uint32_t now_ms; log_notice(LD_GENERAL, "We're low on memory. Killing circuits with " @@ -2013,12 +2051,46 @@ circuits_handle_oom(size_t current_allocation) /* This is O(n log n); there are faster algorithms we could use instead. * Let's hope this doesn't happen enough to be in the critical path. */ smartlist_sort(circlist, circuits_compare_by_oldest_queued_item_); - - /* Okay, now the worst circuits are at the front of the list. Let's mark - * them, and reclaim their storage aggressively. */ + now_ms_for_buf_cmp = now_ms; + smartlist_sort(connection_array, conns_compare_by_buffer_age_); + now_ms_for_buf_cmp = 0; + + /* Fix up the connection array to its new order. */ + SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) { + conn->conn_array_index = conn_sl_idx; + } SMARTLIST_FOREACH_END(conn); + + /* Okay, now the worst circuits and connections are at the front of their + * respective lists. Let's mark them, and reclaim their storage + * aggressively. */ + conn_idx = 0; SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) { - size_t n = n_cells_in_circ_queues(circ); + size_t n; size_t freed; + + /* Free storage in any non-linked directory connections that have buffered + * data older than this circuit. */ + while (conn_idx < smartlist_len(connection_array)) { + connection_t *conn = smartlist_get(connection_array, conn_idx); + uint32_t conn_age = conn_get_buffer_age(conn, now_ms); + if (conn_age < circ->age_tmp) { + break; + } + if (conn->type == CONN_TYPE_DIR && conn->linked_conn == NULL) { + if (!conn->marked_for_close) + connection_mark_for_close(conn); + mem_recovered += single_conn_free_bytes(conn); + + ++n_dirconns_killed; + + if (mem_recovered >= mem_to_recover) + goto done_recovering_mem; + } + ++conn_idx; + } + + /* Now, kill the circuit. */ + n = n_cells_in_circ_queues(circ); if (! circ->marked_for_close) { circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT); } @@ -2031,9 +2103,11 @@ circuits_handle_oom(size_t current_allocation) mem_recovered += freed; if (mem_recovered >= mem_to_recover) - break; + goto done_recovering_mem; } SMARTLIST_FOREACH_END(circ); + done_recovering_mem: + #ifdef ENABLE_MEMPOOLS clean_cell_pool(); /* In case this helps. */ #endif /* ENABLE_MEMPOOLS */ @@ -2041,10 +2115,12 @@ circuits_handle_oom(size_t current_allocation) chunks. */ log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; " - "%d circuits remain alive.", + "%d circuits remain alive. Also killed %d non-linked directory " + "connections.", U64_PRINTF_ARG(mem_recovered), n_circuits_killed, - smartlist_len(circlist) - n_circuits_killed); + smartlist_len(circlist) - n_circuits_killed, + n_dirconns_killed); smartlist_free(circlist); } -- cgit v1.2.3-54-g00ecf From d6033843a49ff029d9f8d070b3c1f446a6edc9fa Mon Sep 17 00:00:00 2001 From: Nick Mathewson Date: Tue, 19 Aug 2014 11:27:52 -0400 Subject: When looking for conns to close, count the age of linked queued data Specifically, count the age of the data queued in a linked directory connection's buffers when counting a stream's age. --- src/or/circuitlist.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/or') diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index bcdd858be0..68534208c6 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1925,6 +1925,11 @@ circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now) age2 = conn_get_buffer_age(conn, now); if (age2 > age) age = age2; + if (conn->linked_conn) { + age2 = conn_get_buffer_age(conn->linked_conn, now); + if (age2 > age) + age = age2; + } } return age; } -- cgit v1.2.3-54-g00ecf