Merge branch 'bug11792_1_squashed'

Conflicts:
	src/or/circuitlist.c

4 files changed, 191 insertions, 23 deletions

diff --git a/src/common/torgzip.c b/src/common/torgzip.c
index 239cab9cac..26ce2183ed 100644
--- a/src/common/torgzip.c
+++ b/src/common/torgzip.c
@@ -46,6 +46,12 @@
 
 #include <zlib.h>
 
+static size_t tor_zlib_state_size_precalc(int inflate,
+                                          int windowbits, int memlevel);
+
+/** Total number of bytes allocated for zlib state */
+static size_t total_zlib_allocation = 0;
+
 /** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't;
  * set to -1 if we haven't checked yet. */
 static int gzip_is_supported = -1;
@@ -411,6 +417,9 @@ struct tor_zlib_state_t {
   size_t input_so_far;
   /** Number of bytes written so far.  Used to detect zlib bombs. */
   size_t output_so_far;
+
+  /** Approximate number of bytes allocated for this object. */
+  size_t allocation;
 };
 
 /** Construct and return a tor_zlib_state_t object using <b>method</b>.  If
@@ -420,6 +429,7 @@ tor_zlib_state_t *
 tor_zlib_new(int compress, compress_method_t method)
 {
   tor_zlib_state_t *out;
+  int bits;
 
   if (method == GZIP_METHOD && !is_gzip_supported()) {
     /* Old zlib version don't support gzip in inflateInit2 */
@@ -432,14 +442,19 @@ tor_zlib_new(int compress, compress_method_t method)
  out->stream.zfree = Z_NULL;
  out->stream.opaque = NULL;
  out->compress = compress;
+ bits = method_bits(method);
  if (compress) {
    if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
-                    method_bits(method), 8, Z_DEFAULT_STRATEGY) != Z_OK)
+                    bits, 8, Z_DEFAULT_STRATEGY) != Z_OK)
      goto err;
  } else {
-   if (inflateInit2(&out->stream, method_bits(method)) != Z_OK)
+   if (inflateInit2(&out->stream, bits) != Z_OK)
      goto err;
  }
+ out->allocation = tor_zlib_state_size_precalc(!compress, bits, 8);
+
+ total_zlib_allocation += out->allocation;
+
  return out;
 
  err:
@@ -517,6 +532,8 @@ tor_zlib_free(tor_zlib_state_t *state)
   if (!state)
     return;
 
+ total_zlib_allocation -= state->allocation;
+
   if (state->compress)
     deflateEnd(&state->stream);
   else
@@ -525,3 +542,47 @@ tor_zlib_free(tor_zlib_state_t *state)
   tor_free(state);
 }
 
+/** Return an approximate number of bytes used in RAM to hold a state with
+ * window bits <b>windowBits</b> and compression level 'memlevel' */
+static size_t
+tor_zlib_state_size_precalc(int inflate, int windowbits, int memlevel)
+{
+  windowbits &= 15;
+
+#define A_FEW_KILOBYTES 2048
+
+  if (inflate) {
+    /* From zconf.h:
+
+       "The memory requirements for inflate are (in bytes) 1 << windowBits
+       that is, 32K for windowBits=15 (default value) plus a few kilobytes
+       for small objects."
+    */
+    return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
+      (1 << 15) + A_FEW_KILOBYTES;
+  } else {
+    /* Also from zconf.h:
+
+       "The memory requirements for deflate are (in bytes):
+            (1 << (windowBits+2)) +  (1 << (memLevel+9))
+        ... plus a few kilobytes for small objects."
+    */
+    return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) +
+      (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
+  }
+#undef A_FEW_KILOBYTES
+}
+
+/** Return the approximate number of bytes allocated for <b>state</b>. */
+size_t
+tor_zlib_state_size(const tor_zlib_state_t *state)
+{
+  return state->allocation;
+}
+
+/** Return the approximate number of bytes allocated for all zlib states. */
+size_t
+tor_zlib_get_total_allocation(void)
+{
+  return total_zlib_allocation;
+}
diff --git a/src/common/torgzip.h b/src/common/torgzip.h
index 5db03fe6e0..d407bf48c6 100644
--- a/src/common/torgzip.h
+++ b/src/common/torgzip.h
@@ -55,5 +55,8 @@ tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state,
                                    int finish);
 void tor_zlib_free(tor_zlib_state_t *state);
 
+size_t tor_zlib_state_size(const tor_zlib_state_t *state);
+size_t tor_zlib_get_total_allocation(void);
+
 #endif
 
diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c
index d6bc67431f..e04b4b9348 100644
--- a/src/or/circuitlist.c
+++ b/src/or/circuitlist.c
@@ -21,6 +21,7 @@
 #include "connection_edge.h"
 #include "connection_or.h"
 #include "control.h"
+#include "main.h"
 #include "networkstatus.h"
 #include "nodelist.h"
 #include "onion.h"
@@ -1824,6 +1825,29 @@ marked_circuit_free_cells(circuit_t *circ)
     cell_queue_clear(& TO_OR_CIRCUIT(circ)->p_chan_cells);
 }
 
+static size_t
+single_conn_free_bytes(connection_t *conn)
+{
+  size_t result = 0;
+  if (conn->inbuf) {
+    result += buf_allocation(conn->inbuf);
+    buf_clear(conn->inbuf);
+  }
+  if (conn->outbuf) {
+    result += buf_allocation(conn->outbuf);
+    buf_clear(conn->outbuf);
+  }
+  if (conn->type == CONN_TYPE_DIR) {
+    dir_connection_t *dir_conn = TO_DIR_CONN(conn);
+    if (dir_conn->zlib_state) {
+      result += tor_zlib_state_size(dir_conn->zlib_state);
+      tor_zlib_free(dir_conn->zlib_state);
+      dir_conn->zlib_state = NULL;
+    }
+  }
+  return result;
+}
+
 /** Aggressively free buffer contents on all the buffers of all streams in the
  * list starting at <b>stream</b>. Return the number of bytes recovered. */
 static size_t
@@ -1832,13 +1856,9 @@ marked_circuit_streams_free_bytes(edge_connection_t *stream)
   size_t result = 0;
   for ( ; stream; stream = stream->next_stream) {
     connection_t *conn = TO_CONN(stream);
-    if (conn->inbuf) {
-      result += buf_allocation(conn->inbuf);
-      buf_clear(conn->inbuf);
-    }
-    if (conn->outbuf) {
-      result += buf_allocation(conn->outbuf);
-      buf_clear(conn->outbuf);
+    result += single_conn_free_bytes(conn);
+    if (conn->linked_conn) {
+      result += single_conn_free_bytes(conn->linked_conn);
     }
   }
   return result;
@@ -1896,6 +1916,28 @@ circuit_max_queued_cell_age(const circuit_t *c, uint32_t now)
   return age;
 }
 
+/** Return the age in milliseconds of the oldest buffer chunk on <b>conn</b>,
+ * where age is taken in milliseconds before the time <b>now</b> (in truncated
+ * milliseconds since the epoch).  If the connection has no data, treat
+ * it as having age zero.
+ **/
+static uint32_t
+conn_get_buffer_age(const connection_t *conn, uint32_t now)
+{
+  uint32_t age = 0, age2;
+  if (conn->outbuf) {
+    age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now);
+    if (age2 > age)
+      age = age2;
+  }
+  if (conn->inbuf) {
+    age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now);
+    if (age2 > age)
+      age = age2;
+  }
+  return age;
+}
+
 /** Return the age in milliseconds of the oldest buffer chunk on any stream in
  * the linked list <b>stream</b>, where age is taken in milliseconds before
  * the time <b>now</b> (in truncated milliseconds since the epoch). */
@@ -1905,18 +1947,15 @@ circuit_get_streams_max_data_age(const edge_connection_t *stream, uint32_t now)
   uint32_t age = 0, age2;
   for (; stream; stream = stream->next_stream) {
     const connection_t *conn = TO_CONN(stream);
-    if (conn->outbuf) {
-      age2 = buf_get_oldest_chunk_timestamp(conn->outbuf, now);
-      if (age2 > age)
-        age = age2;
-    }
-    if (conn->inbuf) {
-      age2 = buf_get_oldest_chunk_timestamp(conn->inbuf, now);
+    age2 = conn_get_buffer_age(conn, now);
+    if (age2 > age)
+      age = age2;
+    if (conn->linked_conn) {
+      age2 = conn_get_buffer_age(conn->linked_conn, now);
       if (age2 > age)
         age = age2;
     }
   }
-
   return age;
 }
 
@@ -1967,6 +2006,26 @@ circuits_compare_by_oldest_queued_item_(const void **a_, const void **b_)
     return -1;
 }
 
+static uint32_t now_ms_for_buf_cmp;
+
+/** Helper to sort a list of circuit_t by age of oldest item, in descending
+ * order. */
+static int
+conns_compare_by_buffer_age_(const void **a_, const void **b_)
+{
+  const connection_t *a = *a_;
+  const connection_t *b = *b_;
+  time_t age_a = conn_get_buffer_age(a, now_ms_for_buf_cmp);
+  time_t age_b = conn_get_buffer_age(b, now_ms_for_buf_cmp);
+
+  if (age_a < age_b)
+    return 1;
+  else if (age_a == age_b)
+    return 0;
+  else
+    return -1;
+}
+
 #define FRACTION_OF_DATA_TO_RETAIN_ON_OOM 0.90
 
 /** We're out of memory for cells, having allocated <b>current_allocation</b>
@@ -1976,9 +2035,12 @@ void
 circuits_handle_oom(size_t current_allocation)
 {
   smartlist_t *circlist;
+  smartlist_t *connection_array = get_connection_array();
+  int conn_idx;
   size_t mem_to_recover;
   size_t mem_recovered=0;
   int n_circuits_killed=0;
+  int n_dirconns_killed=0;
   struct timeval now;
   uint32_t now_ms;
   log_notice(LD_GENERAL, "We're low on memory.  Killing circuits with "
@@ -2021,11 +2083,48 @@ circuits_handle_oom(size_t current_allocation)
     circ->global_circuitlist_idx = circ_sl_idx;
   } SMARTLIST_FOREACH_END(circ);
 
-  /* Okay, now the worst circuits are at the front of the list. Let's mark
-   * them, and reclaim their storage aggressively. */
+
+  /* Now sort the connection array ... */
+  now_ms_for_buf_cmp = now_ms;
+  smartlist_sort(connection_array, conns_compare_by_buffer_age_);
+  now_ms_for_buf_cmp = 0;
+
+  /* Fix up the connection array to its new order. */
+  SMARTLIST_FOREACH_BEGIN(connection_array, connection_t *, conn) {
+    conn->conn_array_index = conn_sl_idx;
+  } SMARTLIST_FOREACH_END(conn);
+
+  /* Okay, now the worst circuits and connections are at the front of their
+   * respective lists. Let's mark them, and reclaim their storage
+   * aggressively. */
+  conn_idx = 0;
   SMARTLIST_FOREACH_BEGIN(circlist, circuit_t *, circ) {
-    size_t n = n_cells_in_circ_queues(circ);
+    size_t n;
     size_t freed;
+
+    /* Free storage in any non-linked directory connections that have buffered
+     * data older than this circuit. */
+    while (conn_idx < smartlist_len(connection_array)) {
+      connection_t *conn = smartlist_get(connection_array, conn_idx);
+      uint32_t conn_age = conn_get_buffer_age(conn, now_ms);
+      if (conn_age < circ->age_tmp) {
+        break;
+      }
+      if (conn->type == CONN_TYPE_DIR && conn->linked_conn == NULL) {
+        if (!conn->marked_for_close)
+          connection_mark_for_close(conn);
+        mem_recovered += single_conn_free_bytes(conn);
+
+        ++n_dirconns_killed;
+
+        if (mem_recovered >= mem_to_recover)
+          goto done_recovering_mem;
+      }
+      ++conn_idx;
+    }
+
+    /* Now, kill the circuit. */
+    n = n_cells_in_circ_queues(circ);
     if (! circ->marked_for_close) {
       circuit_mark_for_close(circ, END_CIRC_REASON_RESOURCELIMIT);
     }
@@ -2038,9 +2137,11 @@ circuits_handle_oom(size_t current_allocation)
     mem_recovered += freed;
 
     if (mem_recovered >= mem_to_recover)
-      break;
+      goto done_recovering_mem;
   } SMARTLIST_FOREACH_END(circ);
 
+ done_recovering_mem:
+
 #ifdef ENABLE_MEMPOOLS
   clean_cell_pool(); /* In case this helps. */
 #endif /* ENABLE_MEMPOOLS */
@@ -2048,10 +2149,12 @@ circuits_handle_oom(size_t current_allocation)
                               chunks. */
 
   log_notice(LD_GENERAL, "Removed "U64_FORMAT" bytes by killing %d circuits; "
-             "%d circuits remain alive.",
+             "%d circuits remain alive. Also killed %d non-linked directory "
+             "connections.",
              U64_PRINTF_ARG(mem_recovered),
              n_circuits_killed,
-             smartlist_len(circlist) - n_circuits_killed);
+             smartlist_len(circlist) - n_circuits_killed,
+             n_dirconns_killed);
 }
 
 /** Verify that cpath layer <b>cp</b> has all of its invariants
diff --git a/src/or/relay.c b/src/or/relay.c
index bb60469066..d97c84fb07 100644
--- a/src/or/relay.c
+++ b/src/or/relay.c
@@ -2439,6 +2439,7 @@ cell_queues_check_size(void)
 {
   size_t alloc = cell_queues_get_total_allocation();
   alloc += buf_get_total_allocation();
+  alloc += tor_zlib_get_total_allocation();
   if (alloc >= get_options()->MaxMemInQueues) {
     circuits_handle_oom(alloc);
     return 1;