23 files changed, 1606 insertions, 131 deletions

diff --git a/src/common/aes.c b/src/common/aes.c
index 73abef143e..94d33d4392 100644
--- a/src/common/aes.c
+++ b/src/common/aes.c
@@ -66,7 +66,7 @@ ENABLE_GCC_WARNING(redundant-decls)
 #elif OPENSSL_VERSION_NUMBER >= OPENSSL_V_NOPATCH(1,0,1) &&               \
   (defined(__i386) || defined(__i386__) || defined(_M_IX86) ||          \
    defined(__x86_64) || defined(__x86_64__) ||                          \
-   defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__))          \
+   defined(_M_AMD64) || defined(_M_X64) || defined(__INTEL__))
 
 #define USE_EVP_AES_CTR
 
diff --git a/src/common/buffers.c b/src/common/buffers.c
new file mode 100644
index 0000000000..50673646df
--- /dev/null
+++ b/src/common/buffers.c
@@ -0,0 +1,1069 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers.c
+ * \brief Implements a generic buffer interface.
+ *
+ * A buf_t is a (fairly) opaque byte-oriented FIFO that can read to or flush
+ * from memory, sockets, file descriptors, TLS connections, or another buf_t.
+ * Buffers are implemented as linked lists of memory chunks.
+ *
+ * All socket-backed and TLS-based connection_t objects have a pair of
+ * buffers: one for incoming data, and one for outcoming data.  These are fed
+ * and drained from functions in connection.c, trigged by events that are
+ * monitored in main.c.
+ **/
+
+#define BUFFERS_PRIVATE
+#include "orconfig.h"
+#include <stddef.h>
+#include "buffers.h"
+#include "compat.h"
+#include "compress.h"
+#include "util.h"
+#include "torint.h"
+#include "torlog.h"
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+//#define PARANOIA
+
+#ifdef PARANOIA
+/** Helper: If PARANOIA is defined, assert that the buffer in local variable
+ * <b>buf</b> is well-formed. */
+#define check() STMT_BEGIN buf_assert_ok(buf); STMT_END
+#else
+#define check() STMT_NIL
+#endif
+
+/* Implementation notes:
+ *
+ * After flirting with memmove, and dallying with ring-buffers, we're finally
+ * getting up to speed with the 1970s and implementing buffers as a linked
+ * list of small chunks.  Each buffer has such a list; data is removed from
+ * the head of the list, and added at the tail.  The list is singly linked,
+ * and the buffer keeps a pointer to the head and the tail.
+ *
+ * Every chunk, except the tail, contains at least one byte of data.  Data in
+ * each chunk is contiguous.
+ *
+ * When you need to treat the first N characters on a buffer as a contiguous
+ * string, use the buf_pullup function to make them so.  Don't do this more
+ * than necessary.
+ *
+ * The major free Unix kernels have handled buffers like this since, like,
+ * forever.
+ */
+
+/* Chunk manipulation functions */
+
+#define CHUNK_HEADER_LEN offsetof(chunk_t, mem[0])
+
+/* We leave this many NUL bytes at the end of the buffer. */
+#ifdef DISABLE_MEMORY_SENTINELS
+#define SENTINEL_LEN 0
+#else
+#define SENTINEL_LEN 4
+#endif
+
+/* Header size plus NUL bytes at the end */
+#define CHUNK_OVERHEAD (CHUNK_HEADER_LEN + SENTINEL_LEN)
+
+/** Return the number of bytes needed to allocate a chunk to hold
+ * <b>memlen</b> bytes. */
+#define CHUNK_ALLOC_SIZE(memlen) (CHUNK_OVERHEAD + (memlen))
+/** Return the number of usable bytes in a chunk allocated with
+ * malloc(<b>memlen</b>). */
+#define CHUNK_SIZE_WITH_ALLOC(memlen) ((memlen) - CHUNK_OVERHEAD)
+
+#define DEBUG_SENTINEL
+
+#if defined(DEBUG_SENTINEL) && !defined(DISABLE_MEMORY_SENTINELS)
+#define DBG_S(s) s
+#else
+#define DBG_S(s) (void)0
+#endif
+
+#ifdef DISABLE_MEMORY_SENTINELS
+#define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL
+#else
+#define CHUNK_SET_SENTINEL(chunk, alloclen) do {                        \
+    uint8_t *a = (uint8_t*) &(chunk)->mem[(chunk)->memlen];             \
+    DBG_S(uint8_t *b = &((uint8_t*)(chunk))[(alloclen)-SENTINEL_LEN]);  \
+    DBG_S(tor_assert(a == b));                                          \
+    memset(a,0,SENTINEL_LEN);                                           \
+  } while (0)
+#endif
+
+/** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem,
+ * to free up space at the end. */
+static inline void
+chunk_repack(chunk_t *chunk)
+{
+  if (chunk->datalen && chunk->data != &chunk->mem[0]) {
+    memmove(chunk->mem, chunk->data, chunk->datalen);
+  }
+  chunk->data = &chunk->mem[0];
+}
+
+/** Keep track of total size of allocated chunks for consistency asserts */
+static size_t total_bytes_allocated_in_chunks = 0;
+static void
+buf_chunk_free_unchecked(chunk_t *chunk)
+{
+  if (!chunk)
+    return;
+#ifdef DEBUG_CHUNK_ALLOC
+  tor_assert(CHUNK_ALLOC_SIZE(chunk->memlen) == chunk->DBG_alloc);
+#endif
+  tor_assert(total_bytes_allocated_in_chunks >=
+             CHUNK_ALLOC_SIZE(chunk->memlen));
+  total_bytes_allocated_in_chunks -= CHUNK_ALLOC_SIZE(chunk->memlen);
+  tor_free(chunk);
+}
+static inline chunk_t *
+chunk_new_with_alloc_size(size_t alloc)
+{
+  chunk_t *ch;
+  ch = tor_malloc(alloc);
+  ch->next = NULL;
+  ch->datalen = 0;
+#ifdef DEBUG_CHUNK_ALLOC
+  ch->DBG_alloc = alloc;
+#endif
+  ch->memlen = CHUNK_SIZE_WITH_ALLOC(alloc);
+  total_bytes_allocated_in_chunks += alloc;
+  ch->data = &ch->mem[0];
+  CHUNK_SET_SENTINEL(ch, alloc);
+  return ch;
+}
+
+/** Expand <b>chunk</b> until it can hold <b>sz</b> bytes, and return a
+ * new pointer to <b>chunk</b>.  Old pointers are no longer valid. */
+static inline chunk_t *
+chunk_grow(chunk_t *chunk, size_t sz)
+{
+  off_t offset;
+  const size_t memlen_orig = chunk->memlen;
+  const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
+  const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
+  tor_assert(sz > chunk->memlen);
+  offset = chunk->data - chunk->mem;
+  chunk = tor_realloc(chunk, new_alloc);
+  chunk->memlen = sz;
+  chunk->data = chunk->mem + offset;
+#ifdef DEBUG_CHUNK_ALLOC
+  tor_assert(chunk->DBG_alloc == orig_alloc);
+  chunk->DBG_alloc = new_alloc;
+#endif
+  total_bytes_allocated_in_chunks += new_alloc - orig_alloc;
+  CHUNK_SET_SENTINEL(chunk, new_alloc);
+  return chunk;
+}
+
+/** Every chunk should take up at least this many bytes. */
+#define MIN_CHUNK_ALLOC 256
+/** No chunk should take up more than this many bytes. */
+#define MAX_CHUNK_ALLOC 65536
+
+/** Return the allocation size we'd like to use to hold <b>target</b>
+ * bytes. */
+size_t
+buf_preferred_chunk_size(size_t target)
+{
+  tor_assert(target <= SIZE_T_CEILING - CHUNK_OVERHEAD);
+  if (CHUNK_ALLOC_SIZE(target) >= MAX_CHUNK_ALLOC)
+    return CHUNK_ALLOC_SIZE(target);
+  size_t sz = MIN_CHUNK_ALLOC;
+  while (CHUNK_SIZE_WITH_ALLOC(sz) < target) {
+    sz <<= 1;
+  }
+  return sz;
+}
+
+/** Collapse data from the first N chunks from <b>buf</b> into buf->head,
+ * growing it as necessary, until buf->head has the first <b>bytes</b> bytes
+ * of data from the buffer, or until buf->head has all the data in <b>buf</b>.
+ *
+ * Set *<b>head_out</b> to point to the first byte of available data, and
+ * *<b>len_out</b> to the number of bytes of data available at
+ * *<b>head_out</b>. Note that *<b>len_out</b> may be more or less than
+ * <b>bytes</b>, depending on the number of bytes available.
+ */
+void
+buf_pullup(buf_t *buf, size_t bytes, const char **head_out, size_t *len_out)
+{
+  chunk_t *dest, *src;
+  size_t capacity;
+  if (!buf->head) {
+    *head_out = NULL;
+    *len_out = 0;
+    return;
+  }
+
+  check();
+  if (buf->datalen < bytes)
+    bytes = buf->datalen;
+
+  capacity = bytes;
+  if (buf->head->datalen >= bytes) {
+    *head_out = buf->head->data;
+    *len_out = buf->head->datalen;
+    return;
+  }
+
+  if (buf->head->memlen >= capacity) {
+    /* We don't need to grow the first chunk, but we might need to repack it.*/
+    size_t needed = capacity - buf->head->datalen;
+    if (CHUNK_REMAINING_CAPACITY(buf->head) < needed)
+      chunk_repack(buf->head);
+    tor_assert(CHUNK_REMAINING_CAPACITY(buf->head) >= needed);
+  } else {
+    chunk_t *newhead;
+    size_t newsize;
+    /* We need to grow the chunk. */
+    chunk_repack(buf->head);
+    newsize = CHUNK_SIZE_WITH_ALLOC(buf_preferred_chunk_size(capacity));
+    newhead = chunk_grow(buf->head, newsize);
+    tor_assert(newhead->memlen >= capacity);
+    if (newhead != buf->head) {
+      if (buf->tail == buf->head)
+        buf->tail = newhead;
+      buf->head = newhead;
+    }
+  }
+
+  dest = buf->head;
+  while (dest->datalen < bytes) {
+    size_t n = bytes - dest->datalen;
+    src = dest->next;
+    tor_assert(src);
+    if (n >= src->datalen) {
+      memcpy(CHUNK_WRITE_PTR(dest), src->data, src->datalen);
+      dest->datalen += src->datalen;
+      dest->next = src->next;
+      if (buf->tail == src)
+        buf->tail = dest;
+      buf_chunk_free_unchecked(src);
+    } else {
+      memcpy(CHUNK_WRITE_PTR(dest), src->data, n);
+      dest->datalen += n;
+      src->data += n;
+      src->datalen -= n;
+      tor_assert(dest->datalen == bytes);
+    }
+  }
+
+  check();
+  *head_out = buf->head->data;
+  *len_out = buf->head->datalen;
+}
+
+#ifdef TOR_UNIT_TESTS
+/* Write sz bytes from cp into a newly allocated buffer buf.
+ * Returns NULL when passed a NULL cp or zero sz.
+ * Asserts on failure: only for use in unit tests.
+ * buf must be freed using buf_free(). */
+buf_t *
+buf_new_with_data(const char *cp, size_t sz)
+{
+  /* Validate arguments */
+  if (!cp || sz <= 0) {
+    return NULL;
+  }
+
+  tor_assert(sz < SSIZE_T_CEILING);
+
+  /* Allocate a buffer */
+  buf_t *buf = buf_new_with_capacity(sz);
+  tor_assert(buf);
+  buf_assert_ok(buf);
+  tor_assert(!buf->head);
+
+  /* Allocate a chunk that is sz bytes long */
+  buf->head = chunk_new_with_alloc_size(CHUNK_ALLOC_SIZE(sz));
+  buf->tail = buf->head;
+  tor_assert(buf->head);
+  buf_assert_ok(buf);
+  tor_assert(buf_allocation(buf) >= sz);
+
+  /* Copy the data and size the buffers */
+  tor_assert(sz <= buf_slack(buf));
+  tor_assert(sz <= CHUNK_REMAINING_CAPACITY(buf->head));
+  memcpy(&buf->head->mem[0], cp, sz);
+  buf->datalen = sz;
+  buf->head->datalen = sz;
+  buf->head->data = &buf->head->mem[0];
+  buf_assert_ok(buf);
+
+  /* Make sure everything is large enough */
+  tor_assert(buf_allocation(buf) >= sz);
+  tor_assert(buf_allocation(buf) >= buf_datalen(buf) + buf_slack(buf));
+  /* Does the buffer implementation allocate more than the requested size?
+   * (for example, by rounding up). If so, these checks will fail. */
+  tor_assert(buf_datalen(buf) == sz);
+  tor_assert(buf_slack(buf) == 0);
+
+  return buf;
+}
+#endif
+
+/** Remove the first <b>n</b> bytes from buf. */
+void
+buf_drain(buf_t *buf, size_t n)
+{
+  tor_assert(buf->datalen >= n);
+  while (n) {
+    tor_assert(buf->head);
+    if (buf->head->datalen > n) {
+      buf->head->datalen -= n;
+      buf->head->data += n;
+      buf->datalen -= n;
+      return;
+    } else {
+      chunk_t *victim = buf->head;
+      n -= victim->datalen;
+      buf->datalen -= victim->datalen;
+      buf->head = victim->next;
+      if (buf->tail == victim)
+        buf->tail = NULL;
+      buf_chunk_free_unchecked(victim);
+    }
+  }
+  check();
+}
+
+/** Create and return a new buf with default chunk capacity <b>size</b>.
+ */
+buf_t *
+buf_new_with_capacity(size_t size)
+{
+  buf_t *b = buf_new();
+  b->default_chunk_size = buf_preferred_chunk_size(size);
+  return b;
+}
+
+/** Allocate and return a new buffer with default capacity. */
+buf_t *
+buf_new(void)
+{
+  buf_t *buf = tor_malloc_zero(sizeof(buf_t));
+  buf->magic = BUFFER_MAGIC;
+  buf->default_chunk_size = 4096;
+  return buf;
+}
+
+size_t
+buf_get_default_chunk_size(const buf_t *buf)
+{
+  return buf->default_chunk_size;
+}
+
+/** Remove all data from <b>buf</b>. */
+void
+buf_clear(buf_t *buf)
+{
+  chunk_t *chunk, *next;
+  buf->datalen = 0;
+  for (chunk = buf->head; chunk; chunk = next) {
+    next = chunk->next;
+    buf_chunk_free_unchecked(chunk);
+  }
+  buf->head = buf->tail = NULL;
+}
+
+/** Return the number of bytes stored in <b>buf</b> */
+MOCK_IMPL(size_t,
+buf_datalen, (const buf_t *buf))
+{
+  return buf->datalen;
+}
+
+/** Return the total length of all chunks used in <b>buf</b>. */
+size_t
+buf_allocation(const buf_t *buf)
+{
+  size_t total = 0;
+  const chunk_t *chunk;
+  for (chunk = buf->head; chunk; chunk = chunk->next) {
+    total += CHUNK_ALLOC_SIZE(chunk->memlen);
+  }
+  return total;
+}
+
+/** Return the number of bytes that can be added to <b>buf</b> without
+ * performing any additional allocation. */
+size_t
+buf_slack(const buf_t *buf)
+{
+  if (!buf->tail)
+    return 0;
+  else
+    return CHUNK_REMAINING_CAPACITY(buf->tail);
+}
+
+/** Release storage held by <b>buf</b>. */
+void
+buf_free(buf_t *buf)
+{
+  if (!buf)
+    return;
+
+  buf_clear(buf);
+  buf->magic = 0xdeadbeef;
+  tor_free(buf);
+}
+
+/** Return a new copy of <b>in_chunk</b> */
+static chunk_t *
+chunk_copy(const chunk_t *in_chunk)
+{
+  chunk_t *newch = tor_memdup(in_chunk, CHUNK_ALLOC_SIZE(in_chunk->memlen));
+  total_bytes_allocated_in_chunks += CHUNK_ALLOC_SIZE(in_chunk->memlen);
+#ifdef DEBUG_CHUNK_ALLOC
+  newch->DBG_alloc = CHUNK_ALLOC_SIZE(in_chunk->memlen);
+#endif
+  newch->next = NULL;
+  if (in_chunk->data) {
+    off_t offset = in_chunk->data - in_chunk->mem;
+    newch->data = newch->mem + offset;
+  }
+  return newch;
+}
+
+/** Return a new copy of <b>buf</b> */
+buf_t *
+buf_copy(const buf_t *buf)
+{
+  chunk_t *ch;
+  buf_t *out = buf_new();
+  out->default_chunk_size = buf->default_chunk_size;
+  for (ch = buf->head; ch; ch = ch->next) {
+    chunk_t *newch = chunk_copy(ch);
+    if (out->tail) {
+      out->tail->next = newch;
+      out->tail = newch;
+    } else {
+      out->head = out->tail = newch;
+    }
+  }
+  out->datalen = buf->datalen;
+  return out;
+}
+
+/** Append a new chunk with enough capacity to hold <b>capacity</b> bytes to
+ * the tail of <b>buf</b>.  If <b>capped</b>, don't allocate a chunk bigger
+ * than MAX_CHUNK_ALLOC. */
+chunk_t *
+buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped)
+{
+  chunk_t *chunk;
+
+  if (CHUNK_ALLOC_SIZE(capacity) < buf->default_chunk_size) {
+    chunk = chunk_new_with_alloc_size(buf->default_chunk_size);
+  } else if (capped && CHUNK_ALLOC_SIZE(capacity) > MAX_CHUNK_ALLOC) {
+    chunk = chunk_new_with_alloc_size(MAX_CHUNK_ALLOC);
+  } else {
+    chunk = chunk_new_with_alloc_size(buf_preferred_chunk_size(capacity));
+  }
+
+  chunk->inserted_time = (uint32_t)monotime_coarse_absolute_msec();
+
+  if (buf->tail) {
+    tor_assert(buf->head);
+    buf->tail->next = chunk;
+    buf->tail = chunk;
+  } else {
+    tor_assert(!buf->head);
+    buf->head = buf->tail = chunk;
+  }
+  check();
+  return chunk;
+}
+
+/** Return the age of the oldest chunk in the buffer <b>buf</b>, in
+ * milliseconds.  Requires the current monotonic time, in truncated msec,
+ * as its input <b>now</b>.
+ */
+uint32_t
+buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now)
+{
+  if (buf->head) {
+    return now - buf->head->inserted_time;
+  } else {
+    return 0;
+  }
+}
+
+size_t
+buf_get_total_allocation(void)
+{
+  return total_bytes_allocated_in_chunks;
+}
+
+/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
+ * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
+ * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
+ * and the number of bytes read otherwise. */
+static inline int
+read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
+              int *reached_eof, int *socket_error)
+{
+  ssize_t read_result;
+  if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
+    at_most = CHUNK_REMAINING_CAPACITY(chunk);
+  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+
+  if (read_result < 0) {
+    int e = tor_socket_errno(fd);
+    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
+#ifdef _WIN32
+      if (e == WSAENOBUFS)
+        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
+#endif
+      *socket_error = e;
+      return -1;
+    }
+    return 0; /* would block. */
+  } else if (read_result == 0) {
+    log_debug(LD_NET,"Encountered eof on fd %d", (int)fd);
+    *reached_eof = 1;
+    return 0;
+  } else { /* actually got bytes. */
+    buf->datalen += read_result;
+    chunk->datalen += read_result;
+    log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result,
+              (int)buf->datalen);
+    tor_assert(read_result < INT_MAX);
+    return (int)read_result;
+  }
+}
+
+/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+/* XXXX indicate "read blocked" somehow? */
+int
+buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                     int *reached_eof,
+                     int *socket_error)
+{
+  /* XXXX It's stupid to overload the return values for these functions:
+   * "error status" and "number of bytes read" are not mutually exclusive.
+   */
+  int r = 0;
+  size_t total_read = 0;
+
+  check();
+  tor_assert(reached_eof);
+  tor_assert(SOCKET_OK(s));
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - at_most))
+    return -1;
+
+  while (at_most > total_read) {
+    size_t readlen = at_most - total_read;
+    chunk_t *chunk;
+    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
+      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
+      if (readlen > chunk->memlen)
+        readlen = chunk->memlen;
+    } else {
+      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
+      chunk = buf->tail;
+      if (cap < readlen)
+        readlen = cap;
+    }
+
+    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
+    check();
+    if (r < 0)
+      return r; /* Error */
+    tor_assert(total_read+r < INT_MAX);
+    total_read += r;
+    if ((size_t)r < readlen) { /* eof, block, or no more to read. */
+      break;
+    }
+  }
+  return (int)total_read;
+}
+
+/** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk
+ * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
+ * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
+ * written on success, 0 on blocking, -1 on failure.
+ */
+static inline int
+flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
+            size_t *buf_flushlen)
+{
+  ssize_t write_result;
+
+  if (sz > chunk->datalen)
+    sz = chunk->datalen;
+  write_result = tor_socket_send(s, chunk->data, sz, 0);
+
+  if (write_result < 0) {
+    int e = tor_socket_errno(s);
+    if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
+#ifdef _WIN32
+      if (e == WSAENOBUFS)
+        log_warn(LD_NET,"write() failed: WSAENOBUFS. Not enough ram?");
+#endif
+      return -1;
+    }
+    log_debug(LD_NET,"write() would block, returning.");
+    return 0;
+  } else {
+    *buf_flushlen -= write_result;
+    buf_drain(buf, write_result);
+    tor_assert(write_result < INT_MAX);
+    return (int)write_result;
+  }
+}
+
+/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                    size_t *buf_flushlen)
+{
+  /* XXXX It's stupid to overload the return values for these functions:
+   * "error status" and "number of bytes flushed" are not mutually exclusive.
+   */
+  int r;
+  size_t flushed = 0;
+  tor_assert(buf_flushlen);
+  tor_assert(SOCKET_OK(s));
+  tor_assert(*buf_flushlen <= buf->datalen);
+  tor_assert(sz <= *buf_flushlen);
+
+  check();
+  while (sz) {
+    size_t flushlen0;
+    tor_assert(buf->head);
+    if (buf->head->datalen >= sz)
+      flushlen0 = sz;
+    else
+      flushlen0 = buf->head->datalen;
+
+    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
+    check();
+    if (r < 0)
+      return r;
+    flushed += r;
+    sz -= r;
+    if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */
+      break;
+  }
+  tor_assert(flushed < INT_MAX);
+  return (int)flushed;
+}
+
+/** Append <b>string_len</b> bytes from <b>string</b> to the end of
+ * <b>buf</b>.
+ *
+ * Return the new length of the buffer on success, -1 on failure.
+ */
+int
+buf_add(buf_t *buf, const char *string, size_t string_len)
+{
+  if (!string_len)
+    return (int)buf->datalen;
+  check();
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - string_len))
+    return -1;
+
+  while (string_len) {
+    size_t copy;
+    if (!buf->tail || !CHUNK_REMAINING_CAPACITY(buf->tail))
+      buf_add_chunk_with_capacity(buf, string_len, 1);
+
+    copy = CHUNK_REMAINING_CAPACITY(buf->tail);
+    if (copy > string_len)
+      copy = string_len;
+    memcpy(CHUNK_WRITE_PTR(buf->tail), string, copy);
+    string_len -= copy;
+    string += copy;
+    buf->datalen += copy;
+    buf->tail->datalen += copy;
+  }
+
+  check();
+  tor_assert(buf->datalen < INT_MAX);
+  return (int)buf->datalen;
+}
+
+/** Helper: copy the first <b>string_len</b> bytes from <b>buf</b>
+ * onto <b>string</b>.
+ */
+void
+buf_peek(const buf_t *buf, char *string, size_t string_len)
+{
+  chunk_t *chunk;
+
+  tor_assert(string);
+  /* make sure we don't ask for too much */
+  tor_assert(string_len <= buf->datalen);
+  /* buf_assert_ok(buf); */
+
+  chunk = buf->head;
+  while (string_len) {
+    size_t copy = string_len;
+    tor_assert(chunk);
+    if (chunk->datalen < copy)
+      copy = chunk->datalen;
+    memcpy(string, chunk->data, copy);
+    string_len -= copy;
+    string += copy;
+    chunk = chunk->next;
+  }
+}
+
+/** Remove <b>string_len</b> bytes from the front of <b>buf</b>, and store
+ * them into <b>string</b>.  Return the new buffer size.  <b>string_len</b>
+ * must be \<= the number of bytes on the buffer.
+ */
+int
+buf_get_bytes(buf_t *buf, char *string, size_t string_len)
+{
+  /* There must be string_len bytes in buf; write them onto string,
+   * then memmove buf back (that is, remove them from buf).
+   *
+   * Return the number of bytes still on the buffer. */
+
+  check();
+  buf_peek(buf, string, string_len);
+  buf_drain(buf, string_len);
+  check();
+  tor_assert(buf->datalen < INT_MAX);
+  return (int)buf->datalen;
+}
+
+/** Move up to *<b>buf_flushlen</b> bytes from <b>buf_in</b> to
+ * <b>buf_out</b>, and modify *<b>buf_flushlen</b> appropriately.
+ * Return the number of bytes actually copied.
+ */
+int
+buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
+{
+  /* We can do way better here, but this doesn't turn up in any profiles. */
+  char b[4096];
+  size_t cp, len;
+
+  if (BUG(buf_out->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen))
+    return -1;
+
+  len = *buf_flushlen;
+  if (len > buf_in->datalen)
+    len = buf_in->datalen;
+
+  cp = len; /* Remember the number of bytes we intend to copy. */
+  tor_assert(cp < INT_MAX);
+  while (len) {
+    /* This isn't the most efficient implementation one could imagine, since
+     * it does two copies instead of 1, but I kinda doubt that this will be
+     * critical path. */
+    size_t n = len > sizeof(b) ? sizeof(b) : len;
+    buf_get_bytes(buf_in, b, n);
+    buf_add(buf_out, b, n);
+    len -= n;
+  }
+  *buf_flushlen -= cp;
+  return (int)cp;
+}
+
+/** Internal structure: represents a position in a buffer. */
+typedef struct buf_pos_t {
+  const chunk_t *chunk; /**< Which chunk are we pointing to? */
+  int pos;/**< Which character inside the chunk's data are we pointing to? */
+  size_t chunk_pos; /**< Total length of all previous chunks. */
+} buf_pos_t;
+
+/** Initialize <b>out</b> to point to the first character of <b>buf</b>.*/
+static void
+buf_pos_init(const buf_t *buf, buf_pos_t *out)
+{
+  out->chunk = buf->head;
+  out->pos = 0;
+  out->chunk_pos = 0;
+}
+
+/** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
+ * position of <b>out</b>, or later.  Return -1 if no instances are found;
+ * otherwise returns the absolute position of the character. */
+static off_t
+buf_find_pos_of_char(char ch, buf_pos_t *out)
+{
+  const chunk_t *chunk;
+  int pos;
+  tor_assert(out);
+  if (out->chunk) {
+    if (out->chunk->datalen) {
+      tor_assert(out->pos < (off_t)out->chunk->datalen);
+    } else {
+      tor_assert(out->pos == 0);
+    }
+  }
+  pos = out->pos;
+  for (chunk = out->chunk; chunk; chunk = chunk->next) {
+    char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos);
+    if (cp) {
+      out->chunk = chunk;
+      tor_assert(cp - chunk->data < INT_MAX);
+      out->pos = (int)(cp - chunk->data);
+      return out->chunk_pos + out->pos;
+    } else {
+      out->chunk_pos += chunk->datalen;
+      pos = 0;
+    }
+  }
+  return -1;
+}
+
+/** Advance <b>pos</b> by a single character, if there are any more characters
+ * in the buffer.  Returns 0 on success, -1 on failure. */
+static inline int
+buf_pos_inc(buf_pos_t *pos)
+{
+  ++pos->pos;
+  if (pos->pos == (off_t)pos->chunk->datalen) {
+    if (!pos->chunk->next)
+      return -1;
+    pos->chunk_pos += pos->chunk->datalen;
+    pos->chunk = pos->chunk->next;
+    pos->pos = 0;
+  }
+  return 0;
+}
+
+/** Return true iff the <b>n</b>-character string in <b>s</b> appears
+ * (verbatim) at <b>pos</b>. */
+static int
+buf_matches_at_pos(const buf_pos_t *pos, const char *s, size_t n)
+{
+  buf_pos_t p;
+  if (!n)
+    return 1;
+
+  memcpy(&p, pos, sizeof(p));
+
+  while (1) {
+    char ch = p.chunk->data[p.pos];
+    if (ch != *s)
+      return 0;
+    ++s;
+    /* If we're out of characters that don't match, we match.  Check this
+     * _before_ we test incrementing pos, in case we're at the end of the
+     * string. */
+    if (--n == 0)
+      return 1;
+    if (buf_pos_inc(&p)<0)
+      return 0;
+  }
+}
+
+/** Return the first position in <b>buf</b> at which the <b>n</b>-character
+ * string <b>s</b> occurs, or -1 if it does not occur. */
+int
+buf_find_string_offset(const buf_t *buf, const char *s, size_t n)
+{
+  buf_pos_t pos;
+  buf_pos_init(buf, &pos);
+  while (buf_find_pos_of_char(*s, &pos) >= 0) {
+    if (buf_matches_at_pos(&pos, s, n)) {
+      tor_assert(pos.chunk_pos + pos.pos < INT_MAX);
+      return (int)(pos.chunk_pos + pos.pos);
+    } else {
+      if (buf_pos_inc(&pos)<0)
+        return -1;
+    }
+  }
+  return -1;
+}
+
+/** Return 1 iff <b>buf</b> starts with <b>cmd</b>. <b>cmd</b> must be a null
+ * terminated string, of no more than PEEK_BUF_STARTSWITH_MAX bytes. */
+int
+buf_peek_startswith(const buf_t *buf, const char *cmd)
+{
+  char tmp[PEEK_BUF_STARTSWITH_MAX];
+  size_t clen = strlen(cmd);
+  if (BUG(clen > sizeof(tmp)))
+    return 0;
+  if (buf->datalen < clen)
+    return 0;
+  buf_peek(buf, tmp, clen);
+  return fast_memeq(tmp, cmd, clen);
+}
+
+/** Return the index within <b>buf</b> at which <b>ch</b> first appears,
+ * or -1 if <b>ch</b> does not appear on buf. */
+static off_t
+buf_find_offset_of_char(buf_t *buf, char ch)
+{
+  chunk_t *chunk;
+  off_t offset = 0;
+  for (chunk = buf->head; chunk; chunk = chunk->next) {
+    char *cp = memchr(chunk->data, ch, chunk->datalen);
+    if (cp)
+      return offset + (cp - chunk->data);
+    else
+      offset += chunk->datalen;
+  }
+  return -1;
+}
+
+/** Try to read a single LF-terminated line from <b>buf</b>, and write it
+ * (including the LF), NUL-terminated, into the *<b>data_len</b> byte buffer
+ * at <b>data_out</b>.  Set *<b>data_len</b> to the number of bytes in the
+ * line, not counting the terminating NUL.  Return 1 if we read a whole line,
+ * return 0 if we don't have a whole line yet, and return -1 if the line
+ * length exceeds *<b>data_len</b>.
+ */
+int
+buf_get_line(buf_t *buf, char *data_out, size_t *data_len)
+{
+  size_t sz;
+  off_t offset;
+
+  if (!buf->head)
+    return 0;
+
+  offset = buf_find_offset_of_char(buf, '\n');
+  if (offset < 0)
+    return 0;
+  sz = (size_t) offset;
+  if (sz+2 > *data_len) {
+    *data_len = sz + 2;
+    return -1;
+  }
+  buf_get_bytes(buf, data_out, sz+1);
+  data_out[sz+1] = '\0';
+  *data_len = sz+1;
+  return 1;
+}
+
+/** Compress on uncompress the <b>data_len</b> bytes in <b>data</b> using the
+ * compression state <b>state</b>, appending the result to <b>buf</b>.  If
+ * <b>done</b> is true, flush the data in the state and finish the
+ * compression/uncompression.  Return -1 on failure, 0 on success. */
+int
+buf_add_compress(buf_t *buf, tor_compress_state_t *state,
+                      const char *data, size_t data_len,
+                      const int done)
+{
+  char *next;
+  size_t old_avail, avail;
+  int over = 0;
+
+  do {
+    int need_new_chunk = 0;
+    if (!buf->tail || ! CHUNK_REMAINING_CAPACITY(buf->tail)) {
+      size_t cap = data_len / 4;
+      buf_add_chunk_with_capacity(buf, cap, 1);
+    }
+    next = CHUNK_WRITE_PTR(buf->tail);
+    avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail);
+    switch (tor_compress_process(state, &next, &avail,
+                                 &data, &data_len, done)) {
+      case TOR_COMPRESS_DONE:
+        over = 1;
+        break;
+      case TOR_COMPRESS_ERROR:
+        return -1;
+      case TOR_COMPRESS_OK:
+        if (data_len == 0) {
+          tor_assert_nonfatal(!done);
+          over = 1;
+        }
+        break;
+      case TOR_COMPRESS_BUFFER_FULL:
+        if (avail) {
+          /* The compression module says we need more room
+           * (TOR_COMPRESS_BUFFER_FULL).  Start a new chunk automatically,
+           * whether were going to or not. */
+          need_new_chunk = 1;
+        }
+        if (data_len == 0 && !done) {
+          /* We've consumed all the input data, though, so there's no
+           * point in forging ahead right now. */
+          over = 1;
+        }
+        break;
+    }
+    buf->datalen += old_avail - avail;
+    buf->tail->datalen += old_avail - avail;
+    if (need_new_chunk) {
+      buf_add_chunk_with_capacity(buf, data_len/4, 1);
+    }
+
+  } while (!over);
+  check();
+  return 0;
+}
+
+/** Set *<b>output</b> to contain a copy of the data in *<b>input</b> */
+int
+buf_set_to_copy(buf_t **output,
+                const buf_t *input)
+{
+  if (*output)
+    buf_free(*output);
+  *output = buf_copy(input);
+  return 0;
+}
+
+/** Log an error and exit if <b>buf</b> is corrupted.
+ */
+void
+buf_assert_ok(buf_t *buf)
+{
+  tor_assert(buf);
+  tor_assert(buf->magic == BUFFER_MAGIC);
+
+  if (! buf->head) {
+    tor_assert(!buf->tail);
+    tor_assert(buf->datalen == 0);
+  } else {
+    chunk_t *ch;
+    size_t total = 0;
+    tor_assert(buf->tail);
+    for (ch = buf->head; ch; ch = ch->next) {
+      total += ch->datalen;
+      tor_assert(ch->datalen <= ch->memlen);
+      tor_assert(ch->data >= &ch->mem[0]);
+      tor_assert(ch->data <= &ch->mem[0]+ch->memlen);
+      if (ch->data == &ch->mem[0]+ch->memlen) {
+        static int warned = 0;
+        if (! warned) {
+          log_warn(LD_BUG, "Invariant violation in buf.c related to #15083");
+          warned = 1;
+        }
+      }
+      tor_assert(ch->data+ch->datalen <= &ch->mem[0] + ch->memlen);
+      if (!ch->next)
+        tor_assert(ch == buf->tail);
+    }
+    tor_assert(buf->datalen == total);
+  }
+}
+
diff --git a/src/common/buffers.h b/src/common/buffers.h
new file mode 100644
index 0000000000..5a52b2a81c
--- /dev/null
+++ b/src/common/buffers.h
@@ -0,0 +1,125 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file buffers.h
+ * \brief Header file for buffers.c.
+ **/
+
+#ifndef TOR_BUFFERS_H
+#define TOR_BUFFERS_H
+
+#include "compat.h"
+#include "compat.h"
+#include "torint.h"
+#include "testsupport.h"
+
+typedef struct buf_t buf_t;
+
+struct tor_compress_state_t;
+
+buf_t *buf_new(void);
+buf_t *buf_new_with_capacity(size_t size);
+size_t buf_get_default_chunk_size(const buf_t *buf);
+void buf_free(buf_t *buf);
+void buf_clear(buf_t *buf);
+buf_t *buf_copy(const buf_t *buf);
+
+MOCK_DECL(size_t, buf_datalen, (const buf_t *buf));
+size_t buf_allocation(const buf_t *buf);
+size_t buf_slack(const buf_t *buf);
+
+uint32_t buf_get_oldest_chunk_timestamp(const buf_t *buf, uint32_t now);
+size_t buf_get_total_allocation(void);
+
+int buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                         int *reached_eof,
+                         int *socket_error);
+
+int buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                        size_t *buf_flushlen);
+
+int buf_add(buf_t *buf, const char *string, size_t string_len);
+int buf_add_compress(buf_t *buf, struct tor_compress_state_t *state,
+                          const char *data, size_t data_len, int done);
+int buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen);
+void buf_peek(const buf_t *buf, char *string, size_t string_len);
+void buf_drain(buf_t *buf, size_t n);
+int buf_get_bytes(buf_t *buf, char *string, size_t string_len);
+int buf_get_line(buf_t *buf, char *data_out, size_t *data_len);
+
+#define PEEK_BUF_STARTSWITH_MAX 16
+int buf_peek_startswith(const buf_t *buf, const char *cmd);
+
+int buf_set_to_copy(buf_t **output,
+                    const buf_t *input);
+
+void buf_assert_ok(buf_t *buf);
+
+int buf_find_string_offset(const buf_t *buf, const char *s, size_t n);
+void buf_pullup(buf_t *buf, size_t bytes,
+                const char **head_out, size_t *len_out);
+
+#ifdef BUFFERS_PRIVATE
+#ifdef TOR_UNIT_TESTS
+buf_t *buf_new_with_data(const char *cp, size_t sz);
+#endif
+size_t buf_preferred_chunk_size(size_t target);
+
+#define DEBUG_CHUNK_ALLOC
+/** A single chunk on a buffer. */
+typedef struct chunk_t {
+  struct chunk_t *next; /**< The next chunk on the buffer. */
+  size_t datalen; /**< The number of bytes stored in this chunk */
+  size_t memlen; /**< The number of usable bytes of storage in <b>mem</b>. */
+#ifdef DEBUG_CHUNK_ALLOC
+  size_t DBG_alloc;
+#endif
+  char *data; /**< A pointer to the first byte of data stored in <b>mem</b>. */
+  uint32_t inserted_time; /**< Timestamp in truncated ms since epoch
+                           * when this chunk was inserted. */
+  char mem[FLEXIBLE_ARRAY_MEMBER]; /**< The actual memory used for storage in
+                * this chunk. */
+} chunk_t;
+
+/** Magic value for buf_t.magic, to catch pointer errors. */
+#define BUFFER_MAGIC 0xB0FFF312u
+/** A resizeable buffer, optimized for reading and writing. */
+struct buf_t {
+  uint32_t magic; /**< Magic cookie for debugging: Must be set to
+                   *   BUFFER_MAGIC. */
+  size_t datalen; /**< How many bytes is this buffer holding right now? */
+  size_t default_chunk_size; /**< Don't allocate any chunks smaller than
+                              * this for this buffer. */
+  chunk_t *head; /**< First chunk in the list, or NULL for none. */
+  chunk_t *tail; /**< Last chunk in the list, or NULL for none. */
+};
+
+chunk_t *buf_add_chunk_with_capacity(buf_t *buf, size_t capacity, int capped);
+/** If a read onto the end of a chunk would be smaller than this number, then
+ * just start a new chunk. */
+#define MIN_READ_LEN 8
+
+/** Return the number of bytes that can be written onto <b>chunk</b> without
+ * running out of space. */
+static inline size_t
+CHUNK_REMAINING_CAPACITY(const chunk_t *chunk)
+{
+  return (chunk->mem + chunk->memlen) - (chunk->data + chunk->datalen);
+}
+
+/** Return the next character in <b>chunk</b> onto which data can be appended.
+ * If the chunk is full, this might be off the end of chunk->mem. */
+static inline char *
+CHUNK_WRITE_PTR(chunk_t *chunk)
+{
+  return chunk->data + chunk->datalen;
+}
+
+#endif
+
+#endif
+
diff --git a/src/common/buffers_tls.c b/src/common/buffers_tls.c
new file mode 100644
index 0000000000..8dbd4bcc8e
--- /dev/null
+++ b/src/common/buffers_tls.c
@@ -0,0 +1,175 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define BUFFERS_PRIVATE
+#include "orconfig.h"
+#include <stddef.h>
+#include "buffers.h"
+#include "buffers_tls.h"
+#include "compat.h"
+#include "compress.h"
+#include "util.h"
+#include "torint.h"
+#include "torlog.h"
+#include "tortls.h"
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/** As read_to_chunk(), but return (negative) error code on error, blocking,
+ * or TLS, and the number of bytes read otherwise. */
+static inline int
+read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls,
+                  size_t at_most)
+{
+  int read_result;
+
+  tor_assert(CHUNK_REMAINING_CAPACITY(chunk) >= at_most);
+  read_result = tor_tls_read(tls, CHUNK_WRITE_PTR(chunk), at_most);
+  if (read_result < 0)
+    return read_result;
+  buf->datalen += read_result;
+  chunk->datalen += read_result;
+  return read_result;
+}
+
+/** As read_to_buf, but reads from a TLS connection, and returns a TLS
+ * status value rather than the number of bytes read.
+ *
+ * Using TLS on OR connections complicates matters in two ways.
+ *
+ * First, a TLS stream has its own read buffer independent of the
+ * connection's read buffer.  (TLS needs to read an entire frame from
+ * the network before it can decrypt any data.  Thus, trying to read 1
+ * byte from TLS can require that several KB be read from the network
+ * and decrypted.  The extra data is stored in TLS's decrypt buffer.)
+ * Because the data hasn't been read by Tor (it's still inside the TLS),
+ * this means that sometimes a connection "has stuff to read" even when
+ * poll() didn't return POLLIN. The tor_tls_get_pending_bytes function is
+ * used in connection.c to detect TLS objects with non-empty internal
+ * buffers and read from them again.
+ *
+ * Second, the TLS stream's events do not correspond directly to network
+ * events: sometimes, before a TLS stream can read, the network must be
+ * ready to write -- or vice versa.
+ */
+int
+buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most)
+{
+  int r = 0;
+  size_t total_read = 0;
+
+  check_no_tls_errors();
+
+  if (BUG(buf->datalen >= INT_MAX))
+    return -1;
+  if (BUG(buf->datalen >= INT_MAX - at_most))
+    return -1;
+
+  while (at_most > total_read) {
+    size_t readlen = at_most - total_read;
+    chunk_t *chunk;
+    if (!buf->tail || CHUNK_REMAINING_CAPACITY(buf->tail) < MIN_READ_LEN) {
+      chunk = buf_add_chunk_with_capacity(buf, at_most, 1);
+      if (readlen > chunk->memlen)
+        readlen = chunk->memlen;
+    } else {
+      size_t cap = CHUNK_REMAINING_CAPACITY(buf->tail);
+      chunk = buf->tail;
+      if (cap < readlen)
+        readlen = cap;
+    }
+
+    r = read_to_chunk_tls(buf, chunk, tls, readlen);
+    if (r < 0)
+      return r; /* Error */
+    tor_assert(total_read+r < INT_MAX);
+    total_read += r;
+    if ((size_t)r < readlen) /* eof, block, or no more to read. */
+      break;
+  }
+  return (int)total_read;
+}
+
+/** Helper for buf_flush_to_tls(): try to write <b>sz</b> bytes from chunk
+ * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  (Tries to write
+ * more if there is a forced pending write size.)  On success, deduct the
+ * bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
+ * written on success, and a TOR_TLS error code on failure or blocking.
+ */
+static inline int
+flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk,
+                size_t sz, size_t *buf_flushlen)
+{
+  int r;
+  size_t forced;
+  char *data;
+
+  forced = tor_tls_get_forced_write_size(tls);
+  if (forced > sz)
+    sz = forced;
+  if (chunk) {
+    data = chunk->data;
+    tor_assert(sz <= chunk->datalen);
+  } else {
+    data = NULL;
+    tor_assert(sz == 0);
+  }
+  r = tor_tls_write(tls, data, sz);
+  if (r < 0)
+    return r;
+  if (*buf_flushlen > (size_t)r)
+    *buf_flushlen -= r;
+  else
+    *buf_flushlen = 0;
+  buf_drain(buf, r);
+  log_debug(LD_NET,"flushed %d bytes, %d ready to flush, %d remain.",
+            r,(int)*buf_flushlen,(int)buf->datalen);
+  return r;
+}
+
+/** As buf_flush_to_socket(), but writes data to a TLS connection.  Can write
+ * more than <b>flushlen</b> bytes.
+ */
+int
+buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen,
+              size_t *buf_flushlen)
+{
+  int r;
+  size_t flushed = 0;
+  ssize_t sz;
+  tor_assert(buf_flushlen);
+  tor_assert(*buf_flushlen <= buf->datalen);
+  tor_assert(flushlen <= *buf_flushlen);
+  sz = (ssize_t) flushlen;
+
+  /* we want to let tls write even if flushlen is zero, because it might
+   * have a partial record pending */
+  check_no_tls_errors();
+
+  do {
+    size_t flushlen0;
+    if (buf->head) {
+      if ((ssize_t)buf->head->datalen >= sz)
+        flushlen0 = sz;
+      else
+        flushlen0 = buf->head->datalen;
+    } else {
+      flushlen0 = 0;
+    }
+
+    r = flush_chunk_tls(tls, buf, buf->head, flushlen0, buf_flushlen);
+    if (r < 0)
+      return r;
+    flushed += r;
+    sz -= r;
+    if (r == 0) /* Can't flush any more now. */
+      break;
+  } while (sz > 0);
+  tor_assert(flushed < INT_MAX);
+  return (int)flushed;
+}
+
diff --git a/src/common/buffers_tls.h b/src/common/buffers_tls.h
new file mode 100644
index 0000000000..e91dfeba0b
--- /dev/null
+++ b/src/common/buffers_tls.h
@@ -0,0 +1,19 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2017, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_BUFFERS_TLS_H
+#define TOR_BUFFERS_TLS_H
+
+struct buf_t;
+struct tor_tls_t;
+
+int buf_read_from_tls(struct buf_t *buf,
+                      struct tor_tls_t *tls, size_t at_most);
+int buf_flush_to_tls(struct buf_t *buf, struct tor_tls_t *tls,
+                     size_t sz, size_t *buf_flushlen);
+
+#endif
+
diff --git a/src/common/compat.c b/src/common/compat.c
index 4d110aba35..836b3813e0 100644
--- a/src/common/compat.c
+++ b/src/common/compat.c
@@ -2349,15 +2349,34 @@ get_parent_directory(char *fname)
 static char *
 alloc_getcwd(void)
 {
-#ifdef PATH_MAX
-#define MAX_CWD PATH_MAX
+#ifdef HAVE_GET_CURRENT_DIR_NAME
+  /* Glibc makes this nice and simple for us. */
+  char *cwd = get_current_dir_name();
+  char *result = NULL;
+  if (cwd) {
+    /* We make a copy here, in case tor_malloc() is not malloc(). */
+    result = tor_strdup(cwd);
+    raw_free(cwd); // alias for free to avoid tripping check-spaces.
+  }
+  return result;
 #else
-#define MAX_CWD 4096
-#endif
+  size_t size = 1024;
+  char *buf = NULL;
+  char *ptr = NULL;
 
-  char path_buf[MAX_CWD];
-  char *path = getcwd(path_buf, sizeof(path_buf));
-  return path ? tor_strdup(path) : NULL;
+  while (ptr == NULL) {
+    buf = tor_realloc(buf, size);
+    ptr = getcwd(buf, size);
+
+    if (ptr == NULL && errno != ERANGE) {
+      tor_free(buf);
+      return NULL;
+    }
+
+    size *= 2;
+  }
+  return buf;
+#endif
 }
 #endif
 
@@ -2600,7 +2619,7 @@ tor_inet_pton(int af, const char *src, void *dst)
         long r = strtol(src, &next, 16);
         if (next == NULL || next == src) {
           /* The 'next == src' error case can happen on versions of openbsd
-           * where treats "0xfoo" as an error, rather than as "0" followed by
+           * which treat "0xfoo" as an error, rather than as "0" followed by
            * "xfoo". */
           return 0;
         }
diff --git a/src/common/compat_time.c b/src/common/compat_time.c
index 2ccaa36e49..3c00c96f94 100644
--- a/src/common/compat_time.c
+++ b/src/common/compat_time.c
@@ -251,7 +251,7 @@ ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out)
 {
   /* must hold lock */
   timeradd(timeval_raw, &timeofday_offset, out);
-  if (PREDICT_UNLIKELY(timercmp(out, &last_timeofday, <))) {
+  if (PREDICT_UNLIKELY(timercmp(out, &last_timeofday, OP_LT))) {
     /* time ran backwards. Instead, declare that no time occurred. */
     timersub(&last_timeofday, timeval_raw, &timeofday_offset);
     memcpy(out, &last_timeofday, sizeof(struct timeval));
diff --git a/src/common/compress.c b/src/common/compress.c
index 7926faaa60..472268a439 100644
--- a/src/common/compress.c
+++ b/src/common/compress.c
@@ -574,6 +574,12 @@ tor_compress_process(tor_compress_state_t *state,
   if (BUG((rv == TOR_COMPRESS_OK) &&
           *in_len == in_len_orig &&
           *out_len == out_len_orig)) {
+    log_warn(LD_GENERAL,
+             "More info on the bug: method == %s, finish == %d, "
+             " *in_len == in_len_orig == %lu, "
+             "*out_len == out_len_orig == %lu",
+             compression_method_get_human_name(state->method), finish,
+             (unsigned long)in_len_orig, (unsigned long)out_len_orig);
     return TOR_COMPRESS_ERROR;
   }
 
diff --git a/src/common/container.c b/src/common/container.c
index 689e7e55e9..8645cb4826 100644
--- a/src/common/container.c
+++ b/src/common/container.c
@@ -843,13 +843,13 @@ smartlist_sort_pointers(smartlist_t *sl)
  *   }
  *
  *   void timer_heap_insert(smartlist_t *heap, timer_t *timer) {
- *      smartlist_pqueue_add(heap, compare, STRUCT_OFFSET(timer_t, heap_index),
+ *      smartlist_pqueue_add(heap, compare, offsetof(timer_t, heap_index),
  *         timer);
  *   }
  *
  *   void timer_heap_pop(smartlist_t *heap) {
  *      return smartlist_pqueue_pop(heap, compare,
- *         STRUCT_OFFSET(timer_t, heap_index));
+ *         offsetof(timer_t, heap_index));
  *   }
  */
 
diff --git a/src/common/crypto.c b/src/common/crypto.c
index 0fc8474832..4d6a70bc49 100644
--- a/src/common/crypto.c
+++ b/src/common/crypto.c
@@ -1238,9 +1238,12 @@ crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
  *   - The beginning of the source data prefixed with a 16-byte symmetric key,
  *     padded and encrypted with the public key; followed by the rest of
  *     the source data encrypted in AES-CTR mode with the symmetric key.
+ *
+ * NOTE that this format does not authenticate the symmetrically encrypted
+ * part of the data, and SHOULD NOT BE USED for new protocols.
  */
 int
-crypto_pk_public_hybrid_encrypt(crypto_pk_t *env,
+crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env,
                                 char *to, size_t tolen,
                                 const char *from,
                                 size_t fromlen,
@@ -1302,10 +1305,14 @@ crypto_pk_public_hybrid_encrypt(crypto_pk_t *env,
   return -1;
 }
 
-/** Invert crypto_pk_public_hybrid_encrypt. Returns the number of bytes
- * written on success, -1 on failure. */
+/** Invert crypto_pk_obsolete_public_hybrid_encrypt. Returns the number of
+ * bytes written on success, -1 on failure.
+ *
+ * NOTE that this format does not authenticate the symmetrically encrypted
+ * part of the data, and SHOULD NOT BE USED for new protocols.
+ */
 int
-crypto_pk_private_hybrid_decrypt(crypto_pk_t *env,
+crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env,
                                  char *to,
                                  size_t tolen,
                                  const char *from,
@@ -1854,6 +1861,18 @@ struct crypto_digest_t {
   } d;
 };
 
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif
+
 /**
  * Return the number of bytes we need to malloc in order to get a
  * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
@@ -1865,7 +1884,7 @@ crypto_digest_alloc_bytes(digest_algorithm_t alg)
   /* Helper: returns the number of bytes in the 'f' field of 'st' */
 #define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
   /* Gives the length of crypto_digest_t through the end of the field 'd' */
-#define END_OF_FIELD(f) (STRUCT_OFFSET(crypto_digest_t, f) + \
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
                          STRUCT_FIELD_SIZE(crypto_digest_t, f))
   switch (alg) {
     case DIGEST_SHA1:
diff --git a/src/common/crypto.h b/src/common/crypto.h
index c70d91c262..cde241dd37 100644
--- a/src/common/crypto.h
+++ b/src/common/crypto.h
@@ -20,6 +20,9 @@
 #include "testsupport.h"
 #include "compat.h"
 
+#include <openssl/engine.h>
+#include "keccak-tiny/keccak-tiny.h"
+
 /*
   Macro to create an arbitrary OpenSSL version number as used by
   OPENSSL_VERSION_NUMBER or SSLeay(), since the actual numbers are a bit hard
@@ -194,11 +197,11 @@ int crypto_pk_private_sign(const crypto_pk_t *env, char *to, size_t tolen,
                            const char *from, size_t fromlen);
 int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen,
                                   const char *from, size_t fromlen);
-int crypto_pk_public_hybrid_encrypt(crypto_pk_t *env, char *to,
+int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to,
                                     size_t tolen,
                                     const char *from, size_t fromlen,
                                     int padding, int force);
-int crypto_pk_private_hybrid_decrypt(crypto_pk_t *env, char *to,
+int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to,
                                      size_t tolen,
                                      const char *from, size_t fromlen,
                                      int padding, int warnOnFailure);
@@ -335,6 +338,7 @@ struct dh_st *crypto_dh_get_dh_(crypto_dh_t *dh);
 void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in);
 
 #ifdef CRYPTO_PRIVATE
+
 STATIC int crypto_force_rand_ssleay(void);
 STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
 
@@ -346,6 +350,7 @@ extern int break_strongest_rng_fallback;
 
 #ifdef TOR_UNIT_TESTS
 void crypto_pk_assign_(crypto_pk_t *dest, const crypto_pk_t *src);
+digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest);
 #endif
 
 #endif
diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c
index 188e18c710..abf23c4b38 100644
--- a/src/common/crypto_ed25519.c
+++ b/src/common/crypto_ed25519.c
@@ -28,6 +28,7 @@
 #include "crypto_format.h"
 #include "torlog.h"
 #include "util.h"
+#include "util_format.h"
 
 #include "ed25519/ref10/ed25519_ref10.h"
 #include "ed25519/donna/ed25519_donna_tor.h"
@@ -57,6 +58,9 @@ typedef struct {
 
   int (*pubkey_from_curve25519_pubkey)(unsigned char *, const unsigned char *,
                                        int);
+
+  int (*ed25519_scalarmult_with_group_order)(unsigned char *,
+                                             const unsigned char *);
 } ed25519_impl_t;
 
 /** The Ref10 Ed25519 implementation. This one is pure C and lightly
@@ -77,6 +81,7 @@ static const ed25519_impl_t impl_ref10 = {
   ed25519_ref10_blind_public_key,
 
   ed25519_ref10_pubkey_from_curve25519_pubkey,
+  ed25519_ref10_scalarmult_with_group_order,
 };
 
 /** The Ref10 Ed25519 implementation. This one is heavily optimized, but still
@@ -97,6 +102,7 @@ static const ed25519_impl_t impl_donna = {
   ed25519_donna_blind_public_key,
 
   ed25519_donna_pubkey_from_curve25519_pubkey,
+  ed25519_donna_scalarmult_with_group_order,
 };
 
 /** Which Ed25519 implementation are we using?  NULL if we haven't decided
@@ -462,7 +468,6 @@ ed25519_keypair_from_curve25519_keypair(ed25519_keypair_t *out,
   tor_assert(fast_memeq(pubkey_check.pubkey, out->pubkey.pubkey, 32));
 
   memwipe(&pubkey_check, 0, sizeof(pubkey_check));
-  memwipe(&ctx, 0, sizeof(ctx));
   memwipe(sha512_output, 0, sizeof(sha512_output));
 
   return 0;
@@ -491,7 +496,8 @@ ed25519_public_key_from_curve25519_public_key(ed25519_public_key_t *pubkey,
  * service descriptors are encrypted with a key derived from the service's
  * long-term public key, and then signed with (and stored at a position
  * indexed by) a short-term key derived by blinding the long-term keys.
- */
+ *
+ * Return 0 if blinding was successful, else return -1. */
 int
 ed25519_keypair_blind(ed25519_keypair_t *out,
                       const ed25519_keypair_t *inp,
@@ -502,7 +508,9 @@ ed25519_keypair_blind(ed25519_keypair_t *out,
   get_ed_impl()->blind_secret_key(out->seckey.seckey,
                                   inp->seckey.seckey, param);
 
-  ed25519_public_blind(&pubkey_check, &inp->pubkey, param);
+  if (ed25519_public_blind(&pubkey_check, &inp->pubkey, param) < 0) {
+    return -1;
+  }
   ed25519_public_key_generate(&out->pubkey, &out->seckey);
 
   tor_assert(fast_memeq(pubkey_check.pubkey, out->pubkey.pubkey, 32));
@@ -522,8 +530,7 @@ ed25519_public_blind(ed25519_public_key_t *out,
                      const ed25519_public_key_t *inp,
                      const uint8_t *param)
 {
-  get_ed_impl()->blind_public_key(out->pubkey, inp->pubkey, param);
-  return 0;
+  return get_ed_impl()->blind_public_key(out->pubkey, inp->pubkey, param);
 }
 
 /**
@@ -754,3 +761,47 @@ ed25519_init(void)
   pick_ed25519_impl();
 }
 
+/* Return true if <b>point</b> is the identity element of the ed25519 group. */
+static int
+ed25519_point_is_identity_element(const uint8_t *point)
+{
+  /* The identity element in ed25159 is the point with coordinates (0,1). */
+  static const uint8_t ed25519_identity[32] = {
+    0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+  tor_assert(sizeof(ed25519_identity) == ED25519_PUBKEY_LEN);
+  return tor_memeq(point, ed25519_identity, sizeof(ed25519_identity));
+}
+
+/** Validate <b>pubkey</b> to ensure that it has no torsion component.
+ *  Return 0 if <b>pubkey</b> is valid, else return -1. */
+int
+ed25519_validate_pubkey(const ed25519_public_key_t *pubkey)
+{
+  uint8_t result[32] = {9};
+
+  /* First check that we were not given the identity element */
+  if (ed25519_point_is_identity_element(pubkey->pubkey)) {
+    log_warn(LD_CRYPTO, "ed25519 pubkey is the identity");
+    return -1;
+  }
+
+  /* For any point on the curve, doing l*point should give the identity element
+   * (where l is the group order). Do the computation and check that the
+   * identity element is returned. */
+  if (get_ed_impl()->ed25519_scalarmult_with_group_order(result,
+                                                         pubkey->pubkey) < 0) {
+    log_warn(LD_CRYPTO, "ed25519 group order scalarmult failed");
+    return -1;
+  }
+
+  if (!ed25519_point_is_identity_element(result)) {
+    log_warn(LD_CRYPTO, "ed25519 validation failed");
+    return -1;
+  }
+
+  return 0;
+}
+
diff --git a/src/common/crypto_ed25519.h b/src/common/crypto_ed25519.h
index 77a3313adc..3a439207b3 100644
--- a/src/common/crypto_ed25519.h
+++ b/src/common/crypto_ed25519.h
@@ -127,6 +127,8 @@ void ed25519_pubkey_copy(ed25519_public_key_t *dest,
 void ed25519_set_impl_params(int use_donna);
 void ed25519_init(void);
 
+int ed25519_validate_pubkey(const ed25519_public_key_t *pubkey);
+
 #ifdef TOR_UNIT_TESTS
 void crypto_ed25519_testing_force_impl(const char *name);
 void crypto_ed25519_testing_restore_impl(void);
diff --git a/src/common/include.am b/src/common/include.am
index 1253888815..cd5eea3404 100644
--- a/src/common/include.am
+++ b/src/common/include.am
@@ -81,6 +81,7 @@ src_common_libor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS)
 LIBOR_A_SRC = \
   src/common/address.c					\
   src/common/backtrace.c				\
+  src/common/buffers.c					\
   src/common/compat.c					\
   src/common/compat_threads.c				\
   src/common/compat_time.c				\
@@ -110,6 +111,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \
 
 LIBOR_CRYPTO_A_SRC = \
   src/common/aes.c		\
+  src/common/buffers_tls.c	\
   src/common/compress.c	\
   src/common/compress_lzma.c	\
   src/common/compress_none.c	\
@@ -147,6 +149,8 @@ src_common_libor_event_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 COMMONHEADERS = \
   src/common/address.h				\
   src/common/backtrace.h			\
+  src/common/buffers.h				\
+  src/common/buffers_tls.h			\
   src/common/aes.h				\
   src/common/ciphers.inc			\
   src/common/compat.h				\
diff --git a/src/common/memarea.c b/src/common/memarea.c
index 659d1edf54..4e2a5e5fc5 100644
--- a/src/common/memarea.c
+++ b/src/common/memarea.c
@@ -7,6 +7,7 @@
  */
 
 #include "orconfig.h"
+#include <stddef.h>
 #include <stdlib.h>
 #include "memarea.h"
 #include "util.h"
@@ -101,7 +102,7 @@ typedef struct memarea_chunk_t {
 
 /** How many bytes are needed for overhead before we get to the memory part
  * of a chunk? */
-#define CHUNK_HEADER_SIZE STRUCT_OFFSET(memarea_chunk_t, U_MEM)
+#define CHUNK_HEADER_SIZE offsetof(memarea_chunk_t, U_MEM)
 
 /** What's the smallest that we'll allocate a chunk? */
 #define CHUNK_SIZE 4096
diff --git a/src/common/procmon.c b/src/common/procmon.c
index d49e7f18f5..5ef16c7268 100644
--- a/src/common/procmon.c
+++ b/src/common/procmon.c
@@ -71,7 +71,7 @@ parse_process_specifier(const char *process_spec,
 
   /* If we're lucky, long will turn out to be large enough to hold a
    * PID everywhere that Tor runs. */
-  pid_l = tor_parse_long(process_spec, 0, 1, LONG_MAX, &pid_ok, &pspec_next);
+  pid_l = tor_parse_long(process_spec, 10, 1, LONG_MAX, &pid_ok, &pspec_next);
 
   /* Reserve room in the ‘process specifier’ for additional
    * (platform-specific) identifying information beyond the PID, to
diff --git a/src/common/sandbox.c b/src/common/sandbox.c
index 5063717355..a85b1406fa 100644
--- a/src/common/sandbox.c
+++ b/src/common/sandbox.c
@@ -289,37 +289,6 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   return rc;
 }
 
-#if 0
-/**
- * Function responsible for setting up the execve syscall for
- * the seccomp filter sandbox.
- */
-static int
-sb_execve(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
-{
-  int rc;
-  sandbox_cfg_t *elem = NULL;
-
-  // for each dynamic parameter filters
-  for (elem = filter; elem != NULL; elem = elem->next) {
-    smp_param_t *param = elem->param;
-
-    if (param != NULL && param->prot == 1 && param->syscall
-        == SCMP_SYS(execve)) {
-      rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(execve),
-               SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
-      if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add execve syscall, received "
-            "libseccomp error %d", rc);
-        return rc;
-      }
-    }
-  }
-
-  return 0;
-}
-#endif
-
 /**
  * Function responsible for setting up the time syscall for
  * the seccomp filter sandbox.
@@ -480,7 +449,7 @@ sb_chmod(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chmod),
             SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
       if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add open syscall, received "
+        log_err(LD_BUG,"(Sandbox) failed to add chmod syscall, received "
             "libseccomp error %d", rc);
         return rc;
       }
@@ -505,7 +474,7 @@ sb_chown(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(chown),
             SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
       if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add open syscall, received "
+        log_err(LD_BUG,"(Sandbox) failed to add chown syscall, received "
             "libseccomp error %d", rc);
         return rc;
       }
@@ -1045,8 +1014,8 @@ sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
       rc = seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(stat64),
           SCMP_CMP_STR(0, SCMP_CMP_EQ, param->value));
       if (rc != 0) {
-        log_err(LD_BUG,"(Sandbox) failed to add open syscall, received "
-            "libseccomp  error %d", rc);
+        log_err(LD_BUG,"(Sandbox) failed to add stat64 syscall, received "
+            "libseccomp error %d", rc);
         return rc;
       }
     }
@@ -1063,9 +1032,6 @@ sb_stat64(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 static sandbox_filter_func_t filter_func[] = {
     sb_rt_sigaction,
     sb_rt_sigprocmask,
-#if 0
-    sb_execve,
-#endif
     sb_time,
     sb_accept4,
 #ifdef __NR_mmap2
@@ -1417,26 +1383,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
   return 0;
 }
 
-#if 0
-int
-sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com)
-{
-  sandbox_cfg_t *elem = NULL;
-
-  elem = new_element(SCMP_SYS(execve), com);
-  if (!elem) {
-    log_err(LD_BUG,"(Sandbox) failed to register parameter!");
-    return -1;
-  }
-
-  elem->next = *cfg;
-  *cfg = elem;
-
-  return 0;
-}
-
-#endif
-
 /** Cache entry for getaddrinfo results; used when sandboxing is implemented
  * so that we can consult the cache when the sandbox prevents us from doing
  * getaddrinfo.
@@ -1692,7 +1638,9 @@ install_syscall_filter(sandbox_cfg_t* cfg)
 
   // loading the seccomp2 filter
   if ((rc = seccomp_load(ctx))) {
-    log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)!", rc,
+    log_err(LD_BUG, "(Sandbox) failed to load: %d (%s)! "
+            "Are you sure that your kernel has seccomp2 support? The "
+            "sandbox won't work without it.", rc,
             strerror(-rc));
     goto end;
   }
@@ -1910,15 +1858,6 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
   return 0;
 }
 
-#if 0
-int
-sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com)
-{
-  (void)cfg; (void)com;
-  return 0;
-}
-#endif
-
 int
 sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file)
 {
diff --git a/src/common/sandbox.h b/src/common/sandbox.h
index a6b83153af..55454eaa2b 100644
--- a/src/common/sandbox.h
+++ b/src/common/sandbox.h
@@ -156,14 +156,6 @@ int sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2);
  */
 int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file);
 
-#if 0
-/**
- * Function used to add a execve allowed filename to a supplied configuration.
- * The (char*) specifies the path to the allowed file; that pointer is stolen.
- */
-int sandbox_cfg_allow_execve(sandbox_cfg_t **cfg, const char *com);
-#endif
-
 /**
  * Function used to add a stat/stat64 allowed filename to a configuration.
  * The (char*) specifies the path to the allowed file; that pointer is stolen.
diff --git a/src/common/tortls.c b/src/common/tortls.c
index 1c47cf9882..5f3cd92a63 100644
--- a/src/common/tortls.c
+++ b/src/common/tortls.c
@@ -1181,17 +1181,20 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
     SSL_CTX_set_options(result->ctx,
                         SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION);
   }
+
+  /* Don't actually allow compression; it uses RAM and time, it makes TLS
+   * vulnerable to CRIME-style attacks, and most of the data we transmit over
+   * TLS is encrypted (and therefore uncompressible) anyway. */
 #ifdef SSL_OP_NO_COMPRESSION
   SSL_CTX_set_options(result->ctx, SSL_OP_NO_COMPRESSION);
 #endif
 #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,1,0)
 #ifndef OPENSSL_NO_COMP
-  /* Don't actually allow compression; it uses ram and time, but the data
-   * we transmit is all encrypted anyway. */
   if (result->ctx->comp_methods)
     result->ctx->comp_methods = NULL;
 #endif
 #endif
+
 #ifdef SSL_MODE_RELEASE_BUFFERS
   SSL_CTX_set_mode(result->ctx, SSL_MODE_RELEASE_BUFFERS);
 #endif
diff --git a/src/common/util.c b/src/common/util.c
index 5b47028097..36d0f4d068 100644
--- a/src/common/util.c
+++ b/src/common/util.c
@@ -1172,7 +1172,7 @@ tor_parse_long(const char *s, int base, long min, long max,
   char *endptr;
   long r;
 
-  if (base < 0) {
+  if (BUG(base < 0)) {
     if (ok)
       *ok = 0;
     return 0;
@@ -1191,7 +1191,7 @@ tor_parse_ulong(const char *s, int base, unsigned long min,
   char *endptr;
   unsigned long r;
 
-  if (base < 0) {
+  if (BUG(base < 0)) {
     if (ok)
       *ok = 0;
     return 0;
@@ -1223,7 +1223,7 @@ tor_parse_uint64(const char *s, int base, uint64_t min,
   char *endptr;
   uint64_t r;
 
-  if (base < 0) {
+  if (BUG(base < 0)) {
     if (ok)
       *ok = 0;
     return 0;
@@ -1233,15 +1233,7 @@ tor_parse_uint64(const char *s, int base, uint64_t min,
 #ifdef HAVE_STRTOULL
   r = (uint64_t)strtoull(s, &endptr, base);
 #elif defined(_WIN32)
-#if defined(_MSC_VER) && _MSC_VER < 1300
-  tor_assert(base <= 10);
-  r = (uint64_t)_atoi64(s);
-  endptr = (char*)s;
-  while (TOR_ISSPACE(*endptr)) endptr++;
-  while (TOR_ISDIGIT(*endptr)) endptr++;
-#else
   r = (uint64_t)_strtoui64(s, &endptr, base);
-#endif
 #elif SIZEOF_LONG == 8
   r = (uint64_t)strtoul(s, &endptr, base);
 #else
@@ -3699,8 +3691,9 @@ finish_daemon(const char *cp)
 #endif
 
 /** Write the current process ID, followed by NL, into <b>filename</b>.
+ * Return 0 on success, -1 on failure.
  */
-void
+int
 write_pidfile(const char *filename)
 {
   FILE *pidfile;
@@ -3708,13 +3701,19 @@ write_pidfile(const char *filename)
   if ((pidfile = fopen(filename, "w")) == NULL) {
     log_warn(LD_FS, "Unable to open \"%s\" for writing: %s", filename,
              strerror(errno));
+    return -1;
   } else {
 #ifdef _WIN32
-    fprintf(pidfile, "%d\n", (int)_getpid());
+    int pid = (int)_getpid();
 #else
-    fprintf(pidfile, "%d\n", (int)getpid());
+    int pid = (int)getpid();
 #endif
-    fclose(pidfile);
+    int rv = 0;
+    if (fprintf(pidfile, "%d\n", pid) < 0)
+      rv = -1;
+    if (fclose(pidfile) < 0)
+      rv = -1;
+    return rv;
   }
 }
 
@@ -4142,6 +4141,20 @@ process_handle_waitpid_cb(int status, void *arg)
 #define CHILD_STATE_EXEC 8
 #define CHILD_STATE_FAILEXEC 9
 /** @} */
+/**
+ * Boolean.  If true, then Tor may call execve or CreateProcess via
+ * tor_spawn_background.
+ **/
+static int may_spawn_background_process = 1;
+/**
+ * Turn off may_spawn_background_process, so that all future calls to
+ * tor_spawn_background are guaranteed to fail.
+ **/
+void
+tor_disable_spawning_background_processes(void)
+{
+  may_spawn_background_process = 0;
+}
 /** Start a program in the background. If <b>filename</b> contains a '/', then
  * it will be treated as an absolute or relative path.  Otherwise, on
  * non-Windows systems, the system path will be searched for <b>filename</b>.
@@ -4166,6 +4179,12 @@ tor_spawn_background(const char *const filename, const char **argv,
                      process_environment_t *env,
                      process_handle_t **process_handle_out)
 {
+  if (BUG(may_spawn_background_process == 0)) {
+    /* We should never reach this point if we're forbidden to spawn
+     * processes. Instead we should have caught the attempt earlier. */
+    return PROCESS_STATUS_ERROR;
+  }
+
 #ifdef _WIN32
   HANDLE stdout_pipe_read = NULL;
   HANDLE stdout_pipe_write = NULL;
diff --git a/src/common/util.h b/src/common/util.h
index d56abcee2e..073fb82aed 100644
--- a/src/common/util.h
+++ b/src/common/util.h
@@ -109,19 +109,11 @@ extern int dmalloc_free(const char *file, const int line, void *pnt,
 
 void tor_log_mallinfo(int severity);
 
-/** Return the offset of <b>member</b> within the type <b>tp</b>, in bytes */
-#if defined(__GNUC__) && __GNUC__ > 3
-#define STRUCT_OFFSET(tp, member) __builtin_offsetof(tp, member)
-#else
- #define STRUCT_OFFSET(tp, member) \
-   ((off_t) (((char*)&((tp*)0)->member)-(char*)0))
-#endif
-
 /** Macro: yield a pointer to the field at position <b>off</b> within the
  * structure <b>st</b>.  Example:
  * <pre>
  *   struct a { int foo; int bar; } x;
- *   off_t bar_offset = STRUCT_OFFSET(struct a, bar);
+ *   off_t bar_offset = offsetof(struct a, bar);
  *   int *bar_p = STRUCT_VAR_P(&x, bar_offset);
  *   *bar_p = 3;
  * </pre>
@@ -138,7 +130,7 @@ void tor_log_mallinfo(int severity);
  * </pre>
  */
 #define SUBTYPE_P(p, subtype, basemember) \
-  ((void*) ( ((char*)(p)) - STRUCT_OFFSET(subtype, basemember) ))
+  ((void*) ( ((char*)(p)) - offsetof(subtype, basemember) ))
 
 /* Logic */
 /** Macro: true if two values have the same boolean value. */
@@ -397,13 +389,15 @@ int path_is_relative(const char *filename);
 /* Process helpers */
 void start_daemon(void);
 void finish_daemon(const char *desired_cwd);
-void write_pidfile(const char *filename);
+int write_pidfile(const char *filename);
 
 /* Port forwarding */
 void tor_check_port_forwarding(const char *filename,
                                struct smartlist_t *ports_to_forward,
                                time_t now);
 
+void tor_disable_spawning_background_processes(void);
+
 typedef struct process_handle_t process_handle_t;
 typedef struct process_environment_t process_environment_t;
 int tor_spawn_background(const char *const filename, const char **argv,
diff --git a/src/common/util_bug.c b/src/common/util_bug.c
index 3d990e3700..cc1ac2ff84 100644
--- a/src/common/util_bug.c
+++ b/src/common/util_bug.c
@@ -13,6 +13,10 @@
 #include "backtrace.h"
 #include "container.h"
 
+#ifdef __COVERITY__
+int bug_macro_deadcode_dummy__ = 0;
+#endif
+
 #ifdef TOR_UNIT_TESTS
 static void (*failed_assertion_cb)(void) = NULL;
 static int n_bugs_to_capture = 0;
diff --git a/src/common/util_bug.h b/src/common/util_bug.h
index ae7e7a37fd..ce6565d92d 100644
--- a/src/common/util_bug.h
+++ b/src/common/util_bug.h
@@ -5,6 +5,32 @@
 
 /**
  * \file util_bug.h
+ *
+ * \brief Macros to manage assertions, fatal and non-fatal.
+ *
+ * Guidelines: All the different kinds of assertion in this file are for
+ * bug-checking only. Don't write code that can assert based on bad inputs.
+ *
+ * We provide two kinds of assertion here: "fatal" and "nonfatal". Use
+ * nonfatal assertions for any bug you can reasonably recover from -- and
+ * please, try to recover!  Many severe bugs in Tor have been caused by using
+ * a regular assertion when a nonfatal assertion would have been better.
+ *
+ * If you need to check a condition with a nonfatal assertion, AND recover
+ * from that same condition, consider using the BUG() macro inside a
+ * conditional.  For example:
+ *
+ * <code>
+ *  // wrong -- use tor_assert_nonfatal() if you just want an assertion.
+ *  BUG(ptr == NULL);
+ *
+ *  // okay, but needlessly verbose
+ *  tor_assert_nonfatal(ptr != NULL);
+ *  if (ptr == NULL) { ... }
+ *
+ *  // this is how we do it:
+ *  if (BUG(ptr == NULL)) { ... }
+ * </code>
  **/
 
 #ifndef TOR_UTIL_BUG_H
@@ -59,10 +85,13 @@
  */
 
 #ifdef __COVERITY__
+extern int bug_macro_deadcode_dummy__;
 #undef BUG
 // Coverity defines this in global headers; let's override it.  This is a
 // magic coverity-only preprocessor thing.
-#nodef BUG(x) ((x)?(__coverity_panic__(),1):0)
+// We use this "deadcode_dummy__" trick to prevent coverity from
+// complaining about unreachable bug cases.
+#nodef BUG(x) ((x)?(__coverity_panic__(),1):(0+bug_macro_deadcode_dummy__))
 #endif
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)