Move zlib compression code into its own module.

This patch refactors the `torgzip` module to allow us to extend a common
compression API to support multiple compression backends.

Additionally we move the gzip/zlib code into its own module under the
name `compress_zlib`.

See https://bugs.torproject.org/21664

1 files changed, 45 insertions, 413 deletions

diff --git a/src/common/torgzip.c b/src/common/torgzip.c
index d6f6225251..d760cd9546 100644
--- a/src/common/torgzip.c
+++ b/src/common/torgzip.c
@@ -23,69 +23,7 @@
 #include "util.h"
 #include "torlog.h"
 #include "torgzip.h"
-
-/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros.  Instead of
-   saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory
-   that nobody will care if the compile outputs a no-such-identifier warning.
-
-   Sorry, but we like -Werror over here, so I guess we need to define these.
-   I hope that zlib 1.2.6 doesn't break these too.
-*/
-#ifndef _LARGEFILE64_SOURCE
-#define _LARGEFILE64_SOURCE 0
-#endif
-#ifndef _LFS64_LARGEFILE
-#define _LFS64_LARGEFILE 0
-#endif
-#ifndef _FILE_OFFSET_BITS
-#define _FILE_OFFSET_BITS 0
-#endif
-#ifndef off64_t
-#define off64_t int64_t
-#endif
-
-#include <zlib.h>
-
-#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200
-#error "We require zlib version 1.2 or later."
-#endif
-
-static size_t tor_zlib_state_size_precalc(int inflate,
-                                          int windowbits, int memlevel);
-
-/** Total number of bytes allocated for zlib state */
-static size_t total_zlib_allocation = 0;
-
-/** Return a string representation of the version of the currently running
- * version of zlib. */
-const char *
-tor_zlib_get_version_str(void)
-{
-  return zlibVersion();
-}
-
-/** Return a string representation of the version of the version of zlib
-* used at compilation. */
-const char *
-tor_zlib_get_header_version_str(void)
-{
-  return ZLIB_VERSION;
-}
-
-/** Return the 'bits' value to tell zlib to use <b>method</b>.*/
-static inline int
-method_bits(compress_method_t method, compression_level_t level)
-{
-  /* Bits+16 means "use gzip" in zlib >= 1.2 */
-  const int flag = method == GZIP_METHOD ? 16 : 0;
-  switch (level) {
-    default:
-    case HIGH_COMPRESSION: return flag + 15;
-    case MEDIUM_COMPRESSION: return flag + 13;
-    case LOW_COMPRESSION: return flag + 11;
-  }
-}
-
+#include "compress_zlib.h"
 
 /** @{ */
 /* These macros define the maximum allowable compression factor.  Anything of
@@ -140,107 +78,9 @@ tor_compress(char **out, size_t *out_len,
              const char *in, size_t in_len,
              compress_method_t method)
 {
-  struct z_stream_s *stream = NULL;
-  size_t out_size, old_size;
-  off_t offset;
-
-  tor_assert(out);
-  tor_assert(out_len);
-  tor_assert(in);
-  tor_assert(in_len < UINT_MAX);
-
-  *out = NULL;
-
-  stream = tor_malloc_zero(sizeof(struct z_stream_s));
-  stream->zalloc = Z_NULL;
-  stream->zfree = Z_NULL;
-  stream->opaque = NULL;
-  stream->next_in = (unsigned char*) in;
-  stream->avail_in = (unsigned int)in_len;
-
-  if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED,
-                   method_bits(method, HIGH_COMPRESSION),
-                   get_memlevel(HIGH_COMPRESSION),
-                   Z_DEFAULT_STRATEGY) != Z_OK) {
-    //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments.
-    log_warn(LD_GENERAL, "Error from deflateInit2: %s",
-             stream->msg?stream->msg:"<no message>");
-    goto err;
-    //LCOV_EXCL_STOP
-  }
-
-  /* Guess 50% compression. */
-  out_size = in_len / 2;
-  if (out_size < 1024) out_size = 1024;
-  *out = tor_malloc(out_size);
-  stream->next_out = (unsigned char*)*out;
-  stream->avail_out = (unsigned int)out_size;
-
-  while (1) {
-    switch (deflate(stream, Z_FINISH))
-      {
-      case Z_STREAM_END:
-        goto done;
-      case Z_OK:
-        /* In case zlib doesn't work as I think .... */
-        if (stream->avail_out >= stream->avail_in+16)
-          break;
-      case Z_BUF_ERROR:
-        offset = stream->next_out - ((unsigned char*)*out);
-        old_size = out_size;
-        out_size *= 2;
-        if (out_size < old_size) {
-          log_warn(LD_GENERAL, "Size overflow in compression.");
-          goto err;
-        }
-        *out = tor_realloc(*out, out_size);
-        stream->next_out = (unsigned char*)(*out + offset);
-        if (out_size - offset > UINT_MAX) {
-          log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
-                   "uncompressing.");
-          goto err;
-        }
-        stream->avail_out = (unsigned int)(out_size - offset);
-        break;
-      default:
-        log_warn(LD_GENERAL, "Gzip compression didn't finish: %s",
-                 stream->msg ? stream->msg : "<no message>");
-        goto err;
-      }
-  }
- done:
-  *out_len = stream->total_out;
-#if defined(OpenBSD)
-  /* "Hey Rocky!  Watch me change an unsigned field to a signed field in a
-   *    third-party API!"
-   * "Oh, that trick will just make people do unsafe casts to the unsigned
-   *    type in their cross-platform code!"
-   * "Don't be foolish.  I'm _sure_ they'll have the good sense to make sure
-   *    the newly unsigned field isn't negative." */
-  tor_assert(stream->total_out >= 0);
-#endif
-  if (deflateEnd(stream)!=Z_OK) {
-    // LCOV_EXCL_START -- unreachable if we handled the zlib structure right
-    tor_assert_nonfatal_unreached();
-    log_warn(LD_BUG, "Error freeing gzip structures");
-    goto err;
-    // LCOV_EXCL_STOP
-  }
-  tor_free(stream);
+  if (method == GZIP_METHOD || method == ZLIB_METHOD)
+    return tor_zlib_compress(out, out_len, in, in_len, method);
 
-  if (is_compression_bomb(*out_len, in_len)) {
-    log_warn(LD_BUG, "We compressed something and got an insanely high "
-          "compression factor; other Tors would think this was a zlib bomb.");
-    goto err;
-  }
-
-  return 0;
- err:
-  if (stream) {
-    deflateEnd(stream);
-    tor_free(stream);
-  }
-  tor_free(*out);
   return -1;
 }
 
@@ -262,127 +102,12 @@ tor_uncompress(char **out, size_t *out_len,
                int complete_only,
                int protocol_warn_level)
 {
-  struct z_stream_s *stream = NULL;
-  size_t out_size, old_size;
-  off_t offset;
-  int r;
-
-  tor_assert(out);
-  tor_assert(out_len);
-  tor_assert(in);
-  tor_assert(in_len < UINT_MAX);
-
-  *out = NULL;
-
-  stream = tor_malloc_zero(sizeof(struct z_stream_s));
-  stream->zalloc = Z_NULL;
-  stream->zfree = Z_NULL;
-  stream->opaque = NULL;
-  stream->next_in = (unsigned char*) in;
-  stream->avail_in = (unsigned int)in_len;
-
-  if (inflateInit2(stream,
-                   method_bits(method, HIGH_COMPRESSION)) != Z_OK) {
-    // LCOV_EXCL_START -- can only hit this if we give bad inputs.
-    log_warn(LD_GENERAL, "Error from inflateInit2: %s",
-             stream->msg?stream->msg:"<no message>");
-    goto err;
-    // LCOV_EXCL_STOP
-  }
+  if (method == GZIP_METHOD || method == ZLIB_METHOD)
+    return tor_zlib_uncompress(out, out_len, in, in_len,
+                               method,
+                               complete_only,
+                               protocol_warn_level);
 
-  out_size = in_len * 2;  /* guess 50% compression. */
-  if (out_size < 1024) out_size = 1024;
-  if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX)
-    goto err;
-
-  *out = tor_malloc(out_size);
-  stream->next_out = (unsigned char*)*out;
-  stream->avail_out = (unsigned int)out_size;
-
-  while (1) {
-    switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH))
-      {
-      case Z_STREAM_END:
-        if (stream->avail_in == 0)
-          goto done;
-        /* There may be more compressed data here. */
-        if ((r = inflateEnd(stream)) != Z_OK) {
-          log_warn(LD_BUG, "Error freeing gzip structures");
-          goto err;
-        }
-        if (inflateInit2(stream,
-                         method_bits(method,HIGH_COMPRESSION)) != Z_OK) {
-          log_warn(LD_GENERAL, "Error from second inflateInit2: %s",
-                   stream->msg?stream->msg:"<no message>");
-          goto err;
-        }
-        break;
-      case Z_OK:
-        if (!complete_only && stream->avail_in == 0)
-          goto done;
-        /* In case zlib doesn't work as I think.... */
-        if (stream->avail_out >= stream->avail_in+16)
-          break;
-      case Z_BUF_ERROR:
-        if (stream->avail_out > 0) {
-          log_fn(protocol_warn_level, LD_PROTOCOL,
-                 "possible truncated or corrupt zlib data");
-          goto err;
-        }
-        offset = stream->next_out - (unsigned char*)*out;
-        old_size = out_size;
-        out_size *= 2;
-        if (out_size < old_size) {
-          log_warn(LD_GENERAL, "Size overflow in uncompression.");
-          goto err;
-        }
-        if (is_compression_bomb(in_len, out_size)) {
-          log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; "
-                   "not proceeding.");
-          goto err;
-        }
-        if (out_size >= SIZE_T_CEILING) {
-          log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing.");
-          goto err;
-        }
-        *out = tor_realloc(*out, out_size);
-        stream->next_out = (unsigned char*)(*out + offset);
-        if (out_size - offset > UINT_MAX) {
-          log_warn(LD_BUG,  "Ran over unsigned int limit of zlib while "
-                   "uncompressing.");
-          goto err;
-        }
-        stream->avail_out = (unsigned int)(out_size - offset);
-        break;
-      default:
-        log_warn(LD_GENERAL, "Gzip decompression returned an error: %s",
-                 stream->msg ? stream->msg : "<no message>");
-        goto err;
-      }
-  }
- done:
-  *out_len = stream->next_out - (unsigned char*)*out;
-  r = inflateEnd(stream);
-  tor_free(stream);
-  if (r != Z_OK) {
-    log_warn(LD_BUG, "Error freeing gzip structures");
-    goto err;
-  }
-
-  /* NUL-terminate output. */
-  if (out_size == *out_len)
-    *out = tor_realloc(*out, out_size + 1);
-  (*out)[*out_len] = '\0';
-
-  return 0;
- err:
-  if (stream) {
-    inflateEnd(stream);
-    tor_free(stream);
-  }
-  if (*out) {
-    tor_free(*out);
-  }
   return -1;
 }
 
@@ -406,58 +131,39 @@ detect_compression_method(const char *in, size_t in_len)
 /** Internal state for an incremental compression/decompression.  The body of
  * this struct is not exposed. */
 struct tor_compress_state_t {
-  struct z_stream_s stream; /**< The zlib stream */
-  int compress; /**< True if we are compressing; false if we are inflating */
+  compress_method_t method; /**< The compression method. */
 
-  /** Number of bytes read so far.  Used to detect zlib bombs. */
-  size_t input_so_far;
-  /** Number of bytes written so far.  Used to detect zlib bombs. */
-  size_t output_so_far;
-
-  /** Approximate number of bytes allocated for this object. */
-  size_t allocation;
+  union {
+    tor_zlib_compress_state_t *zlib_state;
+  } u; /**< Compression backend state. */
 };
 
 /** Construct and return a tor_compress_state_t object using <b>method</b>.  If
  * <b>compress</b>, it's for compression; otherwise it's for decompression. */
 tor_compress_state_t *
-tor_compress_new(int compress_, compress_method_t method,
+tor_compress_new(int compress, compress_method_t method,
                  compression_level_t compression_level)
 {
-  tor_compress_state_t *out;
-  int bits, memlevel;
-
- if (! compress_) {
-   /* use this setting for decompression, since we might have the
-    * max number of window bits */
-   compression_level = HIGH_COMPRESSION;
- }
-
- out = tor_malloc_zero(sizeof(tor_compress_state_t));
- out->stream.zalloc = Z_NULL;
- out->stream.zfree = Z_NULL;
- out->stream.opaque = NULL;
- out->compress = compress_;
- bits = method_bits(method, compression_level);
- memlevel = get_memlevel(compression_level);
- if (compress_) {
-   if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED,
-                    bits, memlevel,
-                    Z_DEFAULT_STRATEGY) != Z_OK)
-     goto err; // LCOV_EXCL_LINE
- } else {
-   if (inflateInit2(&out->stream, bits) != Z_OK)
-     goto err; // LCOV_EXCL_LINE
- }
- out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel);
-
- total_zlib_allocation += out->allocation;
-
- return out;
+  tor_compress_state_t *state;
+
+  state = tor_malloc_zero(sizeof(tor_compress_state_t));
+  state->method = method;
+
+  if (method == GZIP_METHOD || method == ZLIB_METHOD) {
+    tor_zlib_compress_state_t *zlib_state =
+      tor_zlib_compress_new(compress, method, compression_level);
+
+    if (zlib_state == NULL)
+      goto err;
+
+    state->u.zlib_state = zlib_state;
+  }
+
+  return state;
 
  err:
- tor_free(out);
- return NULL;
+  tor_free(state);
+  return NULL;
 }
 
 /** Compress/decompress some bytes using <b>state</b>.  Read up to
@@ -477,112 +183,38 @@ tor_compress_process(tor_compress_state_t *state,
                      const char **in, size_t *in_len,
                      int finish)
 {
-  int err;
-  tor_assert(*in_len <= UINT_MAX);
-  tor_assert(*out_len <= UINT_MAX);
-  state->stream.next_in = (unsigned char*) *in;
-  state->stream.avail_in = (unsigned int)*in_len;
-  state->stream.next_out = (unsigned char*) *out;
-  state->stream.avail_out = (unsigned int)*out_len;
-
-  if (state->compress) {
-    err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH);
-  } else {
-    err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH);
-  }
-
-  state->input_so_far += state->stream.next_in - ((unsigned char*)*in);
-  state->output_so_far += state->stream.next_out - ((unsigned char*)*out);
+  tor_assert(state != NULL);
 
-  *out = (char*) state->stream.next_out;
-  *out_len = state->stream.avail_out;
-  *in = (const char *) state->stream.next_in;
-  *in_len = state->stream.avail_in;
+  if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD)
+    return tor_zlib_compress_process(state->u.zlib_state,
+                                     out, out_len, in, in_len,
+                                     finish);
 
-  if (! state->compress &&
-      is_compression_bomb(state->input_so_far, state->output_so_far)) {
-    log_warn(LD_DIR, "Possible zlib bomb; abandoning stream.");
-    return TOR_COMPRESS_ERROR;
-  }
-
-  switch (err)
-    {
-    case Z_STREAM_END:
-      return TOR_COMPRESS_DONE;
-    case Z_BUF_ERROR:
-      if (state->stream.avail_in == 0 && !finish)
-        return TOR_COMPRESS_OK;
-      return TOR_COMPRESS_BUFFER_FULL;
-    case Z_OK:
-      if (state->stream.avail_out == 0 || finish)
-        return TOR_COMPRESS_BUFFER_FULL;
-      return TOR_COMPRESS_OK;
-    default:
-      log_warn(LD_GENERAL, "Gzip returned an error: %s",
-               state->stream.msg ? state->stream.msg : "<no message>");
-      return TOR_COMPRESS_ERROR;
-    }
+  return TOR_COMPRESS_ERROR;
 }
 
 /** Deallocate <b>state</b>. */
 void
 tor_compress_free(tor_compress_state_t *state)
 {
-  if (!state)
+  if (state == NULL)
     return;
 
- total_zlib_allocation -= state->allocation;
-
-  if (state->compress)
-    deflateEnd(&state->stream);
-  else
-    inflateEnd(&state->stream);
+  if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD)
+    tor_zlib_compress_free(state->u.zlib_state);
 
   tor_free(state);
 }
 
-/** Return an approximate number of bytes used in RAM to hold a state with
- * window bits <b>windowBits</b> and compression level 'memlevel' */
-static size_t
-tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel)
-{
-  windowbits &= 15;
-
-#define A_FEW_KILOBYTES 2048
-
-  if (inflate_) {
-    /* From zconf.h:
-
-       "The memory requirements for inflate are (in bytes) 1 << windowBits
-       that is, 32K for windowBits=15 (default value) plus a few kilobytes
-       for small objects."
-    */
-    return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) +
-      (1 << 15) + A_FEW_KILOBYTES;
-  } else {
-    /* Also from zconf.h:
-
-       "The memory requirements for deflate are (in bytes):
-            (1 << (windowBits+2)) +  (1 << (memLevel+9))
-        ... plus a few kilobytes for small objects."
-    */
-    return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) +
-      (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES;
-  }
-#undef A_FEW_KILOBYTES
-}
-
 /** Return the approximate number of bytes allocated for <b>state</b>. */
 size_t
 tor_compress_state_size(const tor_compress_state_t *state)
 {
-  return state->allocation;
-}
+  tor_assert(state != NULL);
 
-/** Return the approximate number of bytes allocated for all zlib states. */
-size_t
-tor_zlib_get_total_allocation(void)
-{
-  return total_zlib_allocation;
+  if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD)
+    return tor_zlib_compress_state_size(state->u.zlib_state);
+
+  return 0;
 }