diff options
Diffstat (limited to 'src/common/compress_zstd.c')
| -rw-r--r-- | src/common/compress_zstd.c | 75 |
1 files changed, 66 insertions, 9 deletions
diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index deaefc1779..4a8f03e30a 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -20,7 +20,6 @@ #ifdef HAVE_ZSTD #include <zstd.h> -#include <zstd_errors.h> #endif /** Total number of bytes allocated for Zstandard state. */ @@ -109,27 +108,86 @@ struct tor_zstd_compress_state_t { size_t allocation; }; +#ifdef HAVE_ZSTD +/** Return an approximate number of bytes stored in memory to hold the + * Zstandard compression/decompression state. */ +static size_t +tor_zstd_state_size_precalc(int compress, int preset) +{ + tor_assert(preset > 0); + + size_t memory_usage = sizeof(tor_zstd_compress_state_t); + + // The Zstandard library provides a number of functions that would be useful + // here, but they are, unfortunately, still considered experimental and are + // thus only available in libzstd if we link against the library statically. + // + // The code in this function tries to approximate the calculations without + // being able to use the following: + // + // - We do not have access to neither the internal members of ZSTD_CStream + // and ZSTD_DStream and their internal context objects. + // + // - We cannot use ZSTD_sizeof_CStream() and ZSTD_sizeof_DStream() since they + // are unexposed. + // + // In the future it might be useful to check if libzstd have started + // providing these functions in a stable manner and simplify this function. + if (compress) { + // We try to approximate the ZSTD_sizeof_CStream(ZSTD_CStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_CStream): Around 192 bytes on a 64-bit machine: + memory_usage += 192; + + // - ZSTD_sizeof_CCtx(stream->cctx): This function requires access to + // variables that are not exposed via the public API. We use a _very_ + // simplified function to calculate the estimated amount of bytes used in + // this struct. + memory_usage += (preset - 0.5) * 1024 * 1024; + // - ZSTD_sizeof_CDict(stream->cdictLocal): Unused in Tor: 0 bytes. + // - stream->outBuffSize: 128 KB: + memory_usage += 128 * 1024; + // - stream->inBuffSize: 2048 KB: + memory_usage += 2048 * 1024; + } else { + // We try to approximate the ZSTD_sizeof_DStream(ZSTD_DStream *stream) + // function here. This function uses the following fields to make its + // estimate: + + // - sizeof(ZSTD_DStream): Around 208 bytes on a 64-bit machine: + memory_usage += 208; + // - ZSTD_sizeof_DCtx(stream->dctx): Around 150 KB. + memory_usage += 150 * 1024; + + // - ZSTD_sizeof_DDict(stream->ddictLocal): Unused in Tor: 0 bytes. + // - stream->inBuffSize: 0 KB. + // - stream->outBuffSize: 0 KB. + } + + return memory_usage; +} +#endif // HAVE_ZSTD. + /** Construct and return a tor_zstd_compress_state_t object using * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for * decompression. */ tor_zstd_compress_state_t * tor_zstd_compress_new(int compress, compress_method_t method, - compression_level_t compression_level) + compression_level_t level) { tor_assert(method == ZSTD_METHOD); #ifdef HAVE_ZSTD + const int preset = memory_level(level); tor_zstd_compress_state_t *result; size_t retval; result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t)); result->compress = compress; - - // FIXME(ahf): We should either try to do the pre-calculation that is done - // with the zlib backend or use a custom allocator here where we pass our - // tor_zstd_compress_state_t as the opaque value. - result->allocation = 0; + result->allocation = tor_zstd_state_size_precalc(compress, preset); if (compress) { result->u.compress_stream = ZSTD_createCStream(); @@ -139,8 +197,7 @@ tor_zstd_compress_new(int compress, goto err; } - retval = ZSTD_initCStream(result->u.compress_stream, - memory_level(compression_level)); + retval = ZSTD_initCStream(result->u.compress_stream, preset); if (ZSTD_isError(retval)) { log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", |