From a8821d83660c5ed6244e3ad414b348890da93050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 17 Apr 2017 14:02:16 +0200 Subject: [PATCH 01/28] Use tt_int_op() over tt_assert() and do explicit NULL checks in test_util_gzip(). This patch changes some of the tt_assert() usage in test_util_gzip() to use tt_int_op() to get better error messages upon failure. Additionally we move to use explicit NULL checks. See https://bugs.torproject.org/21663 --- src/test/test_util.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/test_util.c b/src/test/test_util.c index 203f9dd1c4..4ad4ffa363 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2257,13 +2257,13 @@ test_util_gzip(void *arg) tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, GZIP_METHOD)); - tt_assert(buf2); - tt_assert(len1 < strlen(buf1)); - tt_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); + tt_assert(buf2 != NULL); + tt_int_op(len1, OP_LT, strlen(buf1)); + tt_int_op(detect_compression_method(buf2, len1), OP_EQ, GZIP_METHOD); tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, GZIP_METHOD, 1, LOG_INFO)); - tt_assert(buf3); + tt_assert(buf3 != NULL); tt_int_op(strlen(buf1) + 1,OP_EQ, len2); tt_str_op(buf1,OP_EQ, buf3); @@ -2273,11 +2273,11 @@ test_util_gzip(void *arg) tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, ZLIB_METHOD)); tt_assert(buf2); - tt_assert(detect_compression_method(buf2, len1) == ZLIB_METHOD); + tt_int_op(detect_compression_method(buf2, len1), OP_EQ, ZLIB_METHOD); tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, ZLIB_METHOD, 1, LOG_INFO)); - tt_assert(buf3); + tt_assert(buf3 != NULL); tt_int_op(strlen(buf1) + 1,OP_EQ, len2); tt_str_op(buf1,OP_EQ, buf3); @@ -2302,7 +2302,7 @@ test_util_gzip(void *arg) tor_strdup("String with low redundancy that won't be compressed much."); tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, ZLIB_METHOD)); - tt_assert(len1>16); + tt_int_op(len1, OP_GT, 16); /* when we allow an incomplete string, we should succeed.*/ tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, ZLIB_METHOD, 0, LOG_INFO)); @@ -2314,7 +2314,7 @@ test_util_gzip(void *arg) tor_free(buf3); tt_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, ZLIB_METHOD, 1, LOG_INFO)); - tt_assert(!buf3); + tt_assert(buf3 == NULL); /* Now, try streaming compression. */ tor_free(buf1); From 4b834e0d5e0374586823048b9c35daf079a59950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 17 Apr 2017 14:07:23 +0200 Subject: [PATCH 02/28] Fix whitespace in test_util_gzip() around `OP_*``. See https://bugs.torproject.org/21663 --- src/test/test_util.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/test_util.c b/src/test/test_util.c index 4ad4ffa363..19d1c79e79 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2264,8 +2264,8 @@ test_util_gzip(void *arg) tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, GZIP_METHOD, 1, LOG_INFO)); tt_assert(buf3 != NULL); - tt_int_op(strlen(buf1) + 1,OP_EQ, len2); - tt_str_op(buf1,OP_EQ, buf3); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); tor_free(buf2); tor_free(buf3); @@ -2278,8 +2278,8 @@ test_util_gzip(void *arg) tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, ZLIB_METHOD, 1, LOG_INFO)); tt_assert(buf3 != NULL); - tt_int_op(strlen(buf1) + 1,OP_EQ, len2); - tt_str_op(buf1,OP_EQ, buf3); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); /* Check whether we can uncompress concatenated, compressed strings. */ tor_free(buf3); @@ -2287,8 +2287,8 @@ test_util_gzip(void *arg) memcpy(buf2+len1, buf2, len1); tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, ZLIB_METHOD, 1, LOG_INFO)); - tt_int_op((strlen(buf1)+1)*2,OP_EQ, len2); - tt_mem_op(buf3,OP_EQ, + tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); + tt_mem_op(buf3, OP_EQ, "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", (strlen(buf1)+1)*2); @@ -2328,21 +2328,21 @@ test_util_gzip(void *arg) len2 = 21; tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) == TOR_ZLIB_OK); - tt_int_op(0,OP_EQ, len2); /* Make sure we compressed it all. */ + tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ tt_assert(cp1 > buf1); len2 = 0; cp2 = cp1; tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) == TOR_ZLIB_DONE); - tt_int_op(0,OP_EQ, len2); + tt_int_op(0, OP_EQ, len2); tt_assert(cp1 > cp2); /* Make sure we really added something. */ tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, ZLIB_METHOD, 1, LOG_WARN)); /* Make sure it compressed right. */ tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); - tt_int_op(21,OP_EQ, len2); + tt_int_op(21, OP_EQ, len2); done: if (state) From 7460b9755aa0b9b4a54c5c80e47dfdf8c8fc513a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 17 Apr 2017 14:11:35 +0200 Subject: [PATCH 03/28] Remove unused function `is_gzip_supported()`. This patch removes the unused `is_gzip_supported()` and changes the documentation string around the `compress_method_t` enumeration to explicitly state that both `ZLIB_METHOD` and `GZIP_METHOD` are both always supported. Zlib version 1.2.0 was released on the 9'th of March, 2003 according to their ChangeLog. See https://bugs.torproject.org/21663 --- src/common/torgzip.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 430b152fe4..42b45ab08a 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -11,10 +11,9 @@ #ifndef TOR_TORGZIP_H #define TOR_TORGZIP_H -/** Enumeration of what kind of compression to use. Only ZLIB_METHOD is - * guaranteed to be supported by the compress/uncompress functions here; - * GZIP_METHOD may be supported if we built against zlib version 1.2 or later - * and is_gzip_supported() returns true. */ +/** Enumeration of what kind of compression to use. Only ZLIB_METHOD and + * GZIP_METHOD is guaranteed to be supported by the compress/uncompress + * functions here. */ typedef enum { NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3 } compress_method_t; @@ -39,8 +38,6 @@ tor_gzip_uncompress(char **out, size_t *out_len, int complete_only, int protocol_warn_level); -int is_gzip_supported(void); - const char * tor_zlib_get_version_str(void); From e8b025dfc30ef0008cb44feb56ab3c740d2bae9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 17 Apr 2017 14:22:13 +0200 Subject: [PATCH 04/28] Rename `zlib_compression_level_t` to `compression_level_t`. See https://bugs.torproject.org/21663 --- src/common/torgzip.c | 6 +++--- src/common/torgzip.h | 4 ++-- src/or/directory.c | 2 +- src/or/directory.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 223cd307a7..a6909cefea 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -74,7 +74,7 @@ tor_zlib_get_header_version_str(void) /** Return the 'bits' value to tell zlib to use method.*/ static inline int -method_bits(compress_method_t method, zlib_compression_level_t level) +method_bits(compress_method_t method, compression_level_t level) { /* Bits+16 means "use gzip" in zlib >= 1.2 */ const int flag = method == GZIP_METHOD ? 16 : 0; @@ -87,7 +87,7 @@ method_bits(compress_method_t method, zlib_compression_level_t level) } static inline int -get_memlevel(zlib_compression_level_t level) +get_memlevel(compression_level_t level) { switch (level) { default: @@ -419,7 +419,7 @@ struct tor_zlib_state_t { * decompression. */ tor_zlib_state_t * tor_zlib_new(int compress_, compress_method_t method, - zlib_compression_level_t compression_level) + compression_level_t compression_level) { tor_zlib_state_t *out; int bits, memlevel; diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 42b45ab08a..cbcbb9978f 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -25,7 +25,7 @@ typedef enum { **/ typedef enum { HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION -} zlib_compression_level_t; +} compression_level_t; int tor_gzip_compress(char **out, size_t *out_len, @@ -54,7 +54,7 @@ typedef enum { /** Internal state for an incremental zlib compression/decompression. */ typedef struct tor_zlib_state_t tor_zlib_state_t; tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method, - zlib_compression_level_t level); + compression_level_t level); tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, char **out, size_t *out_len, diff --git a/src/or/directory.c b/src/or/directory.c index 1b999ee7c3..bbe071b558 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2845,7 +2845,7 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) /** Return the compression level we should use for sending a compressed * response of size n_bytes. */ -STATIC zlib_compression_level_t +STATIC compression_level_t choose_compression_level(ssize_t n_bytes) { if (! have_been_under_memory_pressure()) { diff --git a/src/or/directory.h b/src/or/directory.h index 0c5db3e070..4c52c24049 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -181,7 +181,7 @@ STATIC int handle_post_hs_descriptor(const char *url, const char *body); STATIC char* authdir_type_to_string(dirinfo_type_t auth); STATIC const char * dir_conn_purpose_to_string(int purpose); STATIC int should_use_directory_guards(const or_options_t *options); -STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes); +STATIC compression_level_t choose_compression_level(ssize_t n_bytes); STATIC const smartlist_t *find_dl_schedule(download_status_t *dls, const or_options_t *options); STATIC void find_dl_min_and_max_delay(download_status_t *dls, From 44cb86adbe2e9b9fef7bf5fd64b52a8e44441be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 17 Apr 2017 14:29:10 +0200 Subject: [PATCH 05/28] Rename `tor_gzip_{compress,uncompress}` to `tor_{compress,uncompress}`. To allow us to use the API name `tor_compress` and `tor_uncompress` as the main entry-point for all compression/uncompression and not just gzip and zlib. See https://bugs.torproject.org/21663 --- src/common/torgzip.c | 16 ++++++------ src/common/torgzip.h | 16 ++++++------ src/or/directory.c | 8 +++--- src/or/dirserv.c | 4 +-- src/test/test_buffers.c | 17 ++++++------ src/test/test_dir_handle_get.c | 4 +-- src/test/test_util.c | 48 +++++++++++++++++----------------- 7 files changed, 57 insertions(+), 56 deletions(-) diff --git a/src/common/torgzip.c b/src/common/torgzip.c index a6909cefea..cbfcabd27e 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -132,9 +132,9 @@ is_compression_bomb(size_t size_in, size_t size_out) * Return 0 on success, -1 on failure. */ int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method) +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) { struct z_stream_s *stream = NULL; size_t out_size, old_size; @@ -252,11 +252,11 @@ tor_gzip_compress(char **out, size_t *out_len, * or corrupt inputs at protocol_warn_level. */ int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level) +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) { struct z_stream_s *stream = NULL; size_t out_size, old_size; diff --git a/src/common/torgzip.h b/src/common/torgzip.h index cbcbb9978f..ac9763eabe 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -28,15 +28,15 @@ typedef enum { } compression_level_t; int -tor_gzip_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); +tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); int -tor_gzip_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); +tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); const char * tor_zlib_get_version_str(void); diff --git a/src/or/directory.c b/src/or/directory.c index bbe071b558..e7a71dde15 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -2096,15 +2096,15 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } /* Try declared compression first if we can. */ if (compression == GZIP_METHOD || compression == ZLIB_METHOD) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression, - !allow_partial, LOG_PROTOCOL_WARN); + tor_uncompress(&new_body, &new_len, body, body_len, compression, + !allow_partial, LOG_PROTOCOL_WARN); /* Okay, if that didn't work, and we think that it was compressed * differently, try that. */ if (!new_body && (guessed == GZIP_METHOD || guessed == ZLIB_METHOD) && compression != guessed) - tor_gzip_uncompress(&new_body, &new_len, body, body_len, guessed, - !allow_partial, LOG_PROTOCOL_WARN); + tor_uncompress(&new_body, &new_len, body, body_len, guessed, + !allow_partial, LOG_PROTOCOL_WARN); /* If we're pretty sure that we have a compressed directory, and * we didn't manage to uncompress it, then warn and bail. */ if (!plausible && !new_body) { diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 70b0b22f25..87afd69aba 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -1176,8 +1176,8 @@ new_cached_dir(char *s, time_t published) d->dir = s; d->dir_len = strlen(s); d->published = published; - if (tor_gzip_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, - ZLIB_METHOD)) { + if (tor_compress(&(d->dir_z), &(d->dir_z_len), d->dir, d->dir_len, + ZLIB_METHOD)) { log_warn(LD_BUG, "Error compressing directory"); } return d; diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index d14165c45c..f0edd42a88 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -607,10 +607,10 @@ test_buffers_zlib_impl(int finalize_with_nil) tt_int_op(fetch_from_buf(contents, in_len, buf), OP_EQ, 0); - tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len, - contents, in_len, - ZLIB_METHOD, 1, - LOG_WARN)); + tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len, + contents, in_len, + ZLIB_METHOD, 1, + LOG_WARN)); tt_int_op(out_len, OP_GE, 128); tt_mem_op(msg, OP_EQ, expanded, 128); @@ -676,10 +676,11 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tt_uint_op(in_len, OP_GT, headerjunk); - tt_int_op(0, OP_EQ, tor_gzip_uncompress(&expanded, &out_len, - contents + headerjunk, in_len - headerjunk, - ZLIB_METHOD, 1, - LOG_WARN)); + tt_int_op(0, OP_EQ, tor_uncompress(&expanded, &out_len, + contents + headerjunk, + in_len - headerjunk, + ZLIB_METHOD, 1, + LOG_WARN)); tt_int_op(out_len, OP_EQ, 0); tt_assert(expanded); diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index cfda314693..ed8ea2f809 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -1832,8 +1832,8 @@ test_dir_handle_get_status_vote_current_consensus_ns(void* data) comp_body_used); tt_int_op(ZLIB_METHOD, OP_EQ, compression); - tor_gzip_uncompress(&body, &body_used, comp_body, comp_body_used, - compression, 0, LOG_PROTOCOL_WARN); + tor_uncompress(&body, &body_used, comp_body, comp_body_used, + compression, 0, LOG_PROTOCOL_WARN); tt_str_op(NETWORK_STATUS, OP_EQ, body); tt_int_op(strlen(NETWORK_STATUS), OP_EQ, body_used); diff --git a/src/test/test_util.c b/src/test/test_util.c index 19d1c79e79..7e24279591 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2255,14 +2255,14 @@ test_util_gzip(void *arg) buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + GZIP_METHOD)); tt_assert(buf2 != NULL); tt_int_op(len1, OP_LT, strlen(buf1)); tt_int_op(detect_compression_method(buf2, len1), OP_EQ, GZIP_METHOD); - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + GZIP_METHOD, 1, LOG_INFO)); tt_assert(buf3 != NULL); tt_int_op(strlen(buf1) + 1, OP_EQ, len2); tt_str_op(buf1, OP_EQ, buf3); @@ -2270,13 +2270,13 @@ test_util_gzip(void *arg) tor_free(buf2); tor_free(buf3); - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); tt_assert(buf2); tt_int_op(detect_compression_method(buf2, len1), OP_EQ, ZLIB_METHOD); - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - ZLIB_METHOD, 1, LOG_INFO)); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + ZLIB_METHOD, 1, LOG_INFO)); tt_assert(buf3 != NULL); tt_int_op(strlen(buf1) + 1, OP_EQ, len2); tt_str_op(buf1, OP_EQ, buf3); @@ -2285,8 +2285,8 @@ test_util_gzip(void *arg) tor_free(buf3); buf2 = tor_reallocarray(buf2, len1, 2); memcpy(buf2+len1, buf2, len1); - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1*2, - ZLIB_METHOD, 1, LOG_INFO)); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, + ZLIB_METHOD, 1, LOG_INFO)); tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); tt_mem_op(buf3, OP_EQ, "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" @@ -2300,20 +2300,20 @@ test_util_gzip(void *arg) /* Check whether we can uncompress partial strings. */ buf1 = tor_strdup("String with low redundancy that won't be compressed much."); - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - ZLIB_METHOD)); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZLIB_METHOD)); tt_int_op(len1, OP_GT, 16); /* when we allow an incomplete string, we should succeed.*/ - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 0, LOG_INFO)); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 0, LOG_INFO)); tt_assert(len2 > 5); buf3[len2]='\0'; tt_assert(!strcmpstart(buf1, buf3)); /* when we demand a complete string, this must fail. */ tor_free(buf3); - tt_assert(tor_gzip_uncompress(&buf3, &len2, buf2, len1-16, - ZLIB_METHOD, 1, LOG_INFO)); + tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16, + ZLIB_METHOD, 1, LOG_INFO)); tt_assert(buf3 == NULL); /* Now, try streaming compression. */ @@ -2338,8 +2338,8 @@ test_util_gzip(void *arg) tt_int_op(0, OP_EQ, len2); tt_assert(cp1 > cp2); /* Make sure we really added something. */ - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf1, 1024-len1, - ZLIB_METHOD, 1, LOG_WARN)); + tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1, + ZLIB_METHOD, 1, LOG_WARN)); /* Make sure it compressed right. */ tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); tt_int_op(21, OP_EQ, len2); @@ -2368,9 +2368,9 @@ test_util_gzip_compression_bomb(void *arg) /* Make sure we can't produce a compression bomb */ setup_full_capture_of_logs(LOG_WARN); - tt_int_op(-1, OP_EQ, tor_gzip_compress(&result, &result_len, - one_mb, one_million, - ZLIB_METHOD)); + tt_int_op(-1, OP_EQ, tor_compress(&result, &result_len, + one_mb, one_million, + ZLIB_METHOD)); expect_single_log_msg_containing( "We compressed something and got an insanely high " "compression factor; other Tors would think this " @@ -2381,9 +2381,9 @@ test_util_gzip_compression_bomb(void *arg) const char compression_bomb[1039] = { 0x78, 0xDA, 0xED, 0xC1, 0x31, 0x01, 0x00, 0x00, 0x00, 0xC2, 0xA0, 0xF5, 0x4F, 0x6D, 0x08, 0x5F, 0xA0 /* .... */ }; - tt_int_op(-1, OP_EQ, tor_gzip_uncompress(&result, &result_len, - compression_bomb, 1039, - ZLIB_METHOD, 0, LOG_WARN)); + tt_int_op(-1, OP_EQ, tor_uncompress(&result, &result_len, + compression_bomb, 1039, + ZLIB_METHOD, 0, LOG_WARN)); /* Now try streaming that. */ state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); From 3c4459bcbf1d3a9da4a8b3a8bfed10d2e045af74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 17 Apr 2017 14:57:37 +0200 Subject: [PATCH 06/28] Refactor the streaming compression code. This patch refactors our streaming compression code to allow us to extend it with non-zlib/non-gzip based compression schemas. See https://bugs.torproject.org/21663 --- src/common/torgzip.c | 62 ++++++++++++++++++++--------------------- src/common/torgzip.h | 30 +++++++++++--------- src/or/buffers.c | 20 +++++++------ src/or/buffers.h | 2 +- src/or/circuitlist.c | 4 +-- src/or/connection.c | 2 +- src/or/directory.c | 14 +++++----- src/or/dirserv.c | 2 +- src/or/or.h | 4 +-- src/test/test_buffers.c | 12 ++++---- src/test/test_util.c | 28 +++++++++---------- 11 files changed, 93 insertions(+), 87 deletions(-) diff --git a/src/common/torgzip.c b/src/common/torgzip.c index cbfcabd27e..af4bbc5931 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -399,9 +399,9 @@ detect_compression_method(const char *in, size_t in_len) } } -/** Internal state for an incremental zlib compression/decompression. The - * body of this struct is not exposed. */ -struct tor_zlib_state_t { +/** Internal state for an incremental compression/decompression. The body of + * this struct is not exposed. */ +struct tor_compress_state_t { struct z_stream_s stream; /**< The zlib stream */ int compress; /**< True if we are compressing; false if we are inflating */ @@ -414,14 +414,13 @@ struct tor_zlib_state_t { size_t allocation; }; -/** Construct and return a tor_zlib_state_t object using method. If - * compress, it's for compression; otherwise it's for - * decompression. */ -tor_zlib_state_t * -tor_zlib_new(int compress_, compress_method_t method, - compression_level_t compression_level) +/** Construct and return a tor_compress_state_t object using method. If + * compress, it's for compression; otherwise it's for decompression. */ +tor_compress_state_t * +tor_compress_new(int compress_, compress_method_t method, + compression_level_t compression_level) { - tor_zlib_state_t *out; + tor_compress_state_t *out; int bits, memlevel; if (! compress_) { @@ -430,7 +429,7 @@ tor_zlib_new(int compress_, compress_method_t method, compression_level = HIGH_COMPRESSION; } - out = tor_malloc_zero(sizeof(tor_zlib_state_t)); + out = tor_malloc_zero(sizeof(tor_compress_state_t)); out->stream.zalloc = Z_NULL; out->stream.zfree = Z_NULL; out->stream.opaque = NULL; @@ -462,16 +461,17 @@ tor_zlib_new(int compress_, compress_method_t method, * to *out, adjusting the values as we go. If finish is true, * we've reached the end of the input. * - * Return TOR_ZLIB_DONE if we've finished the entire compression/decompression. - * Return TOR_ZLIB_OK if we're processed everything from the input. - * Return TOR_ZLIB_BUF_FULL if we're out of space on out. - * Return TOR_ZLIB_ERR if the stream is corrupt. + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on out. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. */ -tor_zlib_output_t -tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish) +tor_compress_output_t +tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) { int err; tor_assert(*in_len <= UINT_MAX); @@ -498,31 +498,31 @@ tor_zlib_process(tor_zlib_state_t *state, if (! state->compress && is_compression_bomb(state->input_so_far, state->output_so_far)) { log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); - return TOR_ZLIB_ERR; + return TOR_COMPRESS_ERROR; } switch (err) { case Z_STREAM_END: - return TOR_ZLIB_DONE; + return TOR_COMPRESS_DONE; case Z_BUF_ERROR: if (state->stream.avail_in == 0 && !finish) - return TOR_ZLIB_OK; - return TOR_ZLIB_BUF_FULL; + return TOR_COMPRESS_OK; + return TOR_COMPRESS_BUFFER_FULL; case Z_OK: if (state->stream.avail_out == 0 || finish) - return TOR_ZLIB_BUF_FULL; - return TOR_ZLIB_OK; + return TOR_COMPRESS_BUFFER_FULL; + return TOR_COMPRESS_OK; default: log_warn(LD_GENERAL, "Gzip returned an error: %s", state->stream.msg ? state->stream.msg : ""); - return TOR_ZLIB_ERR; + return TOR_COMPRESS_ERROR; } } /** Deallocate state. */ void -tor_zlib_free(tor_zlib_state_t *state) +tor_compress_free(tor_compress_state_t *state) { if (!state) return; @@ -553,7 +553,7 @@ tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) that is, 32K for windowBits=15 (default value) plus a few kilobytes for small objects." */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + + return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) + (1 << 15) + A_FEW_KILOBYTES; } else { /* Also from zconf.h: @@ -562,7 +562,7 @@ tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) (1 << (windowBits+2)) + (1 << (memLevel+9)) ... plus a few kilobytes for small objects." */ - return sizeof(tor_zlib_state_t) + sizeof(struct z_stream_s) + + return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; } #undef A_FEW_KILOBYTES @@ -570,7 +570,7 @@ tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) /** Return the approximate number of bytes allocated for state. */ size_t -tor_zlib_state_size(const tor_zlib_state_t *state) +tor_compress_state_size(const tor_compress_state_t *state) { return state->allocation; } diff --git a/src/common/torgzip.h b/src/common/torgzip.h index ac9763eabe..2b0504eb13 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -46,23 +46,27 @@ tor_zlib_get_header_version_str(void); compress_method_t detect_compression_method(const char *in, size_t in_len); -/** Return values from tor_zlib_process; see that function's documentation for - * details. */ +/** Return values from tor_compress_process; see that function's documentation + * for details. */ typedef enum { - TOR_ZLIB_OK, TOR_ZLIB_DONE, TOR_ZLIB_BUF_FULL, TOR_ZLIB_ERR -} tor_zlib_output_t; + TOR_COMPRESS_OK, + TOR_COMPRESS_DONE, + TOR_COMPRESS_BUFFER_FULL, + TOR_COMPRESS_ERROR +} tor_compress_output_t; /** Internal state for an incremental zlib compression/decompression. */ -typedef struct tor_zlib_state_t tor_zlib_state_t; -tor_zlib_state_t *tor_zlib_new(int compress, compress_method_t method, - compression_level_t level); +typedef struct tor_compress_state_t tor_compress_state_t; +tor_compress_state_t *tor_compress_new(int compress, + compress_method_t method, + compression_level_t level); -tor_zlib_output_t tor_zlib_process(tor_zlib_state_t *state, - char **out, size_t *out_len, - const char **in, size_t *in_len, - int finish); -void tor_zlib_free(tor_zlib_state_t *state); +tor_compress_output_t tor_compress_process(tor_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); +void tor_compress_free(tor_compress_state_t *state); -size_t tor_zlib_state_size(const tor_zlib_state_t *state); +size_t tor_compress_state_size(const tor_compress_state_t *state); size_t tor_zlib_get_total_allocation(void); #endif diff --git a/src/or/buffers.c b/src/or/buffers.c index e559f80a1e..3490ce48a8 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -2088,11 +2088,11 @@ fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) } /** Compress on uncompress the data_len bytes in data using the - * zlib state state, appending the result to buf. If + * compression state state, appending the result to buf. If * done is true, flush the data in the state and finish the * compression/uncompression. Return -1 on failure, 0 on success. */ int -write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, +write_to_buf_zlib(buf_t *buf, tor_compress_state_t *state, const char *data, size_t data_len, int done) { @@ -2108,20 +2108,22 @@ write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, } next = CHUNK_WRITE_PTR(buf->tail); avail = old_avail = CHUNK_REMAINING_CAPACITY(buf->tail); - switch (tor_zlib_process(state, &next, &avail, &data, &data_len, done)) { - case TOR_ZLIB_DONE: + switch (tor_compress_process(state, &next, &avail, + &data, &data_len, done)) { + case TOR_COMPRESS_DONE: over = 1; break; - case TOR_ZLIB_ERR: + case TOR_COMPRESS_ERROR: return -1; - case TOR_ZLIB_OK: + case TOR_COMPRESS_OK: if (data_len == 0) over = 1; break; - case TOR_ZLIB_BUF_FULL: + case TOR_COMPRESS_BUFFER_FULL: if (avail) { - /* Zlib says we need more room (ZLIB_BUF_FULL). Start a new chunk - * automatically, whether were going to or not. */ + /* The compression module says we need more room + * (TOR_COMPRESS_BUFFER_FULL). Start a new chunk automatically, + * whether were going to or not. */ need_new_chunk = 1; } break; diff --git a/src/or/buffers.h b/src/or/buffers.h index c6a5ffaad5..49a2ae49bc 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -36,7 +36,7 @@ int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen); int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_zlib_state_t *state, +int write_to_buf_zlib(buf_t *buf, tor_compress_state_t *state, const char *data, size_t data_len, int done); int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); int fetch_from_buf(char *string, size_t string_len, buf_t *buf); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 80bb7f69f3..4f11586373 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1991,8 +1991,8 @@ single_conn_free_bytes(connection_t *conn) if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); if (dir_conn->zlib_state) { - result += tor_zlib_state_size(dir_conn->zlib_state); - tor_zlib_free(dir_conn->zlib_state); + result += tor_compress_state_size(dir_conn->zlib_state); + tor_compress_free(dir_conn->zlib_state); dir_conn->zlib_state = NULL; } } diff --git a/src/or/connection.c b/src/or/connection.c index 09e316d214..ad97c3bc2a 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -628,7 +628,7 @@ connection_free_(connection_t *conn) dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - tor_zlib_free(dir_conn->zlib_state); + tor_compress_free(dir_conn->zlib_state); if (dir_conn->spool) { SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled, spooled_resource_free(spooled)); diff --git a/src/or/directory.c b/src/or/directory.c index e7a71dde15..5ae61e470f 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3178,7 +3178,7 @@ handle_get_current_consensus(dir_connection_t *conn, write_http_response_header(conn, -1, compressed, smartlist_len(conn->spool) == 1 ? lifetime : 0); if (! compressed) - conn->zlib_state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + conn->zlib_state = tor_compress_new(0, ZLIB_METHOD, HIGH_COMPRESSION); /* Prime the connection with some data. */ const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -3276,8 +3276,8 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) if (smartlist_len(items)) { if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(estimated_len)); + conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, + choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf_zlib(c, strlen(c), conn, 0)); connection_write_to_buf_zlib("", 0, conn, 1); @@ -3335,7 +3335,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(size_guess)); const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -3428,7 +3428,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) } write_http_response_header(conn, -1, compressed, cache_lifetime); if (compressed) - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, + conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(size_guess)); clear_spool = 0; /* Prime the connection with some data. */ @@ -3519,8 +3519,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, compressed?-1:len, compressed, 60*60); if (compressed) { - conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD, - choose_compression_level(len)); + conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, + choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, c->cache_info.signed_descriptor_len, diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 87afd69aba..186478c1c7 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -3792,7 +3792,7 @@ connection_dirserv_flushed_some(dir_connection_t *conn) /* Flush the zlib state: there could be more bytes pending in there, and * we don't want to omit bytes. */ connection_write_to_buf_zlib("", 0, conn, 1); - tor_zlib_free(conn->zlib_state); + tor_compress_free(conn->zlib_state); conn->zlib_state = NULL; } return 0; diff --git a/src/or/or.h b/src/or/or.h index a7b3a66561..6aec5882b0 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1773,8 +1773,8 @@ typedef struct dir_connection_t { /** List of spooled_resource_t for objects that we're spooling. We use * it from back to front. */ smartlist_t *spool; - /** The zlib object doing on-the-fly compression for spooled data. */ - tor_zlib_state_t *zlib_state; + /** The compression object doing on-the-fly compression for spooled data. */ + tor_compress_state_t *zlib_state; /** What rendezvous service are we querying for? */ rend_data_t *rend_data; diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index f0edd42a88..a6f0309427 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -584,12 +584,12 @@ test_buffers_zlib_impl(int finalize_with_nil) char *contents = NULL; char *expanded = NULL; buf_t *buf = NULL; - tor_zlib_state_t *zlib_state = NULL; + tor_compress_state_t *zlib_state = NULL; size_t out_len, in_len; int done; buf = buf_new_with_capacity(128); /* will round up */ - zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + zlib_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); msg = tor_malloc(512); crypto_rand(msg, 512); @@ -621,7 +621,7 @@ test_buffers_zlib_impl(int finalize_with_nil) done: buf_free(buf); - tor_zlib_free(zlib_state); + tor_compress_free(zlib_state); tor_free(contents); tor_free(expanded); tor_free(msg); @@ -647,7 +647,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) char *contents = NULL; char *expanded = NULL; buf_t *buf = NULL; - tor_zlib_state_t *zlib_state = NULL; + tor_compress_state_t *zlib_state = NULL; size_t out_len, in_len; size_t sz, headerjunk; (void) arg; @@ -666,7 +666,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tt_uint_op(buf->head->datalen, OP_EQ, headerjunk); tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk); /* Write an empty string, with finalization on. */ - zlib_state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + zlib_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0); in_len = buf_datalen(buf); @@ -687,7 +687,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) done: buf_free(buf); - tor_zlib_free(zlib_state); + tor_compress_free(zlib_state); tor_free(contents); tor_free(expanded); tor_free(msg); diff --git a/src/test/test_util.c b/src/test/test_util.c index 7e24279591..dacf56f868 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2249,7 +2249,7 @@ test_util_gzip(void *arg) char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; const char *ccp2; size_t len1, len2; - tor_zlib_state_t *state = NULL; + tor_compress_state_t *state = NULL; (void)arg; buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); @@ -2320,21 +2320,21 @@ test_util_gzip(void *arg) tor_free(buf1); tor_free(buf2); tor_free(buf3); - state = tor_zlib_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); tt_assert(state); cp1 = buf1 = tor_malloc(1024); len1 = 1024; ccp2 = "ABCDEFGHIJABCDEFGHIJ"; len2 = 21; - tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 0) - == TOR_ZLIB_OK); + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0), + OP_EQ, TOR_COMPRESS_OK); tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ tt_assert(cp1 > buf1); len2 = 0; cp2 = cp1; - tt_assert(tor_zlib_process(state, &cp1, &len1, &ccp2, &len2, 1) - == TOR_ZLIB_DONE); + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1), + OP_EQ, TOR_COMPRESS_DONE); tt_int_op(0, OP_EQ, len2); tt_assert(cp1 > cp2); /* Make sure we really added something. */ @@ -2346,7 +2346,7 @@ test_util_gzip(void *arg) done: if (state) - tor_zlib_free(state); + tor_compress_free(state); tor_free(buf2); tor_free(buf3); tor_free(buf1); @@ -2364,7 +2364,7 @@ test_util_gzip_compression_bomb(void *arg) char *one_mb = tor_malloc_zero(one_million); char *result = NULL; size_t result_len = 0; - tor_zlib_state_t *state = NULL; + tor_compress_state_t *state = NULL; /* Make sure we can't produce a compression bomb */ setup_full_capture_of_logs(LOG_WARN); @@ -2386,22 +2386,22 @@ test_util_gzip_compression_bomb(void *arg) ZLIB_METHOD, 0, LOG_WARN)); /* Now try streaming that. */ - state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); - tor_zlib_output_t r; + state = tor_compress_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + tor_compress_output_t r; const char *inp = compression_bomb; size_t inlen = 1039; do { char *outp = one_mb; size_t outleft = 4096; /* small on purpose */ - r = tor_zlib_process(state, &outp, &outleft, &inp, &inlen, 0); + r = tor_compress_process(state, &outp, &outleft, &inp, &inlen, 0); tt_int_op(inlen, OP_NE, 0); - } while (r == TOR_ZLIB_BUF_FULL); + } while (r == TOR_COMPRESS_BUFFER_FULL); - tt_int_op(r, OP_EQ, TOR_ZLIB_ERR); + tt_int_op(r, OP_EQ, TOR_COMPRESS_ERROR); done: tor_free(one_mb); - tor_zlib_free(state); + tor_compress_free(state); } /** Run unit tests for mmap() wrapper functionality. */ From 4b9349192d4bd2e9c598b13d74bc826c8f16d40e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 01:10:25 +0200 Subject: [PATCH 07/28] Rename `zlib_state` to `compress_state` in `dir_connection_t`. This patch renames the `zlib_state` field in `dir_connection_t` to `compress_state`. See https://bugs.torproject.org/21663 --- src/or/circuitlist.c | 8 ++++---- src/or/connection.c | 4 ++-- src/or/directory.c | 13 +++++++------ src/or/dirserv.c | 10 +++++----- src/or/or.h | 2 +- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 4f11586373..365e5b2f89 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -1990,10 +1990,10 @@ single_conn_free_bytes(connection_t *conn) } if (conn->type == CONN_TYPE_DIR) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); - if (dir_conn->zlib_state) { - result += tor_compress_state_size(dir_conn->zlib_state); - tor_compress_free(dir_conn->zlib_state); - dir_conn->zlib_state = NULL; + if (dir_conn->compress_state) { + result += tor_compress_state_size(dir_conn->compress_state); + tor_compress_free(dir_conn->compress_state); + dir_conn->compress_state = NULL; } } return result; diff --git a/src/or/connection.c b/src/or/connection.c index ad97c3bc2a..70b08e2deb 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -628,7 +628,7 @@ connection_free_(connection_t *conn) dir_connection_t *dir_conn = TO_DIR_CONN(conn); tor_free(dir_conn->requested_resource); - tor_compress_free(dir_conn->zlib_state); + tor_compress_free(dir_conn->compress_state); if (dir_conn->spool) { SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled, spooled_resource_free(spooled)); @@ -4061,7 +4061,7 @@ connection_write_to_buf_impl_,(const char *string, size_t len, dir_connection_t *dir_conn = TO_DIR_CONN(conn); int done = zlib < 0; CONN_LOG_PROTECT(conn, r = write_to_buf_zlib(conn->outbuf, - dir_conn->zlib_state, + dir_conn->compress_state, string, len, done)); } else { CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf)); diff --git a/src/or/directory.c b/src/or/directory.c index 5ae61e470f..098683e919 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3178,7 +3178,8 @@ handle_get_current_consensus(dir_connection_t *conn, write_http_response_header(conn, -1, compressed, smartlist_len(conn->spool) == 1 ? lifetime : 0); if (! compressed) - conn->zlib_state = tor_compress_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + conn->compress_state = tor_compress_new(0, ZLIB_METHOD, + HIGH_COMPRESSION); /* Prime the connection with some data. */ const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -3276,7 +3277,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) if (smartlist_len(items)) { if (compressed) { - conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf_zlib(c, strlen(c), conn, 0)); @@ -3335,7 +3336,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME); if (compressed) - conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(size_guess)); const int initial_flush_result = connection_dirserv_flushed_some(conn); @@ -3428,7 +3429,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) } write_http_response_header(conn, -1, compressed, cache_lifetime); if (compressed) - conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(size_guess)); clear_spool = 0; /* Prime the connection with some data. */ @@ -3519,8 +3520,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) write_http_response_header(conn, compressed?-1:len, compressed, 60*60); if (compressed) { - conn->zlib_state = tor_compress_new(1, ZLIB_METHOD, - choose_compression_level(len)); + conn->compress_state = tor_compress_new(1, ZLIB_METHOD, + choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, c->cache_info.signed_descriptor_len, diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 186478c1c7..2746d09543 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -3497,7 +3497,7 @@ spooled_resource_flush_some(spooled_resource_t *spooled, /* Absent objects count as "done". */ return SRFS_DONE; } - if (conn->zlib_state) { + if (conn->compress_state) { connection_write_to_buf_zlib((const char*)body, bodylen, conn, 0); } else { connection_write_to_buf((const char*)body, bodylen, TO_CONN(conn)); @@ -3523,7 +3523,7 @@ spooled_resource_flush_some(spooled_resource_t *spooled, if (BUG(remaining < 0)) return SRFS_ERR; ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining); - if (conn->zlib_state) { + if (conn->compress_state) { connection_write_to_buf_zlib(cached->dir_z + spooled->cached_dir_offset, bytes, conn, 0); } else { @@ -3788,12 +3788,12 @@ connection_dirserv_flushed_some(dir_connection_t *conn) /* If we get here, we're done. */ smartlist_free(conn->spool); conn->spool = NULL; - if (conn->zlib_state) { + if (conn->compress_state) { /* Flush the zlib state: there could be more bytes pending in there, and * we don't want to omit bytes. */ connection_write_to_buf_zlib("", 0, conn, 1); - tor_compress_free(conn->zlib_state); - conn->zlib_state = NULL; + tor_compress_free(conn->compress_state); + conn->compress_state = NULL; } return 0; } diff --git a/src/or/or.h b/src/or/or.h index 6aec5882b0..20c4ebccd0 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -1774,7 +1774,7 @@ typedef struct dir_connection_t { * it from back to front. */ smartlist_t *spool; /** The compression object doing on-the-fly compression for spooled data. */ - tor_compress_state_t *zlib_state; + tor_compress_state_t *compress_state; /** What rendezvous service are we querying for? */ rend_data_t *rend_data; From 40ed68290e69d32d95bc0d0c259d8c76ae3e80ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 01:12:01 +0200 Subject: [PATCH 08/28] Rename `zlib_state` to `compress_state` in the test_buffers. See https://bugs.torproject.org/21663 --- src/test/test_buffers.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index a6f0309427..be757ae628 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -584,22 +584,23 @@ test_buffers_zlib_impl(int finalize_with_nil) char *contents = NULL; char *expanded = NULL; buf_t *buf = NULL; - tor_compress_state_t *zlib_state = NULL; + tor_compress_state_t *compress_state = NULL; size_t out_len, in_len; int done; buf = buf_new_with_capacity(128); /* will round up */ - zlib_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + compress_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); msg = tor_malloc(512); crypto_rand(msg, 512); - tt_int_op(write_to_buf_zlib(buf, zlib_state, msg, 128, 0), OP_EQ, 0); - tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+128, 128, 0), OP_EQ, 0); - tt_int_op(write_to_buf_zlib(buf, zlib_state, msg+256, 256, 0), OP_EQ, 0); + tt_int_op(write_to_buf_zlib(buf, compress_state, msg, 128, 0), OP_EQ, 0); + tt_int_op(write_to_buf_zlib(buf, compress_state, msg+128, 128, 0), OP_EQ, 0); + tt_int_op(write_to_buf_zlib(buf, compress_state, msg+256, 256, 0), OP_EQ, 0); done = !finalize_with_nil; - tt_int_op(write_to_buf_zlib(buf, zlib_state, "all done", 9, done), OP_EQ, 0); + tt_int_op(write_to_buf_zlib(buf, compress_state, + "all done", 9, done), OP_EQ, 0); if (finalize_with_nil) { - tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0); + tt_int_op(write_to_buf_zlib(buf, compress_state, "", 0, 1), OP_EQ, 0); } in_len = buf_datalen(buf); @@ -621,7 +622,7 @@ test_buffers_zlib_impl(int finalize_with_nil) done: buf_free(buf); - tor_compress_free(zlib_state); + tor_compress_free(compress_state); tor_free(contents); tor_free(expanded); tor_free(msg); @@ -647,7 +648,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) char *contents = NULL; char *expanded = NULL; buf_t *buf = NULL; - tor_compress_state_t *zlib_state = NULL; + tor_compress_state_t *compress_state = NULL; size_t out_len, in_len; size_t sz, headerjunk; (void) arg; @@ -666,8 +667,8 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tt_uint_op(buf->head->datalen, OP_EQ, headerjunk); tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk); /* Write an empty string, with finalization on. */ - zlib_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); - tt_int_op(write_to_buf_zlib(buf, zlib_state, "", 0, 1), OP_EQ, 0); + compress_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); + tt_int_op(write_to_buf_zlib(buf, compress_state, "", 0, 1), OP_EQ, 0); in_len = buf_datalen(buf); contents = tor_malloc(in_len); @@ -687,7 +688,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) done: buf_free(buf); - tor_compress_free(zlib_state); + tor_compress_free(compress_state); tor_free(contents); tor_free(expanded); tor_free(msg); From b8c9f229d747d4edc739198113dfb7e3818e23bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 01:23:39 +0200 Subject: [PATCH 09/28] Rename `write_to_buf_zlib()` to `write_to_buf_compress()`. See https://bugs.torproject.org/21663 --- src/or/buffers.c | 6 +++--- src/or/buffers.h | 4 ++-- src/or/connection.c | 6 +++--- src/or/connection.h | 10 +++++----- src/or/directory.c | 13 +++++++------ src/or/dirserv.c | 13 +++++++------ src/test/test_buffers.c | 17 ++++++++++------- 7 files changed, 37 insertions(+), 32 deletions(-) diff --git a/src/or/buffers.c b/src/or/buffers.c index 3490ce48a8..4f22935d26 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -2092,9 +2092,9 @@ fetch_from_buf_line(buf_t *buf, char *data_out, size_t *data_len) * done is true, flush the data in the state and finish the * compression/uncompression. Return -1 on failure, 0 on success. */ int -write_to_buf_zlib(buf_t *buf, tor_compress_state_t *state, - const char *data, size_t data_len, - int done) +write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, + int done) { char *next; size_t old_avail, avail; diff --git a/src/or/buffers.h b/src/or/buffers.h index 49a2ae49bc..23b58a571a 100644 --- a/src/or/buffers.h +++ b/src/or/buffers.h @@ -36,8 +36,8 @@ int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen); int flush_buf_tls(tor_tls_t *tls, buf_t *buf, size_t sz, size_t *buf_flushlen); int write_to_buf(const char *string, size_t string_len, buf_t *buf); -int write_to_buf_zlib(buf_t *buf, tor_compress_state_t *state, - const char *data, size_t data_len, int done); +int write_to_buf_compress(buf_t *buf, tor_compress_state_t *state, + const char *data, size_t data_len, int done); int move_buf_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen); int fetch_from_buf(char *string, size_t string_len, buf_t *buf); int fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto); diff --git a/src/or/connection.c b/src/or/connection.c index 70b08e2deb..5fb2c53677 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -4060,9 +4060,9 @@ connection_write_to_buf_impl_,(const char *string, size_t len, if (zlib) { dir_connection_t *dir_conn = TO_DIR_CONN(conn); int done = zlib < 0; - CONN_LOG_PROTECT(conn, r = write_to_buf_zlib(conn->outbuf, - dir_conn->compress_state, - string, len, done)); + CONN_LOG_PROTECT(conn, r = write_to_buf_compress(conn->outbuf, + dir_conn->compress_state, + string, len, done)); } else { CONN_LOG_PROTECT(conn, r = write_to_buf(string, len, conn->outbuf)); } diff --git a/src/or/connection.h b/src/or/connection.h index df6fc64709..36e45aef38 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -141,17 +141,17 @@ MOCK_DECL(void, connection_write_to_buf_impl_, /* DOCDOC connection_write_to_buf */ static void connection_write_to_buf(const char *string, size_t len, connection_t *conn); -/* DOCDOC connection_write_to_buf_zlib */ -static void connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done); +/* DOCDOC connection_write_to_buf_compress */ +static void connection_write_to_buf_compress(const char *string, size_t len, + dir_connection_t *conn, int done); static inline void connection_write_to_buf(const char *string, size_t len, connection_t *conn) { connection_write_to_buf_impl_(string, len, conn, 0); } static inline void -connection_write_to_buf_zlib(const char *string, size_t len, - dir_connection_t *conn, int done) +connection_write_to_buf_compress(const char *string, size_t len, + dir_connection_t *conn, int done) { connection_write_to_buf_impl_(string, len, TO_CONN(conn), done ? -1 : 1); } diff --git a/src/or/directory.c b/src/or/directory.c index 098683e919..e0409e2021 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -3280,8 +3280,8 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(estimated_len)); SMARTLIST_FOREACH(items, const char *, c, - connection_write_to_buf_zlib(c, strlen(c), conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_write_to_buf_compress(c, strlen(c), conn, 0)); + connection_write_to_buf_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(items, const char *, c, connection_write_to_buf(c, strlen(c), TO_CONN(conn))); @@ -3523,10 +3523,11 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) conn->compress_state = tor_compress_new(1, ZLIB_METHOD, choose_compression_level(len)); SMARTLIST_FOREACH(certs, authority_cert_t *, c, - connection_write_to_buf_zlib(c->cache_info.signed_descriptor_body, - c->cache_info.signed_descriptor_len, - conn, 0)); - connection_write_to_buf_zlib("", 0, conn, 1); + connection_write_to_buf_compress( + c->cache_info.signed_descriptor_body, + c->cache_info.signed_descriptor_len, + conn, 0)); + connection_write_to_buf_compress("", 0, conn, 1); } else { SMARTLIST_FOREACH(certs, authority_cert_t *, c, connection_write_to_buf(c->cache_info.signed_descriptor_body, diff --git a/src/or/dirserv.c b/src/or/dirserv.c index 2746d09543..e76fd932ca 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -3498,7 +3498,7 @@ spooled_resource_flush_some(spooled_resource_t *spooled, return SRFS_DONE; } if (conn->compress_state) { - connection_write_to_buf_zlib((const char*)body, bodylen, conn, 0); + connection_write_to_buf_compress((const char*)body, bodylen, conn, 0); } else { connection_write_to_buf((const char*)body, bodylen, TO_CONN(conn)); } @@ -3524,8 +3524,9 @@ spooled_resource_flush_some(spooled_resource_t *spooled, return SRFS_ERR; ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining); if (conn->compress_state) { - connection_write_to_buf_zlib(cached->dir_z + spooled->cached_dir_offset, - bytes, conn, 0); + connection_write_to_buf_compress( + cached->dir_z + spooled->cached_dir_offset, + bytes, conn, 0); } else { connection_write_to_buf(cached->dir_z + spooled->cached_dir_offset, bytes, TO_CONN(conn)); @@ -3789,9 +3790,9 @@ connection_dirserv_flushed_some(dir_connection_t *conn) smartlist_free(conn->spool); conn->spool = NULL; if (conn->compress_state) { - /* Flush the zlib state: there could be more bytes pending in there, and - * we don't want to omit bytes. */ - connection_write_to_buf_zlib("", 0, conn, 1); + /* Flush the compression state: there could be more bytes pending in there, + * and we don't want to omit bytes. */ + connection_write_to_buf_compress("", 0, conn, 1); tor_compress_free(conn->compress_state); conn->compress_state = NULL; } diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index be757ae628..43582d1b7a 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -593,14 +593,17 @@ test_buffers_zlib_impl(int finalize_with_nil) msg = tor_malloc(512); crypto_rand(msg, 512); - tt_int_op(write_to_buf_zlib(buf, compress_state, msg, 128, 0), OP_EQ, 0); - tt_int_op(write_to_buf_zlib(buf, compress_state, msg+128, 128, 0), OP_EQ, 0); - tt_int_op(write_to_buf_zlib(buf, compress_state, msg+256, 256, 0), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + msg, 128, 0), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + msg+128, 128, 0), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + msg+256, 256, 0), OP_EQ, 0); done = !finalize_with_nil; - tt_int_op(write_to_buf_zlib(buf, compress_state, - "all done", 9, done), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, + "all done", 9, done), OP_EQ, 0); if (finalize_with_nil) { - tt_int_op(write_to_buf_zlib(buf, compress_state, "", 0, 1), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0); } in_len = buf_datalen(buf); @@ -668,7 +671,7 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tt_uint_op(buf_datalen(buf), OP_EQ, headerjunk); /* Write an empty string, with finalization on. */ compress_state = tor_compress_new(1, ZLIB_METHOD, HIGH_COMPRESSION); - tt_int_op(write_to_buf_zlib(buf, compress_state, "", 0, 1), OP_EQ, 0); + tt_int_op(write_to_buf_compress(buf, compress_state, "", 0, 1), OP_EQ, 0); in_len = buf_datalen(buf); contents = tor_malloc(in_len); From 6b5172bcf2018f858d38f5dea13753b8238b15ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 02:23:25 +0200 Subject: [PATCH 10/28] Expose `tor_compress_memory_level()` as a public function in the compression module. This patch makes the internal `get_memlevel()` a part of the public compression API as `tor_compress_memory_level()`. See https://bugs.torproject.org/21663 --- src/common/torgzip.c | 24 ++++++++++++++---------- src/common/torgzip.h | 3 +++ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/common/torgzip.c b/src/common/torgzip.c index af4bbc5931..09f3806895 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -86,16 +86,6 @@ method_bits(compress_method_t method, compression_level_t level) } } -static inline int -get_memlevel(compression_level_t level) -{ - switch (level) { - default: - case HIGH_COMPRESSION: return 8; - case MEDIUM_COMPRESSION: return 7; - case LOW_COMPRESSION: return 6; - } -} /** @{ */ /* These macros define the maximum allowable compression factor. Anything of @@ -126,6 +116,20 @@ is_compression_bomb(size_t size_in, size_t size_out) return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); } +/** Given level return the memory level. The memory level is needed for + * the various compression backends used in Tor. + */ +int +tor_compress_memory_level(compression_level_t level) +{ + switch (level) { + default: + case HIGH_COMPRESSION: return 8; + case MEDIUM_COMPRESSION: return 7; + case LOW_COMPRESSION: return 6; + } +} + /** Given in_len bytes at in, compress them into a newly * allocated buffer, using the method described in method. Store the * compressed string in *out, and its length in *out_len. diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 2b0504eb13..8f2363aa1e 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -46,6 +46,9 @@ tor_zlib_get_header_version_str(void); compress_method_t detect_compression_method(const char *in, size_t in_len); +int +tor_compress_memory_level(compression_level_t level); + /** Return values from tor_compress_process; see that function's documentation * for details. */ typedef enum { From e6c6606a17841eaf263967254990db5e9443942d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 03:12:19 +0200 Subject: [PATCH 11/28] Expose `tor_compress_is_compression_bomb()` as part of the public compression API. This patch exposes the old `is_compression_bomb()` function as a public API as part of the compression module. See https://bugs.torproject.org/21663 --- src/common/torgzip.c | 8 ++++---- src/common/torgzip.h | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 09f3806895..d6f6225251 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -105,10 +105,10 @@ method_bits(compress_method_t method, compression_level_t level) #define CHECK_FOR_COMPRESSION_BOMB_AFTER (1024*64) /** @} */ -/** Return true if uncompressing an input of size in_size to an input - * of size at least size_out looks like a compression bomb. */ -static int -is_compression_bomb(size_t size_in, size_t size_out) +/** Return true if uncompressing an input of size in_size to an input of + * size at least size_out looks like a compression bomb. */ +int +tor_compress_is_compression_bomb(size_t size_in, size_t size_out) { if (size_in == 0 || size_out < CHECK_FOR_COMPRESSION_BOMB_AFTER) return 0; diff --git a/src/common/torgzip.h b/src/common/torgzip.h index 8f2363aa1e..fa566282d7 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -49,6 +49,9 @@ compress_method_t detect_compression_method(const char *in, size_t in_len); int tor_compress_memory_level(compression_level_t level); +int +tor_compress_is_compression_bomb(size_t size_in, size_t size_out); + /** Return values from tor_compress_process; see that function's documentation * for details. */ typedef enum { From 9d5bc1a9354637aa59025f61e577c6d42f8c53ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 03:14:36 +0200 Subject: [PATCH 12/28] Move zlib compression code into its own module. This patch refactors the `torgzip` module to allow us to extend a common compression API to support multiple compression backends. Additionally we move the gzip/zlib code into its own module under the name `compress_zlib`. See https://bugs.torproject.org/21664 --- src/common/compress_zlib.c | 524 +++++++++++++++++++++++++++++++++++++ src/common/compress_zlib.h | 56 ++++ src/common/include.am | 2 + src/common/torgzip.c | 452 +++----------------------------- src/common/torgzip.h | 11 +- src/or/config.c | 1 + src/or/main.c | 1 + src/or/relay.c | 1 + 8 files changed, 630 insertions(+), 418 deletions(-) create mode 100644 src/common/compress_zlib.c create mode 100644 src/common/compress_zlib.h diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c new file mode 100644 index 0000000000..38e500c754 --- /dev/null +++ b/src/common/compress_zlib.c @@ -0,0 +1,524 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.c + * \brief Compression backend for gzip and zlib. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "torgzip.h" +#include "compress_zlib.h" + +/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of + saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory + that nobody will care if the compile outputs a no-such-identifier warning. + + Sorry, but we like -Werror over here, so I guess we need to define these. + I hope that zlib 1.2.6 doesn't break these too. +*/ +#ifndef _LARGEFILE64_SOURCE +#define _LARGEFILE64_SOURCE 0 +#endif +#ifndef _LFS64_LARGEFILE +#define _LFS64_LARGEFILE 0 +#endif +#ifndef _FILE_OFFSET_BITS +#define _FILE_OFFSET_BITS 0 +#endif +#ifndef off64_t +#define off64_t int64_t +#endif + +#include + +#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 +#error "We require zlib version 1.2 or later." +#endif + +static size_t tor_zlib_state_size_precalc(int inflate, + int windowbits, int memlevel); + +/** Total number of bytes allocated for zlib state */ +static size_t total_zlib_allocation = 0; + +/** Return the 'bits' value to tell zlib to use method.*/ +static inline int +method_bits(compress_method_t method, compression_level_t level) +{ + /* Bits+16 means "use gzip" in zlib >= 1.2 */ + const int flag = method == GZIP_METHOD ? 16 : 0; + switch (level) { + default: + case HIGH_COMPRESSION: return flag + 15; + case MEDIUM_COMPRESSION: return flag + 13; + case LOW_COMPRESSION: return flag + 11; + } +} + +/** Return a string representation of the version of the currently running + * version of zlib. */ +const char * +tor_zlib_get_version_str(void) +{ + return zlibVersion(); +} + +/** Return a string representation of the version of the version of zlib +* used at compilation. */ +const char * +tor_zlib_get_header_version_str(void) +{ + return ZLIB_VERSION; +} + +/** Given in_len bytes at in, compress them into a newly + * allocated buffer, using the method described in method. Store the + * compressed string in *out, and its length in *out_len. Return + * 0 on success, -1 on failure. + */ +int +tor_zlib_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ + struct z_stream_s *stream = NULL; + size_t out_size, old_size; + off_t offset; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + + *out = NULL; + + stream = tor_malloc_zero(sizeof(struct z_stream_s)); + stream->zalloc = Z_NULL; + stream->zfree = Z_NULL; + stream->opaque = NULL; + stream->next_in = (unsigned char*) in; + stream->avail_in = (unsigned int)in_len; + + if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, + method_bits(method, HIGH_COMPRESSION), + tor_compress_memory_level(HIGH_COMPRESSION), + Z_DEFAULT_STRATEGY) != Z_OK) { + //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. + log_warn(LD_GENERAL, "Error from deflateInit2: %s", + stream->msg?stream->msg:""); + goto err; + //LCOV_EXCL_STOP + } + + /* Guess 50% compression. */ + out_size = in_len / 2; + if (out_size < 1024) out_size = 1024; + *out = tor_malloc(out_size); + stream->next_out = (unsigned char*)*out; + stream->avail_out = (unsigned int)out_size; + + while (1) { + switch (deflate(stream, Z_FINISH)) + { + case Z_STREAM_END: + goto done; + case Z_OK: + /* In case zlib doesn't work as I think .... */ + if (stream->avail_out >= stream->avail_in+16) + break; + case Z_BUF_ERROR: + offset = stream->next_out - ((unsigned char*)*out); + old_size = out_size; + out_size *= 2; + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in compression."); + goto err; + } + *out = tor_realloc(*out, out_size); + stream->next_out = (unsigned char*)(*out + offset); + if (out_size - offset > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " + "uncompressing."); + goto err; + } + stream->avail_out = (unsigned int)(out_size - offset); + break; + default: + log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", + stream->msg ? stream->msg : ""); + goto err; + } + } + done: + *out_len = stream->total_out; +#if defined(OpenBSD) + /* "Hey Rocky! Watch me change an unsigned field to a signed field in a + * third-party API!" + * "Oh, that trick will just make people do unsafe casts to the unsigned + * type in their cross-platform code!" + * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure + * the newly unsigned field isn't negative." */ + tor_assert(stream->total_out >= 0); +#endif + if (deflateEnd(stream)!=Z_OK) { + // LCOV_EXCL_START -- unreachable if we handled the zlib structure right + tor_assert_nonfatal_unreached(); + log_warn(LD_BUG, "Error freeing gzip structures"); + goto err; + // LCOV_EXCL_STOP + } + tor_free(stream); + + if (tor_compress_is_compression_bomb(*out_len, in_len)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tors would think this was a zlib bomb."); + goto err; + } + + return 0; + err: + if (stream) { + deflateEnd(stream); + tor_free(stream); + } + tor_free(*out); + return -1; +} + +/** Given an Zlib/Gzip compressed string of total length in_len bytes + * at in, uncompress them into a newly allocated buffer. Store the + * uncompressed string in *out, and its length in *out_len. + * Return 0 on success, -1 on failure. + * + * If complete_only is true, we consider a truncated input as a failure; + * otherwise we decompress as much as we can. Warn about truncated or corrupt + * inputs at protocol_warn_level. + */ +int +tor_zlib_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ + struct z_stream_s *stream = NULL; + size_t out_size, old_size; + off_t offset; + int r; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + + *out = NULL; + + stream = tor_malloc_zero(sizeof(struct z_stream_s)); + stream->zalloc = Z_NULL; + stream->zfree = Z_NULL; + stream->opaque = NULL; + stream->next_in = (unsigned char*) in; + stream->avail_in = (unsigned int)in_len; + + if (inflateInit2(stream, + method_bits(method, HIGH_COMPRESSION)) != Z_OK) { + // LCOV_EXCL_START -- can only hit this if we give bad inputs. + log_warn(LD_GENERAL, "Error from inflateInit2: %s", + stream->msg?stream->msg:""); + goto err; + // LCOV_EXCL_STOP + } + + out_size = in_len * 2; /* guess 50% compression. */ + if (out_size < 1024) out_size = 1024; + if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) + goto err; + + *out = tor_malloc(out_size); + stream->next_out = (unsigned char*)*out; + stream->avail_out = (unsigned int)out_size; + + while (1) { + switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) + { + case Z_STREAM_END: + if (stream->avail_in == 0) + goto done; + /* There may be more compressed data here. */ + if ((r = inflateEnd(stream)) != Z_OK) { + log_warn(LD_BUG, "Error freeing gzip structures"); + goto err; + } + if (inflateInit2(stream, + method_bits(method,HIGH_COMPRESSION)) != Z_OK) { + log_warn(LD_GENERAL, "Error from second inflateInit2: %s", + stream->msg?stream->msg:""); + goto err; + } + break; + case Z_OK: + if (!complete_only && stream->avail_in == 0) + goto done; + /* In case zlib doesn't work as I think.... */ + if (stream->avail_out >= stream->avail_in+16) + break; + case Z_BUF_ERROR: + if (stream->avail_out > 0) { + log_fn(protocol_warn_level, LD_PROTOCOL, + "possible truncated or corrupt zlib data"); + goto err; + } + offset = stream->next_out - (unsigned char*)*out; + old_size = out_size; + out_size *= 2; + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in uncompression."); + goto err; + } + if (tor_compress_is_compression_bomb(in_len, out_size)) { + log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " + "not proceeding."); + goto err; + } + if (out_size >= SIZE_T_CEILING) { + log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); + goto err; + } + *out = tor_realloc(*out, out_size); + stream->next_out = (unsigned char*)(*out + offset); + if (out_size - offset > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " + "uncompressing."); + goto err; + } + stream->avail_out = (unsigned int)(out_size - offset); + break; + default: + log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", + stream->msg ? stream->msg : ""); + goto err; + } + } + done: + *out_len = stream->next_out - (unsigned char*)*out; + r = inflateEnd(stream); + tor_free(stream); + if (r != Z_OK) { + log_warn(LD_BUG, "Error freeing gzip structures"); + goto err; + } + + /* NUL-terminate output. */ + if (out_size == *out_len) + *out = tor_realloc(*out, out_size + 1); + (*out)[*out_len] = '\0'; + + return 0; + err: + if (stream) { + inflateEnd(stream); + tor_free(stream); + } + if (*out) { + tor_free(*out); + } + return -1; +} + +/** Internal zlib state for an incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zlib_compress_state_t { + struct z_stream_s stream; /**< The zlib stream */ + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect zlib bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect zlib bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Return an approximate number of bytes used in RAM to hold a state with + * window bits windowBits and compression level 'memlevel' */ +static size_t +tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) +{ + windowbits &= 15; + +#define A_FEW_KILOBYTES 2048 + + if (inflate_) { + /* From zconf.h: + + "The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus a few kilobytes + for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << 15) + A_FEW_KILOBYTES; + } else { + /* Also from zconf.h: + + "The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + ... plus a few kilobytes for small objects." + */ + return sizeof(tor_zlib_compress_state_t) + sizeof(struct z_stream_s) + + (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; + } +#undef A_FEW_KILOBYTES +} + +/** Construct and return a tor_zlib_compress_state_t object using + * method. If compress, it's for compression; otherwise it's for + * decompression. */ +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level) +{ + tor_zlib_compress_state_t *out; + int bits, memlevel; + + if (! compress) { + /* use this setting for decompression, since we might have the + * max number of window bits */ + compression_level = HIGH_COMPRESSION; + } + + out = tor_malloc_zero(sizeof(tor_zlib_compress_state_t)); + out->stream.zalloc = Z_NULL; + out->stream.zfree = Z_NULL; + out->stream.opaque = NULL; + out->compress = compress; + bits = method_bits(method, compression_level); + memlevel = tor_compress_memory_level(compression_level); + if (compress) { + if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, + bits, memlevel, + Z_DEFAULT_STRATEGY) != Z_OK) + goto err; // LCOV_EXCL_LINE + } else { + if (inflateInit2(&out->stream, bits) != Z_OK) + goto err; // LCOV_EXCL_LINE + } + out->allocation = tor_zlib_state_size_precalc(!compress, bits, memlevel); + + total_zlib_allocation += out->allocation; + + return out; + + err: + tor_free(out); + return NULL; +} + +/** Compress/decompress some bytes using state. Read up to + * *in_len bytes from *in, and write up to *out_len bytes + * to *out, adjusting the values as we go. If finish is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on out. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ + int err; + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + state->stream.next_in = (unsigned char*) *in; + state->stream.avail_in = (unsigned int)*in_len; + state->stream.next_out = (unsigned char*) *out; + state->stream.avail_out = (unsigned int)*out_len; + + if (state->compress) { + err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); + } else { + err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); + } + + state->input_so_far += state->stream.next_in - ((unsigned char*)*in); + state->output_so_far += state->stream.next_out - ((unsigned char*)*out); + + *out = (char*) state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *) state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (err) + { + case Z_STREAM_END: + return TOR_COMPRESS_DONE; + case Z_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + return TOR_COMPRESS_BUFFER_FULL; + case Z_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + return TOR_COMPRESS_OK; + default: + log_warn(LD_GENERAL, "Gzip returned an error: %s", + state->stream.msg ? state->stream.msg : ""); + return TOR_COMPRESS_ERROR; + } +} + +/** Deallocate state. */ +void +tor_zlib_compress_free(tor_zlib_compress_state_t *state) +{ + if (state == NULL) + return; + + total_zlib_allocation -= state->allocation; + + if (state->compress) + deflateEnd(&state->stream); + else + inflateEnd(&state->stream); + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for state. */ +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all zlib states. */ +size_t +tor_zlib_get_total_allocation(void) +{ + return total_zlib_allocation; +} + diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h new file mode 100644 index 0000000000..0862678da3 --- /dev/null +++ b/src/common/compress_zlib.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zlib.h + * \brief Header for compress_zlib.c + **/ + +#ifndef TOR_COMPRESS_ZLIB_H +#define TOR_COMPRESS_ZLIB_H + +const char * +tor_zlib_get_version_str(void); + +const char * +tor_zlib_get_header_version_str(void); + +int +tor_zlib_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int +tor_zlib_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +/** Internal state for an incremental zlib/gzip compression/decompression. */ +typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; + +tor_zlib_compress_state_t * +tor_zlib_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zlib_compress_process(tor_zlib_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void +tor_zlib_compress_free(tor_zlib_compress_state_t *state); + +size_t +tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); + +size_t +tor_zlib_get_total_allocation(void); + +#endif // TOR_COMPRESS_ZLIB_H. + diff --git a/src/common/include.am b/src/common/include.am index d6504c7466..ea2c46a670 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -105,6 +105,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/compress_zlib.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ @@ -145,6 +146,7 @@ COMMONHEADERS = \ src/common/compat_openssl.h \ src/common/compat_threads.h \ src/common/compat_time.h \ + src/common/compress_zlib.h \ src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ diff --git a/src/common/torgzip.c b/src/common/torgzip.c index d6f6225251..d760cd9546 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -23,69 +23,7 @@ #include "util.h" #include "torlog.h" #include "torgzip.h" - -/* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of - saying "(defined(FOO) ? FOO : 0)" they like to say "FOO-0", on the theory - that nobody will care if the compile outputs a no-such-identifier warning. - - Sorry, but we like -Werror over here, so I guess we need to define these. - I hope that zlib 1.2.6 doesn't break these too. -*/ -#ifndef _LARGEFILE64_SOURCE -#define _LARGEFILE64_SOURCE 0 -#endif -#ifndef _LFS64_LARGEFILE -#define _LFS64_LARGEFILE 0 -#endif -#ifndef _FILE_OFFSET_BITS -#define _FILE_OFFSET_BITS 0 -#endif -#ifndef off64_t -#define off64_t int64_t -#endif - -#include - -#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 -#error "We require zlib version 1.2 or later." -#endif - -static size_t tor_zlib_state_size_precalc(int inflate, - int windowbits, int memlevel); - -/** Total number of bytes allocated for zlib state */ -static size_t total_zlib_allocation = 0; - -/** Return a string representation of the version of the currently running - * version of zlib. */ -const char * -tor_zlib_get_version_str(void) -{ - return zlibVersion(); -} - -/** Return a string representation of the version of the version of zlib -* used at compilation. */ -const char * -tor_zlib_get_header_version_str(void) -{ - return ZLIB_VERSION; -} - -/** Return the 'bits' value to tell zlib to use method.*/ -static inline int -method_bits(compress_method_t method, compression_level_t level) -{ - /* Bits+16 means "use gzip" in zlib >= 1.2 */ - const int flag = method == GZIP_METHOD ? 16 : 0; - switch (level) { - default: - case HIGH_COMPRESSION: return flag + 15; - case MEDIUM_COMPRESSION: return flag + 13; - case LOW_COMPRESSION: return flag + 11; - } -} - +#include "compress_zlib.h" /** @{ */ /* These macros define the maximum allowable compression factor. Anything of @@ -140,107 +78,9 @@ tor_compress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method) { - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; + if (method == GZIP_METHOD || method == ZLIB_METHOD) + return tor_zlib_compress(out, out_len, in, in_len, method); - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, - method_bits(method, HIGH_COMPRESSION), - get_memlevel(HIGH_COMPRESSION), - Z_DEFAULT_STRATEGY) != Z_OK) { - //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. - log_warn(LD_GENERAL, "Error from deflateInit2: %s", - stream->msg?stream->msg:""); - goto err; - //LCOV_EXCL_STOP - } - - /* Guess 50% compression. */ - out_size = in_len / 2; - if (out_size < 1024) out_size = 1024; - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (deflate(stream, Z_FINISH)) - { - case Z_STREAM_END: - goto done; - case Z_OK: - /* In case zlib doesn't work as I think .... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - offset = stream->next_out - ((unsigned char*)*out); - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in compression."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip compression didn't finish: %s", - stream->msg ? stream->msg : ""); - goto err; - } - } - done: - *out_len = stream->total_out; -#if defined(OpenBSD) - /* "Hey Rocky! Watch me change an unsigned field to a signed field in a - * third-party API!" - * "Oh, that trick will just make people do unsafe casts to the unsigned - * type in their cross-platform code!" - * "Don't be foolish. I'm _sure_ they'll have the good sense to make sure - * the newly unsigned field isn't negative." */ - tor_assert(stream->total_out >= 0); -#endif - if (deflateEnd(stream)!=Z_OK) { - // LCOV_EXCL_START -- unreachable if we handled the zlib structure right - tor_assert_nonfatal_unreached(); - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - // LCOV_EXCL_STOP - } - tor_free(stream); - - if (is_compression_bomb(*out_len, in_len)) { - log_warn(LD_BUG, "We compressed something and got an insanely high " - "compression factor; other Tors would think this was a zlib bomb."); - goto err; - } - - return 0; - err: - if (stream) { - deflateEnd(stream); - tor_free(stream); - } - tor_free(*out); return -1; } @@ -262,127 +102,12 @@ tor_uncompress(char **out, size_t *out_len, int complete_only, int protocol_warn_level) { - struct z_stream_s *stream = NULL; - size_t out_size, old_size; - off_t offset; - int r; + if (method == GZIP_METHOD || method == ZLIB_METHOD) + return tor_zlib_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level); - tor_assert(out); - tor_assert(out_len); - tor_assert(in); - tor_assert(in_len < UINT_MAX); - - *out = NULL; - - stream = tor_malloc_zero(sizeof(struct z_stream_s)); - stream->zalloc = Z_NULL; - stream->zfree = Z_NULL; - stream->opaque = NULL; - stream->next_in = (unsigned char*) in; - stream->avail_in = (unsigned int)in_len; - - if (inflateInit2(stream, - method_bits(method, HIGH_COMPRESSION)) != Z_OK) { - // LCOV_EXCL_START -- can only hit this if we give bad inputs. - log_warn(LD_GENERAL, "Error from inflateInit2: %s", - stream->msg?stream->msg:""); - goto err; - // LCOV_EXCL_STOP - } - - out_size = in_len * 2; /* guess 50% compression. */ - if (out_size < 1024) out_size = 1024; - if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) - goto err; - - *out = tor_malloc(out_size); - stream->next_out = (unsigned char*)*out; - stream->avail_out = (unsigned int)out_size; - - while (1) { - switch (inflate(stream, complete_only ? Z_FINISH : Z_SYNC_FLUSH)) - { - case Z_STREAM_END: - if (stream->avail_in == 0) - goto done; - /* There may be more compressed data here. */ - if ((r = inflateEnd(stream)) != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - if (inflateInit2(stream, - method_bits(method,HIGH_COMPRESSION)) != Z_OK) { - log_warn(LD_GENERAL, "Error from second inflateInit2: %s", - stream->msg?stream->msg:""); - goto err; - } - break; - case Z_OK: - if (!complete_only && stream->avail_in == 0) - goto done; - /* In case zlib doesn't work as I think.... */ - if (stream->avail_out >= stream->avail_in+16) - break; - case Z_BUF_ERROR: - if (stream->avail_out > 0) { - log_fn(protocol_warn_level, LD_PROTOCOL, - "possible truncated or corrupt zlib data"); - goto err; - } - offset = stream->next_out - (unsigned char*)*out; - old_size = out_size; - out_size *= 2; - if (out_size < old_size) { - log_warn(LD_GENERAL, "Size overflow in uncompression."); - goto err; - } - if (is_compression_bomb(in_len, out_size)) { - log_warn(LD_GENERAL, "Input looks like a possible zlib bomb; " - "not proceeding."); - goto err; - } - if (out_size >= SIZE_T_CEILING) { - log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing."); - goto err; - } - *out = tor_realloc(*out, out_size); - stream->next_out = (unsigned char*)(*out + offset); - if (out_size - offset > UINT_MAX) { - log_warn(LD_BUG, "Ran over unsigned int limit of zlib while " - "uncompressing."); - goto err; - } - stream->avail_out = (unsigned int)(out_size - offset); - break; - default: - log_warn(LD_GENERAL, "Gzip decompression returned an error: %s", - stream->msg ? stream->msg : ""); - goto err; - } - } - done: - *out_len = stream->next_out - (unsigned char*)*out; - r = inflateEnd(stream); - tor_free(stream); - if (r != Z_OK) { - log_warn(LD_BUG, "Error freeing gzip structures"); - goto err; - } - - /* NUL-terminate output. */ - if (out_size == *out_len) - *out = tor_realloc(*out, out_size + 1); - (*out)[*out_len] = '\0'; - - return 0; - err: - if (stream) { - inflateEnd(stream); - tor_free(stream); - } - if (*out) { - tor_free(*out); - } return -1; } @@ -406,58 +131,39 @@ detect_compression_method(const char *in, size_t in_len) /** Internal state for an incremental compression/decompression. The body of * this struct is not exposed. */ struct tor_compress_state_t { - struct z_stream_s stream; /**< The zlib stream */ - int compress; /**< True if we are compressing; false if we are inflating */ + compress_method_t method; /**< The compression method. */ - /** Number of bytes read so far. Used to detect zlib bombs. */ - size_t input_so_far; - /** Number of bytes written so far. Used to detect zlib bombs. */ - size_t output_so_far; - - /** Approximate number of bytes allocated for this object. */ - size_t allocation; + union { + tor_zlib_compress_state_t *zlib_state; + } u; /**< Compression backend state. */ }; /** Construct and return a tor_compress_state_t object using method. If * compress, it's for compression; otherwise it's for decompression. */ tor_compress_state_t * -tor_compress_new(int compress_, compress_method_t method, +tor_compress_new(int compress, compress_method_t method, compression_level_t compression_level) { - tor_compress_state_t *out; - int bits, memlevel; + tor_compress_state_t *state; - if (! compress_) { - /* use this setting for decompression, since we might have the - * max number of window bits */ - compression_level = HIGH_COMPRESSION; - } + state = tor_malloc_zero(sizeof(tor_compress_state_t)); + state->method = method; - out = tor_malloc_zero(sizeof(tor_compress_state_t)); - out->stream.zalloc = Z_NULL; - out->stream.zfree = Z_NULL; - out->stream.opaque = NULL; - out->compress = compress_; - bits = method_bits(method, compression_level); - memlevel = get_memlevel(compression_level); - if (compress_) { - if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, - bits, memlevel, - Z_DEFAULT_STRATEGY) != Z_OK) - goto err; // LCOV_EXCL_LINE - } else { - if (inflateInit2(&out->stream, bits) != Z_OK) - goto err; // LCOV_EXCL_LINE - } - out->allocation = tor_zlib_state_size_precalc(!compress_, bits, memlevel); + if (method == GZIP_METHOD || method == ZLIB_METHOD) { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); - total_zlib_allocation += out->allocation; + if (zlib_state == NULL) + goto err; - return out; + state->u.zlib_state = zlib_state; + } + + return state; err: - tor_free(out); - return NULL; + tor_free(state); + return NULL; } /** Compress/decompress some bytes using state. Read up to @@ -477,112 +183,38 @@ tor_compress_process(tor_compress_state_t *state, const char **in, size_t *in_len, int finish) { - int err; - tor_assert(*in_len <= UINT_MAX); - tor_assert(*out_len <= UINT_MAX); - state->stream.next_in = (unsigned char*) *in; - state->stream.avail_in = (unsigned int)*in_len; - state->stream.next_out = (unsigned char*) *out; - state->stream.avail_out = (unsigned int)*out_len; + tor_assert(state != NULL); - if (state->compress) { - err = deflate(&state->stream, finish ? Z_FINISH : Z_NO_FLUSH); - } else { - err = inflate(&state->stream, finish ? Z_FINISH : Z_SYNC_FLUSH); - } + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + return tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); - state->input_so_far += state->stream.next_in - ((unsigned char*)*in); - state->output_so_far += state->stream.next_out - ((unsigned char*)*out); - - *out = (char*) state->stream.next_out; - *out_len = state->stream.avail_out; - *in = (const char *) state->stream.next_in; - *in_len = state->stream.avail_in; - - if (! state->compress && - is_compression_bomb(state->input_so_far, state->output_so_far)) { - log_warn(LD_DIR, "Possible zlib bomb; abandoning stream."); - return TOR_COMPRESS_ERROR; - } - - switch (err) - { - case Z_STREAM_END: - return TOR_COMPRESS_DONE; - case Z_BUF_ERROR: - if (state->stream.avail_in == 0 && !finish) - return TOR_COMPRESS_OK; - return TOR_COMPRESS_BUFFER_FULL; - case Z_OK: - if (state->stream.avail_out == 0 || finish) - return TOR_COMPRESS_BUFFER_FULL; - return TOR_COMPRESS_OK; - default: - log_warn(LD_GENERAL, "Gzip returned an error: %s", - state->stream.msg ? state->stream.msg : ""); - return TOR_COMPRESS_ERROR; - } + return TOR_COMPRESS_ERROR; } /** Deallocate state. */ void tor_compress_free(tor_compress_state_t *state) { - if (!state) + if (state == NULL) return; - total_zlib_allocation -= state->allocation; - - if (state->compress) - deflateEnd(&state->stream); - else - inflateEnd(&state->stream); + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + tor_zlib_compress_free(state->u.zlib_state); tor_free(state); } -/** Return an approximate number of bytes used in RAM to hold a state with - * window bits windowBits and compression level 'memlevel' */ -static size_t -tor_zlib_state_size_precalc(int inflate_, int windowbits, int memlevel) -{ - windowbits &= 15; - -#define A_FEW_KILOBYTES 2048 - - if (inflate_) { - /* From zconf.h: - - "The memory requirements for inflate are (in bytes) 1 << windowBits - that is, 32K for windowBits=15 (default value) plus a few kilobytes - for small objects." - */ - return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) + - (1 << 15) + A_FEW_KILOBYTES; - } else { - /* Also from zconf.h: - - "The memory requirements for deflate are (in bytes): - (1 << (windowBits+2)) + (1 << (memLevel+9)) - ... plus a few kilobytes for small objects." - */ - return sizeof(tor_compress_state_t) + sizeof(struct z_stream_s) + - (1 << (windowbits + 2)) + (1 << (memlevel + 9)) + A_FEW_KILOBYTES; - } -#undef A_FEW_KILOBYTES -} - /** Return the approximate number of bytes allocated for state. */ size_t tor_compress_state_size(const tor_compress_state_t *state) { - return state->allocation; -} - -/** Return the approximate number of bytes allocated for all zlib states. */ -size_t -tor_zlib_get_total_allocation(void) -{ - return total_zlib_allocation; + tor_assert(state != NULL); + + if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) + return tor_zlib_compress_state_size(state->u.zlib_state); + + return 0; } diff --git a/src/common/torgzip.h b/src/common/torgzip.h index fa566282d7..0fcac0c946 100644 --- a/src/common/torgzip.h +++ b/src/common/torgzip.h @@ -38,12 +38,6 @@ tor_uncompress(char **out, size_t *out_len, int complete_only, int protocol_warn_level); -const char * -tor_zlib_get_version_str(void); - -const char * -tor_zlib_get_header_version_str(void); - compress_method_t detect_compression_method(const char *in, size_t in_len); int @@ -60,8 +54,10 @@ typedef enum { TOR_COMPRESS_BUFFER_FULL, TOR_COMPRESS_ERROR } tor_compress_output_t; -/** Internal state for an incremental zlib compression/decompression. */ + +/** Internal state for an incremental compression/decompression. */ typedef struct tor_compress_state_t tor_compress_state_t; + tor_compress_state_t *tor_compress_new(int compress, compress_method_t method, compression_level_t level); @@ -73,7 +69,6 @@ tor_compress_output_t tor_compress_process(tor_compress_state_t *state, void tor_compress_free(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); -size_t tor_zlib_get_total_allocation(void); #endif diff --git a/src/or/config.c b/src/or/config.c index 809ff499fc..83e5f2106c 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -69,6 +69,7 @@ #include "circuitmux.h" #include "circuitmux_ewma.h" #include "circuitstats.h" +#include "compress_zlib.h" #include "config.h" #include "connection.h" #include "connection_edge.h" diff --git a/src/or/main.c b/src/or/main.c index 4505879adc..ddd7b82545 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -58,6 +58,7 @@ #include "circuitlist.h" #include "circuituse.h" #include "command.h" +#include "compress_zlib.h" #include "config.h" #include "confparse.h" #include "connection.h" diff --git a/src/or/relay.c b/src/or/relay.c index 5139036327..8a98c50372 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -54,6 +54,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" +#include "compress_zlib.h" #include "config.h" #include "connection.h" #include "connection_edge.h" From 04583df4520705bc171be9b720344167029292cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 03:21:53 +0200 Subject: [PATCH 13/28] Rename the `torgzip` module to `compress`. See https://bugs.torproject.org/21663 --- src/common/Makefile.nmake | 2 +- src/common/{torgzip.c => compress.c} | 6 +++--- src/common/{torgzip.h => compress.h} | 10 +++++----- src/common/compress_zlib.c | 2 +- src/common/include.am | 4 ++-- src/or/config.c | 2 +- src/or/or.h | 2 +- src/test/test.c | 2 +- src/test/test_dir_handle_get.c | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) rename src/common/{torgzip.c => compress.c} (98%) rename src/common/{torgzip.h => compress.h} (94%) diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake index b8c5dd4fea..8667959b52 100644 --- a/src/common/Makefile.nmake +++ b/src/common/Makefile.nmake @@ -7,7 +7,7 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \ log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \ util_codedigest.obj -LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj torgzip.obj tortls.obj \ +LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj tortls.obj \ crypto_curve25519.obj curve25519-donna.obj LIBOR_EVENT_OBJECTS = compat_libevent.obj diff --git a/src/common/torgzip.c b/src/common/compress.c similarity index 98% rename from src/common/torgzip.c rename to src/common/compress.c index d760cd9546..74e235f006 100644 --- a/src/common/torgzip.c +++ b/src/common/compress.c @@ -4,8 +4,8 @@ /* See LICENSE for licensing information */ /** - * \file torgzip.c - * \brief A simple in-memory gzip implementation. + * \file compress.c + * \brief Common compression API. **/ #include "orconfig.h" @@ -22,7 +22,7 @@ #include "util.h" #include "torlog.h" -#include "torgzip.h" +#include "compress.h" #include "compress_zlib.h" /** @{ */ diff --git a/src/common/torgzip.h b/src/common/compress.h similarity index 94% rename from src/common/torgzip.h rename to src/common/compress.h index 0fcac0c946..c347e1c184 100644 --- a/src/common/torgzip.h +++ b/src/common/compress.h @@ -4,12 +4,12 @@ /* See LICENSE for licensing information */ /** - * \file torgzip.h - * \brief Headers for torgzip.h + * \file compress.h + * \brief Headers for compress.c **/ -#ifndef TOR_TORGZIP_H -#define TOR_TORGZIP_H +#ifndef TOR_COMPRESS_H +#define TOR_COMPRESS_H /** Enumeration of what kind of compression to use. Only ZLIB_METHOD and * GZIP_METHOD is guaranteed to be supported by the compress/uncompress @@ -70,5 +70,5 @@ void tor_compress_free(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); -#endif +#endif // TOR_COMPRESS_H. diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index 38e500c754..2c9aba32ee 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -15,7 +15,7 @@ #include "util.h" #include "torlog.h" -#include "torgzip.h" +#include "compress.h" #include "compress_zlib.h" /* zlib 1.2.4 and 1.2.5 do some "clever" things with macros. Instead of diff --git a/src/common/include.am b/src/common/include.am index ea2c46a670..7ed75d9495 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -105,12 +105,12 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ + src/common/compress.c \ src/common/compress_zlib.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ src/common/crypto_format.c \ - src/common/torgzip.c \ src/common/tortls.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c @@ -146,6 +146,7 @@ COMMONHEADERS = \ src/common/compat_openssl.h \ src/common/compat_threads.h \ src/common/compat_time.h \ + src/common/compress.h \ src/common/compress_zlib.h \ src/common/confline.h \ src/common/container.h \ @@ -165,7 +166,6 @@ COMMONHEADERS = \ src/common/storagedir.h \ src/common/testsupport.h \ src/common/timers.h \ - src/common/torgzip.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ diff --git a/src/or/config.c b/src/or/config.c index 83e5f2106c..9af116db1c 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -69,6 +69,7 @@ #include "circuitmux.h" #include "circuitmux_ewma.h" #include "circuitstats.h" +#include "compress.h" #include "compress_zlib.h" #include "config.h" #include "connection.h" @@ -100,7 +101,6 @@ #include "statefile.h" #include "transports.h" #include "ext_orport.h" -#include "torgzip.h" #ifdef _WIN32 #include #endif diff --git a/src/or/or.h b/src/or/or.h index 20c4ebccd0..9e3e409af9 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -71,7 +71,7 @@ #include "tortls.h" #include "torlog.h" #include "container.h" -#include "torgzip.h" +#include "compress.h" #include "address.h" #include "compat_libevent.h" #include "ht.h" diff --git a/src/test/test.c b/src/test/test.c index 77ea44970c..e8a63e6afd 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -44,13 +44,13 @@ double fabs(double x); #include "buffers.h" #include "circuitlist.h" #include "circuitstats.h" +#include "compress.h" #include "config.h" #include "connection_edge.h" #include "geoip.h" #include "rendcommon.h" #include "rendcache.h" #include "test.h" -#include "torgzip.h" #include "main.h" #include "memarea.h" #include "onion.h" diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index ed8ea2f809..392fa4dde0 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -14,6 +14,7 @@ #include "connection.h" #include "directory.h" #include "test.h" +#include "compress.h" #include "connection.h" #include "rendcommon.h" #include "rendcache.h" @@ -28,7 +29,6 @@ #include "networkstatus.h" #include "geoip.h" #include "dirserv.h" -#include "torgzip.h" #include "dirvote.h" #include "log_test_helpers.h" From 300ac496853448eb3eeb3529d3aedaa8cc212ec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 03:23:29 +0200 Subject: [PATCH 14/28] Add `compress_zlib.obj` to `src/common/Makefile.nmake`. See https://bugs.torproject.org/21663 --- src/common/Makefile.nmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/Makefile.nmake b/src/common/Makefile.nmake index 8667959b52..a1c819fffa 100644 --- a/src/common/Makefile.nmake +++ b/src/common/Makefile.nmake @@ -7,8 +7,8 @@ LIBOR_OBJECTS = address.obj backtrace.obj compat.obj container.obj di_ops.obj \ log.obj memarea.obj mempool.obj procmon.obj sandbox.obj util.obj \ util_codedigest.obj -LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj tortls.obj \ - crypto_curve25519.obj curve25519-donna.obj +LIBOR_CRYPTO_OBJECTS = aes.obj crypto.obj crypto_format.obj compress.obj compress_zlib.obj \ + tortls.obj crypto_curve25519.obj curve25519-donna.obj LIBOR_EVENT_OBJECTS = compat_libevent.obj From c171af048731cc00d77e17a1ff0be5c641024f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 03:39:43 +0200 Subject: [PATCH 15/28] Use a switch-statement when checking for compression method. This patch changes the way `tor_compress_new()`, `tor_compress_process()`, and `tor_compress_free()` handles different compression methods. This should give us compiler warnings in case an additional compression method is added, but the developer forgets to add handlers in the three aforementioned functions. See https://bugs.torproject.org/21663 --- src/common/compress.c | 60 +++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index 74e235f006..4c87f292dd 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -149,14 +149,22 @@ tor_compress_new(int compress, compress_method_t method, state = tor_malloc_zero(sizeof(tor_compress_state_t)); state->method = method; - if (method == GZIP_METHOD || method == ZLIB_METHOD) { - tor_zlib_compress_state_t *zlib_state = - tor_zlib_compress_new(compress, method, compression_level); + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: { + tor_zlib_compress_state_t *zlib_state = + tor_zlib_compress_new(compress, method, compression_level); - if (zlib_state == NULL) + if (zlib_state == NULL) + goto err; + + state->u.zlib_state = zlib_state; + break; + } + + case NO_METHOD: + case UNKNOWN_METHOD: goto err; - - state->u.zlib_state = zlib_state; } return state; @@ -185,11 +193,19 @@ tor_compress_process(tor_compress_state_t *state, { tor_assert(state != NULL); - if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) - return tor_zlib_compress_process(state->u.zlib_state, - out, out_len, in, in_len, - finish); + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_compress_process(state->u.zlib_state, + out, out_len, in, in_len, + finish); + case NO_METHOD: + case UNKNOWN_METHOD: + goto err; + } + + err: return TOR_COMPRESS_ERROR; } @@ -200,8 +216,16 @@ tor_compress_free(tor_compress_state_t *state) if (state == NULL) return; - if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) - tor_zlib_compress_free(state->u.zlib_state); + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + tor_zlib_compress_free(state->u.zlib_state); + break; + + case NO_METHOD: + case UNKNOWN_METHOD: + break; + } tor_free(state); } @@ -212,9 +236,17 @@ tor_compress_state_size(const tor_compress_state_t *state) { tor_assert(state != NULL); - if (state->method == GZIP_METHOD || state->method == ZLIB_METHOD) - return tor_zlib_compress_state_size(state->u.zlib_state); + switch (state->method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_compress_state_size(state->u.zlib_state); + case NO_METHOD: + case UNKNOWN_METHOD: + goto err; + } + + err: return 0; } From 157af1d26e3cdfc3159c1085f35e5ddcda7ccd21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 13:56:02 +0200 Subject: [PATCH 16/28] Add --enable-lzma to our configure script. This patch adds support for enabling support for LZMA to our configure script. By default, the --enable-lzma option is set to "auto" which means if liblzma is available we'll build Tor with LZMA support. See: https://bugs.torproject.org/21662 --- Makefile.am | 2 +- configure.ac | 32 ++++++++++++++++++++++++++++++++ src/or/include.am | 6 ++++-- src/test/fuzz/include.am | 3 ++- src/test/include.am | 16 ++++++++++------ 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/Makefile.am b/Makefile.am index aeac0b85c4..0f59ebee4c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -16,7 +16,7 @@ noinst_PROGRAMS= DISTCLEANFILES= bin_SCRIPTS= AM_CPPFLAGS= -AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ +AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ SHELL=@SHELL@ if COVERAGE_ENABLED diff --git a/configure.ac b/configure.ac index 0d4fa9838b..96af8d7f42 100644 --- a/configure.ac +++ b/configure.ac @@ -732,6 +732,38 @@ else fi AC_SUBST(TOR_ZLIB_LIBS) +dnl ------------------------------------------------------ +dnl Where we do we find lzma? + +AC_ARG_ENABLE(lzma, + AS_HELP_STRING(--enable-lzma, [enable support for the Zstandard compression scheme.]), + [case "${enableval}" in + "yes") lzma=true ;; + "no") lzma=false ;; + * ) AC_MSG_ERROR(bad value for --enable-lzma) ;; + esac], [lzma=auto]) + +if test "x$enable_lzma" = "xno"; then + have_lzma=no; +else + PKG_CHECK_MODULES([LZMA], + [liblzma], + have_lzma=yes, + have_lzma=no) + + if test "x$have_lzma" = "xno" ; then + AC_MSG_WARN([Unable to find liblzma.]) + fi +fi + +if test "x$have_lzma" = "xyes"; then + AC_DEFINE(HAVE_LZMA,1,[Have LZMA]) + TOR_LZMA_CFLAGS="${LZMA_CFLAGS}" + TOR_LZMA_LIBS="${LZMA_LIBS}" +fi +AC_SUBST(TOR_LZMA_CFLAGS) +AC_SUBST(TOR_LZMA_LIBS) + dnl ---------------------------------------------------------------------- dnl Check if libcap is available for capabilities. diff --git a/src/or/include.am b/src/or/include.am index 1841bbfe9d..d4dc6a90ce 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -120,7 +120,8 @@ src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ if COVERAGE_ENABLED src_or_tor_cov_SOURCES = src/or/tor_main.c @@ -132,7 +133,8 @@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ endif ORHEADERS = \ diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index 1b608c6885..abffcad602 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -18,7 +18,8 @@ FUZZING_LIBS = \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ + @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ oss-fuzz-prereqs: \ src/or/libtor-testing.a \ diff --git a/src/test/include.am b/src/test/include.am index c92eab13c9..9b5eb146fc 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -179,7 +179,8 @@ src_test_test_switch_id_LDFLAGS = @TOR_LDFLAGS_zlib@ src_test_test_switch_id_LDADD = \ src/common/libor-testing.a \ src/common/libor-ctime-testing.a \ - @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ + @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ + @TOR_LZMA_LIBS@ src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ @@ -193,7 +194,7 @@ src_test_test_LDADD = src/or/libtor-testing.a \ src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) @@ -216,7 +217,7 @@ src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ @@ -226,7 +227,8 @@ src_test_test_workqueue_LDADD = src/or/libtor-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ - @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ + @TOR_LZMA_LIBS@ src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS) @@ -236,7 +238,8 @@ src_test_test_timers_LDADD = \ src/common/libor-event-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ - @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ + @TOR_LZMA_LIBS@ src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS) noinst_HEADERS+= \ @@ -261,7 +264,8 @@ src_test_test_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \ src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ + @TOR_LZMA_LIBS@ src_test_test_ntor_cl_AM_CPPFLAGS = \ -I"$(top_srcdir)/src/or" From bf1c07cb07c27ca65ab66cf41442cfaabd9bb067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 14:30:44 +0200 Subject: [PATCH 17/28] Add LZMA support. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 35 ++- src/common/compress.h | 6 +- src/common/compress_lzma.c | 582 +++++++++++++++++++++++++++++++++++++ src/common/compress_lzma.h | 56 ++++ src/common/include.am | 2 + src/test/test_util.c | 107 +++++++ 6 files changed, 784 insertions(+), 4 deletions(-) create mode 100644 src/common/compress_lzma.c create mode 100644 src/common/compress_lzma.h diff --git a/src/common/compress.c b/src/common/compress.c index 4c87f292dd..3c7a3c6392 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -23,6 +23,7 @@ #include "util.h" #include "torlog.h" #include "compress.h" +#include "compress_lzma.h" #include "compress_zlib.h" /** @{ */ @@ -81,6 +82,9 @@ tor_compress(char **out, size_t *out_len, if (method == GZIP_METHOD || method == ZLIB_METHOD) return tor_zlib_compress(out, out_len, in, in_len, method); + if (method == LZMA_METHOD) + return tor_lzma_compress(out, out_len, in, in_len, method); + return -1; } @@ -108,6 +112,12 @@ tor_uncompress(char **out, size_t *out_len, complete_only, protocol_warn_level); + if (method == LZMA_METHOD) + return tor_lzma_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level); + return -1; } @@ -123,6 +133,9 @@ detect_compression_method(const char *in, size_t in_len) } else if (in_len > 2 && (in[0] & 0x0f) == 8 && (ntohs(get_uint16(in)) % 31) == 0) { return ZLIB_METHOD; + } else if (in_len > 3 && + fast_memeq(in, "\x5d\x00\x00\x00", 4)) { + return LZMA_METHOD; } else { return UNKNOWN_METHOD; } @@ -135,6 +148,7 @@ struct tor_compress_state_t { union { tor_zlib_compress_state_t *zlib_state; + tor_lzma_compress_state_t *lzma_state; } u; /**< Compression backend state. */ }; @@ -161,7 +175,16 @@ tor_compress_new(int compress, compress_method_t method, state->u.zlib_state = zlib_state; break; } + case LZMA_METHOD: { + tor_lzma_compress_state_t *lzma_state = + tor_lzma_compress_new(compress, method, compression_level); + if (lzma_state == NULL) + goto err; + + state->u.lzma_state = lzma_state; + break; + } case NO_METHOD: case UNKNOWN_METHOD: goto err; @@ -199,7 +222,10 @@ tor_compress_process(tor_compress_state_t *state, return tor_zlib_compress_process(state->u.zlib_state, out, out_len, in, in_len, finish); - + case LZMA_METHOD: + return tor_lzma_compress_process(state->u.lzma_state, + out, out_len, in, in_len, + finish); case NO_METHOD: case UNKNOWN_METHOD: goto err; @@ -221,7 +247,9 @@ tor_compress_free(tor_compress_state_t *state) case ZLIB_METHOD: tor_zlib_compress_free(state->u.zlib_state); break; - + case LZMA_METHOD: + tor_lzma_compress_free(state->u.lzma_state); + break; case NO_METHOD: case UNKNOWN_METHOD: break; @@ -240,7 +268,8 @@ tor_compress_state_size(const tor_compress_state_t *state) case GZIP_METHOD: case ZLIB_METHOD: return tor_zlib_compress_state_size(state->u.zlib_state); - + case LZMA_METHOD: + return tor_lzma_compress_state_size(state->u.lzma_state); case NO_METHOD: case UNKNOWN_METHOD: goto err; diff --git a/src/common/compress.h b/src/common/compress.h index c347e1c184..3e3ab3a574 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -15,7 +15,11 @@ * GZIP_METHOD is guaranteed to be supported by the compress/uncompress * functions here. */ typedef enum { - NO_METHOD=0, GZIP_METHOD=1, ZLIB_METHOD=2, UNKNOWN_METHOD=3 + NO_METHOD=0, + GZIP_METHOD=1, + ZLIB_METHOD=2, + LZMA_METHOD=3, + UNKNOWN_METHOD=4 } compress_method_t; /** diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c new file mode 100644 index 0000000000..e1a7a66b0c --- /dev/null +++ b/src/common/compress_lzma.c @@ -0,0 +1,582 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.c + * \brief Compression backend for LZMA. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_lzma.h" + +#ifdef HAVE_LZMA +#include +#endif + +/** Total number of bytes allocated for LZMA state. */ +static size_t total_lzma_allocation = 0; + +#ifdef HAVE_LZMA +/** Convert a given error to a human readable error string. */ +static const char * +lzma_error_str(lzma_ret error) +{ + switch (error) { + case LZMA_OK: + return "Operation completed successfully"; + case LZMA_STREAM_END: + return "End of stream"; + case LZMA_NO_CHECK: + return "Input stream lacks integrity check"; + case LZMA_UNSUPPORTED_CHECK: + return "Unable to calculate integrity check"; + case LZMA_GET_CHECK: + return "Integrity check available"; + case LZMA_MEM_ERROR: + return "Unable to allocate memory"; + case LZMA_MEMLIMIT_ERROR: + return "Memory limit reached"; + case LZMA_FORMAT_ERROR: + return "Unknown file format"; + case LZMA_OPTIONS_ERROR: + return "Unsupported options"; + case LZMA_DATA_ERROR: + return "Corrupt input data"; + case LZMA_BUF_ERROR: + return "Unable to progress"; + case LZMA_PROG_ERROR: + return "Programming error"; + default: + return "Unknown LZMA error"; + } +} +#endif // HAVE_LZMA. + +/** Return a string representation of the version of the currently running + * version of liblzma. */ +const char * +tor_lzma_get_version_str(void) +{ +#ifdef HAVE_LZMA + return lzma_version_string(); +#else + return "N/A"; +#endif +} + +/** Return a string representation of the version of the version of liblzma + * used at compilation. */ +const char * +tor_lzma_get_header_version_str(void) +{ +#ifdef HAVE_LZMA + return LZMA_VERSION_STRING; +#else + return "N/A"; +#endif +} + +/** Given in_len bytes at in, compress them into a newly + * allocated buffer, using the LZMA method. Store the compressed string in + * *out, and its length in *out_len. Return 0 on success, -1 on + * failure. + */ +int +tor_lzma_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ +#ifdef HAVE_LZMA + lzma_stream stream = LZMA_STREAM_INIT; + lzma_options_lzma stream_options; + lzma_ret retval; + lzma_action action; + size_t out_size, old_size; + off_t offset; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + tor_assert(method == LZMA_METHOD); + + stream.next_in = (unsigned char *)in; + stream.avail_in = in_len; + + lzma_lzma_preset(&stream_options, + tor_compress_memory_level(HIGH_COMPRESSION)); + + retval = lzma_alone_encoder(&stream, &stream_options); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + + out_size = in_len / 2; + if (out_size < 1024) + out_size = 1024; + + *out = tor_malloc(out_size); + + stream.next_out = (unsigned char *)*out; + stream.avail_out = out_size; + + action = LZMA_RUN; + + while (1) { + retval = lzma_code(&stream, action); + switch (retval) { + case LZMA_OK: + action = LZMA_FINISH; + break; + case LZMA_STREAM_END: + goto done; + case LZMA_BUF_ERROR: + offset = stream.next_out - ((unsigned char *)*out); + old_size = out_size; + out_size *= 2; + + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in LZMA compression."); + goto err; + } + + *out = tor_realloc(*out, out_size); + stream.next_out = (unsigned char *)(*out + offset); + if (out_size - offset > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of LZMA while " + "compressing."); + goto err; + } + stream.avail_out = (unsigned int)(out_size - offset); + break; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA compression didn't finish: %s.", + lzma_error_str(retval)); + goto err; + } + } + + done: + *out_len = stream.total_out; + lzma_end(&stream); + + if (tor_compress_is_compression_bomb(*out_len, in_len)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tor instances would think " + "this is a compression bomb."); + goto err; + } + + return 0; + + err: + lzma_end(&stream); + tor_free(*out); + return -1; +#else // HAVE_LZMA. + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)method; + + return -1; +#endif // HAVE_LZMA. +} + +/** Given an LZMA compressed string of total length in_len bytes at + * in, uncompress them into a newly allocated buffer. Store the + * uncompressed string in *out, and its length in *out_len. + * Return 0 on success, -1 on failure. + * + * If complete_only is true, we consider a truncated input as a failure; + * otherwise we decompress as much as we can. Warn about truncated or corrupt + * inputs at protocol_warn_level. + */ +int +tor_lzma_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ +#ifdef HAVE_LZMA + lzma_stream stream = LZMA_STREAM_INIT; + lzma_ret retval; + lzma_action action; + size_t out_size, old_size; + off_t offset; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + tor_assert(method == LZMA_METHOD); + + stream.next_in = (unsigned char *)in; + stream.avail_in = in_len; + + // FIXME(ahf): This should be something more sensible than + // UINT64_MAX: See #21665. + retval = lzma_alone_decoder(&stream, UINT64_MAX); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + + out_size = in_len * 2; + if (out_size < 1024) + out_size = 1024; + + if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) + goto err; + + *out = tor_malloc(out_size); + stream.next_out = (unsigned char *)*out; + stream.avail_out = out_size; + + // FIXME(ahf): We should figure out how to use LZMA_FULL_FLUSH to + // make the partial string read tests. + // action = complete_only ? LZMA_FINISH : LZMA_SYNC_FLUSH. // To do this, + // it seems like we have to use LZMA using their "xz" encoder instead of just + // regular LZMA. + (void)complete_only; + action = LZMA_FINISH; + + while (1) { + retval = lzma_code(&stream, action); + switch (retval) { + case LZMA_STREAM_END: + if (stream.avail_in == 0) + goto done; + + // We might have more data here. Reset our stream. + lzma_end(&stream); + + retval = lzma_alone_decoder(&stream, UINT64_MAX); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + break; + case LZMA_OK: + break; + case LZMA_BUF_ERROR: + if (stream.avail_out > 0) { + log_fn(protocol_warn_level, LD_PROTOCOL, + "possible truncated or corrupt LZMA data."); + goto err; + } + + offset = stream.next_out - (unsigned char *)*out; + old_size = out_size; + out_size *= 2; + + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in LZMA uncompression."); + goto err; + } + + if (tor_compress_is_compression_bomb(in_len, out_size)) { + log_warn(LD_GENERAL, "Input looks like a possible LZMA compression " + "bomb. Not proceeding."); + goto err; + } + + if (out_size >= SIZE_T_CEILING) { + log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing " + "LZMA data."); + goto err; + } + + *out = tor_realloc(*out, out_size); + stream.next_out = (unsigned char *)(*out + offset); + + if (out_size - offset > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of LZMA while " + "uncompressing."); + goto err; + } + + stream.avail_out = (unsigned int)(out_size - offset); + break; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA decompression didn't finish: %s.", + lzma_error_str(retval)); + goto err; + } + } + + done: + *out_len = stream.next_out - (unsigned char*)*out; + lzma_end(&stream); + + // NUL-terminate our output. + if (out_size == *out_len) + *out = tor_realloc(*out, out_size + 1); + (*out)[*out_len] = '\0'; + + return 0; + + err: + lzma_end(&stream); + tor_free(*out); + return -1; +#else // HAVE_LZMA. + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)method; + (void)complete_only; + (void)protocol_warn_level; + + return -1; +#endif // HAVE_LZMA. +} + +/** Internal LZMA state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_lzma_compress_state_t { +#ifdef HAVE_LZMA + lzma_stream stream; /**< The LZMA stream. */ +#endif + + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Construct and return a tor_lzma_compress_state_t object using + * method. If compress, it's for compression; otherwise it's for + * decompression. */ +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level) +{ + tor_assert(method == LZMA_METHOD); + +#ifdef HAVE_LZMA + tor_lzma_compress_state_t *result; + lzma_ret retval; + lzma_options_lzma stream_options; + + // Note that we do not explicitly initialize the lzma_stream object here, + // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is + // also what `tor_malloc_zero()` does. + result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t)); + result->compress = compress; + + // FIXME(ahf): We should either try to do the pre-calculation that is done + // with the zlib backend or use a custom allocator here where we pass our + // tor_lzma_compress_state_t as the opaque value. + result->allocation = 0; + + if (compress) { + lzma_lzma_preset(&stream_options, + tor_compress_memory_level(compression_level)); + + retval = lzma_alone_encoder(&result->stream, &stream_options); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + } else { + // FIXME(ahf): This should be something more sensible than + // UINT64_MAX: See #21665. + retval = lzma_alone_decoder(&result->stream, UINT64_MAX); + + if (retval != LZMA_OK) { + log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", + lzma_error_str(retval), retval); + goto err; + } + } + + return result; + + err: + tor_free(result); + return NULL; +#else // HAVE_LZMA. + (void)compress; + (void)method; + (void)compression_level; + + return NULL; +#endif // HAVE_LZMA. +} + +/** Compress/decompress some bytes using state. Read up to + * *in_len bytes from *in, and write up to *out_len bytes + * to *out, adjusting the values as we go. If finish is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on out. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_LZMA + lzma_ret retval; + lzma_action action; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + state->stream.next_in = (unsigned char *)*in; + state->stream.avail_in = *in_len; + state->stream.next_out = (unsigned char *)*out; + state->stream.avail_out = *out_len; + + action = finish ? LZMA_FINISH : LZMA_RUN; + + retval = lzma_code(&state->stream, action); + + state->input_so_far += state->stream.next_in - ((unsigned char *)*in); + state->output_so_far += state->stream.next_out - ((unsigned char *)*out); + + *out = (char *)state->stream.next_out; + *out_len = state->stream.avail_out; + *in = (const char *)state->stream.next_in; + *in_len = state->stream.avail_in; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + switch (retval) { + case LZMA_OK: + if (state->stream.avail_out == 0 || finish) + return TOR_COMPRESS_BUFFER_FULL; + + return TOR_COMPRESS_OK; + + case LZMA_BUF_ERROR: + if (state->stream.avail_in == 0 && !finish) + return TOR_COMPRESS_OK; + + return TOR_COMPRESS_BUFFER_FULL; + + case LZMA_STREAM_END: + return TOR_COMPRESS_DONE; + + // We list all the possible values of `lzma_ret` here to silence the + // `switch-enum` warning and to detect if a new member was added. + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEM_ERROR: + case LZMA_MEMLIMIT_ERROR: + case LZMA_FORMAT_ERROR: + case LZMA_OPTIONS_ERROR: + case LZMA_DATA_ERROR: + case LZMA_PROG_ERROR: + default: + log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + lzma_error_str(retval)); + return TOR_COMPRESS_ERROR; + } +#else // HAVE_LZMA. + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + return TOR_COMPRESS_ERROR; +#endif // HAVE_LZMA. +} + +/** Deallocate state. */ +void +tor_lzma_compress_free(tor_lzma_compress_state_t *state) +{ + if (state == NULL) + return; + + total_lzma_allocation -= state->allocation; + +#ifdef HAVE_LZMA + lzma_end(&state->stream); +#endif + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for state. */ +size_t +tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all LZMA states. */ +size_t +tor_lzma_get_total_allocation(void) +{ + return total_lzma_allocation; +} + diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h new file mode 100644 index 0000000000..2cc3a00660 --- /dev/null +++ b/src/common/compress_lzma.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_lzma.h + * \brief Header for compress_lzma.c + **/ + +#ifndef TOR_COMPRESS_LZMA_H +#define TOR_COMPRESS_LZMA_H + +const char * +tor_lzma_get_version_str(void); + +const char * +tor_lzma_get_header_version_str(void); + +int +tor_lzma_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int +tor_lzma_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +/** Internal state for an incremental LZMA compression/decompression. */ +typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t; + +tor_lzma_compress_state_t * +tor_lzma_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_lzma_compress_process(tor_lzma_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void +tor_lzma_compress_free(tor_lzma_compress_state_t *state); + +size_t +tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state); + +size_t +tor_lzma_get_total_allocation(void); + +#endif // TOR_COMPRESS_LZMA_H. + diff --git a/src/common/include.am b/src/common/include.am index 7ed75d9495..2682c22b32 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -106,6 +106,7 @@ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ src/common/compress.c \ + src/common/compress_lzma.c \ src/common/compress_zlib.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ @@ -147,6 +148,7 @@ COMMONHEADERS = \ src/common/compat_threads.h \ src/common/compat_time.h \ src/common/compress.h \ + src/common/compress_lzma.h \ src/common/compress_zlib.h \ src/common/confline.h \ src/common/container.h \ diff --git a/src/test/test_util.c b/src/test/test_util.c index dacf56f868..287f7e8cee 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2404,6 +2404,112 @@ test_util_gzip_compression_bomb(void *arg) tor_compress_free(state); } +static void +test_util_lzma(void *arg) +{ +#ifdef HAVE_LZMA + char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; + const char *ccp2; + size_t len1, len2; + tor_compress_state_t *state = NULL; + + (void)arg; + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); + tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); + + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + LZMA_METHOD)); + tt_assert(buf2 != NULL); + tt_int_op(len1, OP_LT, strlen(buf1)); + tt_int_op(detect_compression_method(buf2, len1), OP_EQ, LZMA_METHOD); + + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + LZMA_METHOD, 1, LOG_INFO)); + tt_assert(buf3 != NULL); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + +#if 0 + /* Check whether we can uncompress concatenated, compressed strings. */ + tor_free(buf3); + buf2 = tor_reallocarray(buf2, len1, 2); + memcpy(buf2+len1, buf2, len1); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, + LZMA_METHOD, 1, LOG_INFO)); + tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); + tt_mem_op(buf3, OP_EQ, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", + (strlen(buf1)+1)*2); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + + /* Check whether we can uncompress partial strings. */ + buf1 = + tor_strdup("String with low redundancy that won't be compressed much."); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + LZMA_METHOD)); + tt_int_op(len1, OP_GT, 16); + /* when we allow an incomplete string, we should succeed.*/ + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16, + LZMA_METHOD, 0, LOG_INFO)); + tt_assert(len2 > 5); + buf3[len2]='\0'; + tt_assert(!strcmpstart(buf1, buf3)); + + /* when we demand a complete string, this must fail. */ + tor_free(buf3); + tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16, + LZMA_METHOD, 1, LOG_INFO)); + tt_assert(buf3 == NULL); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); +#endif + + /* Now, try streaming compression. */ + state = tor_compress_new(1, LZMA_METHOD, HIGH_COMPRESSION); + tt_assert(state); + cp1 = buf1 = tor_malloc(1024); + len1 = 1024; + ccp2 = "ABCDEFGHIJABCDEFGHIJ"; + len2 = 21; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0), + OP_EQ, TOR_COMPRESS_OK); + tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ + tt_assert(cp1 > buf1); + + len2 = 0; + cp2 = cp1; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1), + OP_EQ, TOR_COMPRESS_DONE); + tt_int_op(0, OP_EQ, len2); + tt_assert(cp1 > cp2); /* Make sure we really added something. */ + + tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1, + LZMA_METHOD, 1, LOG_WARN)); + /* Make sure it compressed right. */ + tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); + tt_int_op(21, OP_EQ, len2); + + done: + if (state) + tor_compress_free(state); + tor_free(buf2); + tor_free(buf3); + tor_free(buf1); +#else + (void)arg; +#endif // HAVE_LZMA. +} + /** Run unit tests for mmap() wrapper functionality. */ static void test_util_mmap(void *arg) @@ -5717,6 +5823,7 @@ struct testcase_t util_tests[] = { UTIL_LEGACY(pow2), UTIL_LEGACY(gzip), UTIL_TEST(gzip_compression_bomb, TT_FORK), + UTIL_LEGACY(lzma), UTIL_LEGACY(datadir), UTIL_LEGACY(memarea), UTIL_LEGACY(control_formats), From ce1feae9d9a4366e3cd93a68416841d0c6a636c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 17:54:59 +0200 Subject: [PATCH 18/28] Add --enable-zstd to our configure script. This patch adds support for enabling support for Zstandard to our configure script. By default, the --enable-zstd option is set to "auto" which means if libzstd is available we'll build Tor with Zstandard support. See: https://bugs.torproject.org/21662 --- Makefile.am | 2 +- configure.ac | 32 ++++++++++++++++++++++++++++++++ src/or/include.am | 4 ++-- src/test/fuzz/include.am | 3 ++- src/test/include.am | 8 ++++---- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/Makefile.am b/Makefile.am index 0f59ebee4c..be1dc7f7ef 100644 --- a/Makefile.am +++ b/Makefile.am @@ -16,7 +16,7 @@ noinst_PROGRAMS= DISTCLEANFILES= bin_SCRIPTS= AM_CPPFLAGS= -AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ +AM_CFLAGS=@TOR_SYSTEMD_CFLAGS@ @CFLAGS_BUGTRAP@ @TOR_LZMA_CFLAGS@ @TOR_ZSTD_CFLAGS@ SHELL=@SHELL@ if COVERAGE_ENABLED diff --git a/configure.ac b/configure.ac index 96af8d7f42..c7960fa4ed 100644 --- a/configure.ac +++ b/configure.ac @@ -764,6 +764,38 @@ fi AC_SUBST(TOR_LZMA_CFLAGS) AC_SUBST(TOR_LZMA_LIBS) +dnl ------------------------------------------------------ +dnl Where we do we find zstd? + +AC_ARG_ENABLE(zstd, + AS_HELP_STRING(--enable-zstd, [enable support for the Zstandard compression scheme.]), + [case "${enableval}" in + "yes") zstd=true ;; + "no") zstd=false ;; + * ) AC_MSG_ERROR(bad value for --enable-zstd) ;; + esac], [zstd=auto]) + +if test "x$enable_zstd" = "xno"; then + have_zstd=no; +else + PKG_CHECK_MODULES([ZSTD], + [libzstd], + have_zstd=yes, + have_zstd=no) + + if test "x$have_zstd" = "xno" ; then + AC_MSG_WARN([Unable to find libzstd.]) + fi +fi + +if test "x$have_zstd" = "xyes"; then + AC_DEFINE(HAVE_ZSTD,1,[Have Zstd]) + TOR_ZSTD_CFLAGS="${ZSTD_CFLAGS}" + TOR_ZSTD_LIBS="${ZSTD_LIBS}" +fi +AC_SUBST(TOR_ZSTD_CFLAGS) +AC_SUBST(TOR_ZSTD_LIBS) + dnl ---------------------------------------------------------------------- dnl Check if libcap is available for capabilities. diff --git a/src/or/include.am b/src/or/include.am index d4dc6a90ce..d001fcfa3a 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -121,7 +121,7 @@ src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ - @TOR_LZMA_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ if COVERAGE_ENABLED src_or_tor_cov_SOURCES = src/or/tor_main.c @@ -134,7 +134,7 @@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \ - @TOR_LZMA_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ endif ORHEADERS = \ diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index abffcad602..500377f6d7 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -19,7 +19,8 @@ FUZZING_LIBS = \ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ @TOR_SYSTEMD_LIBS@ \ - @TOR_LZMA_LIBS@ + @TOR_LZMA_LIBS@ \ + @TOR_ZSTD_LIBS@ oss-fuzz-prereqs: \ src/or/libtor-testing.a \ diff --git a/src/test/include.am b/src/test/include.am index 9b5eb146fc..2e02343246 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -180,7 +180,7 @@ src_test_test_switch_id_LDADD = \ src/common/libor-testing.a \ src/common/libor-ctime-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_LZMA_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ @@ -194,7 +194,7 @@ src_test_test_LDADD = src/or/libtor-testing.a \ src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_slow_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS) @@ -217,7 +217,7 @@ src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ + @TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ @@ -228,7 +228,7 @@ src_test_test_workqueue_LDADD = src/or/libtor-testing.a \ src/common/libor-event-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ \ - @TOR_LZMA_LIBS@ + @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS) src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS) From 380736d045f85299121f044928170cff321ae852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 22:19:29 +0200 Subject: [PATCH 19/28] Add Zstandard support. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 33 +++ src/common/compress.h | 3 +- src/common/compress_zstd.c | 584 +++++++++++++++++++++++++++++++++++++ src/common/compress_zstd.h | 56 ++++ src/common/include.am | 2 + src/test/test_util.c | 107 +++++++ 6 files changed, 784 insertions(+), 1 deletion(-) create mode 100644 src/common/compress_zstd.c create mode 100644 src/common/compress_zstd.h diff --git a/src/common/compress.c b/src/common/compress.c index 3c7a3c6392..dc86be7750 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -25,6 +25,7 @@ #include "compress.h" #include "compress_lzma.h" #include "compress_zlib.h" +#include "compress_zstd.h" /** @{ */ /* These macros define the maximum allowable compression factor. Anything of @@ -85,6 +86,9 @@ tor_compress(char **out, size_t *out_len, if (method == LZMA_METHOD) return tor_lzma_compress(out, out_len, in, in_len, method); + if (method == ZSTD_METHOD) + return tor_zstd_compress(out, out_len, in, in_len, method); + return -1; } @@ -118,6 +122,12 @@ tor_uncompress(char **out, size_t *out_len, complete_only, protocol_warn_level); + if (method == ZSTD_METHOD) + return tor_zstd_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level); + return -1; } @@ -136,6 +146,9 @@ detect_compression_method(const char *in, size_t in_len) } else if (in_len > 3 && fast_memeq(in, "\x5d\x00\x00\x00", 4)) { return LZMA_METHOD; + } else if (in_len > 3 && + fast_memeq(in, "\x28\xb5\x2f\xfd", 4)) { + return ZSTD_METHOD; } else { return UNKNOWN_METHOD; } @@ -149,6 +162,7 @@ struct tor_compress_state_t { union { tor_zlib_compress_state_t *zlib_state; tor_lzma_compress_state_t *lzma_state; + tor_zstd_compress_state_t *zstd_state; } u; /**< Compression backend state. */ }; @@ -185,6 +199,16 @@ tor_compress_new(int compress, compress_method_t method, state->u.lzma_state = lzma_state; break; } + case ZSTD_METHOD: { + tor_zstd_compress_state_t *zstd_state = + tor_zstd_compress_new(compress, method, compression_level); + + if (zstd_state == NULL) + goto err; + + state->u.zstd_state = zstd_state; + break; + } case NO_METHOD: case UNKNOWN_METHOD: goto err; @@ -226,6 +250,10 @@ tor_compress_process(tor_compress_state_t *state, return tor_lzma_compress_process(state->u.lzma_state, out, out_len, in, in_len, finish); + case ZSTD_METHOD: + return tor_zstd_compress_process(state->u.zstd_state, + out, out_len, in, in_len, + finish); case NO_METHOD: case UNKNOWN_METHOD: goto err; @@ -250,6 +278,9 @@ tor_compress_free(tor_compress_state_t *state) case LZMA_METHOD: tor_lzma_compress_free(state->u.lzma_state); break; + case ZSTD_METHOD: + tor_zstd_compress_free(state->u.zstd_state); + break; case NO_METHOD: case UNKNOWN_METHOD: break; @@ -270,6 +301,8 @@ tor_compress_state_size(const tor_compress_state_t *state) return tor_zlib_compress_state_size(state->u.zlib_state); case LZMA_METHOD: return tor_lzma_compress_state_size(state->u.lzma_state); + case ZSTD_METHOD: + return tor_zstd_compress_state_size(state->u.zstd_state); case NO_METHOD: case UNKNOWN_METHOD: goto err; diff --git a/src/common/compress.h b/src/common/compress.h index 3e3ab3a574..59e7a79061 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -19,7 +19,8 @@ typedef enum { GZIP_METHOD=1, ZLIB_METHOD=2, LZMA_METHOD=3, - UNKNOWN_METHOD=4 + ZSTD_METHOD=4, + UNKNOWN_METHOD=5 } compress_method_t; /** diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c new file mode 100644 index 0000000000..e2eb292d5d --- /dev/null +++ b/src/common/compress_zstd.c @@ -0,0 +1,584 @@ +/* Copyright (c) 2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.c + * \brief Compression backend for Zstandard. + * + * This module should never be invoked directly. Use the compress module + * instead. + **/ + +#include "orconfig.h" + +#include "util.h" +#include "torlog.h" +#include "compress.h" +#include "compress_zstd.h" + +#ifdef HAVE_ZSTD +#include +#include +#endif + +/** Total number of bytes allocated for Zstandard state. */ +static size_t total_zstd_allocation = 0; + +/** Return a string representation of the version of the currently running + * version of libzstd. */ +const char * +tor_zstd_get_version_str(void) +{ +#ifdef HAVE_ZSTD + static char version_str[16]; + size_t version_number; + + version_number = ZSTD_versionNumber(); + tor_snprintf(version_str, sizeof(version_str), + "%lu.%lu.%lu", + version_number / 10000 % 100, + version_number / 100 % 100, + version_number % 100); + + return version_str; +#else + return "N/A"; +#endif +} + +/** Return a string representation of the version of the version of libzstd + * used at compilation. */ +const char * +tor_zstd_get_header_version_str(void) +{ +#ifdef HAVE_ZSTD + return ZSTD_VERSION_STRING; +#else + return "N/A"; +#endif +} + +/** Given in_len bytes at in, compress them into a newly + * allocated buffer, using the Zstandard method. Store the compressed string + * in *out, and its length in *out_len. Return 0 on success, -1 + * on failure. + */ +int +tor_zstd_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method) +{ +#ifdef HAVE_ZSTD + ZSTD_CStream *stream = NULL; + size_t out_size, old_size; + size_t retval; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + tor_assert(method == ZSTD_METHOD); + + *out = NULL; + + stream = ZSTD_createCStream(); + + if (stream == NULL) { + // Zstandard does not give us any useful error message to why this + // happened. See https://github.com/facebook/zstd/issues/398 + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initCStream(stream, + tor_compress_memory_level(HIGH_COMPRESSION)); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + + // Assume 50% compression and update our buffer in case we need to. + out_size = in_len / 2; + if (out_size < 1024) + out_size = 1024; + + *out = tor_malloc(out_size); + *out_len = 0; + + ZSTD_inBuffer input = { in, in_len, 0 }; + ZSTD_outBuffer output = { *out, out_size, 0 }; + + while (input.pos < input.size) { + retval = ZSTD_compressStream(stream, &output, &input); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream compression error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + + if (input.pos < input.size && output.pos == output.size) { + old_size = out_size; + out_size *= 2; + + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); + goto err; + } + + if (out_size - output.pos > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " + "compressing."); + goto err; + } + + output.dst = *out = tor_realloc(*out, out_size); + output.size = out_size; + } + } + + while (1) { + retval = ZSTD_endStream(stream, &output); + + if (retval == 0) + break; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + + if (output.pos == output.size) { + old_size = out_size; + out_size *= 2; + + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); + goto err; + } + + if (out_size - output.pos > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " + "compressing."); + goto err; + } + + output.dst = *out = tor_realloc(*out, out_size); + output.size = out_size; + } + } + + *out_len = output.pos; + + if (tor_compress_is_compression_bomb(*out_len, in_len)) { + log_warn(LD_BUG, "We compressed something and got an insanely high " + "compression factor; other Tor instances would think " + "this is a compression bomb."); + goto err; + } + + if (stream != NULL) { + ZSTD_freeCStream(stream); + } + + return 0; + + err: + if (stream != NULL) { + ZSTD_freeCStream(stream); + } + + tor_free(*out); + return -1; +#else // HAVE_ZSTD. + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)method; + + return -1; +#endif // HAVE_ZSTD. +} + +/** Given a Zstandard compressed string of total length in_len bytes at + * in, uncompress them into a newly allocated buffer. Store the + * uncompressed string in *out, and its length in *out_len. + * Return 0 on success, -1 on failure. + * + * If complete_only is true, we consider a truncated input as a failure; + * otherwise we decompress as much as we can. Warn about truncated or corrupt + * inputs at protocol_warn_level. + */ +int +tor_zstd_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level) +{ +#ifdef HAVE_ZSTD + ZSTD_DStream *stream = NULL; + size_t retval; + size_t out_size, old_size; + + tor_assert(out); + tor_assert(out_len); + tor_assert(in); + tor_assert(in_len < UINT_MAX); + tor_assert(method == ZSTD_METHOD); + + // FIXME(ahf): Handle this? + (void)complete_only; + (void)protocol_warn_level; + + *out = NULL; + + stream = ZSTD_createDStream(); + + if (stream == NULL) { + // Zstandard does not give us any useful error message to why this + // happened. See https://github.com/facebook/zstd/issues/398 + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initDStream(stream); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + + out_size = in_len * 2; + if (out_size < 1024) + out_size = 1024; + + if (out_size >= SIZE_T_CEILING || out_size > UINT_MAX) + goto err; + + *out = tor_malloc(out_size); + *out_len = 0; + + ZSTD_inBuffer input = { in, in_len, 0 }; + ZSTD_outBuffer output = { *out, out_size, 0 }; + + while (input.pos < input.size) { + retval = ZSTD_decompressStream(stream, &output, &input); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream decompression error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + + if (input.pos < input.size && output.pos == output.size) { + old_size = out_size; + out_size *= 2; + + if (out_size < old_size) { + log_warn(LD_GENERAL, "Size overflow in Zstandard compression."); + goto err; + } + + if (tor_compress_is_compression_bomb(in_len, out_size)) { + log_warn(LD_GENERAL, "Input looks like a possible Zstandard " + "compression bomb. Not proceeding."); + goto err; + } + + if (out_size >= SIZE_T_CEILING) { + log_warn(LD_BUG, "Hit SIZE_T_CEILING limit while uncompressing " + "Zstandard data."); + goto err; + } + + if (out_size - output.pos > UINT_MAX) { + log_warn(LD_BUG, "Ran over unsigned int limit of Zstandard while " + "decompressing."); + goto err; + } + + output.dst = *out = tor_realloc(*out, out_size); + output.size = out_size; + } + } + + *out_len = output.pos; + + if (stream != NULL) { + ZSTD_freeDStream(stream); + } + + // NUL-terminate our output. + if (out_size == *out_len) + *out = tor_realloc(*out, out_size + 1); + (*out)[*out_len] = '\0'; + + return 0; + + err: + if (stream != NULL) { + ZSTD_freeDStream(stream); + } + + tor_free(*out); + return -1; +#else // HAVE_ZSTD. + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)method; + (void)complete_only; + (void)protocol_warn_level; + + return -1; +#endif // HAVE_ZSTD. +} + +/** Internal Zstandard state for incremental compression/decompression. + * The body of this struct is not exposed. */ +struct tor_zstd_compress_state_t { +#ifdef HAVE_ZSTD + union { + /** Compression stream. Used when compress is true. */ + ZSTD_CStream *compress_stream; + /** Decompression stream. Used when compress is false. */ + ZSTD_DStream *decompress_stream; + } u; /**< Zstandard stream objects. */ +#endif // HAVE_ZSTD. + + int compress; /**< True if we are compressing; false if we are inflating */ + + /** Number of bytes read so far. Used to detect compression bombs. */ + size_t input_so_far; + /** Number of bytes written so far. Used to detect compression bombs. */ + size_t output_so_far; + + /** Approximate number of bytes allocated for this object. */ + size_t allocation; +}; + +/** Construct and return a tor_zstd_compress_state_t object using + * method. If compress, it's for compression; otherwise it's for + * decompression. */ +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level) +{ + tor_assert(method == ZSTD_METHOD); + +#ifdef HAVE_ZSTD + tor_zstd_compress_state_t *result; + size_t retval; + + result = tor_malloc_zero(sizeof(tor_zstd_compress_state_t)); + result->compress = compress; + + // FIXME(ahf): We should either try to do the pre-calculation that is done + // with the zlib backend or use a custom allocator here where we pass our + // tor_zstd_compress_state_t as the opaque value. + result->allocation = 0; + + if (compress) { + result->u.compress_stream = ZSTD_createCStream(); + + if (result->u.compress_stream == NULL) { + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initCStream(result->u.compress_stream, + tor_compress_memory_level(compression_level)); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + } else { + result->u.decompress_stream = ZSTD_createDStream(); + + if (result->u.decompress_stream == NULL) { + log_warn(LD_GENERAL, "Error while creating Zstandard stream"); + goto err; + } + + retval = ZSTD_initDStream(result->u.decompress_stream); + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", + ZSTD_getErrorName(retval)); + goto err; + } + } + + return result; + + err: + if (compress) { + ZSTD_freeCStream(result->u.compress_stream); + } else { + ZSTD_freeDStream(result->u.decompress_stream); + } + + tor_free(result); + return NULL; +#else // HAVE_ZSTD. + (void)compress; + (void)method; + (void)compression_level; + + return NULL; +#endif // HAVE_ZSTD. +} + +/** Compress/decompress some bytes using state. Read up to + * *in_len bytes from *in, and write up to *out_len bytes + * to *out, adjusting the values as we go. If finish is true, + * we've reached the end of the input. + * + * Return TOR_COMPRESS_DONE if we've finished the entire + * compression/decompression. + * Return TOR_COMPRESS_OK if we're processed everything from the input. + * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on out. + * Return TOR_COMPRESS_ERROR if the stream is corrupt. + */ +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish) +{ +#ifdef HAVE_ZSTD + size_t retval; + + tor_assert(state != NULL); + tor_assert(*in_len <= UINT_MAX); + tor_assert(*out_len <= UINT_MAX); + + ZSTD_inBuffer input = { *in, *in_len, 0 }; + ZSTD_outBuffer output = { *out, *out_len, 0 }; + + if (state->compress) { + retval = ZSTD_compressStream(state->u.compress_stream, + &output, &input); + } else { + retval = ZSTD_decompressStream(state->u.decompress_stream, + &output, &input); + } + + state->input_so_far += input.pos; + state->output_so_far += output.pos; + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + *in = (char *)input.src + input.pos; + *in_len = input.size - input.pos; + + if (! state->compress && + tor_compress_is_compression_bomb(state->input_so_far, + state->output_so_far)) { + log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); + return TOR_COMPRESS_ERROR; + } + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard %s didn't finish: %s.", + state->compress ? "compression" : "decompression", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + if (state->compress && !finish) { + retval = ZSTD_flushStream(state->u.compress_stream, &output); + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to flush: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + if (retval > 0) + return TOR_COMPRESS_BUFFER_FULL; + } + + if (state->compress && finish) { + retval = ZSTD_endStream(state->u.compress_stream, &output); + + *out = (char *)output.dst + output.pos; + *out_len = output.size - output.pos; + + if (ZSTD_isError(retval)) { + log_warn(LD_GENERAL, "Zstandard compression unable to write " + "epilogue: %s.", + ZSTD_getErrorName(retval)); + return TOR_COMPRESS_ERROR; + } + + // endStream returns the number of bytes that is needed to write the + // epilogue. + if (retval > 0) + return TOR_COMPRESS_BUFFER_FULL; + } + + return finish ? TOR_COMPRESS_DONE : TOR_COMPRESS_OK; +#else // HAVE_ZSTD. + (void)state; + (void)out; + (void)out_len; + (void)in; + (void)in_len; + (void)finish; + + return TOR_COMPRESS_ERROR; +#endif // HAVE_ZSTD. +} + +/** Deallocate state. */ +void +tor_zstd_compress_free(tor_zstd_compress_state_t *state) +{ + if (state == NULL) + return; + + total_zstd_allocation -= state->allocation; + +#ifdef HAVE_ZSTD + if (state->compress) { + ZSTD_freeCStream(state->u.compress_stream); + } else { + ZSTD_freeDStream(state->u.decompress_stream); + } +#endif // HAVE_ZSTD. + + tor_free(state); +} + +/** Return the approximate number of bytes allocated for state. */ +size_t +tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state) +{ + tor_assert(state != NULL); + return state->allocation; +} + +/** Return the approximate number of bytes allocated for all Zstandard + * states. */ +size_t +tor_zstd_get_total_allocation(void) +{ + return total_zstd_allocation; +} + diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h new file mode 100644 index 0000000000..ec83ba961e --- /dev/null +++ b/src/common/compress_zstd.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file compress_zstd.h + * \brief Header for compress_zstd.c + **/ + +#ifndef TOR_COMPRESS_ZSTD_H +#define TOR_COMPRESS_ZSTD_H + +const char * +tor_zstd_get_version_str(void); + +const char * +tor_zstd_get_header_version_str(void); + +int +tor_zstd_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int +tor_zstd_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); + +/** Internal state for an incremental Zstandard compression/decompression. */ +typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; + +tor_zstd_compress_state_t * +tor_zstd_compress_new(int compress, + compress_method_t method, + compression_level_t compression_level); + +tor_compress_output_t +tor_zstd_compress_process(tor_zstd_compress_state_t *state, + char **out, size_t *out_len, + const char **in, size_t *in_len, + int finish); + +void +tor_zstd_compress_free(tor_zstd_compress_state_t *state); + +size_t +tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); + +size_t +tor_zstd_get_total_allocation(void); + +#endif // TOR_COMPRESS_ZSTD_H. + diff --git a/src/common/include.am b/src/common/include.am index 2682c22b32..e285ef5f86 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -108,6 +108,7 @@ LIBOR_CRYPTO_A_SRC = \ src/common/compress.c \ src/common/compress_lzma.c \ src/common/compress_zlib.c \ + src/common/compress_zstd.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ src/common/crypto_s2k.c \ @@ -150,6 +151,7 @@ COMMONHEADERS = \ src/common/compress.h \ src/common/compress_lzma.h \ src/common/compress_zlib.h \ + src/common/compress_zstd.h \ src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ diff --git a/src/test/test_util.c b/src/test/test_util.c index 287f7e8cee..6a7db3137a 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2510,6 +2510,112 @@ test_util_lzma(void *arg) #endif // HAVE_LZMA. } +static void +test_util_zstd(void *arg) +{ +#ifdef HAVE_ZSTD + char *buf1=NULL, *buf2=NULL, *buf3=NULL, *cp1, *cp2; + const char *ccp2; + size_t len1, len2; + tor_compress_state_t *state = NULL; + + (void)arg; + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); + tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); + + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZSTD_METHOD)); + tt_assert(buf2 != NULL); + tt_int_op(len1, OP_LT, strlen(buf1)); + tt_int_op(detect_compression_method(buf2, len1), OP_EQ, ZSTD_METHOD); + + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1, + ZSTD_METHOD, 1, LOG_INFO)); + tt_assert(buf3 != NULL); + tt_int_op(strlen(buf1) + 1, OP_EQ, len2); + tt_str_op(buf1, OP_EQ, buf3); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + +#if 0 + /* Check whether we can uncompress concatenated, compressed strings. */ + tor_free(buf3); + buf2 = tor_reallocarray(buf2, len1, 2); + memcpy(buf2+len1, buf2, len1); + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1*2, + ZSTD_METHOD, 1, LOG_INFO)); + tt_int_op((strlen(buf1)+1)*2, OP_EQ, len2); + tt_mem_op(buf3, OP_EQ, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ\0", + (strlen(buf1)+1)*2); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); + + /* Check whether we can uncompress partial strings. */ + buf1 = + tor_strdup("String with low redundancy that won't be compressed much."); + tt_assert(!tor_compress(&buf2, &len1, buf1, strlen(buf1)+1, + ZSTD_METHOD)); + tt_int_op(len1, OP_GT, 16); + /* when we allow an incomplete string, we should succeed.*/ + tt_assert(!tor_uncompress(&buf3, &len2, buf2, len1-16, + ZSTD_METHOD, 0, LOG_INFO)); + tt_assert(len2 > 5); + buf3[len2]='\0'; + tt_assert(!strcmpstart(buf1, buf3)); + + /* when we demand a complete string, this must fail. */ + tor_free(buf3); + tt_assert(tor_uncompress(&buf3, &len2, buf2, len1-16, + ZSTD_METHOD, 1, LOG_INFO)); + tt_assert(buf3 == NULL); + + tor_free(buf1); + tor_free(buf2); + tor_free(buf3); +#endif + + /* Now, try streaming compression. */ + state = tor_compress_new(1, ZSTD_METHOD, HIGH_COMPRESSION); + tt_assert(state); + cp1 = buf1 = tor_malloc(1024); + len1 = 1024; + ccp2 = "ABCDEFGHIJABCDEFGHIJ"; + len2 = 21; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 0), + OP_EQ, TOR_COMPRESS_OK); + tt_int_op(0, OP_EQ, len2); /* Make sure we compressed it all. */ +// tt_assert(cp1 > buf1); + + len2 = 0; + cp2 = cp1; + tt_int_op(tor_compress_process(state, &cp1, &len1, &ccp2, &len2, 1), + OP_EQ, TOR_COMPRESS_DONE); + tt_int_op(0, OP_EQ, len2); + tt_assert(cp1 > cp2); /* Make sure we really added something. */ + + tt_assert(!tor_uncompress(&buf3, &len2, buf1, 1024-len1, + ZSTD_METHOD, 1, LOG_WARN)); + /* Make sure it compressed right. */ + tt_str_op(buf3, OP_EQ, "ABCDEFGHIJABCDEFGHIJ"); + tt_int_op(21, OP_EQ, len2); + + done: + if (state) + tor_compress_free(state); + tor_free(buf2); + tor_free(buf3); + tor_free(buf1); +#else + (void)arg; +#endif // HAVE_ZSTD. +} + /** Run unit tests for mmap() wrapper functionality. */ static void test_util_mmap(void *arg) @@ -5824,6 +5930,7 @@ struct testcase_t util_tests[] = { UTIL_LEGACY(gzip), UTIL_TEST(gzip_compression_bomb, TT_FORK), UTIL_LEGACY(lzma), + UTIL_LEGACY(zstd), UTIL_LEGACY(datadir), UTIL_LEGACY(memarea), UTIL_LEGACY(control_formats), From 2fa7b722ce3dcb54dca598d6b1d5a5abfb716f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 22:30:36 +0200 Subject: [PATCH 20/28] Show liblzma and libzstd versions in `tor --library-versions`. See: https://bugs.torproject.org/21662 --- src/or/config.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/or/config.c b/src/or/config.c index 9af116db1c..3549a1d608 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -70,7 +70,9 @@ #include "circuitmux_ewma.h" #include "circuitstats.h" #include "compress.h" +#include "compress_lzma.h" #include "compress_zlib.h" +#include "compress_zstd.h" #include "config.h" #include "connection.h" #include "connection_edge.h" @@ -4953,6 +4955,12 @@ options_init_from_torrc(int argc, char **argv) printf("Zlib \t\t%-15s\t\t%s\n", tor_zlib_get_header_version_str(), tor_zlib_get_version_str()); + printf("Liblzma \t\t%-15s\t\t%s\n", + tor_lzma_get_header_version_str(), + tor_lzma_get_version_str()); + printf("Libzstd \t\t%-15s\t\t%s\n", + tor_zstd_get_header_version_str(), + tor_zstd_get_version_str()); //TODO: Hex versions? exit(0); } From be4dc546345831773c2e16fa3dc12749d925aa25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 18 Apr 2017 22:31:07 +0200 Subject: [PATCH 21/28] Display LZMA and Zstandard versions when starting Tor. See: https://bugs.torproject.org/21662 --- src/or/main.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/or/main.c b/src/or/main.c index ddd7b82545..f3a0d84aea 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -58,7 +58,9 @@ #include "circuitlist.h" #include "circuituse.h" #include "command.h" +#include "compress_lzma.h" #include "compress_zlib.h" +#include "compress_zstd.h" #include "config.h" #include "confparse.h" #include "connection.h" @@ -2999,11 +3001,13 @@ tor_init(int argc, char *argv[]) const char *version = get_version(); log_notice(LD_GENERAL, "Tor %s running on %s with Libevent %s, " - "OpenSSL %s and Zlib %s.", version, + "OpenSSL %s, Zlib %s, Liblzma %s, and Libzstd %s.", version, get_uname(), tor_libevent_get_version_str(), crypto_openssl_get_version_str(), - tor_zlib_get_version_str()); + tor_zlib_get_version_str(), + tor_lzma_get_version_str(), + tor_zstd_get_version_str()); log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! " "Learn how to be safe at " From 04682d302aeeb0c2a0d9859bc0c1feee38daf16a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Apr 2017 15:33:13 +0200 Subject: [PATCH 22/28] Add `tor_compress_get_total_allocation()` function. This patch adds the `tor_compress_get_total_allocation()` which returns an approximate number of bytes currently in use by all the different compression backends. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 10 ++++++++++ src/common/compress.h | 3 +++ src/or/relay.c | 4 ++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index dc86be7750..52ff13258a 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -154,6 +154,16 @@ detect_compression_method(const char *in, size_t in_len) } } +/** Return the approximate number of bytes allocated for all + * supported compression schemas. */ +size_t +tor_compress_get_total_allocation(void) +{ + return tor_zlib_get_total_allocation() + + tor_lzma_get_total_allocation() + + tor_zstd_get_total_allocation(); +} + /** Internal state for an incremental compression/decompression. The body of * this struct is not exposed. */ struct tor_compress_state_t { diff --git a/src/common/compress.h b/src/common/compress.h index 59e7a79061..1fd4325b7e 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -51,6 +51,9 @@ tor_compress_memory_level(compression_level_t level); int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); +size_t +tor_compress_get_total_allocation(void); + /** Return values from tor_compress_process; see that function's documentation * for details. */ typedef enum { diff --git a/src/or/relay.c b/src/or/relay.c index 8a98c50372..8524080939 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -54,7 +54,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" -#include "compress_zlib.h" +#include "compress.h" #include "config.h" #include "connection.h" #include "connection_edge.h" @@ -2454,7 +2454,7 @@ cell_queues_check_size(void) { size_t alloc = cell_queues_get_total_allocation(); alloc += buf_get_total_allocation(); - alloc += tor_zlib_get_total_allocation(); + alloc += tor_compress_get_total_allocation(); const size_t rend_cache_total = rend_cache_get_total_allocation(); alloc += rend_cache_total; if (alloc >= get_options()->MaxMemInQueues_low_threshold) { From 1c77d8690caa1ad80b9d9581cac8eddae95e82d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Apr 2017 15:56:38 +0200 Subject: [PATCH 23/28] Add function to check if a given compression method is supported. This patch adds support for checking if a given `compress_method_t` is supported by the currently running Tor instance using `tor_compress_supports_method()`. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 19 +++++++++++++++++++ src/common/compress.h | 6 +++++- src/common/compress_lzma.c | 11 +++++++++++ src/common/compress_lzma.h | 3 +++ src/common/compress_zlib.c | 10 ++++++++++ src/common/compress_zlib.h | 3 +++ src/common/compress_zstd.c | 11 +++++++++++ src/common/compress_zstd.h | 3 +++ src/test/test_util.c | 15 +++++++++++++++ 9 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/common/compress.c b/src/common/compress.c index 52ff13258a..725dde53e0 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -154,6 +154,25 @@ detect_compression_method(const char *in, size_t in_len) } } +/** Return 1 if a given method is supported; otherwise 0. */ +int +tor_compress_supports_method(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_method_supported(); + case LZMA_METHOD: + return tor_lzma_method_supported(); + case ZSTD_METHOD: + return tor_zstd_method_supported(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return 0; + } +} + /** Return the approximate number of bytes allocated for all * supported compression schemas. */ size_t diff --git a/src/common/compress.h b/src/common/compress.h index 1fd4325b7e..87306f5fb9 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -13,7 +13,8 @@ /** Enumeration of what kind of compression to use. Only ZLIB_METHOD and * GZIP_METHOD is guaranteed to be supported by the compress/uncompress - * functions here. */ + * functions here. Call tor_compress_supports_method() to check if a given + * compression schema is supported by Tor. */ typedef enum { NO_METHOD=0, GZIP_METHOD=1, @@ -51,6 +52,9 @@ tor_compress_memory_level(compression_level_t level); int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); +int +tor_compress_supports_method(compress_method_t method); + size_t tor_compress_get_total_allocation(void); diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c index e1a7a66b0c..c45cb5eb4f 100644 --- a/src/common/compress_lzma.c +++ b/src/common/compress_lzma.c @@ -61,6 +61,17 @@ lzma_error_str(lzma_ret error) } #endif // HAVE_LZMA. +/** Return 1 if LZMA compression is supported; otherwise 0. */ +int +tor_lzma_method_supported(void) +{ +#ifdef HAVE_LZMA + return 1; +#else + return 0; +#endif +} + /** Return a string representation of the version of the currently running * version of liblzma. */ const char * diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h index 2cc3a00660..801e22ddd4 100644 --- a/src/common/compress_lzma.h +++ b/src/common/compress_lzma.h @@ -11,6 +11,9 @@ #ifndef TOR_COMPRESS_LZMA_H #define TOR_COMPRESS_LZMA_H +int +tor_lzma_method_supported(void); + const char * tor_lzma_get_version_str(void); diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index 2c9aba32ee..e1a68c2aa1 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -64,6 +64,16 @@ method_bits(compress_method_t method, compression_level_t level) } } +/** Return 1 if zlib/gzip compression is supported; otherwise 0. */ +int +tor_zlib_method_supported(void) +{ + /* We currently always support zlib/gzip, but we keep this function around in + * case we some day decide to deprecate zlib/gzip support. + */ + return 1; +} + /** Return a string representation of the version of the currently running * version of zlib. */ const char * diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h index 0862678da3..0b1aad8b9c 100644 --- a/src/common/compress_zlib.h +++ b/src/common/compress_zlib.h @@ -11,6 +11,9 @@ #ifndef TOR_COMPRESS_ZLIB_H #define TOR_COMPRESS_ZLIB_H +int +tor_zlib_method_supported(void); + const char * tor_zlib_get_version_str(void); diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index e2eb292d5d..dca4dbdab5 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -26,6 +26,17 @@ /** Total number of bytes allocated for Zstandard state. */ static size_t total_zstd_allocation = 0; +/** Return 1 if Zstandard compression is supported; otherwise 0. */ +int +tor_zstd_method_supported(void) +{ +#ifdef HAVE_ZSTD + return 1; +#else + return 0; +#endif +} + /** Return a string representation of the version of the currently running * version of libzstd. */ const char * diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h index ec83ba961e..b2297bd1df 100644 --- a/src/common/compress_zstd.h +++ b/src/common/compress_zstd.h @@ -11,6 +11,9 @@ #ifndef TOR_COMPRESS_ZSTD_H #define TOR_COMPRESS_ZSTD_H +int +tor_zstd_method_supported(void); + const char * tor_zstd_get_version_str(void); diff --git a/src/test/test_util.c b/src/test/test_util.c index 6a7db3137a..1e33de82ae 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -2252,6 +2252,9 @@ test_util_gzip(void *arg) tor_compress_state_t *state = NULL; (void)arg; + tt_assert(tor_compress_supports_method(GZIP_METHOD)); + tt_assert(tor_compress_supports_method(ZLIB_METHOD)); + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); @@ -2414,6 +2417,8 @@ test_util_lzma(void *arg) tor_compress_state_t *state = NULL; (void)arg; + tt_assert(tor_compress_supports_method(LZMA_METHOD)); + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); @@ -2507,6 +2512,10 @@ test_util_lzma(void *arg) tor_free(buf1); #else (void)arg; + tt_assert(! tor_compress_supports_method(LZMA_METHOD)); + + done: + ; #endif // HAVE_LZMA. } @@ -2520,6 +2529,8 @@ test_util_zstd(void *arg) tor_compress_state_t *state = NULL; (void)arg; + tt_assert(tor_compress_supports_method(ZSTD_METHOD)); + buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); @@ -2613,6 +2624,10 @@ test_util_zstd(void *arg) tor_free(buf1); #else (void)arg; + tt_assert(! tor_compress_supports_method(ZSTD_METHOD)); + + done: + ; #endif // HAVE_ZSTD. } From 6b905b38bb850d0530ba6f6d02c10b1fb19aef9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Apr 2017 16:20:14 +0200 Subject: [PATCH 24/28] Add API entry-point for getting compression method version numbers. This patch adds `tor_compress_version_str()` and `tor_compress_header_version_str()` to get the version strings of the different compression schema providers. Both functions returns `NULL` in case a given `compress_method_t` is unknown or unsupported. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 42 ++++++++++++++++++++++++++++++++++++++ src/common/compress.h | 6 ++++++ src/common/compress_lzma.c | 10 ++++----- src/common/compress_zstd.c | 8 ++++---- 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index 725dde53e0..e64bbca5d1 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -173,6 +173,48 @@ tor_compress_supports_method(compress_method_t method) } } +/** Return a string representation of the version of the library providing the + * compression method given in method. Returns NULL if method is + * unknown or unsupported. */ +const char * +tor_compress_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_version_str(); + case LZMA_METHOD: + return tor_lzma_get_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + +/** Return a string representation of the version of the library, found at + * compile time, providing the compression method given in method. + * Returns NULL if method is unknown or unsupported. */ +const char * +tor_compress_header_version_str(compress_method_t method) +{ + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_get_header_version_str(); + case LZMA_METHOD: + return tor_lzma_get_header_version_str(); + case ZSTD_METHOD: + return tor_zstd_get_header_version_str(); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return NULL; + } +} + /** Return the approximate number of bytes allocated for all * supported compression schemas. */ size_t diff --git a/src/common/compress.h b/src/common/compress.h index 87306f5fb9..182530fc11 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -55,6 +55,12 @@ tor_compress_is_compression_bomb(size_t size_in, size_t size_out); int tor_compress_supports_method(compress_method_t method); +const char * +tor_compress_version_str(compress_method_t method); + +const char * +tor_compress_header_version_str(compress_method_t method); + size_t tor_compress_get_total_allocation(void); diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c index c45cb5eb4f..f2952cccda 100644 --- a/src/common/compress_lzma.c +++ b/src/common/compress_lzma.c @@ -73,26 +73,26 @@ tor_lzma_method_supported(void) } /** Return a string representation of the version of the currently running - * version of liblzma. */ + * version of liblzma. Returns NULL if LZMA is unsupported. */ const char * tor_lzma_get_version_str(void) { #ifdef HAVE_LZMA return lzma_version_string(); #else - return "N/A"; + return NULL; #endif } -/** Return a string representation of the version of the version of liblzma - * used at compilation. */ +/** Return a string representation of the version of liblzma used at + * compilation time. Returns NULL if LZMA is unsupported. */ const char * tor_lzma_get_header_version_str(void) { #ifdef HAVE_LZMA return LZMA_VERSION_STRING; #else - return "N/A"; + return NULL; #endif } diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index dca4dbdab5..c838cd9364 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -38,7 +38,7 @@ tor_zstd_method_supported(void) } /** Return a string representation of the version of the currently running - * version of libzstd. */ + * version of libzstd. Returns NULL if Zstandard is unsupported. */ const char * tor_zstd_get_version_str(void) { @@ -55,19 +55,19 @@ tor_zstd_get_version_str(void) return version_str; #else - return "N/A"; + return NULL; #endif } /** Return a string representation of the version of the version of libzstd - * used at compilation. */ + * used at compilation time. Returns NULL if Zstandard is unsupported. */ const char * tor_zstd_get_header_version_str(void) { #ifdef HAVE_ZSTD return ZSTD_VERSION_STRING; #else - return "N/A"; + return NULL; #endif } From c2d1d949dec1db9fb413a02be422dfede3bb53aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Apr 2017 16:23:06 +0200 Subject: [PATCH 25/28] Use `tor_compress_supports_method()` before printing library versions. This patch ensures that Tor checks if a given compression method is supported before printing the version string when calling `tor --library-versions`. Additionally, we use the `tor_compress_supports_method()` to check if a given version is supported for Tor's start-up version string, but here we print "N/A" if a given compression method is unavailable. See: https://bugs.torproject.org/21662 --- src/or/config.c | 27 +++++++++++++++------------ src/or/main.c | 13 +++++++------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/or/config.c b/src/or/config.c index 3549a1d608..a73f397073 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -70,9 +70,6 @@ #include "circuitmux_ewma.h" #include "circuitstats.h" #include "compress.h" -#include "compress_lzma.h" -#include "compress_zlib.h" -#include "compress_zstd.h" #include "config.h" #include "connection.h" #include "connection_edge.h" @@ -4952,15 +4949,21 @@ options_init_from_torrc(int argc, char **argv) printf("OpenSSL \t\t%-15s\t\t%s\n", crypto_openssl_get_header_version_str(), crypto_openssl_get_version_str()); - printf("Zlib \t\t%-15s\t\t%s\n", - tor_zlib_get_header_version_str(), - tor_zlib_get_version_str()); - printf("Liblzma \t\t%-15s\t\t%s\n", - tor_lzma_get_header_version_str(), - tor_lzma_get_version_str()); - printf("Libzstd \t\t%-15s\t\t%s\n", - tor_zstd_get_header_version_str(), - tor_zstd_get_version_str()); + if (tor_compress_supports_method(ZLIB_METHOD)) { + printf("Zlib \t\t%-15s\t\t%s\n", + tor_compress_version_str(ZLIB_METHOD), + tor_compress_header_version_str(ZLIB_METHOD)); + } + if (tor_compress_supports_method(LZMA_METHOD)) { + printf("Liblzma \t\t%-15s\t\t%s\n", + tor_compress_version_str(LZMA_METHOD), + tor_compress_header_version_str(LZMA_METHOD)); + } + if (tor_compress_supports_method(ZSTD_METHOD)) { + printf("Libzstd \t\t%-15s\t\t%s\n", + tor_compress_version_str(ZSTD_METHOD), + tor_compress_header_version_str(ZSTD_METHOD)); + } //TODO: Hex versions? exit(0); } diff --git a/src/or/main.c b/src/or/main.c index f3a0d84aea..1ba6554d3e 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -58,9 +58,7 @@ #include "circuitlist.h" #include "circuituse.h" #include "command.h" -#include "compress_lzma.h" -#include "compress_zlib.h" -#include "compress_zstd.h" +#include "compress.h" #include "config.h" #include "confparse.h" #include "connection.h" @@ -3005,9 +3003,12 @@ tor_init(int argc, char *argv[]) get_uname(), tor_libevent_get_version_str(), crypto_openssl_get_version_str(), - tor_zlib_get_version_str(), - tor_lzma_get_version_str(), - tor_zstd_get_version_str()); + tor_compress_supports_method(ZLIB_METHOD) ? + tor_compress_version_str(ZLIB_METHOD) : "N/A", + tor_compress_supports_method(LZMA_METHOD) ? + tor_compress_version_str(LZMA_METHOD) : "N/A", + tor_compress_supports_method(ZSTD_METHOD) ? + tor_compress_version_str(ZSTD_METHOD) : "N/A"); log_notice(LD_GENERAL, "Tor can't help you if you use it wrong! " "Learn how to be safe at " From 69a41e8bc6e148c471ae8ee860e1a43548729db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 20 Apr 2017 16:37:39 +0200 Subject: [PATCH 26/28] Use switch-statement in tor_{compress,uncompress}. Use a switch-statement in `tor_compress()` and `tor_uncompress()` for the given `compress_method_t` parameter. This allows us to have the compiler detect if we forgot add a handler in these functions for a newly added enumeration value. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 64 +++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index e64bbca5d1..8b49af8220 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -80,16 +80,19 @@ tor_compress(char **out, size_t *out_len, const char *in, size_t in_len, compress_method_t method) { - if (method == GZIP_METHOD || method == ZLIB_METHOD) - return tor_zlib_compress(out, out_len, in, in_len, method); - - if (method == LZMA_METHOD) - return tor_lzma_compress(out, out_len, in, in_len, method); - - if (method == ZSTD_METHOD) - return tor_zstd_compress(out, out_len, in, in_len, method); - - return -1; + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_compress(out, out_len, in, in_len, method); + case LZMA_METHOD: + return tor_lzma_compress(out, out_len, in, in_len, method); + case ZSTD_METHOD: + return tor_zstd_compress(out, out_len, in, in_len, method); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return -1; + } } /** Given zero or more zlib-compressed or gzip-compressed strings of @@ -110,25 +113,28 @@ tor_uncompress(char **out, size_t *out_len, int complete_only, int protocol_warn_level) { - if (method == GZIP_METHOD || method == ZLIB_METHOD) - return tor_zlib_uncompress(out, out_len, in, in_len, - method, - complete_only, - protocol_warn_level); - - if (method == LZMA_METHOD) - return tor_lzma_uncompress(out, out_len, in, in_len, - method, - complete_only, - protocol_warn_level); - - if (method == ZSTD_METHOD) - return tor_zstd_uncompress(out, out_len, in, in_len, - method, - complete_only, - protocol_warn_level); - - return -1; + switch (method) { + case GZIP_METHOD: + case ZLIB_METHOD: + return tor_zlib_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level); + case LZMA_METHOD: + return tor_lzma_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level); + case ZSTD_METHOD: + return tor_zstd_uncompress(out, out_len, in, in_len, + method, + complete_only, + protocol_warn_level); + case NO_METHOD: + case UNKNOWN_METHOD: + default: + return -1; + } } /** Try to tell whether the in_len-byte string in in is likely From cf912259ba491e51f6f211e186ff67605ff269c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 24 Apr 2017 14:20:16 +0200 Subject: [PATCH 27/28] Remove `tor_compress_memory_level()`. This patch splits up `tor_compress_memory_level()` into static functions in the individual compression backends, which allows us to tune the values per compression backend rather than globally. See: https://bugs.torproject.org/21662 --- src/common/compress.c | 14 -------------- src/common/compress.h | 3 --- src/common/compress_lzma.c | 16 ++++++++++++++-- src/common/compress_zlib.c | 16 ++++++++++++++-- src/common/compress_zstd.c | 18 ++++++++++++++++-- 5 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/common/compress.c b/src/common/compress.c index 8b49af8220..38b8184573 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -56,20 +56,6 @@ tor_compress_is_compression_bomb(size_t size_in, size_t size_out) return (size_out / size_in > MAX_UNCOMPRESSION_FACTOR); } -/** Given level return the memory level. The memory level is needed for - * the various compression backends used in Tor. - */ -int -tor_compress_memory_level(compression_level_t level) -{ - switch (level) { - default: - case HIGH_COMPRESSION: return 8; - case MEDIUM_COMPRESSION: return 7; - case LOW_COMPRESSION: return 6; - } -} - /** Given in_len bytes at in, compress them into a newly * allocated buffer, using the method described in method. Store the * compressed string in *out, and its length in *out_len. diff --git a/src/common/compress.h b/src/common/compress.h index 182530fc11..1e6e9fd329 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -46,9 +46,6 @@ tor_uncompress(char **out, size_t *out_len, compress_method_t detect_compression_method(const char *in, size_t in_len); -int -tor_compress_memory_level(compression_level_t level); - int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); diff --git a/src/common/compress_lzma.c b/src/common/compress_lzma.c index f2952cccda..ae0327f581 100644 --- a/src/common/compress_lzma.c +++ b/src/common/compress_lzma.c @@ -26,6 +26,18 @@ static size_t total_lzma_allocation = 0; #ifdef HAVE_LZMA +/** Given level return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case HIGH_COMPRESSION: return 9; + case MEDIUM_COMPRESSION: return 6; + case LOW_COMPRESSION: return 3; + } +} + /** Convert a given error to a human readable error string. */ static const char * lzma_error_str(lzma_ret error) @@ -124,7 +136,7 @@ tor_lzma_compress(char **out, size_t *out_len, stream.avail_in = in_len; lzma_lzma_preset(&stream_options, - tor_compress_memory_level(HIGH_COMPRESSION)); + memory_level(HIGH_COMPRESSION)); retval = lzma_alone_encoder(&stream, &stream_options); @@ -432,7 +444,7 @@ tor_lzma_compress_new(int compress, if (compress) { lzma_lzma_preset(&stream_options, - tor_compress_memory_level(compression_level)); + memory_level(compression_level)); retval = lzma_alone_encoder(&result->stream, &stream_options); diff --git a/src/common/compress_zlib.c b/src/common/compress_zlib.c index e1a68c2aa1..50bdf9ece1 100644 --- a/src/common/compress_zlib.c +++ b/src/common/compress_zlib.c @@ -50,6 +50,18 @@ static size_t tor_zlib_state_size_precalc(int inflate, /** Total number of bytes allocated for zlib state */ static size_t total_zlib_allocation = 0; +/** Given level return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case HIGH_COMPRESSION: return 8; + case MEDIUM_COMPRESSION: return 7; + case LOW_COMPRESSION: return 6; + } +} + /** Return the 'bits' value to tell zlib to use method.*/ static inline int method_bits(compress_method_t method, compression_level_t level) @@ -120,7 +132,7 @@ tor_zlib_compress(char **out, size_t *out_len, if (deflateInit2(stream, Z_BEST_COMPRESSION, Z_DEFLATED, method_bits(method, HIGH_COMPRESSION), - tor_compress_memory_level(HIGH_COMPRESSION), + memory_level(HIGH_COMPRESSION), Z_DEFAULT_STRATEGY) != Z_OK) { //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. log_warn(LD_GENERAL, "Error from deflateInit2: %s", @@ -413,7 +425,7 @@ tor_zlib_compress_new(int compress, out->stream.opaque = NULL; out->compress = compress; bits = method_bits(method, compression_level); - memlevel = tor_compress_memory_level(compression_level); + memlevel = memory_level(compression_level); if (compress) { if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, bits, memlevel, diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index c838cd9364..664cce1700 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -26,6 +26,20 @@ /** Total number of bytes allocated for Zstandard state. */ static size_t total_zstd_allocation = 0; +#ifdef HAVE_ZSTD +/** Given level return the memory level. */ +static int +memory_level(compression_level_t level) +{ + switch (level) { + default: + case HIGH_COMPRESSION: return 9; + case MEDIUM_COMPRESSION: return 8; + case LOW_COMPRESSION: return 7; + } +} +#endif // HAVE_ZSTD. + /** Return 1 if Zstandard compression is supported; otherwise 0. */ int tor_zstd_method_supported(void) @@ -104,7 +118,7 @@ tor_zstd_compress(char **out, size_t *out_len, } retval = ZSTD_initCStream(stream, - tor_compress_memory_level(HIGH_COMPRESSION)); + memory_level(HIGH_COMPRESSION)); if (ZSTD_isError(retval)) { log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", @@ -408,7 +422,7 @@ tor_zstd_compress_new(int compress, } retval = ZSTD_initCStream(result->u.compress_stream, - tor_compress_memory_level(compression_level)); + memory_level(compression_level)); if (ZSTD_isError(retval)) { log_warn(LD_GENERAL, "Zstandard stream initialization error: %s", From 2210b330e757d234346191e5d05af761fe8561e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 25 Apr 2017 03:36:30 +0200 Subject: [PATCH 28/28] Fix newlines in compression headers. See: https://bugs.torproject.org/21662 --- src/common/compress.h | 34 ++++++++++++++-------------------- src/common/compress_lzma.h | 36 ++++++++++++++---------------------- src/common/compress_zlib.h | 36 ++++++++++++++---------------------- src/common/compress_zstd.h | 36 ++++++++++++++---------------------- 4 files changed, 56 insertions(+), 86 deletions(-) diff --git a/src/common/compress.h b/src/common/compress.h index 1e6e9fd329..2d812e4430 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -33,33 +33,27 @@ typedef enum { HIGH_COMPRESSION, MEDIUM_COMPRESSION, LOW_COMPRESSION } compression_level_t; -int -tor_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); -int -tor_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); +int tor_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); + +int tor_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); compress_method_t detect_compression_method(const char *in, size_t in_len); -int -tor_compress_is_compression_bomb(size_t size_in, size_t size_out); +int tor_compress_is_compression_bomb(size_t size_in, size_t size_out); -int -tor_compress_supports_method(compress_method_t method); +int tor_compress_supports_method(compress_method_t method); -const char * -tor_compress_version_str(compress_method_t method); +const char *tor_compress_version_str(compress_method_t method); -const char * -tor_compress_header_version_str(compress_method_t method); +const char *tor_compress_header_version_str(compress_method_t method); -size_t -tor_compress_get_total_allocation(void); +size_t tor_compress_get_total_allocation(void); /** Return values from tor_compress_process; see that function's documentation * for details. */ diff --git a/src/common/compress_lzma.h b/src/common/compress_lzma.h index 801e22ddd4..71de56a1c6 100644 --- a/src/common/compress_lzma.h +++ b/src/common/compress_lzma.h @@ -11,26 +11,21 @@ #ifndef TOR_COMPRESS_LZMA_H #define TOR_COMPRESS_LZMA_H -int -tor_lzma_method_supported(void); +int tor_lzma_method_supported(void); -const char * -tor_lzma_get_version_str(void); +const char *tor_lzma_get_version_str(void); -const char * -tor_lzma_get_header_version_str(void); +const char *tor_lzma_get_header_version_str(void); -int -tor_lzma_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); +int tor_lzma_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); -int -tor_lzma_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); +int tor_lzma_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); /** Internal state for an incremental LZMA compression/decompression. */ typedef struct tor_lzma_compress_state_t tor_lzma_compress_state_t; @@ -46,14 +41,11 @@ tor_lzma_compress_process(tor_lzma_compress_state_t *state, const char **in, size_t *in_len, int finish); -void -tor_lzma_compress_free(tor_lzma_compress_state_t *state); +void tor_lzma_compress_free(tor_lzma_compress_state_t *state); -size_t -tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state); +size_t tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state); -size_t -tor_lzma_get_total_allocation(void); +size_t tor_lzma_get_total_allocation(void); #endif // TOR_COMPRESS_LZMA_H. diff --git a/src/common/compress_zlib.h b/src/common/compress_zlib.h index 0b1aad8b9c..6e8e5c5136 100644 --- a/src/common/compress_zlib.h +++ b/src/common/compress_zlib.h @@ -11,26 +11,21 @@ #ifndef TOR_COMPRESS_ZLIB_H #define TOR_COMPRESS_ZLIB_H -int -tor_zlib_method_supported(void); +int tor_zlib_method_supported(void); -const char * -tor_zlib_get_version_str(void); +const char *tor_zlib_get_version_str(void); -const char * -tor_zlib_get_header_version_str(void); +const char *tor_zlib_get_header_version_str(void); -int -tor_zlib_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); +int tor_zlib_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); -int -tor_zlib_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); +int tor_zlib_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); /** Internal state for an incremental zlib/gzip compression/decompression. */ typedef struct tor_zlib_compress_state_t tor_zlib_compress_state_t; @@ -46,14 +41,11 @@ tor_zlib_compress_process(tor_zlib_compress_state_t *state, const char **in, size_t *in_len, int finish); -void -tor_zlib_compress_free(tor_zlib_compress_state_t *state); +void tor_zlib_compress_free(tor_zlib_compress_state_t *state); -size_t -tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); +size_t tor_zlib_compress_state_size(const tor_zlib_compress_state_t *state); -size_t -tor_zlib_get_total_allocation(void); +size_t tor_zlib_get_total_allocation(void); #endif // TOR_COMPRESS_ZLIB_H. diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h index b2297bd1df..663cbdd2ff 100644 --- a/src/common/compress_zstd.h +++ b/src/common/compress_zstd.h @@ -11,26 +11,21 @@ #ifndef TOR_COMPRESS_ZSTD_H #define TOR_COMPRESS_ZSTD_H -int -tor_zstd_method_supported(void); +int tor_zstd_method_supported(void); -const char * -tor_zstd_get_version_str(void); +const char *tor_zstd_get_version_str(void); -const char * -tor_zstd_get_header_version_str(void); +const char *tor_zstd_get_header_version_str(void); -int -tor_zstd_compress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method); +int tor_zstd_compress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method); -int -tor_zstd_uncompress(char **out, size_t *out_len, - const char *in, size_t in_len, - compress_method_t method, - int complete_only, - int protocol_warn_level); +int tor_zstd_uncompress(char **out, size_t *out_len, + const char *in, size_t in_len, + compress_method_t method, + int complete_only, + int protocol_warn_level); /** Internal state for an incremental Zstandard compression/decompression. */ typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; @@ -46,14 +41,11 @@ tor_zstd_compress_process(tor_zstd_compress_state_t *state, const char **in, size_t *in_len, int finish); -void -tor_zstd_compress_free(tor_zstd_compress_state_t *state); +void tor_zstd_compress_free(tor_zstd_compress_state_t *state); -size_t -tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); +size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); -size_t -tor_zstd_get_total_allocation(void); +size_t tor_zstd_get_total_allocation(void); #endif // TOR_COMPRESS_ZSTD_H.