compress_lzma.c (10701B)
1 /* Copyright (c) 2004, Roger Dingledine. 2 * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. 3 * Copyright (c) 2007-2021, The Tor Project, Inc. */ 4 /* See LICENSE for licensing information */ 5 6 /** 7 * \file compress_lzma.c 8 * \brief Compression backend for LZMA. 9 * 10 * This module should never be invoked directly. Use the compress module 11 * instead. 12 **/ 13 14 #include "orconfig.h" 15 16 #include "lib/compress/compress.h" 17 #include "lib/compress/compress_lzma.h" 18 #include "lib/log/log.h" 19 #include "lib/log/util_bug.h" 20 #include "lib/malloc/malloc.h" 21 #include "lib/thread/threads.h" 22 23 #ifdef HAVE_LZMA 24 #include <lzma.h> 25 #endif 26 27 /** The maximum amount of memory we allow the LZMA decoder to use, in bytes. */ 28 #define MEMORY_LIMIT (16 * 1024 * 1024) 29 30 /** Total number of bytes allocated for LZMA state. */ 31 static atomic_counter_t total_lzma_allocation; 32 33 #ifdef HAVE_LZMA 34 /** Given <b>level</b> return the memory level. */ 35 static int 36 memory_level(compression_level_t level) 37 { 38 switch (level) { 39 default: 40 case BEST_COMPRESSION: 41 case HIGH_COMPRESSION: return 6; 42 case MEDIUM_COMPRESSION: return 4; 43 case LOW_COMPRESSION: return 2; 44 } 45 } 46 47 /** Convert a given <b>error</b> to a human readable error string. */ 48 static const char * 49 lzma_error_str(lzma_ret error) 50 { 51 switch (error) { 52 case LZMA_OK: 53 return "Operation completed successfully"; 54 case LZMA_STREAM_END: 55 return "End of stream"; 56 case LZMA_NO_CHECK: 57 return "Input stream lacks integrity check"; 58 case LZMA_UNSUPPORTED_CHECK: 59 return "Unable to calculate integrity check"; 60 case LZMA_GET_CHECK: 61 return "Integrity check available"; 62 case LZMA_MEM_ERROR: 63 return "Unable to allocate memory"; 64 case LZMA_MEMLIMIT_ERROR: 65 return "Memory limit reached"; 66 case LZMA_FORMAT_ERROR: 67 return "Unknown file format"; 68 case LZMA_OPTIONS_ERROR: 69 return "Unsupported options"; 70 case LZMA_DATA_ERROR: 71 return "Corrupt input data"; 72 case LZMA_BUF_ERROR: 73 return "Unable to progress"; 74 case LZMA_PROG_ERROR: 75 return "Programming error"; 76 #if LZMA_VERSION >= 50030010 77 case LZMA_SEEK_NEEDED: 78 // This can be returned by the .xz file_info decoder but with 79 // lzma_alone_decoder/encoder as we use, it should never be seen. 80 return "Seek needed"; 81 #endif 82 #if LZMA_VERSION >= 50030020 83 case LZMA_RET_INTERNAL1: 84 case LZMA_RET_INTERNAL2: 85 case LZMA_RET_INTERNAL3: 86 case LZMA_RET_INTERNAL4: 87 case LZMA_RET_INTERNAL5: 88 case LZMA_RET_INTERNAL6: 89 case LZMA_RET_INTERNAL7: 90 case LZMA_RET_INTERNAL8: 91 FALLTHROUGH; 92 #endif 93 default: 94 return "Unknown LZMA error"; 95 } 96 } 97 #endif /* defined(HAVE_LZMA) */ 98 99 /** Return 1 if LZMA compression is supported; otherwise 0. */ 100 int 101 tor_lzma_method_supported(void) 102 { 103 #ifdef HAVE_LZMA 104 return 1; 105 #else 106 return 0; 107 #endif 108 } 109 110 /** Return a string representation of the version of the currently running 111 * version of liblzma. Returns NULL if LZMA is unsupported. */ 112 const char * 113 tor_lzma_get_version_str(void) 114 { 115 #ifdef HAVE_LZMA 116 return lzma_version_string(); 117 #else 118 return NULL; 119 #endif 120 } 121 122 /** Return a string representation of the version of liblzma used at 123 * compilation time. Returns NULL if LZMA is unsupported. */ 124 const char * 125 tor_lzma_get_header_version_str(void) 126 { 127 #ifdef HAVE_LZMA 128 return LZMA_VERSION_STRING; 129 #else 130 return NULL; 131 #endif 132 } 133 134 /** Internal LZMA state for incremental compression/decompression. 135 * The body of this struct is not exposed. */ 136 struct tor_lzma_compress_state_t { 137 #ifdef HAVE_LZMA 138 lzma_stream stream; /**< The LZMA stream. */ 139 #endif 140 141 int compress; /**< True if we are compressing; false if we are inflating */ 142 143 /** Number of bytes read so far. Used to detect compression bombs. */ 144 size_t input_so_far; 145 /** Number of bytes written so far. Used to detect compression bombs. */ 146 size_t output_so_far; 147 148 /** Approximate number of bytes allocated for this object. */ 149 size_t allocation; 150 }; 151 152 #ifdef HAVE_LZMA 153 /** Return an approximate number of bytes stored in memory to hold the LZMA 154 * encoder/decoder state. */ 155 static size_t 156 tor_lzma_state_size_precalc(int compress, compression_level_t level) 157 { 158 uint64_t memory_usage; 159 160 if (compress) 161 memory_usage = lzma_easy_encoder_memusage(memory_level(level)); 162 else 163 memory_usage = lzma_easy_decoder_memusage(memory_level(level)); 164 165 if (memory_usage == UINT64_MAX) { 166 // LCOV_EXCL_START 167 log_warn(LD_GENERAL, "Unsupported compression level passed to LZMA %s", 168 compress ? "encoder" : "decoder"); 169 goto err; 170 // LCOV_EXCL_STOP 171 } 172 173 if (memory_usage + sizeof(tor_lzma_compress_state_t) > SIZE_MAX) 174 memory_usage = SIZE_MAX; 175 else 176 memory_usage += sizeof(tor_lzma_compress_state_t); 177 178 return (size_t)memory_usage; 179 180 // LCOV_EXCL_START 181 err: 182 return 0; 183 // LCOV_EXCL_STOP 184 } 185 #endif /* defined(HAVE_LZMA) */ 186 187 /** Construct and return a tor_lzma_compress_state_t object using 188 * <b>method</b>. If <b>compress</b>, it's for compression; otherwise it's for 189 * decompression. */ 190 tor_lzma_compress_state_t * 191 tor_lzma_compress_new(int compress, 192 compress_method_t method, 193 compression_level_t level) 194 { 195 tor_assert(method == LZMA_METHOD); 196 197 #ifdef HAVE_LZMA 198 tor_lzma_compress_state_t *result; 199 lzma_ret retval; 200 lzma_options_lzma stream_options; 201 202 // Note that we do not explicitly initialize the lzma_stream object here, 203 // since the LZMA_STREAM_INIT "just" initializes all members to 0, which is 204 // also what `tor_malloc_zero()` does. 205 result = tor_malloc_zero(sizeof(tor_lzma_compress_state_t)); 206 result->compress = compress; 207 result->allocation = tor_lzma_state_size_precalc(compress, level); 208 209 if (compress) { 210 lzma_lzma_preset(&stream_options, memory_level(level)); 211 212 retval = lzma_alone_encoder(&result->stream, &stream_options); 213 214 if (retval != LZMA_OK) { 215 // LCOV_EXCL_START 216 log_warn(LD_GENERAL, "Error from LZMA encoder: %s (%u).", 217 lzma_error_str(retval), retval); 218 goto err; 219 // LCOV_EXCL_STOP 220 } 221 } else { 222 retval = lzma_alone_decoder(&result->stream, MEMORY_LIMIT); 223 224 if (retval != LZMA_OK) { 225 // LCOV_EXCL_START 226 log_warn(LD_GENERAL, "Error from LZMA decoder: %s (%u).", 227 lzma_error_str(retval), retval); 228 goto err; 229 // LCOV_EXCL_STOP 230 } 231 } 232 233 atomic_counter_add(&total_lzma_allocation, result->allocation); 234 return result; 235 236 /* LCOV_EXCL_START */ 237 err: 238 tor_free(result); 239 return NULL; 240 /* LCOV_EXCL_STOP */ 241 #else /* !defined(HAVE_LZMA) */ 242 (void)compress; 243 (void)method; 244 (void)level; 245 246 return NULL; 247 #endif /* defined(HAVE_LZMA) */ 248 } 249 250 /** Compress/decompress some bytes using <b>state</b>. Read up to 251 * *<b>in_len</b> bytes from *<b>in</b>, and write up to *<b>out_len</b> bytes 252 * to *<b>out</b>, adjusting the values as we go. If <b>finish</b> is true, 253 * we've reached the end of the input. 254 * 255 * Return TOR_COMPRESS_DONE if we've finished the entire 256 * compression/decompression. 257 * Return TOR_COMPRESS_OK if we're processed everything from the input. 258 * Return TOR_COMPRESS_BUFFER_FULL if we're out of space on <b>out</b>. 259 * Return TOR_COMPRESS_ERROR if the stream is corrupt. 260 */ 261 tor_compress_output_t 262 tor_lzma_compress_process(tor_lzma_compress_state_t *state, 263 char **out, size_t *out_len, 264 const char **in, size_t *in_len, 265 int finish) 266 { 267 #ifdef HAVE_LZMA 268 lzma_ret retval; 269 lzma_action action; 270 271 tor_assert(state != NULL); 272 tor_assert(*in_len <= UINT_MAX); 273 tor_assert(*out_len <= UINT_MAX); 274 275 state->stream.next_in = (unsigned char *)*in; 276 state->stream.avail_in = *in_len; 277 state->stream.next_out = (unsigned char *)*out; 278 state->stream.avail_out = *out_len; 279 280 action = finish ? LZMA_FINISH : LZMA_RUN; 281 282 retval = lzma_code(&state->stream, action); 283 284 state->input_so_far += state->stream.next_in - ((unsigned char *)*in); 285 state->output_so_far += state->stream.next_out - ((unsigned char *)*out); 286 287 *out = (char *)state->stream.next_out; 288 *out_len = state->stream.avail_out; 289 *in = (const char *)state->stream.next_in; 290 *in_len = state->stream.avail_in; 291 292 if (! state->compress && 293 tor_compress_is_compression_bomb(state->input_so_far, 294 state->output_so_far)) { 295 log_warn(LD_DIR, "Possible compression bomb; abandoning stream."); 296 return TOR_COMPRESS_ERROR; 297 } 298 299 switch (retval) { 300 case LZMA_OK: 301 if (state->stream.avail_out == 0 || finish) 302 return TOR_COMPRESS_BUFFER_FULL; 303 304 return TOR_COMPRESS_OK; 305 306 case LZMA_BUF_ERROR: 307 if (state->stream.avail_in == 0 && !finish) 308 return TOR_COMPRESS_OK; 309 310 return TOR_COMPRESS_BUFFER_FULL; 311 312 case LZMA_STREAM_END: 313 return TOR_COMPRESS_DONE; 314 315 // We list all the possible values of `lzma_ret` here to silence the 316 // `switch-enum` warning and to detect if a new member was added. 317 case LZMA_NO_CHECK: 318 case LZMA_UNSUPPORTED_CHECK: 319 case LZMA_GET_CHECK: 320 case LZMA_MEM_ERROR: 321 case LZMA_MEMLIMIT_ERROR: 322 case LZMA_FORMAT_ERROR: 323 case LZMA_OPTIONS_ERROR: 324 case LZMA_DATA_ERROR: 325 case LZMA_PROG_ERROR: 326 #if LZMA_VERSION >= 50030010 327 case LZMA_SEEK_NEEDED: 328 #endif 329 #if LZMA_VERSION >= 50030020 330 case LZMA_RET_INTERNAL1: 331 case LZMA_RET_INTERNAL2: 332 case LZMA_RET_INTERNAL3: 333 case LZMA_RET_INTERNAL4: 334 case LZMA_RET_INTERNAL5: 335 case LZMA_RET_INTERNAL6: 336 case LZMA_RET_INTERNAL7: 337 case LZMA_RET_INTERNAL8: 338 #endif 339 default: 340 log_warn(LD_GENERAL, "LZMA %s didn't finish: %s.", 341 state->compress ? "compression" : "decompression", 342 lzma_error_str(retval)); 343 return TOR_COMPRESS_ERROR; 344 } 345 #else /* !defined(HAVE_LZMA) */ 346 (void)state; 347 (void)out; 348 (void)out_len; 349 (void)in; 350 (void)in_len; 351 (void)finish; 352 return TOR_COMPRESS_ERROR; 353 #endif /* defined(HAVE_LZMA) */ 354 } 355 356 /** Deallocate <b>state</b>. */ 357 void 358 tor_lzma_compress_free_(tor_lzma_compress_state_t *state) 359 { 360 if (state == NULL) 361 return; 362 363 atomic_counter_sub(&total_lzma_allocation, state->allocation); 364 365 #ifdef HAVE_LZMA 366 lzma_end(&state->stream); 367 #endif 368 369 tor_free(state); 370 } 371 372 /** Return the approximate number of bytes allocated for <b>state</b>. */ 373 size_t 374 tor_lzma_compress_state_size(const tor_lzma_compress_state_t *state) 375 { 376 tor_assert(state != NULL); 377 return state->allocation; 378 } 379 380 /** Return the approximate number of bytes allocated for all LZMA states. */ 381 size_t 382 tor_lzma_get_total_allocation(void) 383 { 384 return atomic_counter_get(&total_lzma_allocation); 385 } 386 387 /** Initialize the lzma module */ 388 void 389 tor_lzma_init(void) 390 { 391 atomic_counter_init(&total_lzma_allocation); 392 }