nsHTTPCompressConv.cpp (37434B)
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim: set sw=2 ts=8 et tw=80 : */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #include "nsHTTPCompressConv.h" 8 #include "ErrorList.h" 9 #include "nsCOMPtr.h" 10 #include "nsCRT.h" 11 #include "nsError.h" 12 #include "nsIChannel.h" 13 #include "nsIForcePendingChannel.h" 14 #include "nsIHttpChannel.h" 15 #include "nsIRequest.h" 16 #include "nsIThreadRetargetableRequest.h" 17 #include "nsIThreadRetargetableStreamListener.h" 18 #include "nsThreadUtils.h" 19 #include "nsStreamUtils.h" 20 #include "nsStringStream.h" 21 #include "nsComponentManagerUtils.h" 22 #include "mozilla/net/Dictionary.h" 23 #include "mozilla/Preferences.h" 24 #include "mozilla/StaticPrefs_network.h" 25 #include "mozilla/Logging.h" 26 #include "mozilla/UniquePtrExtensions.h" 27 28 // brotli headers 29 #undef assert 30 #include "assert.h" 31 #include "state.h" 32 #include "brotli/decode.h" 33 34 #define ZSTD_STATIC_LINKING_ONLY 1 35 #include "zstd/zstd.h" 36 37 namespace mozilla { 38 namespace net { 39 40 class DictionaryCacheEntry; 41 42 extern LazyLogModule gHttpLog; 43 #define LOG(args) \ 44 MOZ_LOG(mozilla::net::gHttpLog, mozilla::LogLevel::Debug, args) 45 46 extern LazyLogModule gDictionaryLog; 47 #define DICTIONARY_LOG(args) \ 48 MOZ_LOG(mozilla::net::gDictionaryLog, mozilla::LogLevel::Debug, args) 49 50 class BrotliWrapper { 51 public: 52 BrotliWrapper() {} 53 ~BrotliWrapper() { BrotliDecoderStateCleanup(&mState); } 54 55 bool Init(nsIRequest* aRequest) { 56 if (!BrotliDecoderStateInit(&mState, nullptr, nullptr, nullptr)) { 57 return false; 58 } 59 60 nsCOMPtr<nsIHttpChannel> httpchannel(do_QueryInterface(aRequest)); 61 if (!httpchannel) { 62 return false; 63 } 64 // XXX Wait for dictionary to be read into RAM!! 65 if (NS_SUCCEEDED(httpchannel->GetDecompressDictionary( 66 getter_AddRefs(mDictionary))) && 67 mDictionary) { 68 size_t length = mDictionary->GetDictionary().length(); 69 DICTIONARY_LOG(("Brotli: dictionary %zu bytes", length)); 70 if (length > 0) { 71 BROTLI_BOOL result = BrotliDecoderAttachDictionary( 72 &mState, BROTLI_SHARED_DICTIONARY_RAW, length, 73 mDictionary->GetDictionary().begin()); 74 if (!result) { 75 DICTIONARY_LOG(("Brotli: AttachDictionary failed")); 76 return false; 77 } 78 } 79 } 80 return true; 81 } 82 83 BrotliDecoderState mState{}; 84 Atomic<size_t, Relaxed> mTotalOut{0}; 85 nsresult mStatus = NS_OK; 86 Atomic<bool, Relaxed> mBrotliStateIsStreamEnd{false}; 87 88 nsIRequest* mRequest{nullptr}; 89 nsISupports* mContext{nullptr}; 90 uint64_t mSourceOffset{0}; 91 92 RefPtr<DictionaryCacheEntry> mDictionary; 93 94 uint8_t mEaten{0}; 95 uint8_t mHeader[36]; // \FF\44\43\42 + 32-byte SHA-256 96 }; 97 98 #ifdef ZSTD_INFALLIBLE 99 // zstd can grab large blocks; use an infallible alloctor 100 static void* zstd_malloc(void*, size_t size) { return moz_xmalloc(size); } 101 102 static void zstd_free(void*, void* address) { free(address); } 103 104 ZSTD_customMem const zstd_allocators = {zstd_malloc, zstd_free, nullptr}; 105 #endif 106 107 class ZstdWrapper { 108 public: 109 ZstdWrapper(nsIRequest* aRequest, nsHTTPCompressConv::CompressMode aMode) { 110 size_t length = 0; 111 if (aMode == nsHTTPCompressConv::HTTP_COMPRESS_ZSTD_DICTIONARY) { 112 nsCOMPtr<nsIHttpChannel> httpchannel(do_QueryInterface(aRequest)); 113 if (httpchannel) { 114 // XXX Wait for dictionary to be read into RAM!! 115 if (NS_FAILED(httpchannel->GetDecompressDictionary( 116 getter_AddRefs(mDictionary))) || 117 !mDictionary) { 118 return; 119 } 120 length = mDictionary->GetDictionary().length(); 121 } else { 122 // Can't decode without a dictionary 123 return; 124 } 125 } 126 127 #ifdef ZSTD_INFALLIBLE 128 mDStream = ZSTD_createDStream_advanced(zstd_allocators); // infallible 129 #else 130 mDStream = ZSTD_createDStream(); // fallible 131 if (!mDStream) { 132 MOZ_RELEASE_ASSERT(ZSTD_defaultCMem.customAlloc == NULL && 133 ZSTD_defaultCMem.customFree == NULL && 134 ZSTD_defaultCMem.opaque == NULL); 135 return; 136 } 137 #endif 138 if (mDictionary) { 139 DICTIONARY_LOG(("zstd: dictionary %zu bytes", length)); 140 ZSTD_DCtx_reset(mDStream, ZSTD_reset_session_only); 141 if (ZSTD_isError(ZSTD_DCtx_loadDictionary( 142 mDStream, mDictionary->GetDictionary().begin(), length))) { 143 return; 144 } 145 } 146 147 ZSTD_DCtx_setParameter(mDStream, ZSTD_d_windowLogMax, 23 /*8*1024*1024*/); 148 } 149 ~ZstdWrapper() { 150 if (mDStream) { 151 ZSTD_freeDStream(mDStream); 152 } 153 } 154 155 UniquePtr<uint8_t[]> mOutBuffer; 156 nsresult mStatus = NS_OK; 157 nsIRequest* mRequest{nullptr}; 158 nsISupports* mContext{nullptr}; 159 uint64_t mSourceOffset{0}; 160 ZSTD_DStream* mDStream{nullptr}; 161 162 RefPtr<DictionaryCacheEntry> mDictionary; 163 }; 164 165 // nsISupports implementation 166 NS_IMPL_ISUPPORTS(nsHTTPCompressConv, nsIStreamConverter, nsIStreamListener, 167 nsIRequestObserver, nsICompressConvStats, 168 nsIThreadRetargetableStreamListener) 169 170 // nsFTPDirListingConv methods 171 nsHTTPCompressConv::nsHTTPCompressConv() { 172 LOG(("nsHttpCompresssConv %p ctor\n", this)); 173 if (NS_IsMainThread()) { 174 mFailUncleanStops = 175 Preferences::GetBool("network.http.enforce-framing.http", false); 176 } else { 177 mFailUncleanStops = false; 178 } 179 } 180 181 nsHTTPCompressConv::~nsHTTPCompressConv() { 182 LOG(("nsHttpCompresssConv %p dtor\n", this)); 183 if (mInpBuffer) { 184 free(mInpBuffer); 185 } 186 187 if (mOutBuffer) { 188 free(mOutBuffer); 189 } 190 191 // For some reason we are not getting Z_STREAM_END. But this was also seen 192 // for mozilla bug 198133. Need to handle this case. 193 if (mStreamInitialized && !mStreamEnded) { 194 inflateEnd(&d_stream); 195 } 196 } 197 198 NS_IMETHODIMP 199 nsHTTPCompressConv::GetDecodedDataLength(uint64_t* aDecodedDataLength) { 200 *aDecodedDataLength = mDecodedDataLength; 201 return NS_OK; 202 } 203 204 NS_IMETHODIMP 205 nsHTTPCompressConv::AsyncConvertData(const char* aFromType, const char* aToType, 206 nsIStreamListener* aListener, 207 nsISupports* aCtxt) { 208 if (!nsCRT::strncasecmp(aFromType, HTTP_COMPRESS_TYPE, 209 sizeof(HTTP_COMPRESS_TYPE) - 1) || 210 !nsCRT::strncasecmp(aFromType, HTTP_X_COMPRESS_TYPE, 211 sizeof(HTTP_X_COMPRESS_TYPE) - 1)) { 212 mMode = HTTP_COMPRESS_COMPRESS; 213 } else if (!nsCRT::strncasecmp(aFromType, HTTP_GZIP_TYPE, 214 sizeof(HTTP_GZIP_TYPE) - 1) || 215 !nsCRT::strncasecmp(aFromType, HTTP_X_GZIP_TYPE, 216 sizeof(HTTP_X_GZIP_TYPE) - 1)) { 217 mMode = HTTP_COMPRESS_GZIP; 218 } else if (!nsCRT::strncasecmp(aFromType, HTTP_DEFLATE_TYPE, 219 sizeof(HTTP_DEFLATE_TYPE) - 1)) { 220 mMode = HTTP_COMPRESS_DEFLATE; 221 } else if (!nsCRT::strncasecmp(aFromType, HTTP_BROTLI_TYPE, 222 sizeof(HTTP_BROTLI_TYPE) - 1)) { 223 mMode = HTTP_COMPRESS_BROTLI; 224 } else if (!nsCRT::strncasecmp(aFromType, HTTP_ZSTD_TYPE, 225 sizeof(HTTP_ZSTD_TYPE) - 1)) { 226 mMode = HTTP_COMPRESS_ZSTD; 227 } else if (!nsCRT::strncasecmp(aFromType, HTTP_ZST_TYPE, 228 sizeof(HTTP_ZST_TYPE) - 1)) { 229 mMode = HTTP_COMPRESS_ZSTD; 230 } else if (!nsCRT::strncasecmp(aFromType, HTTP_BROTLI_DICTIONARY_TYPE, 231 sizeof(HTTP_BROTLI_DICTIONARY_TYPE) - 1)) { 232 mMode = HTTP_COMPRESS_BROTLI_DICTIONARY; 233 } else if (!nsCRT::strncasecmp(aFromType, HTTP_ZSTD_DICTIONARY_TYPE, 234 sizeof(HTTP_ZSTD_DICTIONARY_TYPE) - 1)) { 235 mMode = HTTP_COMPRESS_ZSTD_DICTIONARY; 236 } 237 LOG(("nsHttpCompresssConv %p AsyncConvertData %s %s mode %d\n", this, 238 aFromType, aToType, (CompressMode)mMode)); 239 240 MutexAutoLock lock(mMutex); 241 // hook ourself up with the receiving listener. 242 mListener = aListener; 243 244 return NS_OK; 245 } 246 247 NS_IMETHODIMP 248 nsHTTPCompressConv::GetConvertedType(const nsACString& aFromType, 249 nsIChannel* aChannel, 250 nsACString& aToType) { 251 return NS_ERROR_NOT_IMPLEMENTED; 252 } 253 254 NS_IMETHODIMP 255 nsHTTPCompressConv::MaybeRetarget(nsIRequest* request) { 256 MOZ_ASSERT(NS_IsMainThread()); 257 nsresult rv; 258 nsCOMPtr<nsIThreadRetargetableRequest> req = do_QueryInterface(request); 259 if (!req) { 260 return NS_ERROR_NO_INTERFACE; 261 } 262 if (!StaticPrefs::network_decompression_off_mainthread2()) { 263 return NS_OK; 264 } 265 nsCOMPtr<nsISerialEventTarget> target; 266 rv = req->GetDeliveryTarget(getter_AddRefs(target)); 267 if (NS_FAILED(rv) || !target || target->IsOnCurrentThread()) { 268 nsCOMPtr<nsIChannel> channel(do_QueryInterface(request)); 269 int64_t length = -1; 270 if (channel) { 271 channel->GetContentLength(&length); 272 // If this fails we'll retarget 273 } 274 if (length <= 0 || 275 length >= 276 StaticPrefs::network_decompression_off_mainthread_min_size()) { 277 LOG(("MaybeRetarget: Retargeting to background thread: Length %" PRId64, 278 length)); 279 // No retargetting was performed. Decompress off MainThread, 280 // and dispatch results back to MainThread. 281 // Don't do this if the input is small, if we know the length. 282 // If the length is 0 (unknown), always use OMT. 283 nsCOMPtr<nsISerialEventTarget> backgroundThread; 284 rv = NS_CreateBackgroundTaskQueue("nsHTTPCompressConv", 285 getter_AddRefs(backgroundThread)); 286 NS_ENSURE_SUCCESS(rv, rv); 287 rv = req->RetargetDeliveryTo(backgroundThread); 288 NS_ENSURE_SUCCESS(rv, rv); 289 if (NS_SUCCEEDED(rv)) { 290 mDispatchToMainThread = true; 291 } 292 } else { 293 LOG(("MaybeRetarget: Not retargeting: Length %" PRId64, length)); 294 } 295 } else { 296 LOG(("MaybeRetarget: Don't need to retarget")); 297 } 298 299 return NS_OK; 300 } 301 302 NS_IMETHODIMP 303 nsHTTPCompressConv::OnStartRequest(nsIRequest* request) { 304 LOG(("nsHttpCompresssConv %p onstart\n", this)); 305 nsCOMPtr<nsIStreamListener> listener; 306 { 307 MutexAutoLock lock(mMutex); 308 listener = mListener; 309 } 310 nsresult rv = listener->OnStartRequest(request); 311 if (NS_SUCCEEDED(rv)) { 312 if (XRE_IsContentProcess()) { 313 nsCOMPtr<nsIThreadRetargetableStreamListener> retargetlistener = 314 do_QueryInterface(listener); 315 // |nsHTTPCompressConv| should *always* be dispatched off of the main 316 // thread from a content process, even if its listeners don't support it. 317 // 318 // If its listener chain does not support being retargeted off of the 319 // main thread, it will be dispatched back to the main thread in 320 // |do_OnDataAvailable| and |OnStopRequest|. 321 if (!retargetlistener || 322 NS_FAILED(retargetlistener->CheckListenerChain())) { 323 mDispatchToMainThread = true; 324 } 325 } 326 } 327 return rv; 328 } 329 330 NS_IMETHODIMP 331 nsHTTPCompressConv::OnStopRequest(nsIRequest* request, nsresult aStatus) { 332 nsresult status = aStatus; 333 // Bug 1886237 : TRRServiceChannel calls OnStopRequest OMT 334 // MOZ_ASSERT(NS_IsMainThread()); 335 LOG(("nsHttpCompresssConv %p onstop %" PRIx32 " mDispatchToMainThread %d\n", 336 this, static_cast<uint32_t>(aStatus), mDispatchToMainThread)); 337 338 // Framing integrity is enforced for content-encoding: gzip, but not for 339 // content-encoding: deflate. Note that gzip vs deflate is NOT determined 340 // by content sniffing but only via header. 341 if (!mStreamEnded && NS_SUCCEEDED(status) && 342 (mFailUncleanStops && (mMode == HTTP_COMPRESS_GZIP))) { 343 // This is not a clean end of gzip stream: the transfer is incomplete. 344 status = NS_ERROR_NET_PARTIAL_TRANSFER; 345 LOG(("nsHttpCompresssConv %p onstop partial gzip\n", this)); 346 } 347 if (NS_SUCCEEDED(status) && (mMode == HTTP_COMPRESS_BROTLI || 348 mMode == HTTP_COMPRESS_BROTLI_DICTIONARY)) { 349 nsCOMPtr<nsIForcePendingChannel> fpChannel = do_QueryInterface(request); 350 bool isPending = false; 351 if (request) { 352 request->IsPending(&isPending); 353 } 354 if (fpChannel && !isPending) { 355 fpChannel->ForcePending(true); 356 } 357 if (mBrotli && NS_FAILED(mBrotli->mStatus)) { 358 status = NS_ERROR_INVALID_CONTENT_ENCODING; 359 } 360 LOG(("nsHttpCompresssConv %p onstop brotlihandler rv %" PRIx32 "\n", this, 361 static_cast<uint32_t>(status))); 362 if (fpChannel && !isPending) { 363 fpChannel->ForcePending(false); 364 } 365 } 366 // We don't need the dictionary data anymore 367 if (mBrotli || mZstd) { 368 RefPtr<DictionaryCacheEntry> dict; 369 nsCOMPtr<nsIHttpChannel> httpchannel(do_QueryInterface(request)); 370 if (httpchannel) { 371 httpchannel->SetDecompressDictionary(nullptr); 372 } 373 // paranoia 374 mBrotli = nullptr; 375 mZstd = nullptr; 376 } 377 378 nsCOMPtr<nsIStreamListener> listener; 379 { 380 MutexAutoLock lock(mMutex); 381 listener = mListener; 382 } 383 384 return listener->OnStopRequest(request, status); 385 } 386 387 /* static */ 388 nsresult nsHTTPCompressConv::BrotliHandler(nsIInputStream* stream, 389 void* closure, const char* dataIn, 390 uint32_t, uint32_t aAvail, 391 uint32_t* countRead) { 392 MOZ_ASSERT(stream); 393 nsHTTPCompressConv* self = static_cast<nsHTTPCompressConv*>(closure); 394 *countRead = 0; 395 396 const size_t kOutSize = 128 * 1024; // just a chunk size, we call in a loop 397 uint8_t* outPtr; 398 size_t outSize; 399 size_t avail = aAvail; 400 BrotliDecoderResult res; 401 402 if (!self->mBrotli) { 403 *countRead = aAvail; 404 return NS_OK; 405 } 406 407 // Dictionary-encoded brotli has a 36-byte header (4 byte fixed + 32 byte 408 // SHA-256) 409 if (self->mBrotli->mDictionary && self->mBrotli->mEaten < 36) { 410 uint8_t header_needed = 36 - self->mBrotli->mEaten; 411 if (avail >= header_needed) { 412 memcpy(&self->mBrotli->mHeader[self->mBrotli->mEaten], dataIn, 413 header_needed); 414 avail -= header_needed; 415 dataIn += header_needed; 416 self->mBrotli->mEaten = 36; 417 418 // Validate header 419 // XXX we could verify the SHA-256 matches what we offered 420 static uint8_t brotli_header[4] = {0xff, 0x44, 0x43, 0x42}; 421 if (memcmp(self->mBrotli->mHeader, brotli_header, 4) != 0) { 422 DICTIONARY_LOG( 423 ("!! %p Brotli failed: bad magic header 0x%02x%02x%02x%02x", self, 424 self->mBrotli->mHeader[0], self->mBrotli->mHeader[1], 425 self->mBrotli->mHeader[2], self->mBrotli->mHeader[3])); 426 self->mBrotli->mStatus = NS_ERROR_INVALID_CONTENT_ENCODING; 427 return self->mBrotli->mStatus; 428 } 429 } else { 430 memcpy(&self->mBrotli->mHeader[self->mBrotli->mEaten], dataIn, aAvail); 431 self->mBrotli->mEaten += aAvail; 432 *countRead = aAvail; 433 return NS_OK; 434 } 435 } 436 437 auto outBuffer = MakeUniqueFallible<uint8_t[]>(kOutSize); 438 if (outBuffer == nullptr) { 439 self->mBrotli->mStatus = NS_ERROR_OUT_OF_MEMORY; 440 return self->mBrotli->mStatus; 441 } 442 do { 443 outSize = kOutSize; 444 outPtr = outBuffer.get(); 445 446 // brotli api is documented in brotli/dec/decode.h and brotli/dec/decode.c 447 LOG(("nsHttpCompresssConv %p brotlihandler decompress %zu\n", self, avail)); 448 size_t totalOut = self->mBrotli->mTotalOut; 449 res = ::BrotliDecoderDecompressStream( 450 &self->mBrotli->mState, &avail, 451 reinterpret_cast<const unsigned char**>(&dataIn), &outSize, &outPtr, 452 &totalOut); 453 outSize = kOutSize - outSize; 454 self->mBrotli->mTotalOut = totalOut; 455 self->mBrotli->mBrotliStateIsStreamEnd = 456 BrotliDecoderIsFinished(&self->mBrotli->mState); 457 LOG(("nsHttpCompressConv %p brotlihandler decompress rv=%" PRIx32 458 " out=%zu\n", 459 self, static_cast<uint32_t>(res), outSize)); 460 461 if (res == BROTLI_DECODER_RESULT_ERROR) { 462 DICTIONARY_LOG( 463 ("nsHttpCompressConv %p decoding error: marking invalid encoding " 464 "(%zu)", 465 self, avail)); 466 self->mBrotli->mStatus = NS_ERROR_INVALID_CONTENT_ENCODING; 467 return self->mBrotli->mStatus; 468 } 469 470 // in 'the current implementation' brotli must consume everything before 471 // asking for more input 472 if (res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { 473 MOZ_ASSERT(!avail); 474 if (avail) { 475 LOG(("nsHttpCompressConv %p did not consume all input", self)); 476 self->mBrotli->mStatus = NS_ERROR_UNEXPECTED; 477 return self->mBrotli->mStatus; 478 } 479 } 480 481 auto callOnDataAvailable = [&](uint64_t aSourceOffset, const char* aBuffer, 482 uint32_t aCount) { 483 nsresult rv = self->do_OnDataAvailable(self->mBrotli->mRequest, 484 aSourceOffset, aBuffer, aCount); 485 LOG(("nsHttpCompressConv %p BrotliHandler ODA rv=%" PRIx32, self, 486 static_cast<uint32_t>(rv))); 487 if (NS_FAILED(rv)) { 488 self->mBrotli->mStatus = rv; 489 } 490 491 return rv; 492 }; 493 494 if (outSize > 0) { 495 if (NS_FAILED(callOnDataAvailable( 496 self->mBrotli->mSourceOffset, 497 reinterpret_cast<const char*>(outBuffer.get()), outSize))) { 498 return self->mBrotli->mStatus; 499 } 500 self->mBrotli->mSourceOffset += outSize; 501 } 502 503 // See bug 1759745. If the decoder has more output data, take it. 504 while (::BrotliDecoderHasMoreOutput(&self->mBrotli->mState)) { 505 outSize = kOutSize; 506 const uint8_t* buffer = 507 ::BrotliDecoderTakeOutput(&self->mBrotli->mState, &outSize); 508 if (NS_FAILED(callOnDataAvailable(self->mBrotli->mSourceOffset, 509 reinterpret_cast<const char*>(buffer), 510 outSize))) { 511 return self->mBrotli->mStatus; 512 } 513 self->mBrotli->mSourceOffset += outSize; 514 } 515 516 if (res == BROTLI_DECODER_RESULT_SUCCESS || 517 res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { 518 *countRead = aAvail; 519 return NS_OK; 520 } 521 MOZ_ASSERT(res == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT); 522 } while (res == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT); 523 524 self->mBrotli->mStatus = NS_ERROR_UNEXPECTED; 525 return self->mBrotli->mStatus; 526 } 527 528 /* static */ 529 nsresult nsHTTPCompressConv::ZstdHandler(nsIInputStream* stream, void* closure, 530 const char* dataIn, uint32_t, 531 uint32_t aAvail, uint32_t* countRead) { 532 MOZ_ASSERT(stream); 533 nsHTTPCompressConv* self = static_cast<nsHTTPCompressConv*>(closure); 534 *countRead = 0; 535 536 const size_t kOutSize = ZSTD_DStreamOutSize(); // normally 128K 537 uint8_t* outPtr; 538 size_t avail = aAvail; 539 540 // Stop decompressing after an error 541 if (self->mZstd->mStatus != NS_OK) { 542 *countRead = aAvail; 543 return NS_OK; 544 } 545 546 if (!self->mZstd->mOutBuffer) { 547 self->mZstd->mOutBuffer = MakeUniqueFallible<uint8_t[]>(kOutSize); 548 if (!self->mZstd->mOutBuffer) { 549 self->mZstd->mStatus = NS_ERROR_OUT_OF_MEMORY; 550 return self->mZstd->mStatus; 551 } 552 } 553 ZSTD_inBuffer inBuffer = {.src = dataIn, .size = aAvail, .pos = 0}; 554 uint32_t last_pos = 0; 555 while (inBuffer.pos < inBuffer.size) { 556 outPtr = self->mZstd->mOutBuffer.get(); 557 558 LOG(("nsHttpCompresssConv %p zstdhandler decompress %zu\n", self, avail)); 559 // Use ZSTD_(de)compressStream to (de)compress the input buffer into the 560 // output buffer, and fill aReadCount with the number of bytes consumed. 561 ZSTD_outBuffer outBuffer{.dst = outPtr, .size = kOutSize}; 562 size_t result; 563 bool output_full; 564 do { 565 outBuffer.pos = 0; 566 result = 567 ZSTD_decompressStream(self->mZstd->mDStream, &outBuffer, &inBuffer); 568 569 // If we errored when writing, flag this and abort writing. 570 if (ZSTD_isError(result)) { 571 self->mZstd->mStatus = NS_ERROR_INVALID_CONTENT_ENCODING; 572 return self->mZstd->mStatus; 573 } 574 575 nsresult rv = self->do_OnDataAvailable( 576 self->mZstd->mRequest, self->mZstd->mSourceOffset, 577 reinterpret_cast<const char*>(outPtr), outBuffer.pos); 578 if (NS_FAILED(rv)) { 579 self->mZstd->mStatus = rv; 580 return rv; 581 } 582 self->mZstd->mSourceOffset += inBuffer.pos - last_pos; 583 last_pos = inBuffer.pos; 584 output_full = outBuffer.pos == outBuffer.size; 585 // in the unlikely case that the output buffer was full, loop to 586 // drain it before processing more input 587 } while (output_full); 588 } 589 *countRead = inBuffer.pos; 590 return NS_OK; 591 } 592 593 NS_IMETHODIMP 594 nsHTTPCompressConv::OnDataAvailable(nsIRequest* request, nsIInputStream* iStr, 595 uint64_t aSourceOffset, uint32_t aCount) { 596 nsresult rv = NS_ERROR_INVALID_CONTENT_ENCODING; 597 uint32_t streamLen = aCount; 598 LOG(("nsHttpCompressConv %p OnDataAvailable aSourceOffset:%" PRIu64 599 " count:%u", 600 this, aSourceOffset, aCount)); 601 602 if (streamLen == 0) { 603 NS_ERROR("count of zero passed to OnDataAvailable"); 604 return NS_ERROR_UNEXPECTED; 605 } 606 607 if (mStreamEnded) { 608 // Hmm... this may just indicate that the data stream is done and that 609 // what's left is either metadata or padding of some sort.... throwing 610 // it out is probably the safe thing to do. 611 uint32_t n; 612 return iStr->ReadSegments(NS_DiscardSegment, nullptr, streamLen, &n); 613 } 614 615 switch (mMode) { 616 case HTTP_COMPRESS_GZIP: 617 streamLen = check_header(iStr, streamLen, &rv); 618 619 if (rv != NS_OK) { 620 return rv; 621 } 622 623 if (streamLen == 0) { 624 return NS_OK; 625 } 626 627 [[fallthrough]]; 628 629 case HTTP_COMPRESS_DEFLATE: 630 #if defined(__GNUC__) && (__GNUC__ >= 12) && !defined(__clang__) 631 # pragma GCC diagnostic push 632 # pragma GCC diagnostic ignored "-Wuse-after-free" 633 #endif // __GNUC__ >= 12 634 635 // The return value of realloc is null in case of failure, and the old 636 // buffer will stay valid but GCC isn't smart enough to figure that out. 637 // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110501 638 if (mInpBuffer != nullptr && streamLen > mInpBufferLen) { 639 unsigned char* originalInpBuffer = mInpBuffer; 640 if (!(mInpBuffer = (unsigned char*)realloc( 641 mInpBuffer, mInpBufferLen = streamLen))) { 642 free(originalInpBuffer); 643 mInpBufferLen = 0; 644 } 645 646 if (mOutBufferLen < streamLen * 2) { 647 unsigned char* originalOutBuffer = mOutBuffer; 648 if (!(mOutBuffer = (unsigned char*)realloc( 649 mOutBuffer, mOutBufferLen = streamLen * 3))) { 650 free(originalOutBuffer); 651 mOutBufferLen = 0; 652 } 653 } 654 655 #if defined(__GNUC__) && (__GNUC__ >= 12) && !defined(__clang__) 656 # pragma GCC diagnostic pop 657 #endif // __GNUC__ >= 12 658 659 if (mInpBuffer == nullptr || mOutBuffer == nullptr) { 660 return NS_ERROR_OUT_OF_MEMORY; 661 } 662 } 663 664 if (mInpBuffer == nullptr) { 665 mInpBuffer = (unsigned char*)malloc(mInpBufferLen = streamLen); 666 } 667 668 if (mOutBuffer == nullptr) { 669 mOutBuffer = (unsigned char*)malloc(mOutBufferLen = streamLen * 3); 670 } 671 672 if (mInpBuffer == nullptr || mOutBuffer == nullptr) { 673 return NS_ERROR_OUT_OF_MEMORY; 674 } 675 676 uint32_t unused; 677 iStr->Read((char*)mInpBuffer, streamLen, &unused); 678 679 if (mMode == HTTP_COMPRESS_DEFLATE) { 680 if (!mStreamInitialized) { 681 memset(&d_stream, 0, sizeof(d_stream)); 682 683 if (inflateInit(&d_stream) != Z_OK) { 684 return NS_ERROR_FAILURE; 685 } 686 687 mStreamInitialized = true; 688 } 689 d_stream.next_in = mInpBuffer; 690 d_stream.avail_in = (uInt)streamLen; 691 692 mDummyStreamInitialised = false; 693 for (;;) { 694 d_stream.next_out = mOutBuffer; 695 d_stream.avail_out = (uInt)mOutBufferLen; 696 697 int code = inflate(&d_stream, Z_NO_FLUSH); 698 unsigned bytesWritten = (uInt)mOutBufferLen - d_stream.avail_out; 699 700 if (code == Z_STREAM_END) { 701 if (bytesWritten) { 702 rv = do_OnDataAvailable(request, aSourceOffset, (char*)mOutBuffer, 703 bytesWritten); 704 if (NS_FAILED(rv)) { 705 return rv; 706 } 707 } 708 709 inflateEnd(&d_stream); 710 mStreamEnded = true; 711 break; 712 } 713 if (code == Z_OK) { 714 if (bytesWritten) { 715 rv = do_OnDataAvailable(request, aSourceOffset, (char*)mOutBuffer, 716 bytesWritten); 717 if (NS_FAILED(rv)) { 718 return rv; 719 } 720 } 721 } else if (code == Z_BUF_ERROR) { 722 if (bytesWritten) { 723 rv = do_OnDataAvailable(request, aSourceOffset, (char*)mOutBuffer, 724 bytesWritten); 725 if (NS_FAILED(rv)) { 726 return rv; 727 } 728 } 729 break; 730 } else if (code == Z_DATA_ERROR) { 731 // some servers (notably Apache with mod_deflate) don't generate 732 // zlib headers insert a dummy header and try again 733 static char dummy_head[2] = { 734 0x8 + 0x7 * 0x10, 735 (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, 736 }; 737 inflateReset(&d_stream); 738 d_stream.next_in = (Bytef*)dummy_head; 739 d_stream.avail_in = sizeof(dummy_head); 740 741 code = inflate(&d_stream, Z_NO_FLUSH); 742 if (code != Z_OK) { 743 return NS_ERROR_FAILURE; 744 } 745 746 // stop an endless loop caused by non-deflate data being labelled as 747 // deflate 748 if (mDummyStreamInitialised) { 749 NS_WARNING( 750 "endless loop detected" 751 " - invalid deflate"); 752 return NS_ERROR_INVALID_CONTENT_ENCODING; 753 } 754 mDummyStreamInitialised = true; 755 // reset stream pointers to our original data 756 d_stream.next_in = mInpBuffer; 757 d_stream.avail_in = (uInt)streamLen; 758 } else { 759 return NS_ERROR_INVALID_CONTENT_ENCODING; 760 } 761 } /* for */ 762 } else { 763 if (!mStreamInitialized) { 764 memset(&d_stream, 0, sizeof(d_stream)); 765 766 if (inflateInit2(&d_stream, -MAX_WBITS) != Z_OK) { 767 return NS_ERROR_FAILURE; 768 } 769 770 mStreamInitialized = true; 771 } 772 773 d_stream.next_in = mInpBuffer; 774 d_stream.avail_in = (uInt)streamLen; 775 776 for (;;) { 777 d_stream.next_out = mOutBuffer; 778 d_stream.avail_out = (uInt)mOutBufferLen; 779 780 int code = inflate(&d_stream, Z_NO_FLUSH); 781 unsigned bytesWritten = (uInt)mOutBufferLen - d_stream.avail_out; 782 783 if (code == Z_STREAM_END) { 784 if (bytesWritten) { 785 rv = do_OnDataAvailable(request, aSourceOffset, (char*)mOutBuffer, 786 bytesWritten); 787 if (NS_FAILED(rv)) { 788 return rv; 789 } 790 } 791 792 inflateEnd(&d_stream); 793 mStreamEnded = true; 794 break; 795 } 796 if (code == Z_OK) { 797 if (bytesWritten) { 798 rv = do_OnDataAvailable(request, aSourceOffset, (char*)mOutBuffer, 799 bytesWritten); 800 if (NS_FAILED(rv)) { 801 return rv; 802 } 803 } 804 } else if (code == Z_BUF_ERROR) { 805 if (bytesWritten) { 806 rv = do_OnDataAvailable(request, aSourceOffset, (char*)mOutBuffer, 807 bytesWritten); 808 if (NS_FAILED(rv)) { 809 return rv; 810 } 811 } 812 break; 813 } else { 814 return NS_ERROR_INVALID_CONTENT_ENCODING; 815 } 816 } /* for */ 817 } /* gzip */ 818 break; 819 820 case HTTP_COMPRESS_BROTLI: 821 case HTTP_COMPRESS_BROTLI_DICTIONARY: { 822 if (!mBrotli) { 823 mBrotli = MakeUnique<BrotliWrapper>(); 824 if (!mBrotli->Init(request)) { 825 return NS_ERROR_FAILURE; 826 } 827 } 828 829 mBrotli->mRequest = request; 830 mBrotli->mContext = nullptr; 831 mBrotli->mSourceOffset = aSourceOffset; 832 833 uint32_t countRead; 834 rv = iStr->ReadSegments(BrotliHandler, this, streamLen, &countRead); 835 if (NS_SUCCEEDED(rv)) { 836 rv = mBrotli->mStatus; 837 } 838 if (NS_FAILED(rv)) { 839 return rv; 840 } 841 } break; 842 843 case HTTP_COMPRESS_ZSTD: 844 case HTTP_COMPRESS_ZSTD_DICTIONARY: { 845 if (!mZstd) { 846 mZstd = MakeUnique<ZstdWrapper>(request, mMode); 847 if (!mZstd->mDStream) { 848 return NS_ERROR_OUT_OF_MEMORY; 849 } 850 } 851 852 mZstd->mRequest = request; 853 mZstd->mContext = nullptr; 854 mZstd->mSourceOffset = aSourceOffset; 855 856 uint32_t countRead; 857 rv = iStr->ReadSegments(ZstdHandler, this, streamLen, &countRead); 858 if (NS_SUCCEEDED(rv)) { 859 rv = mZstd->mStatus; 860 } 861 if (NS_FAILED(rv)) { 862 return rv; 863 } 864 } break; 865 866 default: 867 nsCOMPtr<nsIStreamListener> listener; 868 { 869 MutexAutoLock lock(mMutex); 870 listener = mListener; 871 } 872 rv = listener->OnDataAvailable(request, iStr, aSourceOffset, aCount); 873 if (NS_FAILED(rv)) { 874 return rv; 875 } 876 } /* switch */ 877 878 return NS_OK; 879 } /* OnDataAvailable */ 880 881 // XXX/ruslan: need to implement this too 882 883 NS_IMETHODIMP 884 nsHTTPCompressConv::Convert(nsIInputStream* aFromStream, const char* aFromType, 885 const char* aToType, nsISupports* aCtxt, 886 nsIInputStream** _retval) { 887 return NS_ERROR_NOT_IMPLEMENTED; 888 } 889 890 nsresult nsHTTPCompressConv::do_OnDataAvailable(nsIRequest* request, 891 uint64_t offset, 892 const char* buffer, 893 uint32_t count) { 894 LOG( 895 ("nsHttpCompressConv %p do_OnDataAvailable mDispatchToMainThread %d " 896 "count %u", 897 this, mDispatchToMainThread, count)); 898 if (count == 0) { 899 // Never send 0-byte OnDataAvailables; imglib at least barfs on them and 900 // they're not useful 901 return NS_OK; 902 } 903 if (mDispatchToMainThread && !NS_IsMainThread()) { 904 nsCOMPtr<nsIInputStream> stream; 905 MOZ_TRY(NS_NewByteInputStream(getter_AddRefs(stream), Span(buffer, count), 906 nsAssignmentType::NS_ASSIGNMENT_COPY)); 907 908 nsCOMPtr<nsIStreamListener> listener; 909 { 910 MutexAutoLock lock(mMutex); 911 listener = mListener; 912 } 913 914 // This is safe and will always run before OnStopRequest, because 915 // ChanneleventQueue means that we can't enqueue OnStopRequest until after 916 // the OMT OnDataAvailable call has completed. So Dispatching here will 917 // ensure it's in the MainThread event queue before OnStopRequest 918 nsCOMPtr<nsIRunnable> handler = NS_NewRunnableFunction( 919 "nsHTTPCompressConv::do_OnDataAvailable", 920 [request{RefPtr<nsIRequest>(request)}, stream{std::move(stream)}, 921 listener{std::move(listener)}, offset, count]() { 922 LOG(("nsHttpCompressConv Calling OnDataAvailable on Mainthread")); 923 (void)listener->OnDataAvailable(request, stream, offset, count); 924 }); 925 926 mDecodedDataLength += count; 927 return NS_DispatchToMainThread(handler); 928 } 929 930 if (!mStream) { 931 mStream = do_CreateInstance(NS_STRINGINPUTSTREAM_CONTRACTID); 932 NS_ENSURE_STATE(mStream); 933 } 934 935 mStream->ShareData(buffer, count); 936 937 nsCOMPtr<nsIStreamListener> listener; 938 { 939 MutexAutoLock lock(mMutex); 940 listener = mListener; 941 } 942 LOG(("nsHTTPCompressConv::do_OnDataAvailable req:%p offset: offset:%" PRIu64 943 "count:%u", 944 request, offset, count)); 945 nsresult rv = listener->OnDataAvailable(request, mStream, offset, count); 946 947 // Make sure the stream no longer references |buffer| in case our listener 948 // is crazy enough to try to read from |mStream| after ODA. 949 mStream->ShareData("", 0); 950 mDecodedDataLength += count; 951 952 return rv; 953 } 954 955 #define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */ 956 #define HEAD_CRC 0x02 /* bit 1 set: header CRC present */ 957 #define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ 958 #define ORIG_NAME 0x08 /* bit 3 set: original file name present */ 959 #define COMMENT 0x10 /* bit 4 set: file comment present */ 960 #define RESERVED 0xE0 /* bits 5..7: reserved */ 961 962 static unsigned gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */ 963 964 uint32_t nsHTTPCompressConv::check_header(nsIInputStream* iStr, 965 uint32_t streamLen, nsresult* rs) { 966 enum { 967 GZIP_INIT = 0, 968 GZIP_OS, 969 GZIP_EXTRA0, 970 GZIP_EXTRA1, 971 GZIP_EXTRA2, 972 GZIP_ORIG, 973 GZIP_COMMENT, 974 GZIP_CRC 975 }; 976 char c; 977 978 *rs = NS_OK; 979 980 if (mCheckHeaderDone) { 981 return streamLen; 982 } 983 984 while (streamLen) { 985 switch (hMode) { 986 case GZIP_INIT: 987 uint32_t unused; 988 iStr->Read(&c, 1, &unused); 989 streamLen--; 990 991 if (mSkipCount == 0 && ((unsigned)c & 0377) != gz_magic[0]) { 992 *rs = NS_ERROR_INVALID_CONTENT_ENCODING; 993 return 0; 994 } 995 996 if (mSkipCount == 1 && ((unsigned)c & 0377) != gz_magic[1]) { 997 *rs = NS_ERROR_INVALID_CONTENT_ENCODING; 998 return 0; 999 } 1000 1001 if (mSkipCount == 2 && ((unsigned)c & 0377) != Z_DEFLATED) { 1002 *rs = NS_ERROR_INVALID_CONTENT_ENCODING; 1003 return 0; 1004 } 1005 1006 mSkipCount++; 1007 if (mSkipCount == 4) { 1008 mFlags = (unsigned)c & 0377; 1009 if (mFlags & RESERVED) { 1010 *rs = NS_ERROR_INVALID_CONTENT_ENCODING; 1011 return 0; 1012 } 1013 hMode = GZIP_OS; 1014 mSkipCount = 0; 1015 } 1016 break; 1017 1018 case GZIP_OS: 1019 iStr->Read(&c, 1, &unused); 1020 streamLen--; 1021 mSkipCount++; 1022 1023 if (mSkipCount == 6) { 1024 hMode = GZIP_EXTRA0; 1025 } 1026 break; 1027 1028 case GZIP_EXTRA0: 1029 if (mFlags & EXTRA_FIELD) { 1030 iStr->Read(&c, 1, &unused); 1031 streamLen--; 1032 mLen = (uInt)c & 0377; 1033 hMode = GZIP_EXTRA1; 1034 } else { 1035 hMode = GZIP_ORIG; 1036 } 1037 break; 1038 1039 case GZIP_EXTRA1: 1040 iStr->Read(&c, 1, &unused); 1041 streamLen--; 1042 mLen |= ((uInt)c & 0377) << 8; 1043 mSkipCount = 0; 1044 hMode = GZIP_EXTRA2; 1045 break; 1046 1047 case GZIP_EXTRA2: 1048 if (mSkipCount == mLen) { 1049 hMode = GZIP_ORIG; 1050 } else { 1051 iStr->Read(&c, 1, &unused); 1052 streamLen--; 1053 mSkipCount++; 1054 } 1055 break; 1056 1057 case GZIP_ORIG: 1058 if (mFlags & ORIG_NAME) { 1059 iStr->Read(&c, 1, &unused); 1060 streamLen--; 1061 if (c == 0) hMode = GZIP_COMMENT; 1062 } else { 1063 hMode = GZIP_COMMENT; 1064 } 1065 break; 1066 1067 case GZIP_COMMENT: 1068 if (mFlags & COMMENT) { 1069 iStr->Read(&c, 1, &unused); 1070 streamLen--; 1071 if (c == 0) { 1072 hMode = GZIP_CRC; 1073 mSkipCount = 0; 1074 } 1075 } else { 1076 hMode = GZIP_CRC; 1077 mSkipCount = 0; 1078 } 1079 break; 1080 1081 case GZIP_CRC: 1082 if (mFlags & HEAD_CRC) { 1083 iStr->Read(&c, 1, &unused); 1084 streamLen--; 1085 mSkipCount++; 1086 if (mSkipCount == 2) { 1087 mCheckHeaderDone = true; 1088 return streamLen; 1089 } 1090 } else { 1091 mCheckHeaderDone = true; 1092 return streamLen; 1093 } 1094 break; 1095 } 1096 } 1097 return streamLen; 1098 } 1099 1100 NS_IMETHODIMP 1101 nsHTTPCompressConv::CheckListenerChain() { 1102 MOZ_ASSERT(NS_IsMainThread()); 1103 nsCOMPtr<nsIThreadRetargetableStreamListener> listener; 1104 { 1105 MutexAutoLock lock(mMutex); 1106 listener = do_QueryInterface(mListener); 1107 } 1108 1109 nsresult rv = NS_ERROR_NO_INTERFACE; 1110 if (listener) { 1111 rv = listener->CheckListenerChain(); 1112 } 1113 1114 // handle decompression OMT always. If the chain needs to be MT, 1115 // we'll determine that in OnStartRequest and dispatch to MT 1116 bool alwaysOMT = XRE_IsContentProcess() && 1117 StaticPrefs::network_decompression_off_mainthread2(); 1118 1119 if (NS_FAILED(rv) && alwaysOMT) { 1120 mDispatchToMainThread = true; 1121 return NS_OK; 1122 } 1123 1124 return rv; 1125 } 1126 1127 NS_IMETHODIMP 1128 nsHTTPCompressConv::OnDataFinished(nsresult aStatus) { 1129 if (mDispatchToMainThread && !NS_IsMainThread()) { 1130 // If this is called off main thread, but the listener can only 1131 // handle calls on the main thread, then just return. 1132 // Also important - never QI the listener off main thread 1133 // if mDispatchToMainThread is true, because the listener 1134 // might be JS implemented and won't support that. 1135 return NS_OK; 1136 } 1137 1138 nsCOMPtr<nsIThreadRetargetableStreamListener> listener; 1139 { 1140 MutexAutoLock lock(mMutex); 1141 listener = do_QueryInterface(mListener); 1142 } 1143 1144 if (listener) { 1145 return listener->OnDataFinished(aStatus); 1146 } 1147 1148 return NS_OK; 1149 } 1150 1151 } // namespace net 1152 } // namespace mozilla 1153 1154 nsresult NS_NewHTTPCompressConv( 1155 mozilla::net::nsHTTPCompressConv** aHTTPCompressConv) { 1156 MOZ_ASSERT(aHTTPCompressConv != nullptr, "null ptr"); 1157 if (!aHTTPCompressConv) { 1158 return NS_ERROR_NULL_POINTER; 1159 } 1160 1161 RefPtr<mozilla::net::nsHTTPCompressConv> outVal = 1162 new mozilla::net::nsHTTPCompressConv(); 1163 if (!outVal) { 1164 return NS_ERROR_OUT_OF_MEMORY; 1165 } 1166 outVal.forget(aHTTPCompressConv); 1167 return NS_OK; 1168 }