commit 4b80e50e76e2dea98aedd0baeb5f45cbd9ad9e40
parent ff824076445da02a66cb85898c85631361e344f3
Author: Ryan VanderMeulen <rvandermeulen@mozilla.com>
Date: Mon, 10 Nov 2025 13:12:31 +0000
Bug 1998861 - Update Snappy to version 1.2.2. r=dom-storage-reviewers,asuth
Differential Revision: https://phabricator.services.mozilla.com/D271788
Diffstat:
10 files changed, 88 insertions(+), 19 deletions(-)
diff --git a/dom/cache/FileUtils.cpp b/dom/cache/FileUtils.cpp
@@ -32,7 +32,7 @@
namespace mozilla::dom::cache {
-static_assert(SNAPPY_VERSION == 0x010200);
+static_assert(SNAPPY_VERSION == 0x010202);
using mozilla::dom::quota::Client;
using mozilla::dom::quota::CloneFileAndAppend;
diff --git a/dom/indexedDB/ActorsParentCommon.cpp b/dom/indexedDB/ActorsParentCommon.cpp
@@ -66,7 +66,7 @@ class nsIFile;
namespace mozilla::dom::indexedDB {
-static_assert(SNAPPY_VERSION == 0x010200);
+static_assert(SNAPPY_VERSION == 0x010202);
using mozilla::ipc::IsOnBackgroundThread;
diff --git a/dom/localstorage/SnappyUtils.cpp b/dom/localstorage/SnappyUtils.cpp
@@ -17,7 +17,7 @@
namespace mozilla::dom {
-static_assert(SNAPPY_VERSION == 0x010200);
+static_assert(SNAPPY_VERSION == 0x010202);
bool SnappyCompress(const nsACString& aSource, nsACString& aDest) {
MOZ_ASSERT(!aSource.IsVoid());
diff --git a/other-licenses/snappy/README b/other-licenses/snappy/README
@@ -7,7 +7,7 @@ constructor warnings. We have replaced its build system with our own.
Snappy comes from:
https://github.com/google/snappy
-We are currently using revision: 1.2.0
+We are currently using revision: 1.2.2
To upgrade to a newer version:
1. Check out the new code using git.
diff --git a/other-licenses/snappy/snappy-stubs-public.h b/other-licenses/snappy/snappy-stubs-public.h
@@ -39,9 +39,9 @@
#define SNAPPY_MAJOR 1
#define SNAPPY_MINOR 2
-#define SNAPPY_PATCHLEVEL 0
+#define SNAPPY_PATCHLEVEL 2
#define SNAPPY_VERSION \
- ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
+ ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
namespace snappy {
diff --git a/other-licenses/snappy/src/NEWS b/other-licenses/snappy/src/NEWS
@@ -1,3 +1,18 @@
+Snappy v1.2.2, Mar 26th 2025:
+
+ * We added a new compression level in v1.2.1 which compresses a bit
+ denser but slower. Decompression speed should be even faster with it.
+
+ * We fixed a very old issue of data corruption when compressed size
+ exceeds 4GB. This can happen when you compress data close to 4GB
+ and it's incompressible, for example, random data.
+
+ * Started to use minimum CMake 3.10 because older ones are not
+ planned to be supported.
+
+ * Various other small fixes and performance improvements (especially
+ for clang).
+
Snappy v1.1.10, Mar 8th 2023:
* Performance improvements
diff --git a/other-licenses/snappy/src/README.md b/other-licenses/snappy/src/README.md
@@ -140,10 +140,10 @@ explicitly supports the following:
1. C++11
2. Clang (gcc and MSVC are best-effort).
3. Low level optimizations (e.g. assembly or equivalent intrinsics) for:
- 1. [x86](https://en.wikipedia.org/wiki/X86)
- 2. [x86-64](https://en.wikipedia.org/wiki/X86-64)
- 3. ARMv7 (32-bit)
- 4. ARMv8 (AArch64)
+ - [x86](https://en.wikipedia.org/wiki/X86)
+ - [x86-64](https://en.wikipedia.org/wiki/X86-64)
+ - ARMv7 (32-bit)
+ - ARMv8 (AArch64)
4. Supports only the Snappy compression scheme as described in
[format_description.txt](format_description.txt).
5. CMake for building
diff --git a/other-licenses/snappy/src/snappy.cc b/other-licenses/snappy/src/snappy.cc
@@ -74,6 +74,7 @@
#include <cstdint>
#include <cstdio>
#include <cstring>
+#include <functional>
#include <memory>
#include <string>
#include <utility>
@@ -1498,7 +1499,7 @@ class SnappyDecompressor {
// If ip < ip_limit_min_maxtaglen_ it's safe to read kMaxTagLength from
// buffer.
const char* ip_limit_min_maxtaglen_;
- uint32_t peeked_; // Bytes peeked from reader (need to skip)
+ uint64_t peeked_; // Bytes peeked from reader (need to skip)
bool eof_; // Hit end of input without an error?
char scratch_[kMaximumTagLength]; // See RefillTag().
@@ -1689,7 +1690,8 @@ constexpr uint32_t CalculateNeeded(uint8_t tag) {
#if __cplusplus >= 201402L
constexpr bool VerifyCalculateNeeded() {
for (int i = 0; i < 1; i++) {
- if (CalculateNeeded(i) != (char_table[i] >> 11) + 1) return false;
+ if (CalculateNeeded(i) != static_cast<uint32_t>((char_table[i] >> 11)) + 1)
+ return false;
}
return true;
}
@@ -1725,7 +1727,7 @@ bool SnappyDecompressor::RefillTag() {
assert(needed <= sizeof(scratch_));
// Read more bytes from reader if needed
- uint32_t nbuf = ip_limit_ - ip;
+ uint64_t nbuf = ip_limit_ - ip;
if (nbuf < needed) {
// Stitch together bytes from ip and reader to form the word
// contents. We store the needed bytes in "scratch_". They
@@ -1738,7 +1740,7 @@ bool SnappyDecompressor::RefillTag() {
size_t length;
const char* src = reader_->Peek(&length);
if (length == 0) return false;
- uint32_t to_add = std::min<uint32_t>(needed - nbuf, length);
+ uint64_t to_add = std::min<uint64_t>(needed - nbuf, length);
std::memcpy(scratch_ + nbuf, src, to_add);
nbuf += to_add;
reader_->Skip(to_add);
@@ -1792,11 +1794,16 @@ bool GetUncompressedLength(Source* source, uint32_t* result) {
return decompressor.ReadUncompressedLength(result);
}
+size_t Compress(Source* reader, Sink* writer) {
+ return Compress(reader, writer, CompressionOptions{});
+}
+
size_t Compress(Source* reader, Sink* writer, CompressionOptions options) {
assert(options.level == 1 || options.level == 2);
int token = 0;
size_t written = 0;
size_t N = reader->Available();
+ assert(N <= 0xFFFFFFFFu);
const size_t uncompressed_size = N;
char ulength[Varint::kMax32];
char* p = Varint::Encode32(ulength, N);
@@ -2299,6 +2306,12 @@ bool IsValidCompressed(Source* compressed) {
}
void RawCompress(const char* input, size_t input_length, char* compressed,
+ size_t* compressed_length) {
+ RawCompress(input, input_length, compressed, compressed_length,
+ CompressionOptions{});
+}
+
+void RawCompress(const char* input, size_t input_length, char* compressed,
size_t* compressed_length, CompressionOptions options) {
ByteArraySource reader(input, input_length);
UncheckedByteArraySink writer(compressed);
@@ -2309,6 +2322,12 @@ void RawCompress(const char* input, size_t input_length, char* compressed,
}
void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
+ char* compressed, size_t* compressed_length) {
+ RawCompressFromIOVec(iov, uncompressed_length, compressed, compressed_length,
+ CompressionOptions{});
+}
+
+void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
char* compressed, size_t* compressed_length,
CompressionOptions options) {
SnappyIOVecReader reader(iov, uncompressed_length);
@@ -2319,6 +2338,11 @@ void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
*compressed_length = writer.CurrentDestination() - compressed;
}
+size_t Compress(const char* input, size_t input_length,
+ std::string* compressed) {
+ return Compress(input, input_length, compressed, CompressionOptions{});
+}
+
size_t Compress(const char* input, size_t input_length, std::string* compressed,
CompressionOptions options) {
// Pre-grow the buffer to the max length of the compressed output
@@ -2332,6 +2356,11 @@ size_t Compress(const char* input, size_t input_length, std::string* compressed,
}
size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt,
+ std::string* compressed) {
+ return CompressFromIOVec(iov, iov_cnt, compressed, CompressionOptions{});
+}
+
+size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt,
std::string* compressed, CompressionOptions options) {
// Compute the number of bytes to be compressed.
size_t uncompressed_length = 0;
diff --git a/other-licenses/snappy/src/snappy.h b/other-licenses/snappy/src/snappy.h
@@ -78,8 +78,10 @@ namespace snappy {
// Compress the bytes read from "*reader" and append to "*writer". Return the
// number of bytes written.
+ // First version is to preserve ABI.
+ size_t Compress(Source* reader, Sink* writer);
size_t Compress(Source* reader, Sink* writer,
- CompressionOptions options = {});
+ CompressionOptions options);
// Find the uncompressed length of the given stream, as given by the header.
// Note that the true length could deviate from this; the stream could e.g.
@@ -98,16 +100,22 @@ namespace snappy {
// Original contents of *compressed are lost.
//
// REQUIRES: "input[]" is not an alias of "*compressed".
+ // First version is to preserve ABI.
size_t Compress(const char* input, size_t input_length,
- std::string* compressed, CompressionOptions options = {});
+ std::string* compressed);
+ size_t Compress(const char* input, size_t input_length,
+ std::string* compressed, CompressionOptions options);
// Same as `Compress` above but taking an `iovec` array as input. Note that
// this function preprocesses the inputs to compute the sum of
// `iov[0..iov_cnt-1].iov_len` before reading. To avoid this, use
// `RawCompressFromIOVec` below.
+ // First version is to preserve ABI.
+ size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt,
+ std::string* compressed);
size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt,
std::string* compressed,
- CompressionOptions options = {});
+ CompressionOptions options);
// Decompresses "compressed[0..compressed_length-1]" to "*uncompressed".
// Original contents of "*uncompressed" are lost.
@@ -151,14 +159,18 @@ namespace snappy {
// ... Process(output, output_length) ...
// delete [] output;
void RawCompress(const char* input, size_t input_length, char* compressed,
- size_t* compressed_length, CompressionOptions options = {});
+ size_t* compressed_length);
+ void RawCompress(const char* input, size_t input_length, char* compressed,
+ size_t* compressed_length, CompressionOptions options);
// Same as `RawCompress` above but taking an `iovec` array as input. Note that
// `uncompressed_length` is the total number of bytes to be read from the
// elements of `iov` (_not_ the number of elements in `iov`).
void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
+ char* compressed, size_t* compressed_length);
+ void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
char* compressed, size_t* compressed_length,
- CompressionOptions options = {});
+ CompressionOptions options);
// Given data in "compressed[0..compressed_length-1]" generated by
// calling the Snappy::Compress routine, this routine
diff --git a/other-licenses/snappy/src/snappy_unittest.cc b/other-licenses/snappy/src/snappy_unittest.cc
@@ -27,6 +27,7 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <algorithm>
+#include <cinttypes>
#include <cmath>
#include <cstdlib>
#include <random>
@@ -484,6 +485,18 @@ TEST(Snappy, MaxBlowup) {
Verify(input);
}
+// Issue #201, when output is more than 4GB, we had a data corruption bug.
+// We cannot run this test always because of CI constraints.
+TEST(Snappy, DISABLED_MoreThan4GB) {
+ std::mt19937 rng;
+ std::uniform_int_distribution<int> uniform_byte(0, 255);
+ std::string input;
+ input.resize((1ull << 32) - 1);
+ for (uint64_t i = 0; i < ((1ull << 32) - 1); ++i)
+ input[i] = static_cast<char>(uniform_byte(rng));
+ Verify(input);
+}
+
TEST(Snappy, RandomData) {
std::minstd_rand0 rng(snappy::GetFlag(FLAGS_test_random_seed));
std::uniform_int_distribution<int> uniform_0_to_3(0, 3);
