icc_codec.cc (16212B)
1 // Copyright (c) the JPEG XL Project Authors. All rights reserved. 2 // 3 // Use of this source code is governed by a BSD-style 4 // license that can be found in the LICENSE file. 5 6 #include "lib/jxl/icc_codec.h" 7 8 #include <jxl/memory_manager.h> 9 10 #include <cstdint> 11 12 #include "lib/jxl/base/status.h" 13 #include "lib/jxl/dec_ans.h" 14 #include "lib/jxl/fields.h" 15 #include "lib/jxl/icc_codec_common.h" 16 #include "lib/jxl/padded_bytes.h" 17 18 namespace jxl { 19 namespace { 20 21 // Shuffles or interleaves bytes, for example with width 2, turns "ABCDabcd" 22 // into "AaBbCcDd". Transposes a matrix of ceil(size / width) columns and 23 // width rows. There are size elements, size may be < width * height, if so the 24 // last elements of the rightmost column are missing, the missing spots are 25 // transposed along with the filled spots, and the result has the missing 26 // elements at the end of the bottom row. The input is the input matrix in 27 // scanline order but with missing elements skipped (which may occur in multiple 28 // locations), the output is the result matrix in scanline order (with 29 // no need to skip missing elements as they are past the end of the data). 30 Status Shuffle(JxlMemoryManager* memory_manager, uint8_t* data, size_t size, 31 size_t width) { 32 size_t height = (size + width - 1) / width; // amount of rows of output 33 PaddedBytes result(memory_manager); 34 JXL_ASSIGN_OR_RETURN(result, 35 PaddedBytes::WithInitialSpace(memory_manager, size)); 36 // i = output index, j input index 37 size_t s = 0; 38 size_t j = 0; 39 for (size_t i = 0; i < size; i++) { 40 result[i] = data[j]; 41 j += height; 42 if (j >= size) j = ++s; 43 } 44 45 for (size_t i = 0; i < size; i++) { 46 data[i] = result[i]; 47 } 48 return true; 49 } 50 51 // TODO(eustas): should be 20, or even 18, once DecodeVarInt is improved; 52 // currently DecodeVarInt does not signal the errors, and marks 53 // 11 bytes as used even if only 10 are used (and 9 is enough for 54 // 63-bit values). 55 constexpr const size_t kPreambleSize = 22; // enough for reading 2 VarInts 56 57 uint64_t DecodeVarInt(const uint8_t* input, size_t inputSize, size_t* pos) { 58 size_t i; 59 uint64_t ret = 0; 60 for (i = 0; *pos + i < inputSize && i < 10; ++i) { 61 ret |= static_cast<uint64_t>(input[*pos + i] & 127) 62 << static_cast<uint64_t>(7 * i); 63 // If the next-byte flag is not set, stop 64 if ((input[*pos + i] & 128) == 0) break; 65 } 66 // TODO(user): Return a decoding error if i == 10. 67 *pos += i + 1; 68 return ret; 69 } 70 71 } // namespace 72 73 // Mimics the beginning of UnpredictICC for quick validity check. 74 // At least kPreambleSize bytes of data should be valid at invocation time. 75 Status CheckPreamble(const PaddedBytes& data, size_t enc_size) { 76 const uint8_t* enc = data.data(); 77 size_t size = data.size(); 78 size_t pos = 0; 79 uint64_t osize = DecodeVarInt(enc, size, &pos); 80 JXL_RETURN_IF_ERROR(CheckIs32Bit(osize)); 81 if (pos >= size) return JXL_FAILURE("Out of bounds"); 82 uint64_t csize = DecodeVarInt(enc, size, &pos); 83 JXL_RETURN_IF_ERROR(CheckIs32Bit(csize)); 84 JXL_RETURN_IF_ERROR(CheckOutOfBounds(pos, csize, size)); 85 // We expect that UnpredictICC inflates input, not the other way round. 86 if (osize + 65536 < enc_size) return JXL_FAILURE("Malformed ICC"); 87 88 // NB(eustas): 64 MiB ICC should be enough for everything!? 89 const size_t output_limit = 1 << 28; 90 if (output_limit && osize > output_limit) { 91 return JXL_FAILURE("Decoded ICC is too large"); 92 } 93 return true; 94 } 95 96 // Decodes the result of PredictICC back to a valid ICC profile. 97 Status UnpredictICC(const uint8_t* enc, size_t size, PaddedBytes* result) { 98 if (!result->empty()) return JXL_FAILURE("result must be empty initially"); 99 JxlMemoryManager* memory_manager = result->memory_manager(); 100 size_t pos = 0; 101 // TODO(lode): technically speaking we need to check that the entire varint 102 // decoding never goes out of bounds, not just the first byte. This requires 103 // a DecodeVarInt function that returns an error code. It is safe to use 104 // DecodeVarInt with out of bounds values, it silently returns, but the 105 // specification requires an error. Idem for all DecodeVarInt below. 106 if (pos >= size) return JXL_FAILURE("Out of bounds"); 107 uint64_t osize = DecodeVarInt(enc, size, &pos); // Output size 108 JXL_RETURN_IF_ERROR(CheckIs32Bit(osize)); 109 if (pos >= size) return JXL_FAILURE("Out of bounds"); 110 uint64_t csize = DecodeVarInt(enc, size, &pos); // Commands size 111 // Every command is translated to at least on byte. 112 JXL_RETURN_IF_ERROR(CheckIs32Bit(csize)); 113 size_t cpos = pos; // pos in commands stream 114 JXL_RETURN_IF_ERROR(CheckOutOfBounds(pos, csize, size)); 115 size_t commands_end = cpos + csize; 116 pos = commands_end; // pos in data stream 117 118 // Header 119 PaddedBytes header{memory_manager}; 120 JXL_RETURN_IF_ERROR(header.append(ICCInitialHeaderPrediction(osize))); 121 for (size_t i = 0; i <= kICCHeaderSize; i++) { 122 if (result->size() == osize) { 123 if (cpos != commands_end) return JXL_FAILURE("Not all commands used"); 124 if (pos != size) return JXL_FAILURE("Not all data used"); 125 return true; // Valid end 126 } 127 if (i == kICCHeaderSize) break; // Done 128 ICCPredictHeader(result->data(), result->size(), header.data(), i); 129 if (pos >= size) return JXL_FAILURE("Out of bounds"); 130 JXL_RETURN_IF_ERROR(result->push_back(enc[pos++] + header[i])); 131 } 132 if (cpos >= commands_end) return JXL_FAILURE("Out of bounds"); 133 134 // Tag list 135 uint64_t numtags = DecodeVarInt(enc, size, &cpos); 136 137 if (numtags != 0) { 138 numtags--; 139 JXL_RETURN_IF_ERROR(CheckIs32Bit(numtags)); 140 JXL_RETURN_IF_ERROR(AppendUint32(numtags, result)); 141 uint64_t prevtagstart = kICCHeaderSize + numtags * 12; 142 uint64_t prevtagsize = 0; 143 for (;;) { 144 if (result->size() > osize) return JXL_FAILURE("Invalid result size"); 145 if (cpos > commands_end) return JXL_FAILURE("Out of bounds"); 146 if (cpos == commands_end) break; // Valid end 147 uint8_t command = enc[cpos++]; 148 uint8_t tagcode = command & 63; 149 Tag tag; 150 if (tagcode == 0) { 151 break; 152 } else if (tagcode == kCommandTagUnknown) { 153 JXL_RETURN_IF_ERROR(CheckOutOfBounds(pos, 4, size)); 154 tag = DecodeKeyword(enc, size, pos); 155 pos += 4; 156 } else if (tagcode == kCommandTagTRC) { 157 tag = kRtrcTag; 158 } else if (tagcode == kCommandTagXYZ) { 159 tag = kRxyzTag; 160 } else { 161 if (tagcode - kCommandTagStringFirst >= kNumTagStrings) { 162 return JXL_FAILURE("Unknown tagcode"); 163 } 164 tag = *kTagStrings[tagcode - kCommandTagStringFirst]; 165 } 166 JXL_RETURN_IF_ERROR(AppendKeyword(tag, result)); 167 168 uint64_t tagstart; 169 uint64_t tagsize = prevtagsize; 170 if (tag == kRxyzTag || tag == kGxyzTag || tag == kBxyzTag || 171 tag == kKxyzTag || tag == kWtptTag || tag == kBkptTag || 172 tag == kLumiTag) { 173 tagsize = 20; 174 } 175 176 if (command & kFlagBitOffset) { 177 if (cpos >= commands_end) return JXL_FAILURE("Out of bounds"); 178 tagstart = DecodeVarInt(enc, size, &cpos); 179 } else { 180 JXL_RETURN_IF_ERROR(CheckIs32Bit(prevtagstart)); 181 tagstart = prevtagstart + prevtagsize; 182 } 183 JXL_RETURN_IF_ERROR(CheckIs32Bit(tagstart)); 184 JXL_RETURN_IF_ERROR(AppendUint32(tagstart, result)); 185 if (command & kFlagBitSize) { 186 if (cpos >= commands_end) return JXL_FAILURE("Out of bounds"); 187 tagsize = DecodeVarInt(enc, size, &cpos); 188 } 189 JXL_RETURN_IF_ERROR(CheckIs32Bit(tagsize)); 190 JXL_RETURN_IF_ERROR(AppendUint32(tagsize, result)); 191 prevtagstart = tagstart; 192 prevtagsize = tagsize; 193 194 if (tagcode == kCommandTagTRC) { 195 JXL_RETURN_IF_ERROR(AppendKeyword(kGtrcTag, result)); 196 JXL_RETURN_IF_ERROR(AppendUint32(tagstart, result)); 197 JXL_RETURN_IF_ERROR(AppendUint32(tagsize, result)); 198 JXL_RETURN_IF_ERROR(AppendKeyword(kBtrcTag, result)); 199 JXL_RETURN_IF_ERROR(AppendUint32(tagstart, result)); 200 JXL_RETURN_IF_ERROR(AppendUint32(tagsize, result)); 201 } 202 203 if (tagcode == kCommandTagXYZ) { 204 JXL_RETURN_IF_ERROR(CheckIs32Bit(tagstart + tagsize * 2)); 205 JXL_RETURN_IF_ERROR(AppendKeyword(kGxyzTag, result)); 206 JXL_RETURN_IF_ERROR(AppendUint32(tagstart + tagsize, result)); 207 JXL_RETURN_IF_ERROR(AppendUint32(tagsize, result)); 208 JXL_RETURN_IF_ERROR(AppendKeyword(kBxyzTag, result)); 209 JXL_RETURN_IF_ERROR(AppendUint32(tagstart + tagsize * 2, result)); 210 JXL_RETURN_IF_ERROR(AppendUint32(tagsize, result)); 211 } 212 } 213 } 214 215 // Main Content 216 for (;;) { 217 if (result->size() > osize) return JXL_FAILURE("Invalid result size"); 218 if (cpos > commands_end) return JXL_FAILURE("Out of bounds"); 219 if (cpos == commands_end) break; // Valid end 220 uint8_t command = enc[cpos++]; 221 if (command == kCommandInsert) { 222 if (cpos >= commands_end) return JXL_FAILURE("Out of bounds"); 223 uint64_t num = DecodeVarInt(enc, size, &cpos); 224 JXL_RETURN_IF_ERROR(CheckOutOfBounds(pos, num, size)); 225 for (size_t i = 0; i < num; i++) { 226 JXL_RETURN_IF_ERROR(result->push_back(enc[pos++])); 227 } 228 } else if (command == kCommandShuffle2 || command == kCommandShuffle4) { 229 if (cpos >= commands_end) return JXL_FAILURE("Out of bounds"); 230 uint64_t num = DecodeVarInt(enc, size, &cpos); 231 JXL_RETURN_IF_ERROR(CheckOutOfBounds(pos, num, size)); 232 PaddedBytes shuffled(memory_manager); 233 JXL_ASSIGN_OR_RETURN(shuffled, 234 PaddedBytes::WithInitialSpace(memory_manager, num)); 235 for (size_t i = 0; i < num; i++) { 236 shuffled[i] = enc[pos + i]; 237 } 238 if (command == kCommandShuffle2) { 239 JXL_RETURN_IF_ERROR(Shuffle(memory_manager, shuffled.data(), num, 2)); 240 } else if (command == kCommandShuffle4) { 241 JXL_RETURN_IF_ERROR(Shuffle(memory_manager, shuffled.data(), num, 4)); 242 } 243 for (size_t i = 0; i < num; i++) { 244 JXL_RETURN_IF_ERROR(result->push_back(shuffled[i])); 245 pos++; 246 } 247 } else if (command == kCommandPredict) { 248 JXL_RETURN_IF_ERROR(CheckOutOfBounds(cpos, 2, commands_end)); 249 uint8_t flags = enc[cpos++]; 250 251 size_t width = (flags & 3) + 1; 252 if (width == 3) return JXL_FAILURE("Invalid width"); 253 254 int order = (flags & 12) >> 2; 255 if (order == 3) return JXL_FAILURE("Invalid order"); 256 257 uint64_t stride = width; 258 if (flags & 16) { 259 if (cpos >= commands_end) return JXL_FAILURE("Out of bounds"); 260 stride = DecodeVarInt(enc, size, &cpos); 261 if (stride < width) { 262 return JXL_FAILURE("Invalid stride"); 263 } 264 } 265 // If stride * 4 >= result->size(), return failure. The check 266 // "size == 0 || ((size - 1) >> 2) < stride" corresponds to 267 // "stride * 4 >= size", but does not suffer from integer overflow. 268 // This check is more strict than necessary but follows the specification 269 // and the encoder should ensure this is followed. 270 if (result->empty() || ((result->size() - 1u) >> 2u) < stride) { 271 return JXL_FAILURE("Invalid stride"); 272 } 273 274 if (cpos >= commands_end) return JXL_FAILURE("Out of bounds"); 275 uint64_t num = DecodeVarInt(enc, size, &cpos); // in bytes 276 JXL_RETURN_IF_ERROR(CheckOutOfBounds(pos, num, size)); 277 278 PaddedBytes shuffled(memory_manager); 279 JXL_ASSIGN_OR_RETURN(shuffled, 280 PaddedBytes::WithInitialSpace(memory_manager, num)); 281 282 for (size_t i = 0; i < num; i++) { 283 shuffled[i] = enc[pos + i]; 284 } 285 if (width > 1) { 286 JXL_RETURN_IF_ERROR( 287 Shuffle(memory_manager, shuffled.data(), num, width)); 288 } 289 290 size_t start = result->size(); 291 for (size_t i = 0; i < num; i++) { 292 uint8_t predicted = LinearPredictICCValue(result->data(), start, i, 293 stride, width, order); 294 JXL_RETURN_IF_ERROR(result->push_back(predicted + shuffled[i])); 295 } 296 pos += num; 297 } else if (command == kCommandXYZ) { 298 JXL_RETURN_IF_ERROR(AppendKeyword(kXyz_Tag, result)); 299 for (int i = 0; i < 4; i++) { 300 JXL_RETURN_IF_ERROR(result->push_back(0)); 301 } 302 JXL_RETURN_IF_ERROR(CheckOutOfBounds(pos, 12, size)); 303 for (size_t i = 0; i < 12; i++) { 304 JXL_RETURN_IF_ERROR(result->push_back(enc[pos++])); 305 } 306 } else if (command >= kCommandTypeStartFirst && 307 command < kCommandTypeStartFirst + kNumTypeStrings) { 308 JXL_RETURN_IF_ERROR(AppendKeyword( 309 *kTypeStrings[command - kCommandTypeStartFirst], result)); 310 for (size_t i = 0; i < 4; i++) { 311 JXL_RETURN_IF_ERROR(result->push_back(0)); 312 } 313 } else { 314 return JXL_FAILURE("Unknown command"); 315 } 316 } 317 318 if (pos != size) return JXL_FAILURE("Not all data used"); 319 if (result->size() != osize) return JXL_FAILURE("Invalid result size"); 320 321 return true; 322 } 323 324 Status ICCReader::Init(BitReader* reader) { 325 JXL_RETURN_IF_ERROR(CheckEOI(reader)); 326 JxlMemoryManager* memory_manager = decompressed_.memory_manager(); 327 used_bits_base_ = reader->TotalBitsConsumed(); 328 if (bits_to_skip_ == 0) { 329 enc_size_ = U64Coder::Read(reader); 330 if (enc_size_ > 268435456) { 331 // Avoid too large memory allocation for invalid file. 332 return JXL_FAILURE("Too large encoded profile"); 333 } 334 JXL_RETURN_IF_ERROR(DecodeHistograms( 335 memory_manager, reader, kNumICCContexts, &code_, &context_map_)); 336 JXL_ASSIGN_OR_RETURN(ans_reader_, ANSSymbolReader::Create(&code_, reader)); 337 i_ = 0; 338 JXL_RETURN_IF_ERROR( 339 decompressed_.resize(std::min<size_t>(i_ + 0x400, enc_size_))); 340 for (; i_ < std::min<size_t>(2, enc_size_); i_++) { 341 decompressed_[i_] = ans_reader_.ReadHybridUint( 342 ICCANSContext(i_, i_ > 0 ? decompressed_[i_ - 1] : 0, 343 i_ > 1 ? decompressed_[i_ - 2] : 0), 344 reader, context_map_); 345 } 346 if (enc_size_ > kPreambleSize) { 347 for (; i_ < kPreambleSize; i_++) { 348 decompressed_[i_] = ans_reader_.ReadHybridUint( 349 ICCANSContext(i_, decompressed_[i_ - 1], decompressed_[i_ - 2]), 350 reader, context_map_); 351 } 352 JXL_RETURN_IF_ERROR(CheckEOI(reader)); 353 JXL_RETURN_IF_ERROR(CheckPreamble(decompressed_, enc_size_)); 354 } 355 bits_to_skip_ = reader->TotalBitsConsumed() - used_bits_base_; 356 } else { 357 reader->SkipBits(bits_to_skip_); 358 } 359 return true; 360 } 361 362 Status ICCReader::Process(BitReader* reader, PaddedBytes* icc) { 363 ANSSymbolReader::Checkpoint checkpoint; 364 size_t saved_i = 0; 365 auto save = [&]() { 366 ans_reader_.Save(&checkpoint); 367 bits_to_skip_ = reader->TotalBitsConsumed() - used_bits_base_; 368 saved_i = i_; 369 }; 370 save(); 371 auto check_and_restore = [&]() { 372 Status status = CheckEOI(reader); 373 if (!status) { 374 // not enough bytes. 375 ans_reader_.Restore(checkpoint); 376 i_ = saved_i; 377 return status; 378 } 379 return Status(true); 380 }; 381 for (; i_ < enc_size_; i_++) { 382 if (i_ % ANSSymbolReader::kMaxCheckpointInterval == 0 && i_ > 0) { 383 JXL_RETURN_IF_ERROR(check_and_restore()); 384 save(); 385 if ((i_ > 0) && (((i_ & 0xFFFF) == 0))) { 386 float used_bytes = 387 (reader->TotalBitsConsumed() - used_bits_base_) / 8.0f; 388 if (i_ > used_bytes * 256) return JXL_FAILURE("Corrupted stream"); 389 } 390 JXL_RETURN_IF_ERROR( 391 decompressed_.resize(std::min<size_t>(i_ + 0x400, enc_size_))); 392 } 393 JXL_ENSURE(i_ >= 2); 394 decompressed_[i_] = ans_reader_.ReadHybridUint( 395 ICCANSContext(i_, decompressed_[i_ - 1], decompressed_[i_ - 2]), reader, 396 context_map_); 397 } 398 JXL_RETURN_IF_ERROR(check_and_restore()); 399 bits_to_skip_ = reader->TotalBitsConsumed() - used_bits_base_; 400 if (!ans_reader_.CheckANSFinalState()) { 401 return JXL_FAILURE("Corrupted ICC profile"); 402 } 403 404 icc->clear(); 405 return UnpredictICC(decompressed_.data(), decompressed_.size(), icc); 406 } 407 408 Status ICCReader::CheckEOI(BitReader* reader) { 409 if (reader->AllReadsWithinBounds()) return true; 410 return JXL_STATUS(StatusCode::kNotEnoughBytes, 411 "Not enough bytes for reading ICC profile"); 412 } 413 414 } // namespace jxl