lightfield_bitstream_parsing.c (14896B)
1 /* 2 * Copyright (c) 2018, Alliance for Open Media. All rights reserved. 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 // Lightfield Bitstream Parsing 13 // ============================ 14 // 15 // This is a lightfield bitstream parsing example. It takes an input file 16 // containing the whole compressed lightfield bitstream(ivf file) and a text 17 // file containing a stream of tiles to decode and then constructs and outputs 18 // a new bitstream that can be decoded by an AV1 decoder. The output bitstream 19 // contains reference frames(i.e. anchor frames), camera frame header, and 20 // tile list OBUs. num_references is the number of anchor frames coded at the 21 // beginning of the light field file. After running the lightfield encoder, 22 // run lightfield bitstream parsing: 23 // examples/lightfield_bitstream_parsing vase10x10.ivf vase_tile_list.ivf 4 24 // tile_list.txt 25 // 26 // The tile_list.txt is expected to be of the form: 27 // Frame <frame_index0> 28 // <image_index0> <anchor_index0> <tile_col0> <tile_row0> 29 // <image_index1> <anchor_index1> <tile_col1> <tile_row1> 30 // ... 31 // Frame <frame_index1) 32 // ... 33 // 34 // The "Frame" markers indicate a new render frame and thus a new tile list 35 // will be started and the old one flushed. The image_indexN, anchor_indexN, 36 // tile_colN, and tile_rowN identify an individual tile to be decoded and 37 // to use anchor_indexN anchor image for MCP. 38 39 #include <stdio.h> 40 #include <stdlib.h> 41 #include <string.h> 42 43 #include "aom/aom_codec.h" 44 #include "aom/aom_decoder.h" 45 #include "aom/aom_encoder.h" 46 #include "aom/aom_integer.h" 47 #include "aom/aomdx.h" 48 #include "common/tools_common.h" 49 #include "common/video_reader.h" 50 #include "common/video_writer.h" 51 52 #define MAX_TILES 512 53 54 static const char *exec_name; 55 56 void usage_exit(void) { 57 fprintf(stderr, "Usage: %s <infile> <outfile> <num_references> <tile_list>\n", 58 exec_name); 59 exit(EXIT_FAILURE); 60 } 61 62 #define ALIGN_POWER_OF_TWO(value, n) \ 63 (((value) + ((1 << (n)) - 1)) & ~((1 << (n)) - 1)) 64 65 const int output_frame_width = 512; 66 const int output_frame_height = 512; 67 68 // Spec: 69 // typedef struct { 70 // uint8_t anchor_frame_idx; 71 // uint8_t tile_row; 72 // uint8_t tile_col; 73 // uint16_t coded_tile_data_size_minus_1; 74 // uint8_t *coded_tile_data; 75 // } TILE_LIST_ENTRY; 76 77 // Tile list entry provided by the application 78 typedef struct { 79 int image_idx; 80 int reference_idx; 81 int tile_col; 82 int tile_row; 83 } TILE_LIST_INFO; 84 85 static int get_image_bps(aom_img_fmt_t fmt) { 86 switch (fmt) { 87 case AOM_IMG_FMT_I420: return 12; 88 case AOM_IMG_FMT_I422: return 16; 89 case AOM_IMG_FMT_I444: return 24; 90 case AOM_IMG_FMT_I42016: return 24; 91 case AOM_IMG_FMT_I42216: return 32; 92 case AOM_IMG_FMT_I44416: return 48; 93 default: die("Invalid image format"); 94 } 95 } 96 97 static void process_tile_list(const TILE_LIST_INFO *tiles, int num_tiles, 98 aom_codec_pts_t tl_pts, unsigned char **frames, 99 const size_t *frame_sizes, aom_codec_ctx_t *codec, 100 unsigned char *tl_buf, AvxVideoWriter *writer, 101 uint8_t output_frame_width_in_tiles_minus_1, 102 uint8_t output_frame_height_in_tiles_minus_1) { 103 unsigned char *tl = tl_buf; 104 unsigned char *saved_obu_size_loc = NULL; 105 uint32_t tile_list_obu_header_size = 0; 106 uint32_t tile_list_obu_size = 0; 107 int num_tiles_minus_1 = num_tiles - 1; 108 int i; 109 110 // Write the tile list OBU header that is 1 byte long. 111 int obu_type = OBU_TILE_LIST; 112 int obu_has_size_field = 1; 113 *tl++ = (obu_type << 3) | (obu_has_size_field << 1); 114 tile_list_obu_header_size++; 115 116 // Write the OBU size using a fixed length_field_size of 4 bytes. 117 saved_obu_size_loc = tl; 118 for (i = 0; i < 4; i++) { 119 *tl++ = 0; 120 } 121 tile_list_obu_header_size += 4; 122 123 // write_tile_list_obu() 124 *tl++ = output_frame_width_in_tiles_minus_1; 125 *tl++ = output_frame_height_in_tiles_minus_1; 126 *tl++ = (num_tiles_minus_1 >> 8) & 0xff; 127 *tl++ = num_tiles_minus_1 & 0xff; 128 tile_list_obu_size += 4; 129 130 // Write each tile's data 131 for (i = 0; i <= num_tiles_minus_1; i++) { 132 aom_tile_data tile_data = { 0, NULL, 0 }; 133 134 int image_idx = tiles[i].image_idx; 135 int ref_idx = tiles[i].reference_idx; 136 int tc = tiles[i].tile_col; 137 int tr = tiles[i].tile_row; 138 139 size_t frame_size = frame_sizes[image_idx]; 140 const unsigned char *frame = frames[image_idx]; 141 142 AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1_SET_DECODE_TILE_ROW, tr); 143 AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1_SET_DECODE_TILE_COL, tc); 144 145 aom_codec_err_t aom_status = 146 aom_codec_decode(codec, frame, frame_size, NULL); 147 if (aom_status) die_codec(codec, "Failed to decode tile."); 148 149 AOM_CODEC_CONTROL_TYPECHECKED(codec, AV1D_GET_TILE_DATA, &tile_data); 150 151 // Copy over tile info. 152 // uint8_t anchor_frame_idx; 153 // uint8_t tile_row; 154 // uint8_t tile_col; 155 // uint16_t coded_tile_data_size_minus_1; 156 // uint8_t *coded_tile_data; 157 uint32_t tile_info_bytes = 5; 158 *tl++ = ref_idx; 159 *tl++ = tr; 160 *tl++ = tc; 161 int coded_tile_data_size_minus_1 = (int)tile_data.coded_tile_data_size - 1; 162 *tl++ = (coded_tile_data_size_minus_1 >> 8) & 0xff; 163 *tl++ = coded_tile_data_size_minus_1 & 0xff; 164 165 memcpy(tl, (uint8_t *)tile_data.coded_tile_data, 166 tile_data.coded_tile_data_size); 167 tl += tile_data.coded_tile_data_size; 168 169 tile_list_obu_size += 170 tile_info_bytes + (uint32_t)tile_data.coded_tile_data_size; 171 } 172 173 // Write tile list OBU size. 174 size_t bytes_written = 0; 175 if (aom_uleb_encode_fixed_size(tile_list_obu_size, 4, 4, saved_obu_size_loc, 176 &bytes_written)) 177 die_codec(codec, "Failed to encode the tile list obu size."); 178 179 // Copy the tile list. 180 if (!aom_video_writer_write_frame( 181 writer, tl_buf, tile_list_obu_header_size + tile_list_obu_size, 182 tl_pts)) 183 die_codec(codec, "Failed to copy compressed tile list."); 184 } 185 186 int main(int argc, char **argv) { 187 AvxVideoReader *reader = NULL; 188 AvxVideoWriter *writer = NULL; 189 const AvxVideoInfo *info = NULL; 190 int num_references; 191 int i; 192 aom_codec_pts_t pts; 193 const char *tile_list_file = NULL; 194 195 exec_name = argv[0]; 196 if (argc != 5) die("Invalid number of arguments."); 197 198 reader = aom_video_reader_open(argv[1]); 199 if (!reader) die("Failed to open %s for reading.", argv[1]); 200 201 num_references = (int)strtol(argv[3], NULL, 0); 202 info = aom_video_reader_get_info(reader); 203 204 aom_video_reader_set_fourcc(reader, AV1_FOURCC); 205 206 // The writer to write out ivf file in tile list OBU, which can be decoded by 207 // AV1 decoder. 208 writer = aom_video_writer_open(argv[2], kContainerIVF, info); 209 if (!writer) die("Failed to open %s for writing", argv[2]); 210 211 tile_list_file = argv[4]; 212 213 aom_codec_iface_t *decoder = get_aom_decoder_by_fourcc(info->codec_fourcc); 214 if (!decoder) die("Unknown input codec."); 215 printf("Using %s\n", aom_codec_iface_name(decoder)); 216 217 aom_codec_ctx_t codec; 218 if (aom_codec_dec_init(&codec, decoder, NULL, 0)) 219 die("Failed to initialize decoder."); 220 221 // Decode anchor frames. 222 AOM_CODEC_CONTROL_TYPECHECKED(&codec, AV1_SET_TILE_MODE, 0); 223 224 printf("Reading %d reference images.\n", num_references); 225 for (i = 0; i < num_references; ++i) { 226 aom_video_reader_read_frame(reader); 227 228 size_t frame_size = 0; 229 const unsigned char *frame = 230 aom_video_reader_get_frame(reader, &frame_size); 231 pts = (aom_codec_pts_t)aom_video_reader_get_frame_pts(reader); 232 233 // Copy references bitstream directly. 234 if (!aom_video_writer_write_frame(writer, frame, frame_size, pts)) 235 die_codec(&codec, "Failed to copy compressed anchor frame."); 236 237 if (aom_codec_decode(&codec, frame, frame_size, NULL)) 238 die_codec(&codec, "Failed to decode frame."); 239 } 240 241 // Decode camera frames. 242 AOM_CODEC_CONTROL_TYPECHECKED(&codec, AV1_SET_TILE_MODE, 1); 243 AOM_CODEC_CONTROL_TYPECHECKED(&codec, AV1D_EXT_TILE_DEBUG, 1); 244 245 FILE *infile = aom_video_reader_get_file(reader); 246 // Record the offset of the first camera image. 247 const FileOffset camera_frame_pos = ftello(infile); 248 249 printf("Loading compressed frames into memory.\n"); 250 251 // Count the frames in the lightfield. 252 int num_frames = 0; 253 while (aom_video_reader_read_frame(reader)) { 254 ++num_frames; 255 } 256 if (num_frames < 1) die("Input light field has no frames."); 257 258 // Read all of the lightfield frames into memory. 259 unsigned char **frames = 260 (unsigned char **)malloc(num_frames * sizeof(unsigned char *)); 261 size_t *frame_sizes = (size_t *)malloc(num_frames * sizeof(size_t)); 262 if (!(frames && frame_sizes)) die("Failed to allocate frame data."); 263 264 // Seek to the first camera image. 265 fseeko(infile, camera_frame_pos, SEEK_SET); 266 for (int f = 0; f < num_frames; ++f) { 267 aom_video_reader_read_frame(reader); 268 size_t frame_size = 0; 269 const unsigned char *frame = 270 aom_video_reader_get_frame(reader, &frame_size); 271 frames[f] = (unsigned char *)malloc(frame_size * sizeof(unsigned char)); 272 if (!frames[f]) die("Failed to allocate frame data."); 273 memcpy(frames[f], frame, frame_size); 274 frame_sizes[f] = frame_size; 275 } 276 printf("Read %d frames.\n", num_frames); 277 278 // Copy first camera frame for getting camera frame header. This is done 279 // only once. 280 { 281 size_t frame_size = frame_sizes[0]; 282 const unsigned char *frame = frames[0]; 283 pts = num_references; 284 aom_tile_data frame_header_info = { 0, NULL, 0 }; 285 286 // Need to decode frame header to get camera frame header info. So, here 287 // decoding 1 tile is enough. 288 AOM_CODEC_CONTROL_TYPECHECKED(&codec, AV1_SET_DECODE_TILE_ROW, 0); 289 AOM_CODEC_CONTROL_TYPECHECKED(&codec, AV1_SET_DECODE_TILE_COL, 0); 290 291 aom_codec_err_t aom_status = 292 aom_codec_decode(&codec, frame, frame_size, NULL); 293 if (aom_status) die_codec(&codec, "Failed to decode tile."); 294 295 AOM_CODEC_CONTROL_TYPECHECKED(&codec, AV1D_GET_FRAME_HEADER_INFO, 296 &frame_header_info); 297 298 size_t obu_size_offset = 299 (uint8_t *)frame_header_info.coded_tile_data - frame; 300 size_t length_field_size = frame_header_info.coded_tile_data_size; 301 // Remove ext-tile tile info. 302 uint32_t frame_header_size = (uint32_t)frame_header_info.extra_size - 1; 303 size_t bytes_to_copy = 304 obu_size_offset + length_field_size + frame_header_size; 305 306 unsigned char *frame_hdr_buf = (unsigned char *)malloc(bytes_to_copy); 307 if (frame_hdr_buf == NULL) 308 die_codec(&codec, "Failed to allocate frame header buffer."); 309 310 memcpy(frame_hdr_buf, frame, bytes_to_copy); 311 312 // Update frame header OBU size. 313 size_t bytes_written = 0; 314 if (aom_uleb_encode_fixed_size( 315 frame_header_size, length_field_size, length_field_size, 316 frame_hdr_buf + obu_size_offset, &bytes_written)) 317 die_codec(&codec, "Failed to encode the tile list obu size."); 318 319 // Copy camera frame header bitstream. 320 if (!aom_video_writer_write_frame(writer, frame_hdr_buf, bytes_to_copy, 321 pts)) 322 die_codec(&codec, "Failed to copy compressed camera frame header."); 323 free(frame_hdr_buf); 324 } 325 326 // Read out the image format. 327 aom_img_fmt_t ref_fmt = 0; 328 if (AOM_CODEC_CONTROL_TYPECHECKED(&codec, AV1D_GET_IMG_FORMAT, &ref_fmt)) 329 die_codec(&codec, "Failed to get the image format"); 330 const int bps = get_image_bps(ref_fmt); 331 if (!bps) die_codec(&codec, "Invalid image format."); 332 // read out the tile size. 333 unsigned int tile_size = 0; 334 if (AOM_CODEC_CONTROL_TYPECHECKED(&codec, AV1D_GET_TILE_SIZE, &tile_size)) 335 die_codec(&codec, "Failed to get the tile size"); 336 const unsigned int tile_width = tile_size >> 16; 337 const unsigned int tile_height = tile_size & 65535; 338 // Allocate a buffer to store tile list bitstream. 339 const size_t data_sz = MAX_TILES * ALIGN_POWER_OF_TWO(tile_width, 5) * 340 ALIGN_POWER_OF_TWO(tile_height, 5) * bps / 8; 341 342 unsigned char *tl_buf = (unsigned char *)malloc(data_sz); 343 if (tl_buf == NULL) die_codec(&codec, "Failed to allocate tile list buffer."); 344 345 aom_codec_pts_t tl_pts = num_references; 346 const uint8_t output_frame_width_in_tiles_minus_1 = 347 output_frame_width / tile_width - 1; 348 const uint8_t output_frame_height_in_tiles_minus_1 = 349 output_frame_height / tile_height - 1; 350 351 printf("Reading tile list from file.\n"); 352 char line[1024]; 353 FILE *tile_list_fptr = fopen(tile_list_file, "r"); 354 if (!tile_list_fptr) die_codec(&codec, "Failed to open tile list file."); 355 int num_tiles = 0; 356 TILE_LIST_INFO tiles[MAX_TILES]; 357 while ((fgets(line, 1024, tile_list_fptr)) != NULL) { 358 if (line[0] == 'F' || num_tiles >= MAX_TILES) { 359 // Flush existing tile list and start another, either because we hit a 360 // new render frame or because we've hit our max number of tiles per list. 361 if (num_tiles > 0) { 362 process_tile_list(tiles, num_tiles, tl_pts, frames, frame_sizes, &codec, 363 tl_buf, writer, output_frame_width_in_tiles_minus_1, 364 output_frame_height_in_tiles_minus_1); 365 ++tl_pts; 366 } 367 num_tiles = 0; 368 } 369 if (line[0] == 'F') { 370 continue; 371 } 372 if (sscanf(line, "%d %d %d %d", &tiles[num_tiles].image_idx, 373 &tiles[num_tiles].reference_idx, &tiles[num_tiles].tile_col, 374 &tiles[num_tiles].tile_row) == 4) { 375 if (tiles[num_tiles].image_idx >= num_frames) { 376 die("Tile list image_idx out of bounds: %d >= %d.", 377 tiles[num_tiles].image_idx, num_frames); 378 } 379 if (tiles[num_tiles].reference_idx >= num_references) { 380 die("Tile list reference_idx out of bounds: %d >= %d.", 381 tiles[num_tiles].reference_idx, num_references); 382 } 383 ++num_tiles; 384 } 385 } 386 if (num_tiles > 0) { 387 // Flush out the last tile list. 388 process_tile_list(tiles, num_tiles, tl_pts, frames, frame_sizes, &codec, 389 tl_buf, writer, output_frame_width_in_tiles_minus_1, 390 output_frame_height_in_tiles_minus_1); 391 ++tl_pts; 392 } 393 394 const int num_tile_lists = (int)(tl_pts - pts); 395 printf("Finished processing tile lists. Num tile lists: %d.\n", 396 num_tile_lists); 397 free(tl_buf); 398 for (int f = 0; f < num_frames; ++f) { 399 free(frames[f]); 400 } 401 free(frame_sizes); 402 free(frames); 403 if (aom_codec_destroy(&codec)) die_codec(&codec, "Failed to destroy codec"); 404 aom_video_writer_close(writer); 405 aom_video_reader_close(reader); 406 407 return EXIT_SUCCESS; 408 }