mediasource-correct-frames-after-reappend.html (6841B)
1 <!DOCTYPE html> 2 <!-- Copyright © 2019 Igalia. --> 3 <html> 4 <head> 5 <title>Frame checking test for MSE playback in presence of a reappend.</title> 6 <meta name="timeout" content="long"> 7 <meta name="charset" content="UTF-8"> 8 <link rel="author" title="Alicia Boya García" href="mailto:aboya@igalia.com"> 9 <script src="/resources/testharness.js"></script> 10 <script src="/resources/testharnessreport.js"></script> 11 <script src="mediasource-util.js"></script> 12 </head> 13 <body> 14 <div id="log"></div> 15 <canvas id="test-canvas"></canvas> 16 <script> 17 function waitForEventPromise(element, event) { 18 return new Promise(resolve => { 19 function handler(ev) { 20 element.removeEventListener(event, handler); 21 resolve(ev); 22 } 23 element.addEventListener(event, handler); 24 }); 25 } 26 27 function appendBufferPromise(sourceBuffer, data) { 28 sourceBuffer.appendBuffer(data); 29 return waitForEventPromise(sourceBuffer, "update"); 30 } 31 32 function waitForPlayerToReachTimePromise(mediaElement, time) { 33 return new Promise(resolve => { 34 function timeupdate() { 35 if (mediaElement.currentTime < time) 36 return; 37 38 mediaElement.removeEventListener("timeupdate", timeupdate); 39 resolve(); 40 } 41 mediaElement.addEventListener("timeupdate", timeupdate); 42 }); 43 } 44 45 function readPixel(imageData, x, y) { 46 return { 47 r: imageData.data[4 * (y * imageData.width + x)], 48 g: imageData.data[1 + 4 * (y * imageData.width + x)], 49 b: imageData.data[2 + 4 * (y * imageData.width + x)], 50 a: imageData.data[3 + 4 * (y * imageData.width + x)], 51 }; 52 } 53 54 function isPixelLit(pixel) { 55 const threshold = 200; // out of 255 56 return pixel.r >= threshold && pixel.g >= threshold && pixel.b >= threshold; 57 } 58 59 // The test video has a few gray boxes. Each box interval (1 second) a new box is lit white and a different note 60 // is played. This test makes sure the right number of lit boxes and the right note are played at the right time. 61 const totalBoxes = 7; 62 const boxInterval = 1; // seconds 63 64 const videoWidth = 320; 65 const videoHeight = 240; 66 const boxesY = 210; 67 const boxSide = 20; 68 const boxMargin = 20; 69 const allBoxesWidth = totalBoxes * boxSide + (totalBoxes - 1) * boxMargin; 70 const boxesX = new Array(totalBoxes).fill(undefined) 71 .map((_, i) => (videoWidth - allBoxesWidth) / 2 + boxSide / 2 + i * (boxSide + boxMargin)); 72 73 // Sound starts playing A4 (440 Hz) and goes one chromatic note up with every box lit. 74 // By comparing the player position to both the amount of boxes lit and the note played we can detect A/V 75 // synchronization issues automatically. 76 const noteFrequencies = new Array(1 + totalBoxes).fill(undefined) 77 .map((_, i) => 440 * Math.pow(Math.pow(2, 1 / 12), i)); 78 79 // We also check the first second [0, 1) where no boxes are lit, therefore we start counting at -1 to do the check 80 // for zero lit boxes. 81 let boxesLitSoFar = -1; 82 83 mediasource_test(async function (test, mediaElement, mediaSource) { 84 const canvas = document.getElementById("test-canvas"); 85 const canvasCtx = canvas.getContext("2d"); 86 canvas.width = videoWidth; 87 canvas.height = videoHeight; 88 89 const videoData = await (await fetch("mp4/test-boxes-video.mp4")).arrayBuffer(); 90 const audioData = (await (await fetch("mp4/test-boxes-audio.mp4")).arrayBuffer()); 91 92 const videoSb = mediaSource.addSourceBuffer('video/mp4; codecs="avc1.4d401f"'); 93 const audioSb = mediaSource.addSourceBuffer('audio/mp4; codecs="mp4a.40.2"'); 94 95 mediaElement.addEventListener('error', test.unreached_func("Unexpected event 'error'")); 96 mediaElement.addEventListener('ended', onEnded); 97 mediaElement.addEventListener('timeupdate', onTimeUpdate); 98 99 await appendBufferPromise(videoSb, videoData); 100 await appendBufferPromise(audioSb, audioData); 101 mediaElement.play(); 102 103 audioCtx = new (window.AudioContext || window.webkitAudioContext)(); 104 source = audioCtx.createMediaElementSource(mediaElement); 105 analyser = audioCtx.createAnalyser(); 106 analyser.fftSize = 8192; 107 source.connect(analyser); 108 analyser.connect(audioCtx.destination); 109 110 const freqDomainArray = new Float32Array(analyser.frequencyBinCount); 111 112 function checkNoteBeingPlayed() { 113 const expectedNoteFrequency = noteFrequencies[boxesLitSoFar]; 114 115 analyser.getFloatFrequencyData(freqDomainArray); 116 const maxBin = freqDomainArray.reduce((prev, curValue, i) => 117 curValue > prev.value ? {index: i, value: curValue} : prev, 118 {index: -1, value: -Infinity}); 119 const binFrequencyWidth = audioCtx.sampleRate / analyser.fftSize; 120 const binFreq = maxBin.index * binFrequencyWidth; 121 122 assert_true(Math.abs(expectedNoteFrequency - binFreq) <= binFrequencyWidth, 123 `The note being played matches the expected one (boxes lit: ${boxesLitSoFar}, ${expectedNoteFrequency.toFixed(1)} Hz)` + 124 `, found ~${binFreq.toFixed(1)} Hz`); 125 } 126 127 function countLitBoxesInCurrentVideoFrame() { 128 canvasCtx.drawImage(mediaElement, 0, 0); 129 const imageData = canvasCtx.getImageData(0, 0, videoWidth, videoHeight); 130 const lights = boxesX.map(boxX => isPixelLit(readPixel(imageData, boxX, boxesY))); 131 let litBoxes = 0; 132 for (let i = 0; i < lights.length; i++) { 133 if (lights[i]) 134 litBoxes++; 135 } 136 for (let i = litBoxes; i < lights.length; i++) { 137 assert_false(lights[i], 'After the first non-lit box, all boxes must non-lit'); 138 } 139 return litBoxes; 140 } 141 142 await waitForPlayerToReachTimePromise(mediaElement, 2.5); 143 await appendBufferPromise(audioSb, audioData); 144 mediaSource.endOfStream(); 145 146 function onTimeUpdate() { 147 const graceTime = 0.5; 148 if (mediaElement.currentTime >= (1 + boxesLitSoFar) * boxInterval + graceTime && boxesLitSoFar < totalBoxes) { 149 assert_equals(countLitBoxesInCurrentVideoFrame(), boxesLitSoFar + 1, "Num of lit boxes:"); 150 boxesLitSoFar++; 151 checkNoteBeingPlayed(); 152 } 153 } 154 155 function onEnded() { 156 assert_equals(boxesLitSoFar, totalBoxes, "Boxes lit at video ended event"); 157 test.done(); 158 } 159 }, "Test the expected frames are played at the expected times, even in presence of reappends"); 160 </script> 161 </body> 162 </html>