mediasource-correct-frames.html (6257B)
1 <!DOCTYPE html> 2 <!-- Copyright © 2019 Igalia. --> 3 <html> 4 <head> 5 <title>Frame checking test for simple MSE playback.</title> 6 <meta name="timeout" content="long"> 7 <meta name="charset" content="UTF-8"> 8 <link rel="author" title="Alicia Boya García" href="mailto:aboya@igalia.com"> 9 <script src="/resources/testharness.js"></script> 10 <script src="/resources/testharnessreport.js"></script> 11 <script src="mediasource-util.js"></script> 12 </head> 13 <body> 14 <div id="log"></div> 15 <canvas id="test-canvas"></canvas> 16 <script> 17 function waitForEventPromise(element, event) { 18 return new Promise(resolve => { 19 function handler(ev) { 20 element.removeEventListener(event, handler); 21 resolve(ev); 22 } 23 element.addEventListener(event, handler); 24 }); 25 } 26 27 function appendBufferPromise(sourceBuffer, data) { 28 sourceBuffer.appendBuffer(data); 29 return waitForEventPromise(sourceBuffer, "update"); 30 } 31 32 function readPixel(imageData, x, y) { 33 return { 34 r: imageData.data[4 * (y * imageData.width + x)], 35 g: imageData.data[1 + 4 * (y * imageData.width + x)], 36 b: imageData.data[2 + 4 * (y * imageData.width + x)], 37 a: imageData.data[3 + 4 * (y * imageData.width + x)], 38 }; 39 } 40 41 function isPixelLit(pixel) { 42 const threshold = 200; // out of 255 43 return pixel.r >= threshold && pixel.g >= threshold && pixel.b >= threshold; 44 } 45 46 // The test video has a few gray boxes. Each box interval (1 second) a new box is lit white and a different note 47 // is played. This test makes sure the right number of lit boxes and the right note are played at the right time. 48 const totalBoxes = 7; 49 const boxInterval = 1; // seconds 50 51 const videoWidth = 320; 52 const videoHeight = 240; 53 const boxesY = 210; 54 const boxSide = 20; 55 const boxMargin = 20; 56 const allBoxesWidth = totalBoxes * boxSide + (totalBoxes - 1) * boxMargin; 57 const boxesX = new Array(totalBoxes).fill(undefined) 58 .map((_, i) => (videoWidth - allBoxesWidth) / 2 + boxSide / 2 + i * (boxSide + boxMargin)); 59 60 // Sound starts playing A4 (440 Hz) and goes one chromatic note up with every box lit. 61 // By comparing the player position to both the amount of boxes lit and the note played we can detect A/V 62 // synchronization issues automatically. 63 const noteFrequencies = new Array(1 + totalBoxes).fill(undefined) 64 .map((_, i) => 440 * Math.pow(Math.pow(2, 1 / 12), i)); 65 66 // We also check the first second [0, 1) where no boxes are lit, therefore we start counting at -1 to do the check 67 // for zero lit boxes. 68 let boxesLitSoFar = -1; 69 70 mediasource_test(async function (test, mediaElement, mediaSource) { 71 const canvas = document.getElementById("test-canvas"); 72 const canvasCtx = canvas.getContext("2d"); 73 canvas.width = videoWidth; 74 canvas.height = videoHeight; 75 76 const videoData = await (await fetch("mp4/test-boxes-video.mp4")).arrayBuffer(); 77 const audioData = (await (await fetch("mp4/test-boxes-audio.mp4")).arrayBuffer()); 78 79 const videoSb = mediaSource.addSourceBuffer('video/mp4; codecs="avc1.4d401f"'); 80 const audioSb = mediaSource.addSourceBuffer('audio/mp4; codecs="mp4a.40.2"'); 81 82 mediaElement.addEventListener('error', test.unreached_func("Unexpected event 'error'")); 83 mediaElement.addEventListener('ended', onEnded); 84 mediaElement.addEventListener('timeupdate', onTimeUpdate); 85 86 await appendBufferPromise(videoSb, videoData); 87 await appendBufferPromise(audioSb, audioData); 88 mediaSource.endOfStream(); 89 mediaElement.play(); 90 91 const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); 92 const source = audioCtx.createMediaElementSource(mediaElement); 93 const analyser = audioCtx.createAnalyser(); 94 analyser.fftSize = 8192; 95 source.connect(analyser); 96 analyser.connect(audioCtx.destination); 97 98 const freqDomainArray = new Float32Array(analyser.frequencyBinCount); 99 100 function checkNoteBeingPlayed() { 101 const expectedNoteFrequency = noteFrequencies[boxesLitSoFar]; 102 103 analyser.getFloatFrequencyData(freqDomainArray); 104 const maxBin = freqDomainArray.reduce((prev, curValue, i) => 105 curValue > prev.value ? {index: i, value: curValue} : prev, 106 {index: -1, value: -Infinity}); 107 const binFrequencyWidth = audioCtx.sampleRate / analyser.fftSize; 108 const binFreq = maxBin.index * binFrequencyWidth; 109 110 assert_true(Math.abs(expectedNoteFrequency - binFreq) <= binFrequencyWidth, 111 `The note being played matches the expected one (boxes lit: ${boxesLitSoFar}, ${expectedNoteFrequency.toFixed(1)} Hz)` + 112 `, found ~${binFreq.toFixed(1)} Hz`); 113 } 114 115 function countLitBoxesInCurrentVideoFrame() { 116 canvasCtx.drawImage(mediaElement, 0, 0); 117 const imageData = canvasCtx.getImageData(0, 0, videoWidth, videoHeight); 118 const lights = boxesX.map(boxX => isPixelLit(readPixel(imageData, boxX, boxesY))); 119 let litBoxes = 0; 120 for (let i = 0; i < lights.length; i++) { 121 if (lights[i]) 122 litBoxes++; 123 } 124 for (let i = litBoxes; i < lights.length; i++) { 125 assert_false(lights[i], 'After the first non-lit box, all boxes must non-lit'); 126 } 127 return litBoxes; 128 } 129 130 function onTimeUpdate() { 131 const graceTime = 0.5; 132 if (mediaElement.currentTime >= (1 + boxesLitSoFar) * boxInterval + graceTime && boxesLitSoFar < totalBoxes) { 133 assert_equals(countLitBoxesInCurrentVideoFrame(), boxesLitSoFar + 1, "Num of lit boxes:"); 134 boxesLitSoFar++; 135 checkNoteBeingPlayed(); 136 } 137 } 138 139 function onEnded() { 140 assert_equals(boxesLitSoFar, totalBoxes, "Boxes lit at video ended event"); 141 test.done(); 142 } 143 }, "Test the expected frames are played at the expected times"); 144 </script> 145 </body> 146 </html>