Bug 1420322 - Test for seamless looping; r?jwwang,padenot draft
authorChun-Min Chang <chun.m.chang@gmail.com>
Thu, 18 Jan 2018 10:42:03 +0800
changeset 721914 f54dc80578ac8668ba2071b9c6d677fb591bd578
parent 718845 4db166f0442dddc5b9011c722d7499501fedf283
child 746481 1b3f61d15c1ea3d58ea4282520130c3b4fbda42b
push id95996
push userbmo:cchang@mozilla.com
push dateThu, 18 Jan 2018 02:42:28 +0000
reviewersjwwang, padenot
bugs1420322
milestone59.0a1
Bug 1420322 - Test for seamless looping; r?jwwang,padenot MozReview-Commit-ID: Awbs24pb8SB
dom/media/test/mochitest.ini
dom/media/test/sine-3000hz-1s.wav
dom/media/test/test_seamless_loop.html
dom/media/test/test_seamless_loop_helper.js
--- a/dom/media/test/mochitest.ini
+++ b/dom/media/test/mochitest.ini
@@ -547,16 +547,17 @@ support-files =
   seek.webm^headers^
   seek-short.webm
   seek-short.webm^headers^
   seek_support.js
   seekLies.sjs
   seek_with_sound.ogg^headers^
   sequential.vtt
   short-cenc.mp4
+  sine-3000hz-1s.wav
   sine.webm
   sine.webm^headers^
   sintel-short-clearkey-subsample-encrypted-audio.webm
   sintel-short-clearkey-subsample-encrypted-audio.webm^headers^
   sintel-short-clearkey-subsample-encrypted-video.webm
   sintel-short-clearkey-subsample-encrypted-video.webm^headers^
   short.mp4
   short.mp4.gz
@@ -1046,16 +1047,20 @@ skip-if = toolkit == 'android' # bug 131
 skip-if = toolkit == 'android' # android(bug 1232305)
 [test_video_dimensions.html]
 skip-if = toolkit == 'android' # bug 1298238, bug 1304535, android(bug 1232305)
 [test_resolution_change.html]
 skip-if = android_version == '19' # bug 1393866
 tags=capturestream
 [test_resume.html]
 skip-if = true # bug 1021673
+[test_seamless_loop.html]
+skip-if = toolkit == 'android' # bug 1242112, android(bug 1232305)
+support-files =
+  test_seamless_loop_helper.js
 [test_seek_negative.html]
 skip-if = toolkit == 'android' # bug 1295443, bug 1306787, android(bug 1232305)
 [test_seek_nosrc.html]
 [test_seek_out_of_range.html]
 skip-if = toolkit == 'android' # bug 1299382, android(bug 1232305)
 [test_seek_promise_bug1344357.html]
 skip-if = toolkit == 'android' # bug 1299382, android(bug 1232305)
 [test_seek-1.html]
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..a7bff68f663783eda8a13518e400a1a485925f08
GIT binary patch
literal 88278
zc%1Ff%WsTp00!`HX4+2Mxx9^7xda=I4UQ9ug%go5Oe!u#j}1+u5u3wVG**a+#o0)(
z6C!Ho5^P+8g@v6&Vue_UuoQW}nd!7M?X>F8ko@v&o;-P0zoBZS^7e2h`Dfekj^qE1
zuN+8{B(sq1_mkxRg*}@LB)h8z4lY^Pt9vSYs|P9vOACGX=;`B|Hm}>b?#~T>CCS+6
z_~@^9l1%h0Dh`Ja;e6?N@3HL7nTg4ZX(o=OBa^#k3fcd9_m$p+lVPx!_8d+gw$?UY
z$K`RT)@knSl)IMp50^XPb@@wwp?h%d)6|#R_IM_qtG}AwKcDHjQCJzihfk%!zL2ZT
zu5G?c^Kn`FxpBLFB=a%<y?8HN3(ty|^7+jE_C%u-i?N-aYF?im%U$W~E;m9hj1(Sq
zx97&EtMwD{P+VMJGktsRNcXD3w{k12FTd}5kxOSEH&@pd$Io%BG2eQSY|URNUJNh8
ziqh)d$C+k(#pK^<F5XSQO{O!?ve~`|rLSR2_)r+?xide|dQ!g{H^oP_%TtZc)2>_n
zx61S3Y<Z;rSl7)?W$JwGk9ar!s2^=nqC|-jB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6
zM2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$Yi
zQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzBi4rAB
zlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-j
zB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzB
zi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3
zqC|-jB}$YiQKCeN5+zEMC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEM
xC{dzBi4rABlqgZ6M2Qk5N|Y#3qC|-jB}$YiQKCeN5+zEMC{dzBiPG<@^b_%w-uM6j
new file mode 100644
--- /dev/null
+++ b/dom/media/test/test_seamless_loop.html
@@ -0,0 +1,214 @@
+<!DOCTYPE HTML>
+<html>
+<meta charset="utf-8">
+<head>
+  <title>Test for seamless looping</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script type="text/javascript" src="./test_seamless_loop_helper.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<pre id="test">
+<h2>Fast Fourier transform(FFT)</h2>
+<p>Show the frequency-domain data of the audio.</p>
+<div id="fft"></div>
+<h2>Variation of FFT data</h2>
+<p>Compare the current data with last one at the same frequency.</p>
+<p>The variation below is 40x of the original value.</p>
+<div id="variation"></div>
+<script class="testbody" type="text/javascript">
+SimpleTest.waitForExplicitFinish();
+// ============================================================================
+//   How this test works
+// ============================================================================
+// 1) Prepare a high-frequency sine wave
+// 2) Loop the sine wave by an audio element
+// 3) Capture the media stream from the playing audio element while looping
+// 4) Get the frequency-domain data of the captured stream by FFT
+// 5) Check if there is noise in the frequency-domain data
+//
+// Ideally, the frequency-domain data for a seamless sine wave should be:
+//
+//           |
+//           |
+//           |
+//           |
+// ----------+--------------> Frequency
+//           ^
+//        Frequency of the sine wave
+//
+// If there is discontinuity in the sine wave, it may look like:
+//
+//           |
+//           |       noise
+//           |       v
+//   | | | | | | | | |
+// --+-+-+-+-+-+-+-+-+------> Frequency
+//           ^
+//        Frequency of the sine wave
+//
+// However, in practice, our FFT is not perfect. The frequency-domain data for
+// a seamless sine wave will be:
+//
+//           |
+//          |||
+//         |||||
+//         |||||
+// --------+++++------------> Frequency
+//           ^
+//        Frequency of the sine wave
+//
+// The higher frequency is, the more stable of the FFT data is. Thus, we set
+// an acceptable tolerance for the variation of the FFT data to check if there
+// is a discontinuity while looping the sine wave.
+// ============================================================================
+//   Helpers
+// ============================================================================
+// Resolve the Promise after the environment setup.
+function setupEnvironment() {
+  return new Promise((resolve, reject) => {
+    let prefs = {
+      'set': [
+        // Uncomment below to check if the test fails.
+        // ['media.seamless-looping', false],
+      ]
+    };
+    SpecialPowers.pushPrefEnv(prefs, () => { resolve(); });
+  });
+}
+// ============================================================================
+//   Test
+// ============================================================================
+//          peak
+//           v
+//           |
+//          |||    ^
+//         |||||   | tolerance
+//        |||||||  v
+//       |||||||||
+// ------+++++++++----------> Frequency
+//           ^
+//       |---|
+//         ^ |
+//     range |
+//           |
+//        Frequency of the sine wave
+const FREQUENCY = 3000; // The frequency of the sine wave.
+const RANGE = 8;        // A half of an acceptable width of the FFT pulse
+                        // whose peak is FREQUENCY.
+const TOLERANCE = 5;    // A tolerance for the variation of FFT data.
+const TIMES = 3;        // The looping time for the test.
+
+var audioElement = document.createElement("audio");
+audioElement.src = "sine-3000hz-1s.wav";
+audioElement.loop = true;
+ok(audioElement.loop == TIMES > 0, "The loop setting is different.");
+
+var context = new AudioContext();
+// Create an AudioAnalyser to get the FFT data of the audio stream.
+// The FFT data will be used to check if there is discontinuity while looping.
+var analyser = new AudioAnalyser(context, audioElement.mozCaptureStream());
+analyser.node.connect(context.destination);
+
+// Open a canvas to show the frequency-domain data.
+analyser.openSpectrum(document.getElementById("fft"));
+// Open a canvas to show the variation of frequency-domain data.
+// The variation is |current fft value - previous fft value| at the same index.
+analyser.openVariation(document.getElementById("variation"));
+
+// Create a SeamlessSineDetector to check if the sine wave loops seamlessly
+// every 100ms.
+var seamlessSineDetector =
+  new SeamlessSineDetector(analyser, 100, FREQUENCY, RANGE, TOLERANCE);
+
+// Numbers of the received events.
+var eventCount = {
+  play: 0,
+  seeking: 0,
+  seeked: 0,
+  timeupdate: 0,
+};
+
+// Get how much time has passed.
+function getTimePass(ae) {
+  return ae.duration * eventCount.seeked + ae.currentTime;
+}
+
+// Event callback for play and seeking event.
+function eventCounter(evt) {
+  ++eventCount[evt.type];
+  let ae = evt.target;
+  info(ae.currentTime + "(" + getTimePass(ae).toFixed(6) + ")\t" +
+       evt.type + ": " + eventCount[evt.type]);
+}
+
+// Event callback for seeked event.
+function loopCounter(evt) {
+  eventCounter(evt);
+  if (eventCount.seeked >= TIMES) {
+    info("Cancel loop.");
+    evt.target.loop = false;
+    seamlessSineDetector.stop();
+  }
+}
+
+// Event callback for timeupdate event. It will be removed after starting
+// the seamlessSineDetector. The result of the FFT might be not stable at
+// the very beginning, so we start detecting after the audio is played for
+// a while.
+function waitToStartDetector(evt) {
+  eventCounter(evt);
+  let ae = evt.target;
+  let begin = ae.duration / 2;
+  let time = getTimePass(ae);
+  if (time < begin) {
+    return;
+  }
+  seamlessSineDetector.start();
+  ae.removeEventListener(evt.type, waitToStartDetector, false);
+}
+
+// Event callback for end event. It's the endpoint of the test.
+function terminator(evt) {
+  let ae = evt.target;
+  ok(ae.currentTime == ae.duration,
+     "The end time should be same as the duration.");
+  ok(eventCount.play == 1,
+     "The play event must be received only once.");
+  ok(eventCount.seeking == TIMES,
+     "The number of received seeking events must be same as looping-times.");
+  ok(eventCount.seeked == TIMES,
+     "The number of received seeked events must be same as looping-times.");
+  ok(seamlessSineDetector.seamlessness,
+     "FFT Noise Detected. The looping is not seamless.");
+
+  context.close();
+  // analyser.closeSpectrum();
+  // analyser.closeVariation();
+  SimpleTest.finish();
+}
+
+// Register all the event callbacks.
+function registerEventListeners(audio) {
+  let listeners = {
+    play: eventCounter,
+    seeking: eventCounter,
+    seeked: loopCounter,
+    ended: terminator,
+    timeupdate: waitToStartDetector,
+  };
+
+  for (evt in listeners) {
+    audio.addEventListener(evt, listeners[evt], false);
+  }
+}
+
+(async () => {
+  await setupEnvironment();
+  registerEventListeners(audioElement);
+  audioElement.play();
+})();
+</script>
+</pre>
+</body>
+</html>
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/dom/media/test/test_seamless_loop_helper.js
@@ -0,0 +1,336 @@
+/**
+ * This class provides helpers around analysing the audio content in a stream
+ * using WebAudio AnalyserNodes.
+ *
+ * @constructor
+ * @param {AudioContext} audioContext
+ *        AudioContext in which to create the AnalyserNode.
+ */
+function AudioAnalyser(audioContext, audioStream) {
+  this.context = audioContext;
+  this.analyser = this.context.createAnalyser();
+  // this.analyser.smoothingTimeConstant = 0.2;
+  this.analyser.fftSize = 1024;
+  this.fftData = new Uint8Array(this.analyser.frequencyBinCount);
+
+  if (audioStream) {
+    this.stream = audioStream;
+    this.source = this.context.createMediaStreamSource(this.stream);
+    this.source.connect(this.analyser);
+  }
+}
+
+AudioAnalyser.prototype = {
+  /**
+   * Get the internal AnalyserNode for analysing the audio stream.
+   */
+  get node() {
+    return this.analyser;
+  },
+
+  /**
+   * Get an array of frequency domain data for our stream's audio track.
+   *
+   * @returns {array} A Uint8Array containing the frequency domain data.
+   */
+  getByteFrequencyData: function() {
+    this.analyser.getByteFrequencyData(this.fftData);
+    return this.fftData;
+  },
+
+  /**
+   * Get an array containing the indices of the peaks of pulses in FFT data.
+   */
+   getPeaks: function() {
+    let peaks = [];
+    const Direction = {
+      Unknown: 0,
+      Up: 1,
+      Down: 2,
+    };
+    let dir = Direction.Unknown;
+    let data = this.getByteFrequencyData();
+    for (let i = 1 ; i < data.length ; ++i) {
+      if (data[i] > data[i-1]) { // Go up.
+        dir = Direction.Up;
+      }
+      if (data[i] < data[i-1]) { // Go down.
+        if (dir == Direction.Up) {
+          peaks.push(i-1);
+        }
+        dir = Direction.Down;
+      }
+    }
+    return peaks;
+  },
+
+  /**
+   * Return the FFT bin index for a given frequency.
+   *
+   * @param {double} frequency
+   *        The frequency for whicht to return the bin number.
+   * @returns {integer} the index of the bin in the FFT array.
+   */
+  binIndexForFrequency: function(frequency) {
+    return 1 + Math.round(frequency * this.analyser.fftSize / this.context.sampleRate);
+  },
+
+  /**
+   * Reverse operation, get the frequency for a bin index.
+   *
+   * @param {integer} index an index in an FFT array
+   * @returns {double} the frequency for this bin
+   */
+  frequencyForBinIndex: function(index) {
+    return (index - 1) * this.context.sampleRate / this.analyser.fftSize;
+  },
+
+  /**
+   * Stop drawing of and remove the canvas from the DOM.
+   */
+  removeCanvas: function(cvs) {
+    if (!cvs || !cvs.parentElement) {
+      return;
+    }
+
+    cvs.stopDrawing = true;
+    cvs.parentElement.removeChild(cvs);
+  },
+
+  /**
+   * Append a canvas to the DOM where the frequency data are drawn.
+   */
+  openSpectrum: function(content) {
+    let cvs = this.fftCanvas = document.createElement("canvas");
+    content.insertBefore(cvs, content.children[0]);
+
+    // Easy: 1px per bin
+    cvs.width = this.analyser.frequencyBinCount;
+    cvs.height = 200;
+    cvs.style.border = "1px solid red";
+
+    let c = cvs.getContext('2d');
+    c.fillStyle = 'black';
+
+    let self = this;
+    function render() {
+      c.clearRect(0, 0, cvs.width, cvs.height);
+      let array = self.getByteFrequencyData();
+      for (let i = 0; i < array.length; ++i) {
+        c.fillRect(i, (cvs.height - (array[i] / 2)), 1, cvs.height);
+        // if (array[i]) {
+        //   console.log(i + " (" + self.frequencyForBinIndex(i) + "hz): " + array[i]);
+        // }
+      }
+      if (!cvs.stopDrawing) {
+        requestAnimationFrame(render);
+      }
+    }
+    requestAnimationFrame(render);
+  },
+
+  /**
+   * Stop drawing frequency data.
+   */
+  closeSpectrum: function() {
+    this.removeCanvas(this.fftCanvas);
+  },
+
+  /**
+   * Append a canvas to the DOM where the variation of frequency data are drawn.
+   */
+  openVariation: function(content) {
+    let cvs = this.variationCanvas = document.createElement("canvas");
+    content.appendChild(cvs);
+
+    // Easy: 1px per bin
+    cvs.width = this.analyser.frequencyBinCount;
+    cvs.height = 200;
+    cvs.style.border = "1px solid red";
+
+    let c = cvs.getContext('2d');
+    c.fillStyle = 'black';
+
+    let lastArray = [];
+    let self = this;
+    function render() {
+      c.clearRect(0, 0, cvs.width, cvs.height);
+      let array = self.getByteFrequencyData();
+      if (lastArray.length) {
+        for (let i = 0; i < array.length; ++i) {
+          let diff = Math.abs(lastArray[i] - array[i]);
+          c.fillRect(i, cvs.height - 40 * diff, 1, cvs.height);
+          // if (diff) {
+          //   console.log(i + " (" + self.frequencyForBinIndex(i) +
+          //               "hz) - last: " + lastArray[i] +
+          //               ", current: " + array[i]);
+          // }
+        }
+      }
+      if (!cvs.stopDrawing) {
+        requestAnimationFrame(render);
+      }
+      lastArray = array.slice(0);
+    }
+    requestAnimationFrame(render);
+  },
+
+  /**
+   * Stop drawing the variation of frequency data.
+   */
+  closeVariation: function() {
+    this.removeCanvas(this.variationCanvas);
+  },
+};
+
+/**
+ * This class creates a timer to repeatedly check if the stream that is
+ * processed in the given analyser is a seamless sine wave or not.
+ *
+ * @constructor
+ * @param {AudioAnalyser} analyser
+ *        A AudioAnalyser that is processing a sine wave stream.
+ * @param {integer} interval
+ *        The interval in milliseconds for the timer.
+ * @param {integer} frequency
+ *        The frequency of the sine wave that is processed
+ *        by the given analyser.
+ * @param {integer} range
+ *        A half of an acceptable width of the pulse whose peak is the
+ *        frequency of the sine wave that is processed by the given analyser.
+ * @param {integer} tolerance
+ *        The acceptable range of the variation of the FFT data
+ *
+ *          peak
+ *           v
+ *           |
+ *          |||    ^
+ *         |||||   | tolerance
+ *        |||||||  v
+ *       |||||||||
+ * ------+++++++++----------> Frequency
+ *           ^
+ *       |---|
+ *         ^ |
+ *     range |
+ *           |
+ *        Frequency of the sine wave
+ *
+ */
+ function SeamlessSineDetector(analyser, interval, frequency, range = 0, tolerance = 0)
+{
+  this.analyser = analyser;
+  this.interval = interval;
+  this.frequency = frequency;
+  this.range = range;
+  this.tolerance = tolerance;
+  this.peak = this.analyser.binIndexForFrequency(this.frequency);
+  this.data = [];
+  this.seamlessness = true;
+}
+
+SeamlessSineDetector.prototype = {
+  /**
+   * Set the timer and start detecting.
+   */
+  start: function() {
+    this.detector = setInterval(this.detect.bind(this), this.interval);
+  },
+
+  /**
+   * Clear the timer and stop detecting.
+   */
+  stop: function() {
+    clearInterval(this.detector);
+  },
+
+  /**
+   * The detecting function to check if the variation of FFT data is
+   * acceptable.
+   */
+  detect: function() {
+    if (!this.seamlessness) {
+      return;
+    }
+
+    // Make sure all the peaks are in the available range.
+    let peaks = this.analyser.getPeaks();
+    for (let i = 0 ; i < peaks.length ; ++i) {
+      if (!this._isIndexInPulse(peaks[i])) {
+        this.seamlessness = false;
+        // console.log("There are peaks out of range!");
+        return;
+      }
+    }
+
+    // Sort the peaks by the distance to the ideal peak(this.peak).
+    // Make sure the nearer to the ideal peak is, the higher its FFT value is.
+    let sortedPeaks = this._getSortedPeaksByDistance(peaks);
+    for (let i = 1 ; i < sortedPeaks.length ; ++i) {
+      if (sortedPeaks[i-1].value < sortedPeaks[i].value) {
+        this.seamlessness = false;
+        // console.log("The farther peak should not be higher than the nearer peak.");
+        return;
+      }
+    }
+
+    let currentdata = this.analyser.getByteFrequencyData();
+    if (this.data.length) {
+      for (let i = 0 ; i < currentdata.length ; ++i) {
+        // If the data is not in the pulse of a sine wave, then it must be 0.
+        if (!this._isIndexInPulse(i)) {
+          if (currentdata[i] == 0) {
+            continue;
+          }
+          this.seamlessness = false;
+          // console.log("This FFT value should be 0!");
+          return;
+        }
+        // If the data is in the pulse of a sine wave, then check
+        // 1) The FFT value doesn't change from 0/non-0 to non-0/0.
+        if ((currentdata[i] != 0) != (this.data[i] != 0)) {
+          this.seamlessness = false;
+          // console.log("The data changes from 0 to non-0, or non-0 to 0!");
+          return;
+        }
+        // 2) The variation of the FFT data is acceptable.
+        let diff = Math.abs(currentdata[i] - this.data[i]);
+        if (diff > this.tolerance) {
+          this.seamlessness = false;
+          // console.log("The shape of wave is changed!");
+        }
+      }
+    }
+    this.data = currentdata.slice(0); // Copy data.
+  },
+
+  _isIndexInPulse: function(index) {
+    let min = this.peak - this.range;
+    if (min < 0) {
+      min = 0;
+    }
+
+    let max = this.peak + this.range;
+    if (max >= this.analyser.node.frequencyBinCount) {
+      max = this.analyser.node.frequencyBinCount - 1;
+    }
+
+    return min <= index && index <= max;
+  },
+
+  _getSortedPeaksByDistance: function(peaks) {
+    let sorted = [];
+    let data = this.analyser.getByteFrequencyData();
+    for (let i = 0 ; i < peaks.length ; ++i) {
+      sorted.push({
+        index: peaks[i],
+        distance: Math.abs(this.peak - peaks[i]),
+        value: data[peaks[i]],
+      });
+    }
+    sorted.sort(function(x, y) {
+      return x.distance - y.distance;
+    });
+    return sorted;
+  },
+};