Bug 1397390 - Support better thumbnails for image urls draft
authorahillier <ahillier@mozilla.com>
Thu, 07 Sep 2017 21:18:45 -0400
changeset 661141 9a8c9a889a4d
parent 659233 973e8b890a62
child 730465 51332cb9d086
push id78644
push userbmo:ahillier@mozilla.com
push dateFri, 08 Sep 2017 01:21:33 +0000
bugs1397390
milestone57.0a1
Bug 1397390 - Support better thumbnails for image urls MozReview-Commit-ID: Ksxo6Gj2rIO
toolkit/components/thumbnails/BackgroundPageThumbs.jsm
toolkit/components/thumbnails/PageThumbUtils.jsm
toolkit/components/thumbnails/content/backgroundPageThumbsContent.js
toolkit/components/thumbnails/test/browser.ini
toolkit/components/thumbnails/test/browser_thumbnails_bg_image_capture.js
toolkit/components/thumbnails/test/sample_image_blue_300x600.jpg
toolkit/components/thumbnails/test/sample_image_green_1024x1024.jpg
toolkit/components/thumbnails/test/sample_image_red_1920x1080.jpg
--- a/toolkit/components/thumbnails/BackgroundPageThumbs.jsm
+++ b/toolkit/components/thumbnails/BackgroundPageThumbs.jsm
@@ -52,16 +52,19 @@ const BackgroundPageThumbs = {
    *                 properties are the following, and all are optional:
    * @opt onDone     A function that will be asynchronously called when the
    *                 capture is complete or times out.  It's called as
    *                   onDone(url),
    *                 where `url` is the captured URL.
    * @opt timeout    The capture will time out after this many milliseconds have
    *                 elapsed after the capture has progressed to the head of
    *                 the queue and started.  Defaults to 30000 (30 seconds).
+   * @opt isImage    If true, backgroundPageThumbsContent will attempt to render
+   *                 the url directly to canvas. Note that images will mostly get
+   *                 detected and rendered as such anyway, but this will ensure it.
    */
   capture(url, options = {}) {
     if (!PageThumbs._prefEnabled()) {
       if (options.onDone)
         schedule(() => options.onDone(url));
       return;
     }
     this._captureQueue = this._captureQueue || [];
@@ -399,17 +402,17 @@ Capture.prototype = {
                   DEFAULT_CAPTURE_TIMEOUT;
     this._timeoutTimer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
     this._timeoutTimer.initWithCallback(this, timeout,
                                         Ci.nsITimer.TYPE_ONE_SHOT);
 
     // didCapture registration
     this._msgMan = messageManager;
     this._msgMan.sendAsyncMessage("BackgroundPageThumbs:capture",
-                                  { id: this.id, url: this.url });
+                                  { id: this.id, url: this.url, isImage: this.options.isImage });
     this._msgMan.addMessageListener("BackgroundPageThumbs:didCapture", this);
   },
 
   /**
    * The only intended external use of this method is by the service when it's
    * uninitializing and doing things like destroying the thumbnail browser.  In
    * that case the consumer's completion callback will never be called.
    */
--- a/toolkit/components/thumbnails/PageThumbUtils.jsm
+++ b/toolkit/components/thumbnails/PageThumbUtils.jsm
@@ -109,16 +109,57 @@ this.PageThumbUtils = {
     // Even in RTL mode, scrollbars are always on the right.
     // So there's no need to determine a left offset.
     let width = aWindow.innerWidth - sbWidth.value;
     let height = aWindow.innerHeight - sbHeight.value;
 
     return [width, height];
   },
 
+  /**
+   * Renders an image onto a new canvas of a given width and proportional
+   * height. Uses an image that exists in the window and is loaded, or falls
+   * back to loading the url into a new image element.
+   */
+  async createImageThumbnailCanvas(window, url, targetWidth = 448) {
+    // 224px is the width of cards in ActivityStream; capture thumbnails at 2x
+    const doc = (window || Services.appShell.hiddenDOMWindow).document;
+
+    let image = doc.querySelector("img");
+    if (!image || image.src !== url) {
+      image = doc.createElementNS(this.HTML_NAMESPACE, "img");
+    }
+    if (!image.complete) {
+      await new Promise(resolve => {
+        image.onload = () => resolve();
+        image.onerror = () => { throw new Error("Image failed to load"); }
+        image.src = url;
+      });
+    }
+
+    // <img src="*.svg"> has width/height but not naturalWidth/naturalHeight
+    const imageWidth = image.naturalWidth || image.width;
+    const imageHeight = image.naturalHeight || image.height;
+    if (imageWidth === 0 || imageHeight === 0) {
+      throw new Error("Image has zero dimension");
+    }
+    const width = Math.min(targetWidth, imageWidth);
+    const height = imageHeight * width / imageWidth;
+
+    // As we're setting the width and maintaining the aspect ratio, if an image
+    // is very tall we might get a very large thumbnail. Restricting the canvas
+    // size to {width}x{width} solves this problem. Here we choose to clip the
+    // image at the bottom rather than centre it vertically, based on an
+    // estimate that the focus of a tall image is most likely to be near the top
+    // (e.g., the face of a person).
+    const canvas = this.createCanvas(window, width, Math.min(height, width));
+    canvas.getContext("2d").drawImage(image, 0, 0, width, height);
+    return canvas;
+  },
+
   /** *
    * Given a browser window, this creates a snapshot of the content
    * and returns a canvas with the resulting snapshot of the content
    * at the thumbnail size. It has to do this through a two step process:
    *
    * 1) Render the content at the window size to a canvas that is 2x the thumbnail size
    * 2) Downscale the canvas from (1) down to the thumbnail size
    *
--- a/toolkit/components/thumbnails/content/backgroundPageThumbsContent.js
+++ b/toolkit/components/thumbnails/content/backgroundPageThumbsContent.js
@@ -73,16 +73,17 @@ const backgroundPageThumbsContent = {
   get _webNav() {
     return docShell.QueryInterface(Ci.nsIWebNavigation);
   },
 
   _onCapture(msg) {
     this._nextCapture = {
       id: msg.data.id,
       url: msg.data.url,
+      isImage: msg.data.isImage
     };
     if (this._currentCapture) {
       if (this._state == STATE_LOADING) {
         // Cancel the current capture.
         this._state = STATE_CANCELED;
         this._loadAboutBlank();
       }
       // Let the current capture finish capturing, or if it was just canceled,
@@ -158,25 +159,32 @@ const backgroundPageThumbsContent = {
         }
       }
     }
   },
 
   _captureCurrentPage() {
     let win = docShell.QueryInterface(Ci.nsIInterfaceRequestor)
                       .getInterface(Ci.nsIDOMWindow);
-    win.requestIdleCallback(() => {
+    win.requestIdleCallback(async () => {
       let capture = this._currentCapture;
       capture.finalURL = this._webNav.currentURI.spec;
       capture.pageLoadTime = new Date() - capture.pageLoadStartDate;
 
       let canvasDrawDate = new Date();
 
       docShell.isActive = true;
-      let finalCanvas = PageThumbUtils.createSnapshotThumbnail(content, null);
+
+      let finalCanvas;
+      if (capture.isImage || content.document instanceof content.ImageDocument) {
+        finalCanvas = await PageThumbUtils.createImageThumbnailCanvas(content, capture.url);
+      } else {
+        finalCanvas = PageThumbUtils.createSnapshotThumbnail(content, null);
+      }
+
       docShell.isActive = false;
       capture.canvasDrawTime = new Date() - canvasDrawDate;
 
       finalCanvas.toBlob(blob => {
         capture.imageBlob = new Blob([blob]);
         // Load about:blank to finish the capture and wait for onStateChange.
         this._loadAboutBlank();
       });
--- a/toolkit/components/thumbnails/test/browser.ini
+++ b/toolkit/components/thumbnails/test/browser.ini
@@ -1,14 +1,17 @@
 [DEFAULT]
 support-files =
   authenticate.sjs
   background_red.html
   background_red_redirect.sjs
   background_red_scroll.html
+  sample_image_red_1920x1080.jpg
+  sample_image_green_1024x1024.jpg
+  sample_image_blue_300x600.jpg
   head.js
   privacy_cache_control.sjs
   thumbnails_background.sjs
   thumbnails_update.sjs
 
 [browser_thumbnails_bg_bad_url.js]
 [browser_thumbnails_bg_crash_during_capture.js]
 skip-if = !crashreporter
@@ -20,16 +23,17 @@ skip-if = !crashreporter
 [browser_thumbnails_bg_redirect.js]
 [browser_thumbnails_bg_destroy_browser.js]
 [browser_thumbnails_bg_no_cookies_sent.js]
 [browser_thumbnails_bg_no_cookies_stored.js]
 [browser_thumbnails_bg_no_auth_prompt.js]
 [browser_thumbnails_bg_no_alert.js]
 [browser_thumbnails_bg_no_duplicates.js]
 [browser_thumbnails_bg_captureIfMissing.js]
+[browser_thumbnails_bg_image_capture.js]
 [browser_thumbnails_bug726727.js]
 [browser_thumbnails_bug727765.js]
 [browser_thumbnails_bug818225.js]
 [browser_thumbnails_capture.js]
 skip-if = os == "mac" && !debug # bug 1314039
 [browser_thumbnails_expiration.js]
 [browser_thumbnails_privacy.js]
 [browser_thumbnails_redirect.js]
new file mode 100644
--- /dev/null
+++ b/toolkit/components/thumbnails/test/browser_thumbnails_bg_image_capture.js
@@ -0,0 +1,72 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const BASE_URL = "http://mochi.test:8888/browser/toolkit/components/thumbnails/";
+
+/**
+ * These tests ensure that when trying to capture a url that is an image file,
+ * the image itself is captured instead of the the browser window displaying the
+ * image, and that the thumbnail maintains the image aspect ratio.
+ */
+function* runTests() {
+  for (const {url, color, width, height} of [{
+    url: BASE_URL + "test/sample_image_red_1920x1080.jpg",
+    color: [255, 0, 0],
+    width: 1920,
+    height: 1080
+  }, {
+    url: BASE_URL + "test/sample_image_green_1024x1024.jpg",
+    color: [0, 255, 0],
+    width: 1024,
+    height: 1024
+  }, {
+    url: BASE_URL + "test/sample_image_blue_300x600.jpg",
+    color: [0, 0, 255],
+    width: 300,
+    height: 600
+  }]) {
+    dontExpireThumbnailURLs([url]);
+    const capturedPromise = new Promise(resolve => {
+      bgAddPageThumbObserver(url).then(() => {
+        ok(true, `page-thumbnail created for ${url}`);
+        resolve();
+      });
+    });
+    yield bgCapture(url);
+    yield capturedPromise;
+    ok(thumbnailExists(url), "The image thumbnail should exist after capture");
+
+    const thumb = PageThumbs.getThumbnailURL(url);
+    const htmlns = "http://www.w3.org/1999/xhtml";
+    const img = document.createElementNS(htmlns, "img");
+    yield new Promise(resolve => {
+      img.onload = () => resolve();
+      img.src = thumb;
+    });
+
+    // 448px is the default max-width of an image thumbnail
+    const expectedWidth = Math.min(448, width);
+    // Tall images are clipped to {width}x{width}
+    const expectedHeight = Math.min(expectedWidth * height / width, expectedWidth);
+    // Fuzzy equality to account for rounding
+    ok(Math.abs(img.naturalWidth - expectedWidth) <= 1,
+      "The thumbnail should have the right width");
+    ok(Math.abs(img.naturalHeight - expectedHeight) <= 1,
+      "The thumbnail should have the right height");
+
+    // Draw the image to a canvas and compare the pixel color values.
+    const canvas = document.createElementNS(htmlns, "canvas");
+    canvas.width = expectedWidth;
+    canvas.height = expectedHeight;
+    const ctx = canvas.getContext("2d");
+    ctx.drawImage(img, 0, 0, expectedWidth, expectedHeight);
+    const [r, g, b] = ctx.getImageData(0, 0, expectedWidth, expectedHeight).data;
+    // Fuzzy equality to account for image encoding
+    ok((Math.abs(r - color[0]) <= 2 &&
+        Math.abs(g - color[1]) <= 2 &&
+        Math.abs(b - color[2]) <= 2),
+        "The thumbnail should have the right color");
+
+    removeThumbnail(url);
+  }
+}
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ad5b44ecbc721ef793d81716e9e519aaf2c56d7b
GIT binary patch
literal 3581
zc%1ux<NpH&0WUXCHwH#VMur3+WcdG&!Ol6z)iK0B$VwqSMK`M;HC>_1P|rX?qqI0P
zFI~aY%U!`Mz|~!$%)&rZM<FFOEwMDGM4_-WF(<R6lI#C%24@BiHa2!PRt|P{c1}(X
zE*=qH9&T<PNg-i=5m_mDIaw(g83h$Rbp<6IWf>U_b4?usLlYAdd38%$3nLpnV-q8g
zA&i`yoIKn-61=<;Mv5|uMkIs(2N(o7m?9W;m>HEAm;@P_1sVSzVUTBFU}R+k0|qEy
zWMXDvWn<^y<l+V@*ebxl#K_Fd#KO$V%EAJatp&<6un4jWDH=Mm2?r*!D;0_uHBMZ}
zq3pErplHy=4=Tn<MNOPsV&W2#QmSg|8k$-rre@|AmR8O#u5Rugo?gKrp<&?>kx|LO
zz)H`^%qlJ^Ei136tZHs)ZENr7?3y%r%G7DoXUv?nXz`Mz%a*TLxoXqqEnBy3-?4Mo
zp~FXx9y@;G<f%)SuUx%${l?8(4<9{#^7PsB7cXCZ{Pg+D*Kgl{{QL#-7b62RBMe~m
zmmttzOe`$SEbJhEF*22d6bQ1gDjKp0IR>&P778mFHFAhJO<cI~Ag8i%&<D|^qKjN&
zDkcwAKZ3jl_8D;=Ya+{MaE~GUb&G+AnGqOy%z_N|3?ENvEZSdab6`=I2E!;G#iMu>
VkK$20ibwG%9>t@WXw3ZoCIAo+qjCTM
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..7d91079ecd21b337c0b62cd764ae7bc18ce6f68c
GIT binary patch
literal 17077
zc%1FgIZzW(9LMqZvb)*cB#dkb2uv`W1SC<+kPzh<j|nkE(kKzn${-kF7)OqPIAa0c
zba-nA-gc;XRib!ycpzXS-db^Z7p@z|qp?sMo8SM<r+RPR%=bNIP<cWULnWmpgi=b@
zsEa6H$$Wn};45DoE|Q~_vuZY0Mdc>9D_3^J8mc!$<<d}44lM};<)(sMXO3K1wJs8?
zZItU`k=p9U7M=2#_=%v^@>)*dd0rF+oxzk~(CZCp#-u5xjC9MijC8YE&U9tVwj8_J
z?8tZK<hs3HuO+*1UV&$>E6?kRUqVGuH0TYf2??nltJ&)L>rt)}Nni;}7-}OLi82yZ
z`iO-P%F)Ro--l`#mecZrsMD(ldJ>6-GK_|0IF4o2-7V@qVI|IJb^EkQWf9(1XY#am
zbPD$3?%w2Z|7d1jMSYv7OPQLQmYy{|+u@wyojoVNps>ilFi;X)R2o`ZzHIr5m8({-
zRUa$5zPhHNG1k<)sb%xlZQFP3+_k%X&)$9e4;(yn_(<2$W5-XNJazia*>mT6E?m5H
z`O3BHeK&62x_#&Fz55Ry4m=th8XkH2?D>nAuU@}-J2pP??)`_4pFV$ycTu(T$NaWS
zQoA%P%dmXBi)xzV2PBrWy0u1M86T-jvUyqsQ*lRUcduyA3y&sO)c5OBGQA^NWAU`f
zvOf!J`=_#>VZXYDiGiW&=P?plKnA+nXHNLXNmsiA0000000000000000000000000
U00000000000002~A4VDa2G`0Yy#N3J
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..7a2bec4d5e8d2f7169858ea8a5c06173831d5948
GIT binary patch
literal 57332
zc%1FgHH;Kd9Khi>JF~ZU2L+2mL-2zRmxf@6bhtDGmkyVN<Z$V52?cHq!KJ~q!>z%s
z!Yw4rE)Zxa2vj8Bo8rsl{qypEGr3#2d!b72@?Pa3ib9C;PsrU3UGs1MwhD**>lsn*
zM93B=?K2f|lntqD6lbH{nNTl(&-C|!@MA@(I7t^}ic3l><p&N{38^TKQ%RgoCrN&H
zc78u3*>u(VEjkxf>oYvlU}E)_bCzu?ZrEk-$r^nxziiZM#H6_;r8R5Su2Z*hlcr_O
znzwG#wq5%U9lLhxUf!c;uipLo4;VOT@Q|S+M~xmccHH<0lc!9bHhsp-S@Y&ESh#5M
zlBLU6tX#Ev&DwSAH*eXxZTpU$yLRu}f8gMu!$*!DJ9YZZ*>mSFT)cGU>b2`PZr-|m
z=l+9-j~+jH`t13OSFhi^efR#u$4{RNT~UaCrvJ7p{}G~8lEg`-&=sYo7Y4FPx_*nI
zs-62}hEJ^4pyiz6>Rpy?+IzC3VXM9`YmAt5xwK}Z*7qB~Dx`fY`?IjQe^vG~>{r*F
zP&tnBKQGRPPT|Y2J+C+9!tsiYsfyB2761SM000000000000000000000000000000
z00000000000000000000000000000000000000000000000000000000000000000
M0002|*W_-01vO*H4*&oF