Bug 932937 - Images are resized on the server-side for image preview tooltips. r=harth

browser/devtools/markupview/markup-view.js

@@ -14,6 +14,7 @@ const COLLAPSE_ATTRIBUTE_LENGTH = 120;
 const COLLAPSE_DATA_URL_REGEX = /^data.+base64/;
 const COLLAPSE_DATA_URL_LENGTH = 60;
 const CONTAINER_FLASHING_DURATION = 500;
+const IMAGE_PREVIEW_MAX_DIM = 400;
 
 const {UndoStack} = require("devtools/shared/undo");
 const {editableField, InplaceEditor} = require("devtools/shared/inplace-editor");
@@ -99,6 +100,10 @@ function MarkupView(aInspector, aFrame, aControllerWindow) {
   gDevTools.on("pref-changed", this._handlePrefChange);
 
   this._initPreview();
+
+  this.tooltip = new Tooltip(this._inspector.panelDoc);
+  this.tooltip.startTogglingOnHover(this._elt,
+    this._buildTooltipContent.bind(this));
 }
 
 exports.MarkupView = MarkupView;
@@ -148,6 +153,25 @@ MarkupView.prototype = {
     updateChildren(documentElement);
   },
 
+  _buildTooltipContent: function(target) {
+    // From the target passed here, let's find the parent MarkupContainer
+    // and ask it if the tooltip should be shown
+    let parent = target, container;
+    while (parent !== this.doc.body) {
+      if (parent.container) {
+        container = parent.container;
+        break;
+      }
+      parent = parent.parentNode;
+    }
+
+    if (container) {
+      // With the newly found container, delegate the tooltip content creation
+      // and decision to show or not the tooltip
+      return container._buildTooltipContent(target, this.tooltip);
+    }
+  },
+
   /**
    * Highlight the inspector selected node.
    */
@@ -954,6 +978,9 @@ MarkupView.prototype = {
       container.destroy();
     }
     delete this._containers;
+
+    this.tooltip.destroy();
+    delete this.tooltip;
   },
 
   /**
@@ -1099,8 +1126,8 @@ function MarkupContainer(aMarkupView, aNode, aInspector) {
   this._onMouseDown = this._onMouseDown.bind(this);
   this.elt.addEventListener("mousedown", this._onMouseDown, false);
 
-  this.tooltip = null;
-  this._attachTooltipIfNeeded();
+  // Prepare the image preview tooltip data if any
+  this._prepareImagePreview();
 }
 
 MarkupContainer.prototype = {
@@ -1108,36 +1135,43 @@ MarkupContainer.prototype = {
     return "[MarkupContainer for " + this.node + "]";
   },
 
-  _attachTooltipIfNeeded: function() {
+  _prepareImagePreview: function() {
     if (this.node.tagName) {
       let tagName = this.node.tagName.toLowerCase();
-      let isImage = tagName === "img" &&
-        this.editor.getAttributeElement("src");
-      let isCanvas = tagName && tagName === "canvas";
+      let srcAttr = this.editor.getAttributeElement("src");
+      let isImage = tagName === "img" && srcAttr;
+      let isCanvas = tagName === "canvas";
 
       // Get the image data for later so that when the user actually hovers over
       // the element, the tooltip does contain the image
       if (isImage || isCanvas) {
-        this.tooltip = new Tooltip(this._inspector.panelDoc);
+        let def = promise.defer();
 
-        this.node.getImageData().then(data => {
+        this.tooltipData = {
+          target: isImage ? srcAttr : this.editor.tag,
+          data: def.promise
+        };
+
+        this.node.getImageData(IMAGE_PREVIEW_MAX_DIM).then(data => {
           if (data) {
-            data.string().then(str => {
-              this.tooltip.setImageContent(str);
+            data.data.string().then(str => {
+              // Resolving the data promise and, to always keep tooltipData.data
+              // as a promise, create a new one that resolves immediately
+              def.resolve(str, data.size);
+              this.tooltipData.data = promise.resolve(str, data.size);
             });
           }
         });
       }
+    }
+  },
 
-      // If it's an image, show the tooltip on the src attribute
-      if (isImage) {
-        this.tooltip.startTogglingOnHover(this.editor.getAttributeElement("src"));
-      }
-
-      // If it's a canvas, show it on the tag
-      if (isCanvas) {
-        this.tooltip.startTogglingOnHover(this.editor.tag);
-      }
+  _buildTooltipContent: function(target, tooltip) {
+    if (this.tooltipData && target === this.tooltipData.target) {
+      this.tooltipData.data.then((data, size) => {
+        tooltip.setImageContent(data, size);
+      });
+      return true;
     }
   },
 
@@ -1375,12 +1409,6 @@ MarkupContainer.prototype = {
 
     // Destroy my editor
     this.editor.destroy();
-
-    // Destroy the tooltip if any
-    if (this.tooltip) {
-      this.tooltip.destroy();
-      this.tooltip = null;
-    }
   }
 };
 

browser/devtools/markupview/test/browser_inspector_markup_765105_tooltip.js

@@ -88,14 +88,14 @@ function testImageTooltip(index) {
     target = container.editor.getAttributeElement("src");
   }
 
-  assertTooltipShownOn(container.tooltip, target, () => {
-    let images = container.tooltip.panel.getElementsByTagName("image");
+  assertTooltipShownOn(target, () => {
+    let images = markup.tooltip.panel.getElementsByTagName("image");
     is(images.length, 1, "Tooltip for [" + TEST_NODES[index] + "] contains an image");
     if (isImg) {
       compareImageData(node, images[0].src);
     }
 
-    container.tooltip.hide();
+    markup.tooltip.hide();
 
     testImageTooltip(index + 1);
   });
@@ -115,17 +115,17 @@ function compareImageData(img, imgData) {
   is(data, imgData, "Tooltip image has the right content");
 }
 
-function assertTooltipShownOn(tooltip, element, cb) {
+function assertTooltipShownOn(element, cb) {
   // If there is indeed a show-on-hover on element, the xul panel will be shown
-  tooltip.panel.addEventListener("popupshown", function shown() {
-    tooltip.panel.removeEventListener("popupshown", shown, true);
+  markup.tooltip.panel.addEventListener("popupshown", function shown() {
+    markup.tooltip.panel.removeEventListener("popupshown", shown, true);
 
     // Poll until the image gets loaded in the tooltip. This is required because
     // markup containers only load images in their associated tooltips when
     // the image data comes back from the server. However, this test is executed
     // synchronously as soon as "inspector-updated" is fired, which is before
     // the data for images is known.
-    let hasImage = () => tooltip.panel.getElementsByTagName("image").length;
+    let hasImage = () => markup.tooltip.panel.getElementsByTagName("image").length;
     let poll = setInterval(() => {
       if (hasImage()) {
         clearInterval(poll);
@@ -133,5 +133,5 @@ function assertTooltipShownOn(tooltip, element, cb) {
       }
     }, 200);
   }, true);
-  tooltip._showOnHover(element);
+  markup.tooltip._showOnHover(element);
 }

browser/devtools/shared/widgets/Tooltip.js

@@ -288,10 +288,21 @@ Tooltip.prototype = {
 
   /**
    * Fill the tooltip with an image, displayed over a tiled background useful
-   * for transparent images.
-   * Also adds the image dimension as a label at the bottom.
+   * for transparent images. Also adds the image dimension as a label at the
+   * bottom.
+   * @param {string} imageUrl
+   *        The url to load the image from
+   * @param {Object} options
+   *        The following options are supported:
+   *        - resized : whether or not the image identified by imageUrl has been
+   *        resized before this function was called.
+   *        - naturalWidth/naturalHeight : the original size of the image before
+   *        it was resized, if if was resized before this function was called.
+   *        If not provided, will be measured on the loaded image.
+   *        - maxDim : if the image should be resized before being shown, pass
+   *        a number here
    */
-  setImageContent: function(imageUrl, maxDim=400) {
+  setImageContent: function(imageUrl, options={}) {
     // Main container
     let vbox = this.doc.createElement("vbox");
     vbox.setAttribute("align", "center")
@@ -308,9 +319,9 @@ Tooltip.prototype = {
     // Display the image
     let image = this.doc.createElement("image");
     image.setAttribute("src", imageUrl);
-    if (maxDim) {
-      image.style.maxWidth = maxDim + "px";
-      image.style.maxHeight = maxDim + "px";
+    if (options.maxDim) {
+      image.style.maxWidth = options.maxDim + "px";
+      image.style.maxHeight = options.maxDim + "px";
     }
     tiles.appendChild(image);
 
@@ -323,11 +334,9 @@ Tooltip.prototype = {
       imgObj.onload = null;
 
       // Display dimensions
-      label.textContent = imgObj.naturalWidth + " x " + imgObj.naturalHeight;
-      if (imgObj.naturalWidth > maxDim ||
-        imgObj.naturalHeight > maxDim) {
-        label.textContent += " *";
-      }
+      let w = options.naturalWidth || imgObj.naturalWidth;
+      let h = options.naturalHeight || imgObj.naturalHeight;
+      label.textContent = w + " x " + h;
     }
   },
 
@@ -338,7 +347,9 @@ Tooltip.prototype = {
   setCssBackgroundImageContent: function(cssBackground, sheetHref, maxDim=400) {
     let uri = getBackgroundImageUri(cssBackground, sheetHref);
     if (uri) {
-      this.setImageContent(uri, maxDim);
+      this.setImageContent(uri, {
+        maxDim: maxDim
+      });
     }
   },
 

toolkit/devtools/server/actors/inspector.js

@@ -110,6 +110,13 @@ function delayedResolve(value) {
   return deferred.promise;
 }
 
+types.addDictType("imageData", {
+  // The image data
+  data: "nullable:longstring",
+  // The original image dimensions
+  size: "json"
+});
+
 /**
  * We only send nodeValue up to a certain size by default.  This stuff
  * controls that size.
@@ -255,8 +262,12 @@ var NodeActor = protocol.ActorClass({
    * A null return value means the node isn't an image
    * An empty string return value means the node is an image but image data
    * could not be retrieved (missing/broken image).
+   *
+   * Accepts a maxDim request parameter to resize images that are larger. This
+   * is important as the resizing occurs server-side so that image-data being
+   * transfered in the longstring back to the client will be that much smaller
    */
-  getImageData: method(function() {
+  getImageData: method(function(maxDim) {
     let isImg = this.rawNode.tagName.toLowerCase() === "img";
     let isCanvas = this.rawNode.tagName.toLowerCase() === "canvas";
 
@@ -264,29 +275,44 @@ var NodeActor = protocol.ActorClass({
       return null;
     }
 
-    let imageData;
-    if (isImg) {
-      let canvas = this.rawNode.ownerDocument.createElement("canvas");
-      canvas.width = this.rawNode.naturalWidth;
-      canvas.height = this.rawNode.naturalHeight;
-      let ctx = canvas.getContext("2d");
-      try {
-        // This will fail if the image is missing
-        ctx.drawImage(this.rawNode, 0, 0);
-        imageData = canvas.toDataURL("image/png");
-      } catch (e) {
-        imageData = "";
-      }
-    } else if (isCanvas) {
-      imageData = this.rawNode.toDataURL("image/png");
+    // Get the image resize ratio if a maxDim was provided
+    let resizeRatio = 1;
+    let imgWidth = isImg ? this.rawNode.naturalWidth : this.rawNode.width;
+    let imgHeight = isImg ? this.rawNode.naturalHeight : this.rawNode.height;
+    let imgMax = Math.max(imgWidth, imgHeight);
+    if (maxDim && imgMax > maxDim) {
+      resizeRatio = maxDim / imgMax;
     }
 
-    return LongStringActor(this.conn, imageData);
-  }, {
-    request: {},
-    response: {
-      data: RetVal("nullable:longstring")
+    // Create a canvas to copy the rawNode into and get the imageData from
+    let canvas = this.rawNode.ownerDocument.createElement("canvas");
+    canvas.width = imgWidth * resizeRatio;
+    canvas.height = imgHeight * resizeRatio;
+    let ctx = canvas.getContext("2d");
+
+    // Copy the rawNode image or canvas in the new canvas and extract data
+    let imageData;
+    // This may fail if the image is missing
+    try {
+      ctx.drawImage(this.rawNode, 0, 0, canvas.width, canvas.height);
+      imageData = canvas.toDataURL("image/png");
+    } catch (e) {
+      imageData = "";
     }
+
+    return {
+      data: LongStringActor(this.conn, imageData),
+      size: {
+        naturalWidth: imgWidth,
+        naturalHeight: imgHeight,
+        width: canvas.width,
+        height: canvas.height,
+        resized: resizeRatio !== 1
+      }
+    }
+  }, {
+    request: {maxDim: Arg(0, "nullable:number")},
+    response: RetVal("imageData")
   }),
 
   /**

toolkit/devtools/server/tests/mochitest/chrome.ini

@@ -5,6 +5,9 @@ support-files =
   inspector-styles-data.html
   inspector-traversal-data.html
   nonchrome_unsafeDereference.html
+  inspector_getImageData.html
+  large-image.jpg
+  small-image.gif
 
 [test_connection-manager.html]
 [test_device.html]
@@ -30,3 +33,4 @@ support-files =
 [test_styles-svg.html]
 [test_unsafeDereference.html]
 [test_evalInGlobal-outerized_this.html]
+[test_inspector_getImageData.html]

toolkit/devtools/server/tests/mochitest/inspector_getImageData.html

@@ -0,0 +1,19 @@
+<html>
+<head>
+<body>
+  <img class="big-horizontal" src="large-image.jpg" style="width:500px;" />
+  <canvas class="big-vertical" style="width:500px;"></canvas>
+  <img class="small" src="small-image.gif"></img>
+  <script>
+    window.onload = () => {
+      var canvas = document.querySelector("canvas"), ctx = canvas.getContext("2d");
+      canvas.width = 1000;
+      canvas.height = 2000;
+      ctx.fillStyle = "red";
+      ctx.fillRect(0, 0, 1000, 2000);
+
+      window.opener.postMessage('ready', '*')
+    }
+  </script>
+</body>
+</html>

toolkit/devtools/server/tests/mochitest/test_inspector_getImageData.html

@@ -0,0 +1,123 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=932937
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test for Bug 932937</title>
+
+  <script type="application/javascript" src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css">
+  <script type="application/javascript;version=1.8" src="inspector-helpers.js"></script>
+  <script type="application/javascript;version=1.8">
+Components.utils.import("resource://gre/modules/devtools/Loader.jsm");
+
+const promise = devtools.require("sdk/core/promise");
+const inspector = devtools.require("devtools/server/actors/inspector");
+
+window.onload = function() {
+  SimpleTest.waitForExplicitFinish();
+  runNextTest();
+}
+
+var gWalker = null;
+
+addTest(function setup() {
+  let url = document.getElementById("inspectorContent").href;
+  attachURL(url, function(err, client, tab, doc) {
+    let {InspectorFront} = devtools.require("devtools/server/actors/inspector");
+    let inspector = InspectorFront(client, tab);
+
+    promiseDone(inspector.getWalker().then(walker => {
+      gWalker = walker;
+    }).then(runNextTest));
+  });
+});
+
+addTest(function testLargeImage() {
+  // Select the image node from the test page
+  gWalker.querySelector(gWalker.rootNode, ".big-horizontal").then(img => {
+    ok(img, "Image node found in the test page");
+    ok(img.getImageData, "Image node has the getImageData function");
+
+    img.getImageData(100).then(imageData => {
+      ok(imageData.data, "Image data actor was sent back");
+      ok(imageData.size, "Image size info was sent back too");
+      is(imageData.size.naturalWidth, 5333, "Natural width of the image correct");
+      is(imageData.size.naturalHeight, 3000, "Natural width of the image correct");
+      is(imageData.size.width, 100, "Resized image width correct");
+      is(imageData.size.height, 56, "Resized image height correct");
+      ok(imageData.size.resized, "Image was resized");
+
+      imageData.data.string().then(str => {
+        ok(str, "We have an image data string!");
+        runNextTest();
+      });
+    });
+  });
+});
+
+addTest(function testLargeCanvas() {
+  // Select the canvas node from the test page
+  gWalker.querySelector(gWalker.rootNode, ".big-vertical").then(canvas => {
+    ok(canvas, "Image node found in the test page");
+    ok(canvas.getImageData, "Image node has the getImageData function");
+
+    canvas.getImageData(350).then(imageData => {
+      ok(imageData.data, "Image data actor was sent back");
+      ok(imageData.size, "Image size info was sent back too");
+      is(imageData.size.naturalWidth, 1000, "Natural width of the image correct");
+      is(imageData.size.naturalHeight, 2000, "Natural width of the image correct");
+      is(imageData.size.width, 175, "Resized image width correct");
+      is(imageData.size.height, 350, "Resized image height correct");
+      ok(imageData.size.resized, "Image was resized");
+
+      imageData.data.string().then(str => {
+        ok(str, "We have an image data string!");
+        runNextTest();
+      });
+    });
+  });
+});
+
+addTest(function testSmallImage() {
+  // Select the small image node from the test page
+  gWalker.querySelector(gWalker.rootNode, ".small").then(img => {
+    ok(img, "Image node found in the test page");
+    ok(img.getImageData, "Image node has the getImageData function");
+
+    img.getImageData().then(imageData => {
+      ok(imageData.data, "Image data actor was sent back");
+      ok(imageData.size, "Image size info was sent back too");
+      is(imageData.size.naturalWidth, 245, "Natural width of the image correct");
+      is(imageData.size.naturalHeight, 240, "Natural width of the image correct");
+      is(imageData.size.width, 245, "Resized image width correct");
+      is(imageData.size.height, 240, "Resized image height correct");
+      ok(!imageData.size.resized, "Image was NOT resized");
+
+      imageData.data.string().then(str => {
+        ok(str, "We have an image data string!");
+        runNextTest();
+      });
+    });
+  });
+});
+
+addTest(function cleanup() {
+  delete gWalker;
+  runNextTest();
+});
+  </script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=932937">Mozilla Bug 932937</a>
+<a id="inspectorContent" target="_blank" href="inspector_getImageData.html">Test Document</a>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+</pre>
+</body>
+</html>