merge m-c to fx-team

browser/base/content/browser-thumbnails.js

@@ -36,6 +36,7 @@ let gBrowserThumbnails = {
         return;
     } catch (e) {}
 
+    PageThumbs.addExpirationFilter(this);
     gBrowser.addTabsProgressListener(this);
     Services.prefs.addObserver(this.PREF_DISK_CACHE_SSL, this, false);
 
@@ -50,6 +51,7 @@ let gBrowserThumbnails = {
   },
 
   uninit: function Thumbnails_uninit() {
+    PageThumbs.removeExpirationFilter(this);
     gBrowser.removeTabsProgressListener(this);
     Services.prefs.removeObserver(this.PREF_DISK_CACHE_SSL, this);
 
@@ -80,6 +82,11 @@ let gBrowserThumbnails = {
       Services.prefs.getBoolPref(this.PREF_DISK_CACHE_SSL);
   },
 
+  filterForThumbnailExpiration:
+  function Thumbnails_filterForThumbnailExpiration(aCallback) {
+    aCallback([browser.currentURI.spec for (browser of gBrowser.browsers)]);
+  },
+
   /**
    * State change progress listener for all tabs.
    */

browser/components/nsBrowserGlue.js

@@ -32,6 +32,9 @@ XPCOMUtils.defineLazyModuleGetter(this, "webappsUI",
 XPCOMUtils.defineLazyModuleGetter(this, "PageThumbs",
                                   "resource:///modules/PageThumbs.jsm");
 
+XPCOMUtils.defineLazyModuleGetter(this, "NewTabUtils",
+                                  "resource:///modules/NewTabUtils.jsm");
+
 XPCOMUtils.defineLazyModuleGetter(this, "PdfJs",
                                   "resource://pdf.js/PdfJs.jsm");
 
@@ -338,6 +341,7 @@ BrowserGlue.prototype = {
     webappsUI.init();
 
     PageThumbs.init();
+    NewTabUtils.init();
 
     SignInToWebsiteUX.init();
 

browser/components/thumbnails/Makefile.in

@@ -15,6 +15,7 @@ EXTRA_COMPONENTS = \
 	$(NULL)
 
 EXTRA_JS_MODULES = \
+	PageThumbsWorker.js \
 	PageThumbs.jsm \
 	$(NULL)
 

browser/components/thumbnails/PageThumbs.jsm

@@ -12,7 +12,10 @@ const Ci = Components.interfaces;
 
 const HTML_NAMESPACE = "http://www.w3.org/1999/xhtml";
 const PREF_STORAGE_VERSION = "browser.pagethumbnails.storage_version";
-const LATEST_STORAGE_VERSION = 1;
+const LATEST_STORAGE_VERSION = 2;
+
+const EXPIRATION_MIN_CHUNK_SIZE = 50;
+const EXPIRATION_INTERVAL_SECS = 3600;
 
 /**
  * Name of the directory in the profile that contains the thumbnails.
@@ -38,6 +41,9 @@ XPCOMUtils.defineLazyModuleGetter(this, "FileUtils",
 XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
   "resource://gre/modules/PlacesUtils.jsm");
 
+XPCOMUtils.defineLazyServiceGetter(this, "gUpdateTimerManager",
+  "@mozilla.org/updates/timer-manager;1", "nsIUpdateTimerManager");
+
 XPCOMUtils.defineLazyGetter(this, "gCryptoHash", function () {
   return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash);
 });
@@ -84,6 +90,7 @@ let PageThumbs = {
 
       // Migrate the underlying storage, if needed.
       PageThumbsStorageMigrator.migrate();
+      PageThumbsExpiration.init();
     }
   },
 
@@ -198,6 +205,14 @@ let PageThumbs = {
     });
   },
 
+  addExpirationFilter: function PageThumbs_addExpirationFilter(aFilter) {
+    PageThumbsExpiration.addFilter(aFilter);
+  },
+
+  removeExpirationFilter: function PageThumbs_removeExpirationFilter(aFilter) {
+    PageThumbsExpiration.removeFilter(aFilter);
+  },
+
   /**
    * Determines the crop size for a given content window.
    * @param aWindow The content window.
@@ -264,16 +279,23 @@ let PageThumbs = {
 };
 
 let PageThumbsStorage = {
-  getFileForURL: function Storage_getFileForURL(aURL, aOptions) {
+  getDirectory: function Storage_getDirectory(aCreate = true) {
+    return FileUtils.getDir("ProfLD", [THUMBNAIL_DIRECTORY], aCreate);
+  },
+
+  getLeafNameForURL: function Storage_getLeafNameForURL(aURL) {
     let hash = this._calculateMD5Hash(aURL);
-    let parts = [THUMBNAIL_DIRECTORY, hash[0], hash[1]];
-    let file = FileUtils.getDir("ProfLD", parts, aOptions && aOptions.createPath);
-    file.append(hash.slice(2) + ".png");
+    return hash + ".png";
+  },
+
+  getFileForURL: function Storage_getFileForURL(aURL) {
+    let file = this.getDirectory();
+    file.append(this.getLeafNameForURL(aURL));
     return file;
   },
 
   write: function Storage_write(aURL, aDataStream, aCallback) {
-    let file = this.getFileForURL(aURL, {createPath: true});
+    let file = this.getFileForURL(aURL);
     let fos = FileUtils.openSafeFileOutputStream(file);
 
     NetUtil.asyncCopy(aDataStream, fos, function (aResult) {
@@ -294,18 +316,17 @@ let PageThumbsStorage = {
   },
 
   remove: function Storage_remove(aURL) {
-    try {
-      this.getFileForURL(aURL).remove(false);
-    } catch (e) {
-      /* The file might not exist or we're not permitted to remove it. */
-    }
+    let file = this.getFileForURL(aURL);
+    PageThumbsWorker.postMessage({type: "removeFile", path: file.path});
   },
 
   wipe: function Storage_wipe() {
+    let dir = this.getDirectory(false);
+    dir.followLinks = false;
     try {
-      FileUtils.getDir("ProfLD", [THUMBNAIL_DIRECTORY]).remove(true);
+      dir.remove(true);
     } catch (e) {
-      /* The file might not exist or we're not permitted to remove it. */
+      /* The directory might not exist or we're not permitted to remove it. */
     }
   },
 
@@ -323,8 +344,7 @@ let PageThumbsStorage = {
     for (let i = 0; i < aData.length; i++)
       hex += ("0" + aData.charCodeAt(i).toString(16)).slice(-2);
     return hex;
-  },
-
+  }
 };
 
 let PageThumbsStorageMigrator = {
@@ -344,8 +364,12 @@ let PageThumbsStorageMigrator = {
   migrate: function Migrator_migrate() {
     let version = this.currentVersion;
 
-    if (version < 1)
+    if (version < 1) {
       this.removeThumbnailsFromRoamingProfile();
+    }
+    if (version < 2) {
+      this.renameThumbnailsFolder();
+    }
 
     this.currentVersion = LATEST_STORAGE_VERSION;
   },
@@ -356,12 +380,137 @@ let PageThumbsStorageMigrator = {
     let roaming = FileUtils.getDir("ProfD", [THUMBNAIL_DIRECTORY]);
 
     if (!roaming.equals(local) && roaming.exists()) {
+      roaming.followLinks = false;
       try {
         roaming.remove(true);
       } catch (e) {
         // The directory might not exist or we're not permitted to remove it.
       }
     }
+  },
+
+  renameThumbnailsFolder: function Migrator_renameThumbnailsFolder() {
+    let dir = FileUtils.getDir("ProfLD", [THUMBNAIL_DIRECTORY]);
+    try {
+      dir.moveTo(null, dir.leafName + "-old");
+    } catch (e) {
+      // The directory might not exist or we're not permitted to rename it.
+    }
+  }
+};
+
+let PageThumbsExpiration = {
+  _filters: [],
+
+  init: function Expiration_init() {
+    gUpdateTimerManager.registerTimer("browser-cleanup-thumbnails", this,
+                                      EXPIRATION_INTERVAL_SECS);
+  },
+
+  addFilter: function Expiration_addFilter(aFilter) {
+    this._filters.push(aFilter);
+  },
+
+  removeFilter: function Expiration_removeFilter(aFilter) {
+    let index = this._filters.indexOf(aFilter);
+    if (index > -1)
+      this._filters.splice(index, 1);
+  },
+
+  notify: function Expiration_notify(aTimer) {
+    let urls = [];
+    let filtersToWaitFor = this._filters.length;
+
+    let expire = function expire() {
+      this.expireThumbnails(urls);
+    }.bind(this);
+
+    // No registered filters.
+    if (!filtersToWaitFor) {
+      expire();
+      return;
+    }
+
+    function filterCallback(aURLs) {
+      urls = urls.concat(aURLs);
+      if (--filtersToWaitFor == 0)
+        expire();
+    }
+
+    for (let filter of this._filters) {
+      if (typeof filter == "function")
+        filter(filterCallback)
+      else
+        filter.filterForThumbnailExpiration(filterCallback);
+    }
+  },
+
+  expireThumbnails: function Expiration_expireThumbnails(aURLsToKeep) {
+    let keep = {};
+
+    // Transform all these URLs into file names.
+    for (let url of aURLsToKeep) {
+      keep[PageThumbsStorage.getLeafNameForURL(url)] = true;
+    }
+
+    let numFilesRemoved = 0;
+    let dir = PageThumbsStorage.getDirectory().path;
+    let msg = {type: "getFilesInDirectory", path: dir};
+
+    PageThumbsWorker.postMessage(msg, function (aData) {
+      let files = [file for (file of aData.result) if (!(file in keep))];
+      let maxFilesToRemove = Math.max(EXPIRATION_MIN_CHUNK_SIZE,
+                                      Math.round(files.length / 2));
+
+      let fileNames = files.slice(0, maxFilesToRemove);
+      let filePaths = [dir + "/" + fileName for (fileName of fileNames)];
+      PageThumbsWorker.postMessage({type: "removeFiles", paths: filePaths});
+    });
+  }
+};
+
+/**
+ * Interface to a dedicated thread handling I/O
+ */
+let PageThumbsWorker = {
+  /**
+   * A (fifo) queue of callbacks registered for execution
+   * upon completion of calls to the worker.
+   */
+  _callbacks: [],
+
+  /**
+   * Get the worker, spawning it if necessary.
+   * Code of the worker is in companion file PageThumbsWorker.js
+   */
+  get _worker() {
+    delete this._worker;
+    this._worker = new ChromeWorker("resource://gre/modules/PageThumbsWorker.js");
+    this._worker.addEventListener("message", this);
+    return this._worker;
+  },
+
+  /**
+   * Post a message to the dedicated thread, registering a callback
+   * to be executed once the reply has been received.
+   *
+   * See PageThumbsWorker.js for the format of messages and replies.
+   *
+   * @param {*} message A JSON message.
+   * @param {Function=} callback An optional callback.
+   */
+  postMessage: function Worker_postMessage(message, callback) {
+    this._callbacks.push(callback);
+    this._worker.postMessage(message);
+  },
+
+  /**
+   * Handle a message from the dedicated thread.
+   */
+  handleEvent: function Worker_handleEvent(aEvent) {
+    let callback = this._callbacks.shift();
+    if (callback)
+      callback(aEvent.data);
   }
 };
 

browser/components/thumbnails/PageThumbsWorker.js

@@ -0,0 +1,64 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * A worker dedicated for the I/O component of PageThumbs storage.
+ *
+ * Do not rely on the API of this worker. In a future version, it might be
+ * fully replaced by a OS.File global I/O worker.
+ */
+
+"use strict";
+
+importScripts("resource://gre/modules/osfile.jsm");
+
+let PageThumbsWorker = {
+  handleMessage: function Worker_handleMessage(aEvent) {
+    let msg = aEvent.data;
+    let data = {result: null, data: null};
+
+    switch (msg.type) {
+      case "removeFiles":
+        data.result = this.removeFiles(msg);
+        break;
+      case "getFilesInDirectory":
+        data.result = this.getFilesInDirectory(msg);
+        break;
+      default:
+        data.result = false;
+        data.detail = "message not understood";
+        break;
+    }
+
+    self.postMessage(data);
+  },
+
+  getFilesInDirectory: function Worker_getFilesInDirectory(msg) {
+    let iter = new OS.File.DirectoryIterator(msg.path);
+    let entries = [];
+
+    for (let entry in iter) {
+      if (!entry.isDir && !entry.isSymLink) {
+        entries.push(entry.name);
+      }
+    }
+
+    iter.close();
+    return entries;
+  },
+
+  removeFiles: function Worker_removeFiles(msg) {
+    for (let file of msg.paths) {
+      try {
+        OS.File.remove(file);
+      } catch (e) {
+        // We couldn't remove the file for some reason.
+        // Let's just continue with the next one.
+      }
+    }
+    return true;
+  }
+};
+
+self.onmessage = PageThumbsWorker.handleMessage.bind(PageThumbsWorker);

browser/components/thumbnails/test/Makefile.in

@@ -17,7 +17,6 @@ _BROWSER_FILES = \
 	browser_thumbnails_redirect.js \
 	browser_thumbnails_storage.js \
 	browser_thumbnails_bug726727.js \
-	browser_thumbnails_bug753755.js \
 	head.js \
 	background_red.html \
 	background_red_redirect.sjs \

browser/components/thumbnails/test/browser_thumbnails_bug753755.js

@@ -1,15 +0,0 @@
-/* Any copyright is dedicated to the Public Domain.
-   http://creativecommons.org/publicdomain/zero/1.0/ */
-
-/**
- * Make sure that PageThumbsStorage.getFileForURL(url) doesn't implicitly
- * create the file's parent path.
- */
-function runTests() {
-  let url = "http://non.existant.url/";
-  let file = PageThumbsStorage.getFileForURL(url);
-  ok(!file.exists() && !file.parent.exists(), "file and path don't exist");
-
-  let file = PageThumbsStorage.getFileForURL(url, {createPath: true});
-  ok(!file.exists() && file.parent.exists(), "path exists, file doesn't");
-}

browser/modules/NewTabUtils.jsm

@@ -16,6 +16,9 @@ Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
   "resource://gre/modules/PlacesUtils.jsm");
 
+XPCOMUtils.defineLazyModuleGetter(this, "PageThumbs",
+  "resource:///modules/PageThumbs.jsm");
+
 XPCOMUtils.defineLazyGetter(this, "gPrincipal", function () {
   let uri = Services.io.newURI("about:newtab", null, null);
   return Services.scriptSecurityManager.getNoAppCodebasePrincipal(uri);
@@ -611,8 +614,6 @@ let Telemetry = {
   }
 };
 
-Telemetry.init();
-
 /**
  * Singleton that checks if a given link should be displayed on about:newtab
  * or if we should rather not do it for security reasons. URIs that inherit
@@ -644,10 +645,46 @@ let LinkChecker = {
   }
 };
 
+let ExpirationFilter = {
+  init: function ExpirationFilter_init() {
+    PageThumbs.addExpirationFilter(this);
+  },
+
+  filterForThumbnailExpiration:
+  function ExpirationFilter_filterForThumbnailExpiration(aCallback) {
+    if (!AllPages.enabled) {
+      aCallback([]);
+      return;
+    }
+
+    Links.populateCache(function () {
+      let urls = [];
+
+      // Add all URLs to the list that we want to keep thumbnails for.
+      for (let link of Links.getLinks().slice(0, 25)) {
+        if (link && link.url)
+          urls.push(link.url);
+      }
+
+      aCallback(urls);
+    });
+  }
+};
+
 /**
  * Singleton that provides the public API of this JSM.
  */
 let NewTabUtils = {
+  _initialized: false,
+
+  init: function NewTabUtils_init() {
+    if (!this._initialized) {
+      this._initialized = true;
+      ExpirationFilter.init();
+      Telemetry.init();
+    }
+  },
+
   /**
    * Restores all sites that have been removed from the grid.
    */

toolkit/components/places/nsPlacesAutoComplete.js

@@ -130,6 +130,19 @@ function initTempTable(aDatabase)
  * @return the modified uri.
  */
 function fixupSearchText(aURIString)
+{
+  let uri = stripPrefix(aURIString);
+  return gTextURIService.unEscapeURIForUI("UTF-8", uri);
+}
+
+/**
+ * Strip prefixes from the URI that we don't care about for searching.
+ *
+ * @param aURIString
+ *        The text to modify.
+ * @return the modified uri.
+ */
+function stripPrefix(aURIString)
 {
   let uri = aURIString;
 
@@ -146,8 +159,7 @@ function fixupSearchText(aURIString)
   if (uri.indexOf("www.") == 0) {
     uri = uri.slice(4);
   }
-
-  return gTextURIService.unEscapeURIForUI("UTF-8", uri);
+  return uri;
 }
 
 /**
@@ -1487,7 +1499,7 @@ urlInlineComplete.prototype = {
     let value = row.getResultByIndex(0);
     let url = fixupSearchText(value);
 
-    let prefix = value.slice(0, value.length - url.length);
+    let prefix = value.slice(0, value.length - stripPrefix(value).length);
 
     // We must complete the URL up to the next separator (which is /, ? or #).
     let separatorIndex = url.slice(this._currentSearchString.length)

toolkit/components/places/tests/inline/test_trimming.js

@@ -132,3 +132,13 @@ add_autocomplete_test([
     addBookmark({ url: "http://mozilla.co/" });
   },
 ]);
+
+add_autocomplete_test([
+  "Searching for URL with characters that are normally escaped",
+  "https://www.mozilla.org/啊-test",
+  { autoFilled: "https://www.mozilla.org/啊-test", completed: "https://www.mozilla.org/啊-test" },
+  function () {
+    addVisits({ uri: NetUtil.newURI("https://www.mozilla.org/啊-test"),
+                transition: TRANSITION_TYPED });
+  },
+]);

toolkit/devtools/debugger/server/dbg-script-actors.js

@@ -918,15 +918,16 @@ ThreadActor.prototype = {
    *        A Debugger.Object instance whose referent is the global object.
    */
   onNewScript: function TA_onNewScript(aScript, aGlobal) {
-    this._addScript(aScript);
-    // Notify the client.
-    this.conn.send({
-      from: this.actorID,
-      type: "newScript",
-      url: aScript.url,
-      startLine: aScript.startLine,
-      lineCount: aScript.lineCount
-    });
+    if (this._addScript(aScript)) {
+      // Notify the client.
+      this.conn.send({
+        from: this.actorID,
+        type: "newScript",
+        url: aScript.url,
+        startLine: aScript.startLine,
+        lineCount: aScript.lineCount
+      });
+    }
   },
 
   /**
@@ -934,15 +935,16 @@ ThreadActor.prototype = {
    *
    * @param aScript Debugger.Script
    *        The source script that will be stored.
+   * @returns true, if the script was added, false otherwise.
    */
   _addScript: function TA__addScript(aScript) {
     // Ignore XBL bindings for content debugging.
     if (aScript.url.indexOf("chrome://") == 0) {
-      return;
+      return false;
     }
     // Ignore about:* pages for content debugging.
     if (aScript.url.indexOf("about:") == 0) {
-      return;
+      return false;
     }
     // Use a sparse array for storing the scripts for each URL in order to
     // optimize retrieval.
@@ -965,6 +967,7 @@ ThreadActor.prototype = {
         }
       }
     }
+    return true;
   }
 
 };