Bug 1213078 - PlacesProvider.jsm module for remote new tab page r=emtwo

browser/components/newtab/PlacesProvider.jsm

@@ -0,0 +1,271 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* global XPCOMUtils, Services, BinarySearch, PlacesUtils, gPrincipal, EventEmitter */
+/* global gLinks */
+/* exported PlacesProvider */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["PlacesProvider"];
+
+const {interfaces: Ci, utils: Cu} = Components;
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "BinarySearch",
+  "resource://gre/modules/BinarySearch.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
+  "resource://gre/modules/PlacesUtils.jsm");
+
+XPCOMUtils.defineLazyGetter(this, "EventEmitter", function() {
+  const {EventEmitter} = Cu.import("resource://gre/modules/devtools/event-emitter.js", {});
+  return EventEmitter;
+});
+
+XPCOMUtils.defineLazyGetter(this, "gPrincipal", function() {
+  let uri = Services.io.newURI("about:newtab", null, null);
+  return Services.scriptSecurityManager.getNoAppCodebasePrincipal(uri);
+});
+
+// The maximum number of results PlacesProvider retrieves from history.
+const HISTORY_RESULTS_LIMIT = 100;
+
+/**
+ * Singleton that checks if a given link should be displayed on about:newtab
+ * or if we should rather not do it for security reasons. URIs that inherit
+ * their caller's principal will be filtered.
+ */
+let LinkChecker = {
+  _cache: new Map(),
+
+  get flags() {
+    return Ci.nsIScriptSecurityManager.DISALLOW_INHERIT_PRINCIPAL |
+           Ci.nsIScriptSecurityManager.DONT_REPORT_ERRORS;
+  },
+
+  checkLoadURI: function LinkChecker_checkLoadURI(aURI) {
+    if (!this._cache.has(aURI)) {
+      this._cache.set(aURI, this._doCheckLoadURI(aURI));
+    }
+
+    return this._cache.get(aURI);
+  },
+
+  _doCheckLoadURI: function LinkChecker_doCheckLoadURI(aURI) {
+    let result = false;
+    try {
+      Services.scriptSecurityManager.
+        checkLoadURIStrWithPrincipal(gPrincipal, aURI, this.flags);
+      result = true;
+    } catch (e) {
+      // We got a weird URI or one that would inherit the caller's principal.
+      Cu.reportError(e);
+    }
+    return result;
+  }
+};
+
+/**
+ * Singleton that provides utility functions for links.
+ * A link is a plain object that looks like this:
+ *
+ * {
+ *   url: "http://www.mozilla.org/",
+ *   title: "Mozilla",
+ *   frecency: 1337,
+ *   lastVisitDate: 1394678824766431,
+ * }
+ */
+const LinkUtils = {
+  _sortProperties: [
+    "frecency",
+    "lastVisitDate",
+    "url",
+  ],
+
+  /**
+   * Compares two links.
+   *
+   * @param {String} aLink1 The first link.
+   * @param {String} aLink2 The second link.
+   * @return {Number} A negative number if aLink1 is ordered before aLink2, zero if
+   *         aLink1 and aLink2 have the same ordering, or a positive number if
+   *         aLink1 is ordered after aLink2.
+   *         Order is ascending.
+   */
+  compareLinks: function LinkUtils_compareLinks(aLink1, aLink2) {
+    for (let prop of LinkUtils._sortProperties) {
+      if (!aLink1.hasOwnProperty(prop) || !aLink2.hasOwnProperty(prop)) {
+        throw new Error("Comparable link missing required property: " + prop);
+      }
+    }
+    return aLink2.frecency - aLink1.frecency ||
+           aLink2.lastVisitDate - aLink1.lastVisitDate ||
+           aLink1.url.localeCompare(aLink2.url);
+  },
+};
+
+/* Queries history to retrieve the most visited sites. Emits events when the
+ * history changes.
+ * Implements the EventEmitter interface.
+ */
+let Links = function Links() {
+  EventEmitter.decorate(this);
+};
+
+Links.prototype = {
+  /**
+   * Set this to change the maximum number of links the provider will provide.
+   */
+  get maxNumLinks() {
+    // getter, so it can't be replaced dynamically
+    return HISTORY_RESULTS_LIMIT;
+  },
+
+  /**
+   * A set of functions called by @mozilla.org/browser/nav-historyservice
+   * All history events are emitted from this object.
+   */
+  historyObserver: {
+    onDeleteURI: function historyObserver_onDeleteURI(aURI) {
+      // let observers remove sensetive data associated with deleted visit
+      gLinks.emit("deleteURI", {
+        url: aURI.spec,
+      });
+    },
+
+    onClearHistory: function historyObserver_onClearHistory() {
+      gLinks.emit("clearHistory");
+    },
+
+    onFrecencyChanged: function historyObserver_onFrecencyChanged(aURI,
+                           aNewFrecency, aGUID, aHidden, aLastVisitDate) { // jshint ignore:line
+      // The implementation of the query in getLinks excludes hidden and
+      // unvisited pages, so it's important to exclude them here, too.
+      if (!aHidden && aLastVisitDate) {
+        gLinks.emit("linkChanged", {
+          url: aURI.spec,
+          frecency: aNewFrecency,
+          lastVisitDate: aLastVisitDate,
+          type: "history",
+        });
+      }
+    },
+
+    onManyFrecenciesChanged: function historyObserver_onManyFrecenciesChanged() {
+      // Called when frecencies are invalidated and also when clearHistory is called
+      // See toolkit/components/places/tests/unit/test_frecency_observers.js
+      gLinks.emit("manyLinksChanged");
+    },
+
+    onTitleChanged: function historyObserver_onTitleChanged(aURI, aNewTitle) {
+      gLinks.emit("linkChanged", {
+        url: aURI.spec,
+        title: aNewTitle
+      });
+    },
+
+    QueryInterface: XPCOMUtils.generateQI([Ci.nsINavHistoryObserver,
+                        Ci.nsISupportsWeakReference])
+  },
+
+  /**
+   * Must be called before the provider is used.
+   * Makes it easy to disable under pref
+   */
+  init: function PlacesProvider_init() {
+    PlacesUtils.history.addObserver(this.historyObserver, true);
+  },
+
+  /**
+   * Must be called before the provider is unloaded.
+   */
+  destroy: function PlacesProvider_destroy() {
+    PlacesUtils.history.removeObserver(this.historyObserver);
+  },
+
+  /**
+   * Gets the current set of links delivered by this provider.
+   *
+   * @returns {Promise} Returns a promise with the array of links as payload.
+   */
+  getLinks: function PlacesProvider_getLinks() {
+    let getLinksPromise = new Promise((resolve, reject) => {
+      let options = PlacesUtils.history.getNewQueryOptions();
+      options.maxResults = this.maxNumLinks;
+
+      // Sort by frecency, descending.
+      options.sortingMode = Ci.nsINavHistoryQueryOptions
+        .SORT_BY_FRECENCY_DESCENDING;
+
+      let links = [];
+
+      let queryHandlers = {
+        handleResult: function(aResultSet) {
+          for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
+            let url = row.getResultByIndex(1);
+            if (LinkChecker.checkLoadURI(url)) {
+              let link = {
+                url: url,
+                title: row.getResultByIndex(2),
+                frecency: row.getResultByIndex(12),
+                lastVisitDate: row.getResultByIndex(5),
+                type: "history",
+              };
+              links.push(link);
+            }
+          }
+        },
+
+        handleError: function(aError) {
+          reject(aError);
+        },
+
+        handleCompletion: function(aReason) { // jshint ignore:line
+          // The Places query breaks ties in frecency by place ID descending, but
+          // that's different from how Links.compareLinks breaks ties, because
+          // compareLinks doesn't have access to place IDs.  It's very important
+          // that the initial list of links is sorted in the same order imposed by
+          // compareLinks, because Links uses compareLinks to perform binary
+          // searches on the list.  So, ensure the list is so ordered.
+          let i = 1;
+          let outOfOrder = [];
+          while (i < links.length) {
+            if (LinkUtils.compareLinks(links[i - 1], links[i]) > 0) {
+              outOfOrder.push(links.splice(i, 1)[0]);
+            } else {
+              i++;
+            }
+          }
+          for (let link of outOfOrder) {
+            i = BinarySearch.insertionIndexOf(LinkUtils.compareLinks, links, link);
+            links.splice(i, 0, link);
+          }
+
+          resolve(links);
+        }
+      };
+
+      // Execute the query.
+      let query = PlacesUtils.history.getNewQuery();
+      let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase);
+      db.asyncExecuteLegacyQueries([query], 1, options, queryHandlers);
+    });
+
+    return getLinksPromise;
+  }
+};
+
+/**
+ * Singleton that serves as the default link provider for the grid.
+ */
+const gLinks = new Links(); // jshint ignore:line
+
+let PlacesProvider = {
+  LinkChecker: LinkChecker,
+  LinkUtils: LinkUtils,
+  links: gLinks,
+};

browser/components/newtab/moz.build

@@ -12,6 +12,7 @@ XPCSHELL_TESTS_MANIFESTS += [
 
 EXTRA_JS_MODULES += [
     'NewTabURL.jsm',
+    'PlacesProvider.jsm',
     'RemoteAboutNewTab.jsm',
     'RemoteDirectoryLinksProvider.jsm',
     'RemoteNewTabLocation.jsm',

browser/components/newtab/tests/xpcshell/test_PlacesProvider.js

@@ -0,0 +1,304 @@
+"use strict";
+
+/* global XPCOMUtils, PlacesUtils, PlacesTestUtils, PlacesProvider, NetUtil */
+/* global do_get_profile, do_register_cleanup, run_next_test, add_task */
+/* global equal, ok */
+/* exported run_test */
+
+const {
+  utils: Cu,
+  interfaces: Ci,
+} = Components;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "PlacesProvider",
+    "resource:///modules/PlacesProvider.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
+    "resource://gre/modules/PlacesUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "PlacesTestUtils",
+    "resource://testing-common/PlacesTestUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "NetUtil",
+    "resource://gre/modules/NetUtil.jsm");
+
+// ensure a profile exists
+do_get_profile();
+
+function run_test() {
+  run_next_test();
+}
+
+/** Test LinkChecker **/
+
+add_task(function test_LinkChecker_securityCheck() {
+
+  let urls = [
+    {url: "file://home/file/image.png", expected: false},
+    {url: "resource:///modules/PlacesProvider.jsm", expected: false},
+    {url: "javascript:alert('hello')", expected: false}, // jshint ignore:line
+    {url: "data:image/png;base64,XXX", expected: false},
+    {url: "about:newtab", expected: true},
+    {url: "https://example.com", expected: true},
+    {url: "ftp://example.com", expected: true},
+  ];
+  for (let {url, expected} of urls) {
+    let observed = PlacesProvider.LinkChecker.checkLoadURI(url);
+    equal(observed , expected, `can load "${url}"?`);
+  }
+});
+
+/** Test LinkUtils **/
+
+add_task(function test_LinkUtils_compareLinks() {
+
+  let fixtures = {
+    firstOlder: {
+      url: "http://www.mozilla.org/firstolder",
+      title: "Mozilla",
+      frecency: 1337,
+      lastVisitDate: 1394678824766431,
+    },
+    older: {
+      url: "http://www.mozilla.org/older",
+      title: "Mozilla",
+      frecency: 1337,
+      lastVisitDate: 1394678824766431,
+    },
+    newer: {
+      url: "http://www.mozilla.org/newer",
+      title: "Mozilla",
+      frecency: 1337,
+      lastVisitDate: 1494678824766431,
+    },
+    moreFrecent: {
+      url: "http://www.mozilla.org/moreFrecent",
+      title: "Mozilla",
+      frecency: 1337357,
+      lastVisitDate: 1394678824766431,
+    }
+  };
+
+  let links = [
+    // tests string ordering, f is before o
+    {link1: fixtures.firstOlder, link2: fixtures.older, expected: false},
+
+    // test identity
+    {link1: fixtures.older, link2: fixtures.older, expected: false},
+
+    // test ordering by date
+    {link1: fixtures.older, link2: fixtures.newer, expected: true},
+    {link1: fixtures.newer, link2: fixtures.older, expected: false},
+
+    // test frecency
+    {link1: fixtures.moreFrecent, link2: fixtures.older, expected: false},
+  ];
+
+  for (let {link1, link2, expected} of links) {
+    let observed = PlacesProvider.LinkUtils.compareLinks(link1, link2) > 0;
+    equal(observed , expected, `comparing ${link1.url} and ${link2.url}`);
+  }
+
+  // test error scenarios
+
+  let errorFixtures = {
+    missingFrecency: {
+      url: "http://www.mozilla.org/firstolder",
+      title: "Mozilla",
+      lastVisitDate: 1394678824766431,
+    },
+    missingVisitDate: {
+      url: "http://www.mozilla.org/firstolder",
+      title: "Mozilla",
+      frecency: 1337,
+    },
+    missingURL: {
+      title: "Mozilla",
+      frecency: 1337,
+      lastVisitDate: 1394678824766431,
+    }
+  };
+
+  let errorLinks = [
+    {link1: fixtures.older, link2: errorFixtures.missingFrecency},
+    {link2: fixtures.older, link1: errorFixtures.missingFrecency},
+    {link1: fixtures.older, link2: errorFixtures.missingVisitDate},
+    {link1: fixtures.older, link2: errorFixtures.missingURL},
+    {link1: errorFixtures.missingFrecency, link2: errorFixtures.missingVisitDate}
+  ];
+
+  let errorCount = 0;
+  for (let {link1, link2} of errorLinks) {
+    try {
+      let observed = PlacesProvider.LinkUtils.compareLinks(link1, link2) > 0; // jshint ignore:line
+    } catch (e) {
+      ok(true, `exception for comparison of ${link1.url} and ${link2.url}`);
+      errorCount += 1;
+    }
+  }
+  equal(errorCount, errorLinks.length);
+});
+
+/** Test Provider **/
+
+add_task(function* test_Links_getLinks() {
+  yield PlacesTestUtils.clearHistory();
+  let provider = PlacesProvider.links;
+
+  let links = yield provider.getLinks();
+  equal(links.length, 0, "empty history yields empty links");
+
+  // add a visit
+  let testURI = NetUtil.newURI("http://mozilla.com");
+  yield PlacesTestUtils.addVisits(testURI);
+
+  links = yield provider.getLinks();
+  equal(links.length, 1, "adding a visit yields a link");
+  equal(links[0].url, testURI.spec, "added visit corresponds to added url");
+});
+
+add_task(function* test_Links_getLinks_Order() {
+  yield PlacesTestUtils.clearHistory();
+  let provider = PlacesProvider.links;
+  let {
+    TRANSITION_TYPED,
+    TRANSITION_LINK
+  } = PlacesUtils.history;
+
+  function timeDaysAgo(numDays) {
+    let now = new Date();
+    return now.getTime() - (numDays * 24 * 60 * 60 * 1000);
+  }
+
+  let visits = [
+    // frecency 200
+    {uri: NetUtil.newURI("https://mozilla.com/0"), visitDate: timeDaysAgo(0), transition: TRANSITION_TYPED},
+    // sort by url, frecency 200
+    {uri: NetUtil.newURI("https://mozilla.com/1"), visitDate: timeDaysAgo(0), transition: TRANSITION_TYPED},
+    // sort by last visit date, frecency 200
+    {uri: NetUtil.newURI("https://mozilla.com/2"), visitDate: timeDaysAgo(2), transition: TRANSITION_TYPED},
+    // sort by frecency, frecency 10
+    {uri: NetUtil.newURI("https://mozilla.com/3"), visitDate: timeDaysAgo(2), transition: TRANSITION_LINK},
+  ];
+
+  let links = yield provider.getLinks();
+  equal(links.length, 0, "empty history yields empty links");
+  yield PlacesTestUtils.addVisits(visits);
+
+  links = yield provider.getLinks();
+  equal(links.length, visits.length, "number of links added is the same as obtain by getLinks");
+  for (let i = 0; i < links.length; i++) {
+    equal(links[i].url, visits[i].uri.spec, "links are obtained in the expected order");
+  }
+});
+
+add_task(function* test_Links_onLinkChanged() {
+  let provider = PlacesProvider.links;
+  provider.init();
+
+  let url = "https://example.com/onFrecencyChanged1";
+  let linkChangedMsgCount = 0;
+
+  let linkChangedPromise = new Promise(resolve => {
+    let handler = (_, link) => { // jshint ignore:line
+      /* There are 3 linkChanged events:
+       * 1. visit insertion (-1 frecency by default)
+       * 2. frecency score update (after transition type calculation etc)
+       * 3. title change
+       */
+      if (link.url === url) {
+        equal(link.url, url, `expected url on linkChanged event`);
+        linkChangedMsgCount += 1;
+        if (linkChangedMsgCount === 3) {
+          ok(true, `all linkChanged events captured`);
+          provider.off("linkChanged", this);
+          resolve();
+        }
+      }
+    };
+    provider.on("linkChanged", handler);
+  });
+
+  // add a visit
+  let testURI = NetUtil.newURI(url);
+  yield PlacesTestUtils.addVisits(testURI);
+  yield linkChangedPromise;
+
+  yield PlacesTestUtils.clearHistory();
+  provider.destroy();
+});
+
+add_task(function* test_Links_onClearHistory() {
+  let provider = PlacesProvider.links;
+  provider.init();
+
+  let clearHistoryPromise = new Promise(resolve => {
+    let handler = () => {
+      ok(true, `clearHistory event captured`);
+      provider.off("clearHistory", handler);
+      resolve();
+    };
+    provider.on("clearHistory", handler);
+  });
+
+  // add visits
+  for (let i = 0; i <= 10; i++) {
+    let url = `https://example.com/onClearHistory${i}`;
+    let testURI = NetUtil.newURI(url);
+    yield PlacesTestUtils.addVisits(testURI);
+  }
+  yield PlacesTestUtils.clearHistory();
+  yield clearHistoryPromise;
+  provider.destroy();
+});
+
+add_task(function* test_Links_onDeleteURI() {
+  let provider = PlacesProvider.links;
+  provider.init();
+
+  let testURL = "https://example.com/toDelete";
+
+  let deleteURIPromise = new Promise(resolve => {
+    let handler = (_, {url}) => { // jshint ignore:line
+      equal(testURL, url, "deleted url and expected url are the same");
+      provider.off("deleteURI", handler);
+      resolve();
+    };
+
+    provider.on("deleteURI", handler);
+  });
+
+  let testURI = NetUtil.newURI(testURL);
+  yield PlacesTestUtils.addVisits(testURI);
+  yield PlacesUtils.history.remove(testURL);
+  yield deleteURIPromise;
+  provider.destroy();
+});
+
+add_task(function* test_Links_onManyLinksChanged() {
+  let provider = PlacesProvider.links;
+  provider.init();
+
+  let promise = new Promise(resolve => {
+    let handler = () => {
+      ok(true);
+      provider.off("manyLinksChanged", handler);
+      resolve();
+    };
+
+    provider.on("manyLinksChanged", handler);
+  });
+
+  let testURL = "https://example.com/toDelete";
+  let testURI = NetUtil.newURI(testURL);
+  yield PlacesTestUtils.addVisits(testURI);
+
+  // trigger DecayFrecency
+  PlacesUtils.history.QueryInterface(Ci.nsIObserver).
+    observe(null, "idle-daily", "");
+
+  yield promise;
+  provider.destroy();
+});

browser/components/newtab/tests/xpcshell/xpcshell.ini

@@ -5,6 +5,7 @@ firefox-appdir = browser
 skip-if = toolkit == 'android' || toolkit == 'gonk'
 
 [test_NewTabURL.js]
+[test_PlacesProvider.js]
 [test_RemoteDirectoryLinksProvider.js]
 [test_RemoteNewTabLocation.js]
 [test_RemoteNewTabUtils.js]