Bug 1045760 - Enhance tiles more specifically than eTLD+1/baseDomain (e.g., no enhance bugzilla.mozilla.org for mozilla.org) [r=dao]

Use all levels of subdomain except for common subdomains that are actually the same site.

toolkit/modules/DirectoryLinksProvider.jsm

@@ -18,6 +18,8 @@ Cu.import("resource://gre/modules/Task.jsm");
 
 XPCOMUtils.defineLazyModuleGetter(this, "NetUtil",
   "resource://gre/modules/NetUtil.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "NewTabUtils",
+  "resource://gre/modules/NewTabUtils.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "OS",
   "resource://gre/modules/osfile.jsm")
 XPCOMUtils.defineLazyModuleGetter(this, "Promise",
@@ -180,18 +182,6 @@ let DirectoryLinksProvider = {
     }
   },
 
-  /**
-   * Get the eTLD+1 / base domain from a url spec
-   */
-  _extractSite: function DirectoryLinksProvider_extractSite(url) {
-    let linkURI = Services.io.newURI(url, null, null);
-    try {
-      return Services.eTLD.getBaseDomain(linkURI);
-    }
-    catch(ex) {}
-    return linkURI.asciiHost;
-  },
-
   _fetchAndCacheLinks: function DirectoryLinksProvider_fetchAndCacheLinks(uri) {
     let deferred = Promise.defer();
     let xmlHttp = new XMLHttpRequest();
@@ -366,7 +356,7 @@ let DirectoryLinksProvider = {
   getEnhancedLink: function DirectoryLinksProvider_getEnhancedLink(link) {
     // Use the provided link if it's already enhanced
     return link.enhancedImageURI && link ||
-           this._enhancedLinks.get(this._extractSite(link.url));
+           this._enhancedLinks.get(NewTabUtils.extractSite(link.url));
   },
 
   /**
@@ -382,7 +372,7 @@ let DirectoryLinksProvider = {
       aCallback(rawLinks.map((link, position) => {
         // Stash the enhanced image for the site
         if (link.enhancedImageURI) {
-          this._enhancedLinks.set(this._extractSite(link.url), link);
+          this._enhancedLinks.set(NewTabUtils.extractSite(link.url), link);
         }
 
         link.frecency = DIRECTORY_FRECENCY;

toolkit/modules/NewTabUtils.jsm

@@ -829,33 +829,18 @@ let Links = {
     let pinnedLinks = Array.slice(PinnedLinks.links);
     let links = this._getMergedProviderLinks();
 
-    function getBaseDomain(url) {
-      let uri;
-      try {
-        uri = Services.io.newURI(url, null, null);
-      } catch (e) {
-        return null;
-      }
-
-      try {
-        return Services.eTLD.getBaseDomain(uri);
-      } catch (e) {
-        return uri.asciiHost;
-      }
-    }
-
-    let baseDomains = new Set();
+    let sites = new Set();
     for (let link of pinnedLinks) {
       if (link)
-        baseDomains.add(getBaseDomain(link.url));
+        sites.add(NewTabUtils.extractSite(link.url));
     }
 
     // Filter blocked and pinned links and duplicate base domains.
     links = links.filter(function (link) {
-      let baseDomain = getBaseDomain(link.url);
-      if (baseDomain == null || baseDomains.has(baseDomain))
+      let site = NewTabUtils.extractSite(link.url);
+      if (site == null || sites.has(site))
         return false;
-      baseDomains.add(baseDomain);
+      sites.add(site);
 
       return !BlockedLinks.isBlocked(link) && !PinnedLinks.isPinned(link);
     });
@@ -1184,6 +1169,24 @@ let ExpirationFilter = {
 this.NewTabUtils = {
   _initialized: false,
 
+  /**
+   * Extract a "site" from a url in a way that multiple urls of a "site" returns
+   * the same "site."
+   * @param aUrl Url spec string
+   * @return The "site" string or null
+   */
+  extractSite: function Links_extractSite(url) {
+    let uri;
+    try {
+      uri = Services.io.newURI(url, null, null);
+    } catch (ex) {
+      return null;
+    }
+
+    // Strip off common subdomains of the same site (e.g., www, load balancer)
+    return uri.asciiHost.replace(/^(m|mobile|www\d*)\./, "");
+  },
+
   init: function NewTabUtils_init() {
     if (this.initWithoutProviders()) {
       PlacesProvider.init();

toolkit/modules/tests/xpcshell/test_DirectoryLinksProvider.js

@@ -558,9 +558,9 @@ add_task(function test_DirectoryLinksProvider_getEnhancedLink() {
 
   // Get the expected image for the same site
   checkEnhanced("http://example.net/", "net1");
-  checkEnhanced("http://sub.example.net/", "net1");
   checkEnhanced("http://example.net/path", "net1");
   checkEnhanced("https://www.example.net/", "net1");
+  checkEnhanced("https://www3.example.net/", "net1");
 
   // Get the image of the last entry
   checkEnhanced("http://example.com", "com2");
@@ -574,6 +574,7 @@ add_task(function test_DirectoryLinksProvider_getEnhancedLink() {
   do_check_eq(inline.url, "http://example.com/echo");
 
   // Undefined for not enhanced
+  checkEnhanced("http://sub.example.net/", undefined);
   checkEnhanced("http://example.org", undefined);
   checkEnhanced("http://localhost", undefined);
   checkEnhanced("http://127.0.0.1", undefined);

toolkit/modules/tests/xpcshell/test_NewTabUtils.js

@@ -144,6 +144,58 @@ add_task(function newLowRankedLink() {
   NewTabUtils.links.removeProvider(provider);
 });
 
+add_task(function extractSite() {
+  // All these should extract to the same site
+  [ "mozilla.org",
+    "m.mozilla.org",
+    "mobile.mozilla.org",
+    "www.mozilla.org",
+    "www3.mozilla.org",
+  ].forEach(host => {
+    let url = "http://" + host;
+    do_check_eq(NewTabUtils.extractSite(url), "mozilla.org", "extracted same " + host);
+  });
+
+  // All these should extract to the same subdomain
+  [ "bugzilla.mozilla.org",
+    "www.bugzilla.mozilla.org",
+  ].forEach(host => {
+    let url = "http://" + host;
+    do_check_eq(NewTabUtils.extractSite(url), "bugzilla.mozilla.org", "extracted eTLD+2 " + host);
+  });
+
+  // All these should not extract to the same site
+  [ "bugzilla.mozilla.org",
+    "bug123.bugzilla.mozilla.org",
+    "too.many.levels.bugzilla.mozilla.org",
+    "m2.mozilla.org",
+    "mobile30.mozilla.org",
+    "ww.mozilla.org",
+    "ww2.mozilla.org",
+    "wwwww.mozilla.org",
+    "wwwww50.mozilla.org",
+    "wwws.mozilla.org",
+    "secure.mozilla.org",
+    "secure10.mozilla.org",
+    "many.levels.deep.mozilla.org",
+    "just.check.in",
+    "192.168.0.1",
+    "localhost",
+  ].forEach(host => {
+    let url = "http://" + host;
+    do_check_neq(NewTabUtils.extractSite(url), "mozilla.org", "extracted diff " + host);
+  });
+
+  // All these should not extract to the same site
+  [ "about:blank",
+    "file:///Users/user/file",
+    "chrome://browser/something",
+    "ftp://ftp.mozilla.org/",
+  ].forEach(url => {
+    do_check_neq(NewTabUtils.extractSite(url), "mozilla.org", "extracted diff url " + url);
+  });
+});
+
 function TestProvider(getLinksFn) {
   this.getLinks = getLinksFn;
   this._observers = new Set();