Bug 1109120 - use a geoip xhr request for more reliable country detection for search. r=gavin

browser/app/profile/firefox.js

@@ -439,6 +439,12 @@ pref("browser.search.official", true);
 // How many times to show the new search highlight
 pref("browser.search.highlightCount", 5);
 
+// geoip end-point and timeout
+pref("browser.search.geoip.url", "https://location.services.mozilla.com/v1/country?key=%MOZILLA_API_KEY%");
+// NOTE: this timeout figure is also the "high" value for the telemetry probe
+// SEARCH_SERVICE_COUNTRY_FETCH_MS - if you change this also change that probe.
+pref("browser.search.geoip.timeout", 2000);
+
 pref("browser.sessionhistory.max_entries", 50);
 
 // handle links targeting new windows

layout/tools/reftest/reftest-preferences.js

@@ -49,3 +49,8 @@
     // Disable the auto-hide feature of touch caret to avoid potential
     // intermittent issues.
     branch.setIntPref("touchcaret.expiration.time", 0);
+
+    // Tell the search service we are running in the US.  This also has the
+    // desired side-effect of preventing our geoip lookup.
+    branch.setBoolPref("browser.search.isUS", true);
+    branch.setCharPref("browser.search.countryCode", "US");

testing/profiles/prefs_general.js

@@ -272,6 +272,10 @@ user_pref("browser.uitour.url", "http://%(server)s/uitour-dummy/tour");
 
 // Don't show the search first run UI by default
 user_pref("browser.search.highlightCount", 0);
+// Tell the search service we are running in the US.  This also has the desired
+// side-effect of preventing our geoip lookup.
+user_pref("browser.search.isUS", true);
+user_pref("browser.search.countryCode", "US");
 
 user_pref("media.eme.enabled", true);
 

toolkit/components/places/tests/unit/test_PlacesSearchAutocompleteProvider.js

@@ -5,6 +5,10 @@
 Cu.import("resource://gre/modules/PlacesSearchAutocompleteProvider.jsm");
 
 function run_test() {
+  // Tell the search service we are running in the US.  This also has the
+  // desired side-effect of preventing our geoip lookup.
+  Services.prefs.setBoolPref("browser.search.isUS", true);
+  Services.prefs.setCharPref("browser.search.countryCode", "US");
   run_next_test();
 }
 

toolkit/components/search/nsSearchService.js

@@ -30,6 +30,8 @@ XPCOMUtils.defineLazyServiceGetter(this, "gTextToSubURI",
                                    "@mozilla.org/intl/texttosuburi;1",
                                    "nsITextToSubURI");
 
+Cu.importGlobalProperties(["XMLHttpRequest"]);
+
 // A text encoder to UTF8, used whenever we commit the
 // engine metadata to disk.
 XPCOMUtils.defineLazyGetter(this, "gEncoder",
@@ -423,7 +425,12 @@ function getIsUS() {
     Services.prefs.setBoolPref(cachePref, false);
     return false;
   }
+  let isNA = isUSTimezone();
+  Services.prefs.setBoolPref(cachePref, isNA);
+  return isNA;
+}
 
+function isUSTimezone() {
   // Timezone assumptions! We assume that if the system clock's timezone is
   // between Newfoundland and Hawaii, that the user is in North America.
 
@@ -437,11 +444,80 @@ function getIsUS() {
   // Hawaii-Aleutian Standard Time (http://www.timeanddate.com/time/zones/hast)
 
   let UTCOffset = (new Date()).getTimezoneOffset();
-  let isNA = UTCOffset >= 150 && UTCOffset <= 600;
+  return UTCOffset >= 150 && UTCOffset <= 600;
+}
 
-  Services.prefs.setBoolPref(cachePref, isNA);
+// A less hacky method that tries to determine our country-code via an XHR
+// geoip lookup.
+// If this succeeds and we are using an en-US locale, we set the pref used by
+// the hacky method above, so isUS() can avoid the hacky timezone method.
+// If it fails we don't touch that pref so isUS() does its normal thing.
+let ensureKnownCountryCode = Task.async(function* () {
+  // If we have a country-code already stored in our prefs we trust it.
+  try {
+    Services.prefs.getCharPref("browser.search.countryCode");
+    return; // pref exists, so we've done this before.
+  } catch(e) {}
+  // we don't have it cached, so fetch it.
+  let cc = yield fetchCountryCode();
+  if (cc) {
+    // we got one - stash it away
+    Services.prefs.setCharPref("browser.search.countryCode", cc);
+    // and update our "isUS" cache pref if it is US - that will prevent a
+    // fallback to the timezone check.
+    // However, only do this if the locale also matches.
+    if (getLocale() == "en-US") {
+      Services.prefs.setBoolPref("browser.search.isUS", (cc == "US"));
+    }
+    // and telemetry...
+    let isTimezoneUS = isUSTimezone();
+    if (cc == "US" && !isTimezoneUS) {
+      Services.telemetry.getHistogramById("SEARCH_SERVICE_US_COUNTRY_MISMATCHED_TIMEZONE").add(1);
+    }
+    if (cc != "US" && isTimezoneUS) {
+      Services.telemetry.getHistogramById("SEARCH_SERVICE_US_TIMEZONE_MISMATCHED_COUNTRY").add(1);
+    }
+  }
+});
 
-  return isNA;
+// Get the country we are in via a XHR geoip request.
+function fetchCountryCode() {
+  let endpoint = Services.urlFormatter.formatURLPref("browser.search.geoip.url");
+  // As an escape hatch, no endpoint means no geoip.
+  if (!endpoint) {
+    return Promise.resolve(null);
+  }
+  let startTime = Date.now();
+  return new Promise(resolve => {
+    let request = new XMLHttpRequest();
+    request.timeout = Services.prefs.getIntPref("browser.search.geoip.timeout");
+    request.onload = function(event) {
+      let took = Date.now() - startTime;
+      let cc = event.target.response && event.target.response.country_code;
+      LOG("_fetchCountryCode got success response in " + took + "ms: " + cc);
+      Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_FETCH_MS").add(took);
+      Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_SUCCESS").add(cc ? 1 : 0);
+      if (!cc) {
+        Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_SUCCESS_WITHOUT_DATA").add(1);
+      }
+      resolve(cc);
+    };
+    request.onerror = function(event) {
+      LOG("_fetchCountryCode: failed to retrieve country information");
+      Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_SUCCESS").add(0);
+      resolve(null);
+    };
+    request.ontimeout = function(event) {
+      LOG("_fetchCountryCode: timeout fetching country information");
+      Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_FETCH_TIMEOUT").add(1);
+      Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_SUCCESS").add(0);
+      resolve(null);
+    }
+    request.open("POST", endpoint, true);
+    request.setRequestHeader("Content-Type", "application/json");
+    request.responseType = "json";
+    request.send("{}");
+  });
 }
 
 /**
@@ -3010,6 +3086,11 @@ SearchService.prototype = {
   _asyncInit: function SRCH_SVC__asyncInit() {
     return Task.spawn(function() {
       LOG("_asyncInit start");
+      try {
+        yield checkForSyncCompletion(ensureKnownCountryCode());
+      } catch (ex if ex.result != Cr.NS_ERROR_ALREADY_INITIALIZED) {
+        LOG("_asyncInit: failure determining country code: " + ex);
+      }
       try {
         yield checkForSyncCompletion(this._asyncLoadEngines());
       } catch (ex if ex.result != Cr.NS_ERROR_ALREADY_INITIALIZED) {

toolkit/components/search/tests/xpcshell/head_search.js

@@ -111,6 +111,26 @@ function removeCache()
 
 }
 
+/**
+ * isUSTimezone taken from nsSearchService.js
+ */
+function isUSTimezone() {
+  // Timezone assumptions! We assume that if the system clock's timezone is
+  // between Newfoundland and Hawaii, that the user is in North America.
+
+  // This includes all of South America as well, but we have relatively few
+  // en-US users there, so that's OK.
+
+  // 150 minutes = 2.5 hours (UTC-2.5), which is
+  // Newfoundland Daylight Time (http://www.timeanddate.com/time/zones/ndt)
+
+  // 600 minutes = 10 hours (UTC-10), which is
+  // Hawaii-Aleutian Standard Time (http://www.timeanddate.com/time/zones/hast)
+
+  let UTCOffset = (new Date()).getTimezoneOffset();
+  return UTCOffset >= 150 && UTCOffset <= 600;
+}
+
 /**
  * Run some callback once metadata has been committed to disk.
  */
@@ -185,6 +205,9 @@ function isSubObjectOf(expectedObj, actualObj) {
 // Expand the amount of information available in error logs
 Services.prefs.setBoolPref("browser.search.log", true);
 
+// Disable geoip lookups
+Services.prefs.setCharPref("browser.search.geoip.url", "");
+
 /**
  * After useHttpServer() is called, this string contains the URL of the "data"
  * directory, including the final slash.

toolkit/components/search/tests/xpcshell/test_location.js

@@ -0,0 +1,35 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+function run_test() {
+  removeMetadata();
+  removeCacheFile();
+
+  do_check_false(Services.search.isInitialized);
+
+  let engineDummyFile = gProfD.clone();
+  engineDummyFile.append("searchplugins");
+  engineDummyFile.append("test-search-engine.xml");
+  let engineDir = engineDummyFile.parent;
+  engineDir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
+
+  do_get_file("data/engine.xml").copyTo(engineDir, "engine.xml");
+
+  do_register_cleanup(function() {
+    removeMetadata();
+    removeCacheFile();
+  });
+
+  Services.prefs.setCharPref("browser.search.geoip.url", 'data:application/json,{"country_code": "AU"}');
+  Services.search.init(() => {
+    equal(Services.prefs.getCharPref("browser.search.countryCode"), "AU", "got the correct country code.");
+    equal(Services.prefs.getBoolPref("browser.search.isUS"), false, "AU is not in the US.")
+    // check we have "success" recorded in telemetry
+    let histogram = Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_SUCCESS");
+    let snapshot = histogram.snapshot();
+    equal(snapshot.sum, 1)
+    do_test_finished();
+    run_next_test();
+  });
+  do_test_pending();
+}

toolkit/components/search/tests/xpcshell/test_location_error.js

@@ -0,0 +1,45 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+function run_test() {
+  removeMetadata();
+  removeCacheFile();
+
+  do_check_false(Services.search.isInitialized);
+
+  let engineDummyFile = gProfD.clone();
+  engineDummyFile.append("searchplugins");
+  engineDummyFile.append("test-search-engine.xml");
+  let engineDir = engineDummyFile.parent;
+  engineDir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
+
+  do_get_file("data/engine.xml").copyTo(engineDir, "engine.xml");
+
+  do_register_cleanup(function() {
+    removeMetadata();
+    removeCacheFile();
+  });
+
+  // from server-locations.txt, we choose a URL without a cert.
+  let url = "https://nocert.example.com:443";
+  Services.prefs.setCharPref("browser.search.geoip.url", url);
+  Services.search.init(() => {
+    try {
+      Services.prefs.getCharPref("browser.search.countryCode");
+      ok(false, "not expecting countryCode to be set");
+    } catch (ex) {}
+    // should be no success recorded.
+    let histogram = Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_SUCCESS");
+    let snapshot = histogram.snapshot();
+    equal(snapshot.sum, 0);
+
+    // should be no timeout either.
+    histogram = Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_FETCH_TIMEOUT");
+    snapshot = histogram.snapshot();
+    equal(snapshot.sum, 0);
+
+    do_test_finished();
+    run_next_test();
+  });
+  do_test_pending();
+}

toolkit/components/search/tests/xpcshell/test_location_malformed_json.js

@@ -0,0 +1,53 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+function run_test() {
+  removeMetadata();
+  removeCacheFile();
+
+  do_check_false(Services.search.isInitialized);
+
+  let engineDummyFile = gProfD.clone();
+  engineDummyFile.append("searchplugins");
+  engineDummyFile.append("test-search-engine.xml");
+  let engineDir = engineDummyFile.parent;
+  engineDir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
+
+  do_get_file("data/engine.xml").copyTo(engineDir, "engine.xml");
+
+  do_register_cleanup(function() {
+    removeMetadata();
+    removeCacheFile();
+  });
+
+  // Here we have malformed JSON
+  Services.prefs.setCharPref("browser.search.geoip.url", 'data:application/json,{"country_code"');
+  Services.search.init(() => {
+    try {
+      Services.prefs.getCharPref("browser.search.countryCode");
+      ok(false, "should be no countryCode pref");
+    } catch (_) {}
+    try {
+      Services.prefs.getCharPref("browser.search.isUS");
+      ok(false, "should be no isUS pref yet either");
+    } catch (_) {}
+    // fetch the engines - this should force the timezone check
+    Services.search.getEngines();
+    equal(Services.prefs.getBoolPref("browser.search.isUS"),
+          isUSTimezone(),
+          "should have set isUS based on current timezone.");
+    // should have a false value for success.
+    let histogram = Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_SUCCESS");
+    let snapshot = histogram.snapshot();
+    equal(snapshot.sum, 0);
+
+    // and a flag for SEARCH_SERVICE_COUNTRY_SUCCESS_WITHOUT_DATA
+    histogram = Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_SUCCESS_WITHOUT_DATA");
+    snapshot = histogram.snapshot();
+    equal(snapshot.sum, 1);
+
+    do_test_finished();
+    run_next_test();
+  });
+  do_test_pending();
+}

toolkit/components/search/tests/xpcshell/test_location_sync.js

@@ -0,0 +1,78 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+function getCountryCodePref() {
+  try {
+    return Services.prefs.getCharPref("browser.search.countryCode");
+  } catch (_) {
+    return undefined;
+  }
+}
+
+function getIsUSPref() {
+  try {
+    return Services.prefs.getBoolPref("browser.search.isUS");
+  } catch (_) {
+    return undefined;
+  }
+}
+
+function run_test() {
+  removeMetadata();
+  removeCacheFile();
+
+  ok(!Services.search.isInitialized);
+
+  let engineDummyFile = gProfD.clone();
+  engineDummyFile.append("searchplugins");
+  engineDummyFile.append("test-search-engine.xml");
+  let engineDir = engineDummyFile.parent;
+  engineDir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
+
+  do_get_file("data/engine.xml").copyTo(engineDir, "engine.xml");
+
+  do_register_cleanup(function() {
+    removeMetadata();
+    removeCacheFile();
+  });
+
+  run_next_test();
+}
+
+// Force a sync init and ensure the right thing happens (ie, that no xhr
+// request is made and we fall back to the timezone-only trick)
+add_task(function* test_simple() {
+  deepEqual(getCountryCodePref(), undefined, "no countryCode pref");
+  deepEqual(getIsUSPref(), undefined, "no isUS pref");
+
+  // Still set a geoip pref so we can (indirectly) check it wasn't used.
+  Services.prefs.setCharPref("browser.search.geoip.url", 'data:application/json,{"country_code": "AU"}');
+
+  ok(!Services.search.isInitialized);
+
+  // fetching the engines forces a sync init, and should have caused us to
+  // check the timezone.
+  let engines = Services.search.getEngines();
+  ok(Services.search.isInitialized);
+  deepEqual(getIsUSPref(), isUSTimezone(), "isUS pref was set by sync init.");
+
+  // a little wait to check we didn't do the xhr thang.
+  yield new Promise(resolve => {
+    do_timeout(500, resolve);
+  });
+
+  deepEqual(getCountryCodePref(), undefined, "didn't do the geoip xhr");
+  // and no telemetry evidence of geoip.
+  for (let hid of [
+    "SEARCH_SERVICE_COUNTRY_FETCH_MS",
+    "SEARCH_SERVICE_COUNTRY_SUCCESS",
+    "SEARCH_SERVICE_COUNTRY_SUCCESS_WITHOUT_DATA",
+    "SEARCH_SERVICE_COUNTRY_FETCH_TIMEOUT",
+    "SEARCH_SERVICE_US_COUNTRY_MISMATCHED_TIMEZONE",
+    "SEARCH_SERVICE_US_TIMEZONE_MISMATCHED_COUNTRY",
+    ]) {
+      let histogram = Services.telemetry.getHistogramById(hid);
+      let snapshot = histogram.snapshot();
+      equal(snapshot.sum, 0);
+    }
+});

toolkit/components/search/tests/xpcshell/test_location_timeout.js

@@ -0,0 +1,63 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+function startServer() {
+  let srv = new HttpServer();
+  function lookupCountry(metadata, response) {
+    response.processAsync();
+    // wait 200 ms before writing a valid response - the search service
+    // should timeout before the response is written so the response should
+    // be ignored.
+    do_timeout(200, () => {
+      response.setStatusLine("1.1", 200, "OK");
+      response.write('{"country_code" : "AU"}');
+    });
+  }
+  srv.registerPathHandler("/lookup_country", lookupCountry);
+  srv.start(-1);
+  return srv;
+}
+
+function run_test() {
+  removeMetadata();
+  removeCacheFile();
+
+  do_check_false(Services.search.isInitialized);
+
+  let engineDummyFile = gProfD.clone();
+  engineDummyFile.append("searchplugins");
+  engineDummyFile.append("test-search-engine.xml");
+  let engineDir = engineDummyFile.parent;
+  engineDir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
+
+  do_get_file("data/engine.xml").copyTo(engineDir, "engine.xml");
+
+  do_register_cleanup(function() {
+    removeMetadata();
+    removeCacheFile();
+  });
+
+  let server = startServer();
+  let url = "http://localhost:" + server.identity.primaryPort + "/lookup_country";
+  Services.prefs.setCharPref("browser.search.geoip.url", url);
+  Services.prefs.setIntPref("browser.search.geoip.timeout", 50);
+  Services.search.init(() => {
+    try {
+      Services.prefs.getCharPref("browser.search.countryCode");
+      ok(false, "not expecting countryCode to be set");
+    } catch (ex) {}
+    // should be no success recorded.
+    let histogram = Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_SUCCESS");
+    let snapshot = histogram.snapshot();
+    equal(snapshot.sum, 0);
+
+    // should be a timeout.
+    histogram = Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_FETCH_TIMEOUT");
+    snapshot = histogram.snapshot();
+    equal(snapshot.sum, 1);
+
+    do_test_finished();
+    server.stop(run_next_test);
+  });
+  do_test_pending();
+}

toolkit/components/search/tests/xpcshell/xpcshell.ini

@@ -28,6 +28,11 @@ support-files =
 [test_init_async_multiple.js]
 [test_init_async_multiple_then_sync.js]
 [test_json_cache.js]
+[test_location.js]
+[test_location_error.js]
+[test_location_malformed_json.js]
+[test_location_sync.js]
+[test_location_timeout.js]
 [test_nodb.js]
 [test_nodb_pluschanges.js]
 [test_save_sorted_engines.js]

toolkit/components/telemetry/Histograms.json

@@ -4577,6 +4577,44 @@
     "extended_statistics_ok": true,
     "description": "Time (ms) it takes to build the cache of the search service"
   },
+  "SEARCH_SERVICE_COUNTRY_FETCH_MS": {
+    "alert_emails": ["mhammond@mozilla.com", "gavin@mozilla.com"],
+    "expires_in_version": "never",
+    "kind": "linear",
+    "n_buckets": 20,
+    "high": 2000,
+    "description": "Time (ms) it takes to fetch the country code"
+  },
+  "SEARCH_SERVICE_COUNTRY_FETCH_TIMEOUT": {
+    "alert_emails": ["mhammond@mozilla.com", "gavin@mozilla.com"],
+    "expires_in_version": "never",
+    "kind": "flag",
+    "description": "If we saw a timeout fetching the country-code"
+  },
+  "SEARCH_SERVICE_COUNTRY_SUCCESS": {
+    "alert_emails": ["mhammond@mozilla.com", "gavin@mozilla.com"],
+    "expires_in_version": "never",
+    "kind": "boolean",
+    "description": "If we successfully fetched the country-code."
+  },
+  "SEARCH_SERVICE_COUNTRY_SUCCESS_WITHOUT_DATA": {
+    "alert_emails": ["mhammond@mozilla.com", "gavin@mozilla.com"],
+    "expires_in_version": "never",
+    "kind": "flag",
+    "description": "If we got a success response but no country-code"
+  },
+  "SEARCH_SERVICE_US_COUNTRY_MISMATCHED_TIMEZONE": {
+    "alert_emails": ["mhammond@mozilla.com", "gavin@mozilla.com"],
+    "expires_in_version": "never",
+    "kind": "flag",
+    "description": "Set if the fetched country-code indicates US but the time-zone heuristic doesn't"
+  },
+  "SEARCH_SERVICE_US_TIMEZONE_MISMATCHED_COUNTRY": {
+    "alert_emails": ["mhammond@mozilla.com", "gavin@mozilla.com"],
+    "expires_in_version": "never",
+    "kind": "flag",
+    "description": "Set if the time-zone heuristic indicates US but the fetched country code doesn't"
+  },
   "SOCIAL_ENABLED_ON_SESSION": {
     "expires_in_version": "never",
     "kind": "flag",