Bug 1227956 - Implement Kinto.js OneCRL client r=rnewman

browser/app/profile/firefox.js

@@ -62,6 +62,12 @@ pref("extensions.blocklist.url", "https://blocklist.addons.mozilla.org/blocklist
 pref("extensions.blocklist.detailsURL", "https://www.mozilla.org/%LOCALE%/blocklist/");
 pref("extensions.blocklist.itemURL", "https://blocklist.addons.mozilla.org/%LOCALE%/%APP%/blocked/%blockID%");
 
+// Kinto blocklist preferences
+pref("services.kinto.base", "https://firefox.settings.services.mozilla.com/v1");
+pref("services.kinto.bucket", "blocklists");
+pref("services.kinto.onecrl.collection", "certificates");
+pref("services.kinto.onecrl.checked", 0);
+
 pref("extensions.update.autoUpdateDefault", true);
 
 pref("extensions.hotfix.id", "firefox-hotfix@mozilla.org");

mobile/android/app/mobile.js

@@ -238,6 +238,12 @@ pref("extensions.blocklist.interval", 86400);
 pref("extensions.blocklist.url", "https://blocklist.addons.mozilla.org/blocklist/3/%APP_ID%/%APP_VERSION%/%PRODUCT%/%BUILD_ID%/%BUILD_TARGET%/%LOCALE%/%CHANNEL%/%OS_VERSION%/%DISTRIBUTION%/%DISTRIBUTION_VERSION%/%PING_COUNT%/%TOTAL_PING_COUNT%/%DAYS_SINCE_LAST_PING%/");
 pref("extensions.blocklist.detailsURL", "https://www.mozilla.com/%LOCALE%/blocklist/");
 
+// Kinto blocklist preferences
+pref("services.kinto.base", "https://firefox.settings.services.mozilla.com/v1");
+pref("services.kinto.bucket", "blocklists");
+pref("services.kinto.onecrl.collection", "certificates");
+pref("services.kinto.onecrl.checked", 0);
+
 /* Don't let XPIProvider install distribution add-ons; we do our own thing on mobile. */
 pref("extensions.installDistroAddons", false);
 

services/common/KintoCertificateBlocklist.js

@@ -0,0 +1,115 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["OneCRLClient"];
+
+const { classes: Cc, interfaces: Ci, utils: Cu } = Components;
+
+Cu.import("resource://services-common/moz-kinto-client.js");
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyServiceGetter(this, "uuidgen",
+                                   "@mozilla.org/uuid-generator;1",
+                                   "nsIUUIDGenerator");
+
+const PREF_KINTO_BASE = "services.kinto.base";
+const PREF_KINTO_BUCKET = "services.kinto.bucket";
+const PREF_KINTO_ONECRL_COLLECTION = "services.kinto.onecrl.collection";
+const PREF_KINTO_ONECRL_CHECKED_SECONDS = "services.kinto.onecrl.checked";
+
+const RE_UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+// Kinto.js assumes version 4 UUIDs but allows you to specify custom
+// validators and generators. The tooling that generates records in the
+// certificates collection currently uses a version 1 UUID so we must
+// specify a validator that's less strict. We must also supply a generator
+// since Kinto.js does not allow one without the other.
+function makeIDSchema() {
+  return {
+    validate: RE_UUID.test.bind(RE_UUID),
+    generate: function() {
+      return uuidgen.generateUUID().toString();
+    }
+  };
+}
+
+// A Kinto based client to keep the OneCRL certificate blocklist up to date.
+function CertBlocklistClient() {
+  // maybe sync the collection of certificates with remote data.
+  // lastModified - the lastModified date (on the server, milliseconds since
+  // epoch) of data in the remote collection
+  // serverTime - the time on the server (milliseconds since epoch)
+  // returns a promise which rejects on sync failure
+  this.maybeSync = function(lastModified, serverTime) {
+    let base = Services.prefs.getCharPref(PREF_KINTO_BASE);
+    let bucket = Services.prefs.getCharPref(PREF_KINTO_BUCKET);
+
+    let Kinto = loadKinto();
+
+    let FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+
+
+    let certList = Cc["@mozilla.org/security/certblocklist;1"]
+                     .getService(Ci.nsICertBlocklist);
+
+    // Future blocklist clients can extract the sync-if-stale logic. For
+    // now, since this is currently the only client, we'll do this here.
+    let config = {
+      remote: base,
+      bucket: bucket,
+      adapter: FirefoxAdapter,
+    };
+
+    let db = new Kinto(config);
+    let collectionName = Services.prefs.getCharPref(PREF_KINTO_ONECRL_COLLECTION,
+                                                    "certificates");
+    let blocklist = db.collection(collectionName,
+                                  { idSchema: makeIDSchema() });
+
+    let updateLastCheck = function() {
+      let checkedServerTimeInSeconds = Math.round(serverTime / 1000);
+      Services.prefs.setIntPref(PREF_KINTO_ONECRL_CHECKED_SECONDS,
+                                checkedServerTimeInSeconds);
+    }
+
+    return Task.spawn(function* () {
+      try {
+        yield blocklist.db.open();
+        let collectionLastModified = yield blocklist.db.getLastModified();
+        // if the data is up to date, there's no need to sync. We still need
+        // to record the fact that a check happened.
+        if (lastModified <= collectionLastModified) {
+          updateLastCheck();
+          return;
+        }
+        yield blocklist.sync();
+        let list = yield blocklist.list();
+        for (let item of list.data) {
+          if (item.issuerName && item.serialNumber) {
+            certList.revokeCertByIssuerAndSerial(item.issuerName,
+                                                 item.serialNumber);
+          } else if (item.subject && item.pubKeyHash) {
+            certList.revokeCertBySubjectAndPubKey(item.subject,
+                                                  item.pubKeyHash);
+          } else {
+            throw new Error("Cert blocklist record has incomplete data");
+          }
+        }
+        // We explicitly do not want to save entries or update the
+        // last-checked time if sync fails
+        certList.saveEntries();
+        updateLastCheck();
+      } finally {
+        blocklist.db.close()
+      }
+    });
+  }
+}
+
+this.OneCRLClient = new CertBlocklistClient();

services/common/moz.build

@@ -15,6 +15,7 @@ EXTRA_COMPONENTS += [
 
 EXTRA_JS_MODULES['services-common'] += [
     'async.js',
+    'KintoCertificateBlocklist.js',
     'logmanager.js',
     'moz-kinto-client.js',
     'observers.js',

services/common/tests/unit/test_kintoCertBlocklist.js

@@ -0,0 +1,189 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Constructor: CC } = Components;
+
+Cu.import("resource://services-common/KintoCertificateBlocklist.js");
+Cu.import("resource://services-common/moz-kinto-client.js")
+Cu.import("resource://testing-common/httpd.js");
+
+const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
+  "nsIBinaryInputStream", "setInputStream");
+
+var server;
+
+// set up what we need to make storage adapters
+const Kinto = loadKinto();
+const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+const kintoFilename = "kinto.sqlite";
+
+let kintoClient;
+
+function do_get_kinto_collection(collectionName) {
+  if (!kintoClient) {
+    let config = {
+      // Set the remote to be some server that will cause test failure when
+      // hit since we should never hit the server directly, only via maybeSync()
+      remote: "https://firefox.settings.services.mozilla.com/v1/",
+      // Set up the adapter and bucket as normal
+      adapter: FirefoxAdapter,
+      bucket: "blocklists"
+    };
+    kintoClient = new Kinto(config);
+  }
+  return kintoClient.collection(collectionName);
+}
+
+// Some simple tests to demonstrate that the logic inside maybeSync works
+// correctly and that simple kinto operations are working as expected. There
+// are more tests for core Kinto.js (and its storage adapter) in the
+// xpcshell tests under /services/common
+add_task(function* test_something(){
+  const configPath = "/v1/";
+  const recordsPath = "/v1/buckets/blocklists/collections/certificates/records";
+
+  Services.prefs.setCharPref("services.kinto.base",
+                             `http://localhost:${server.identity.primaryPort}/v1`);
+
+  // register a handler
+  function handleResponse (request, response) {
+    try {
+      const sampled = getSampleResponse(request, server.identity.primaryPort);
+      if (!sampled) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sampled.status.status,
+                             sampled.status.statusText);
+      // send the headers
+      for (let headerLine of sampled.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+      response.setHeader("Date", (new Date()).toUTCString());
+
+      response.write(sampled.responseBody);
+    } catch (e) {
+      dump(`${e}\n`);
+    }
+  }
+  server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(recordsPath, handleResponse);
+
+  // Test an empty db populates
+  let result = yield OneCRLClient.maybeSync(2000, Date.now());
+
+  // Open the collection, verify it's been populated:
+  // Our test data has a single record; it should be in the local collection
+  let collection = do_get_kinto_collection("certificates");
+  yield collection.db.open();
+  let list = yield collection.list();
+  do_check_eq(list.data.length, 1);
+  yield collection.db.close();
+
+  // Test the db is updated when we call again with a later lastModified value
+  result = yield OneCRLClient.maybeSync(4000, Date.now());
+
+  // Open the collection, verify it's been updated:
+  // Our test data now has two records; both should be in the local collection
+  collection = do_get_kinto_collection("certificates");
+  yield collection.db.open();
+  list = yield collection.list();
+  do_check_eq(list.data.length, 3);
+  yield collection.db.close();
+
+  // Try to maybeSync with the current lastModified value - no connection
+  // should be attempted.
+  // Clear the kinto base pref so any connections will cause a test failure
+  Services.prefs.clearUserPref("services.kinto.base");
+  yield OneCRLClient.maybeSync(4000, Date.now());
+
+  // Try again with a lastModified value at some point in the past
+  yield OneCRLClient.maybeSync(3000, Date.now());
+
+  // Check the OneCRL check time pref is modified, even if the collection
+  // hasn't changed
+  Services.prefs.setIntPref("services.kinto.onecrl.checked", 0);
+  yield OneCRLClient.maybeSync(3000, Date.now());
+  let newValue = Services.prefs.getIntPref("services.kinto.onecrl.checked");
+  do_check_neq(newValue, 0);
+});
+
+function run_test() {
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(function() { });
+  });
+}
+
+// get a response for a given request from sample data
+function getSampleResponse(req, port) {
+  const responses = {
+    "OPTIONS": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
+        "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
+        "Access-Control-Allow-Origin: *",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": "null"
+    },
+    "GET:/v1/?": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"settings":{"cliquet.batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
+    },
+    "GET:/v1/buckets/blocklists/collections/certificates/records?": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"3000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "issuerName": "MEQxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwx0aGF3dGUsIEluYy4xHjAcBgNVBAMTFXRoYXd0ZSBFViBTU0wgQ0EgLSBHMw==",
+        "serialNumber":"CrTHPEE6AZSfI3jysin2bA==",
+        "id":"78cf8900-fdea-4ce5-f8fb-b78710617718",
+        "last_modified":3000
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_since=3000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"4000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "issuerName":"MFkxCzAJBgNVBAYTAk5MMR4wHAYDVQQKExVTdGFhdCBkZXIgTmVkZXJsYW5kZW4xKjAoBgNVBAMTIVN0YWF0IGRlciBOZWRlcmxhbmRlbiBPdmVyaGVpZCBDQQ",
+        "serialNumber":"ATFpsA==",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02c",
+        "last_modified":4000
+      },{
+        "subject":"MCIxIDAeBgNVBAMMF0Fub3RoZXIgVGVzdCBFbmQtZW50aXR5",
+        "pubKeyHash":"VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02d",
+        "last_modified":4000
+      }]})
+    }
+  };
+  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[req.method];
+
+}

services/common/tests/unit/xpcshell.ini

@@ -10,6 +10,7 @@ support-files =
 [test_load_modules.js]
 
 [test_kinto.js]
+[test_kintoCertBlocklist.js]
 [test_storage_adapter.js]
 
 [test_utils_atob.js]