backout patch for 387196

2024-09-13 09:24:08 -07:00 · 2007-07-25 20:11:43 -07:00 · 2007-07-25 20:11:43 -07:00 · 3bee8fae17
commit 3bee8fae17
parent fe2806ae47
33 changed files with 2705 additions and 2871 deletions
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@ -493,7 +493,7 @@ pref("browser.safebrowsing.enabled", true);
 pref("browser.safebrowsing.remoteLookups", false);

 // Non-enhanced mode (local url lists) URL list to check for updates
-pref("browser.safebrowsing.provider.0.updateURL", "http://sb.google.com/safebrowsing/downloads?client={moz:client}&appver={moz:version}&pver=2.0");
+pref("browser.safebrowsing.provider.0.updateURL", "http://sb.google.com/safebrowsing/update?client={moz:client}&appver={moz:version}&");

 pref("browser.safebrowsing.dataProvider", 0);

--- a/browser/components/safebrowsing/content/list-warden.js
+++ b/browser/components/safebrowsing/content/list-warden.js
@ -188,51 +188,82 @@ function MultiTableQuerier(url, whiteTables, blackTables, callback) {
  this.debugZone = "multitablequerier";
  this.url_ = url;

-  this.whiteTables_ = {};
-  for (var i = 0; i < whiteTables.length; i++) {
-    this.whiteTables_[whiteTables[i]] = true;
-  }
-
-  this.blackTables_ = {};
-  for (var i = 0; i < blackTables.length; i++) {
-    this.blackTables_[blackTables[i]] = true;
-  }
+  this.whiteTables_ = whiteTables;
+  this.blackTables_ = blackTables;
+  this.whiteIdx_ = 0;
+  this.blackIdx_ = 0;

  this.callback_ = callback;
  this.listManager_ = Cc["@mozilla.org/url-classifier/listmanager;1"]
                      .getService(Ci.nsIUrlListManager);
 }

+/**
+ * We first query the white tables in succession.  If any contain
+ * the url, we stop.  If none contain the url, we query the black tables
+ * in succession.  If any contain the url, we call callback and
+ * stop.  If none of the black tables contain the url, then we just stop
+ * (i.e., it's not black url).
+ */
 MultiTableQuerier.prototype.run = function() {
-  /* ask the dbservice for all the tables to which this URL belongs */
-  this.listManager_.safeLookup(this.url_,
-                               BindToObject(this.lookupCallback_, this));
-}
-
-MultiTableQuerier.prototype.lookupCallback_ = function(result) {
-  if (result == "") {
+  var whiteTable = this.whiteTables_[this.whiteIdx_];
+  var blackTable = this.blackTables_[this.blackIdx_];
+  if (whiteTable) {
+    //G_Debug(this, "Looking in whitetable: " + whiteTable);
+    ++this.whiteIdx_;
+    this.listManager_.safeExists(whiteTable, this.url_,
+                                 BindToObject(this.whiteTableCallback_,
+                                              this));
+  } else if (blackTable) {
+    //G_Debug(this, "Looking in blacktable: " + blackTable);
+    ++this.blackIdx_;
+    this.listManager_.safeExists(blackTable, this.url_,
+                                 BindToObject(this.blackTableCallback_,
+                                              this));
+  } else {
+    // No tables left to check, so we quit.
+    G_Debug(this, "Not found in any tables: " + this.url_);
    this.callback_(PROT_ListWarden.NOT_FOUND);
-    return;
+
+    // Break circular ref to callback.
+    this.callback_ = null;
+    this.listManager_ = null;
+  }
+}
+
+/**
+ * After checking a white table, we return here.  If the url is found,
+ * we can stop.  Otherwise, we call run again.
+ */
+MultiTableQuerier.prototype.whiteTableCallback_ = function(isFound) {
+  //G_Debug(this, "whiteTableCallback_: " + isFound);
+  if (!isFound)
+    this.run();
+  else {
+    G_Debug(this, "Found in whitelist: " + this.url_)
+    this.callback_(PROT_ListWarden.IN_WHITELIST);
+
+    // Break circular ref to callback.
+    this.callback_ = null;
+    this.listManager_ = null;
+  }
+}
+
+/**
+ * After checking a black table, we return here.  If the url is found,
+ * we can call the callback and stop.  Otherwise, we call run again.
+ */
+MultiTableQuerier.prototype.blackTableCallback_ = function(isFound) {
+  //G_Debug(this, "blackTableCallback_: " + isFound);
+  if (!isFound) {
+    this.run();
+  } else {
+    // In the blacklist, must be an evil url.
+    G_Debug(this, "Found in blacklist: " + this.url_)
+    this.callback_(PROT_ListWarden.IN_BLACKLIST);
+
+    // Break circular ref to callback.
+    this.callback_ = null;
+    this.listManager_ = null;
  }
-
-  var tableNames = result.split(",");
-
-  /* Check the whitelists */
-  for (var i = 0; i < tableNames.length; i++) {
-    if (tableNames[i] && this.whiteTables_[tableNames[i]]) {
-      this.callback_(PROT_ListWarden.IN_WHITELIST);
-      return;
-    }
-  }
-
-  /* Check the blacklists */
-  for (var i = 0; i < tableNames.length; i++) {
-    if (tableNames[i] && this.blackTables_[tableNames[i]]) {
-      this.callback_(PROT_ListWarden.IN_BLACKLIST);
-      return;
-    }
-  }
-
-  /* Not in any lists we know about */
-  this.callback_(PROT_ListWarden.NOT_FOUND);
 }
--- a/browser/components/safebrowsing/content/sb-loader.js
+++ b/browser/components/safebrowsing/content/sb-loader.js
@ -79,8 +79,10 @@ var safebrowsing = {
    // Register tables
    // XXX: move table names to a pref that we originally will download
    // from the provider (need to workout protocol details)
-    phishWarden.registerWhiteTable("goog-white-exp");
-    phishWarden.registerBlackTable("goog-phish-sha128");
+    phishWarden.registerWhiteTable("goog-white-domain");
+    phishWarden.registerWhiteTable("goog-white-url");
+    phishWarden.registerBlackTable("goog-black-url");
+    phishWarden.registerBlackTable("goog-black-enchash");

    // Download/update lists if we're in non-enhanced mode
    phishWarden.maybeToggleUpdateChecking();
--- a/toolkit/components/build/Makefile.in
+++ b/toolkit/components/build/Makefile.in
@ -129,7 +129,6 @@ endif

 ifdef MOZ_URL_CLASSIFIER
 SHARED_LIBRARY_LIBS += ../url-classifier/src/$(LIB_PREFIX)urlclassifier_s.$(LIB_SUFFIX)
-EXTRA_DSO_LDOPTS += $(ZLIB_LIBS)
 endif

 ifdef MOZ_FEEDS
--- a/toolkit/components/url-classifier/content/enchash-decrypter.js
+++ b/toolkit/components/url-classifier/content/enchash-decrypter.js
@ -0,0 +1,331 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Google Safe Browsing.
+#
+# The Initial Developer of the Original Code is Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Fritz Schneider <fritz@google.com> (original author)
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+
+// This is the code used to interact with data encoded in the
+// goog-black-enchash format. The format is basically a map from
+// hashed hostnames to encrypted sequences of regular expressions
+// where the encryption key is derived from the hashed
+// hostname. Encoding lists like this raises the bar slightly on
+// deriving complete table data from the db. This data format is NOT
+// our idea; we would've raise the bar higher :)
+//
+// Anyway, this code is a port of the original C++ implementation by
+// Garret. To ease verification, I mirrored that code as closely as
+// possible.  As a result, you'll see some C++-style variable naming
+// and roundabout (C++) ways of doing things. Additionally, I've
+// omitted the comments.
+//
+// This code should not change, except to fix bugs.
+//
+// TODO: accommodate other kinds of perl-but-not-javascript qualifiers
+
+/**
+ * This thing knows how to generate lookup keys and decrypt values found in
+ * a table of type enchash.
+ */
+function PROT_EnchashDecrypter() {
+  this.debugZone = "enchashdecrypter";
+  this.REs_ = PROT_EnchashDecrypter.REs;
+  this.hasher_ = new G_CryptoHasher();
+  this.streamCipher_ = Cc["@mozilla.org/security/streamcipher;1"]
+                       .createInstance(Ci.nsIStreamCipher);
+}
+
+PROT_EnchashDecrypter.DATABASE_SALT = "oU3q.72p";
+PROT_EnchashDecrypter.SALT_LENGTH = PROT_EnchashDecrypter.DATABASE_SALT.length;
+
+PROT_EnchashDecrypter.MAX_DOTS = 5;
+
+PROT_EnchashDecrypter.REs = {};
+PROT_EnchashDecrypter.REs.FIND_DODGY_CHARS_GLOBAL =
+  new RegExp("[\x00-\x1f\x7f-\xff]+", "g");
+PROT_EnchashDecrypter.REs.FIND_END_DOTS_GLOBAL =
+  new RegExp("^\\.+|\\.+$", "g");
+PROT_EnchashDecrypter.REs.FIND_MULTIPLE_DOTS_GLOBAL =
+  new RegExp("\\.{2,}", "g");
+PROT_EnchashDecrypter.REs.FIND_TRAILING_SPACE =
+  new RegExp("^(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}) ");
+PROT_EnchashDecrypter.REs.POSSIBLE_IP =
+  new RegExp("^((?:0x[0-9a-f]+|[0-9\\.])+)$", "i");
+PROT_EnchashDecrypter.REs.FIND_BAD_OCTAL = new RegExp("(^|\\.)0\\d*[89]");
+PROT_EnchashDecrypter.REs.IS_OCTAL = new RegExp("^0[0-7]*$");
+PROT_EnchashDecrypter.REs.IS_DECIMAL = new RegExp("^[0-9]+$");
+PROT_EnchashDecrypter.REs.IS_HEX = new RegExp("^0[xX]([0-9a-fA-F]+)$");
+
+// Regexps are given in perl regexp format. Unfortunately, JavaScript's
+// library isn't completely compatible. For example, you can't specify
+// case-insensitive matching by using (?i) in the expression text :(
+// So we manually set this bit with the help of this regular expression.
+PROT_EnchashDecrypter.REs.CASE_INSENSITIVE = /\(\?i\)/g;
+
+/**
+ * Helper function 
+ *
+ * @param str String to get chars from
+ * 
+ * @param n Number of characters to get
+ *
+ * @returns String made up of the last n characters of str
+ */ 
+PROT_EnchashDecrypter.prototype.lastNChars_ = function(str, n) {
+  n = -n;
+  return str.substr(n);
+}
+
+/**
+ * Translate a plaintext enchash value into regular expressions
+ *
+ * @param data String containing a decrypted enchash db entry
+ *
+ * @returns An array of RegExps
+ */
+PROT_EnchashDecrypter.prototype.parseRegExps = function(data) {
+  var res = data.split("\t");
+  
+  G_Debug(this, "Got " + res.length + " regular rexpressions");
+  
+  for (var i = 0; i < res.length; i++) {
+    // Could have leading (?i); if so, set the flag and strip it
+    var flags = (this.REs_.CASE_INSENSITIVE.test(res[i])) ? "i" : "";
+    res[i] = res[i].replace(this.REs_.CASE_INSENSITIVE, "");
+    res[i] = new RegExp(res[i], flags);
+  }
+
+  return res;
+}
+
+/**
+ * Get the canonical version of the given URL for lookup in a table of 
+ * type -url.
+ *
+ * @param url String to canonicalize
+ *
+ * @returns String containing the canonicalized url (maximally url-decoded
+ *          with hostname normalized, then specially url-encoded)
+ */
+PROT_EnchashDecrypter.prototype.getCanonicalUrl = function(url) {
+  var urlUtils = Cc["@mozilla.org/url-classifier/utils;1"]
+                 .getService(Ci.nsIUrlClassifierUtils);
+  var escapedUrl = urlUtils.canonicalizeURL(url);
+  // Normalize the host
+  var host = this.getCanonicalHost(escapedUrl);
+  if (!host) {
+    // Probably an invalid url, return what we have so far.
+    return escapedUrl;
+  }
+
+  // Combine our normalized host with our escaped url.
+  var ioService = Cc["@mozilla.org/network/io-service;1"]
+                  .getService(Ci.nsIIOService);
+  var urlObj = ioService.newURI(escapedUrl, null, null);
+  urlObj.host = host;
+  return urlObj.asciiSpec;
+}
+
+/**
+ * @param opt_maxDots Number maximum number of dots to include.
+ */
+PROT_EnchashDecrypter.prototype.getCanonicalHost = function(str, opt_maxDots) {
+  var ioService = Cc["@mozilla.org/network/io-service;1"]
+                  .getService(Ci.nsIIOService);
+  try {
+    var urlObj = ioService.newURI(str, null, null);
+    var asciiHost = urlObj.asciiHost;
+  } catch (e) {
+    G_Debug(this, "Unable to get hostname: " + str);
+    return "";
+  }
+
+  var unescaped = unescape(asciiHost);
+
+  unescaped = unescaped.replace(this.REs_.FIND_DODGY_CHARS_GLOBAL, "")
+              .replace(this.REs_.FIND_END_DOTS_GLOBAL, "")
+              .replace(this.REs_.FIND_MULTIPLE_DOTS_GLOBAL, ".");
+
+  var temp = this.parseIPAddress_(unescaped);
+  if (temp)
+    unescaped = temp;
+
+  // Escape everything that's not alphanumeric, hyphen, or dot.
+  var urlUtils = Cc["@mozilla.org/url-classifier/utils;1"]
+                 .getService(Ci.nsIUrlClassifierUtils);
+  var escaped = urlUtils.escapeHostname(unescaped);
+
+  if (opt_maxDots) {
+    // Limit the number of dots
+    var k;
+    var index = escaped.length;
+    for (k = 0; k < opt_maxDots + 1; k++) {
+      temp = escaped.lastIndexOf(".", index - 1);
+      if (temp == -1) {
+        break;
+      } else {
+        index = temp;
+      }
+    }
+    
+    if (k == opt_maxDots + 1 && index != -1) {
+      escaped = escaped.substring(index + 1);
+    }
+  }
+
+  escaped = escaped.toLowerCase();
+  return escaped;
+}
+
+PROT_EnchashDecrypter.prototype.parseIPAddress_ = function(host) {
+  if (host.length <= 15) {
+
+    // The Windows resolver allows a 4-part dotted decimal IP address to
+    // have a space followed by any old rubbish, so long as the total length
+    // of the string doesn't get above 15 characters. So, "10.192.95.89 xy"
+    // is resolved to 10.192.95.89.
+    // If the string length is greater than 15 characters, e.g.
+    // "10.192.95.89 xy.wildcard.example.com", it will be resolved through
+    // DNS.
+    var match = this.REs_.FIND_TRAILING_SPACE.exec(host);
+    if (match) {
+      host = match[1];
+    }
+  }
+  
+  if (!this.REs_.POSSIBLE_IP.test(host))
+    return "";
+
+  var parts = host.split(".");
+  if (parts.length > 4)
+    return "";
+
+  var allowOctal = !this.REs_.FIND_BAD_OCTAL.test(host);
+
+  for (var k = 0; k < parts.length; k++) {
+    var canon;
+    if (k == parts.length - 1) {
+      canon = this.canonicalNum_(parts[k], 5 - parts.length, allowOctal);
+    } else {
+      canon = this.canonicalNum_(parts[k], 1, allowOctal);
+    }
+    if (canon != "") 
+      parts[k] = canon;
+    else
+      return "";
+  }
+
+  return parts.join(".");
+}
+
+PROT_EnchashDecrypter.prototype.canonicalNum_ = function(num, bytes, octal) {
+  if (bytes < 0) 
+    return "";
+  var temp_num;
+
+  if (octal && this.REs_.IS_OCTAL.test(num)) {
+
+    num = this.lastNChars_(num, 11);
+
+    temp_num = parseInt(num, 8);
+    if (isNaN(temp_num))
+      temp_num = -1;
+
+  } else if (this.REs_.IS_DECIMAL.test(num)) {
+
+    num = this.lastNChars_(num, 32);
+
+    temp_num = parseInt(num, 10);
+    if (isNaN(temp_num))
+      temp_num = -1;
+
+  } else if (this.REs_.IS_HEX.test(num)) {
+    var matches = this.REs_.IS_HEX.exec(num);
+    if (matches) {
+      num = matches[1];
+    }
+
+    temp_num = parseInt(num, 16);
+    if (isNaN(temp_num))
+      temp_num = -1;
+
+  } else {
+    return "";
+  }
+
+  if (temp_num == -1) 
+    return "";
+
+  // Since we mod the number, we're removing the least significant bits.  We
+  // Want to push them into the front of the array to preserve the order.
+  var parts = [];
+  while (bytes--) {
+    parts.unshift("" + (temp_num % 256));
+    temp_num -= temp_num % 256;
+    temp_num /= 256;
+  }
+
+  return parts.join(".");
+}
+
+PROT_EnchashDecrypter.prototype.getLookupKey = function(host) {
+  var dataKey = PROT_EnchashDecrypter.DATABASE_SALT + host;
+  dataKey = Array.map(dataKey, function(c) { return c.charCodeAt(0); });
+
+  this.hasher_.init(G_CryptoHasher.algorithms.MD5);
+  var lookupDigest = this.hasher_.updateFromArray(dataKey);
+  var lookupKey = this.hasher_.digestHex();
+
+  return lookupKey.toUpperCase();
+}
+
+PROT_EnchashDecrypter.prototype.decryptData = function(data, host) {
+  var ascii = atob(data);
+
+  var random_salt = ascii.slice(0, PROT_EnchashDecrypter.SALT_LENGTH);
+  var encrypted_data = ascii.slice(PROT_EnchashDecrypter.SALT_LENGTH);
+  var temp_decryption_key = PROT_EnchashDecrypter.DATABASE_SALT
+      + random_salt + host;
+  this.hasher_.init(G_CryptoHasher.algorithms.MD5);
+  this.hasher_.updateFromString(temp_decryption_key);
+
+  var keyFactory = Cc["@mozilla.org/security/keyobjectfactory;1"]
+                   .getService(Ci.nsIKeyObjectFactory);
+  var key = keyFactory.keyFromString(Ci.nsIKeyObject.RC4,
+                                     this.hasher_.digestRaw());
+
+  this.streamCipher_.init(key);
+  this.streamCipher_.updateFromString(encrypted_data);
+
+  return this.streamCipher_.finish(false /* no base64 */);
+}
--- a/toolkit/components/url-classifier/content/listmanager.js
+++ b/toolkit/components/url-classifier/content/listmanager.js
@ -38,24 +38,49 @@

 // A class that manages lists, namely white and black lists for
 // phishing or malware protection. The ListManager knows how to fetch,
-// update, and store lists.
+// update, and store lists, and knows the "kind" of list each is (is
+// it a whitelist? a blacklist? etc). However it doesn't know how the
+// lists are serialized or deserialized (the wireformat classes know
+// this) nor the specific format of each list. For example, the list
+// could be a map of domains to "1" if the domain is phishy. Or it
+// could be a map of hosts to regular expressions to match, who knows?
+// Answer: the trtable knows. List are serialized/deserialized by the
+// wireformat reader from/to trtables, and queried by the listmanager.
 //
 // There is a single listmanager for the whole application.
 //
+// The listmanager is used only in privacy mode; in advanced protection
+// mode a remote server is queried.
+//
+// How to add a new table:
+// 1) get it up on the server
+// 2) add it to tablesKnown
+// 3) if it is not a known table type (trtable.js), add an implementation
+//    for it in trtable.js
+// 4) add a check for it in the phishwarden's isXY() method, for example
+//    isBlackURL()
+//
+// TODO: obviously the way this works could use a lot of improvement. In
+//       particular adding a list should just be a matter of adding
+//       its name to the listmanager and an implementation to trtable
+//       (or not if a talbe of that type exists). The format and semantics
+//       of the list comprise its name, so the listmanager should easily
+//       be able to figure out what to do with what list (i.e., no
+//       need for step 4).
 // TODO more comprehensive update tests, for example add unittest check 
 //      that the listmanagers tables are properly written on updates

+/**
+ * The base pref name for where we keep table version numbers.
+ * We add append the table name to this and set the value to
+ * the version.  E.g., tableversion.goog-black-enchash may have
+ * a value of 1.1234.
+ */
+const kTableVersionPrefPrefix = "urlclassifier.tableversion.";
+
 // How frequently we check for updates (30 minutes)
 const kUpdateInterval = 30 * 60 * 1000;

-function QueryAdapter(callback) {
-  this.callback_ = callback;
-};
-
-QueryAdapter.prototype.handleResponse = function(value) {
-  this.callback_.handleEvent(value);
-}
-
 /**
 * A ListManager keeps track of black and white lists and knows
 * how to update them.
@ -71,7 +96,28 @@ function PROT_ListManager() {

  this.updateserverURL_ = null;

+  // The lists we know about and the parses we can use to read
+  // them. Default all to the earlies possible version (1.-1); this
+  // version will get updated when successfully read from disk or
+  // fetch updates.
+  this.tablesKnown_ = {};
  this.isTesting_ = false;
+  
+  if (this.isTesting_) {
+    // populate with some tables for unittesting
+    this.tablesKnown_ = {
+      // A major version of zero means local, so don't ask for updates       
+      "test1-foo-domain" : new PROT_VersionParser("test1-foo-domain", 0, -1),
+      "test2-foo-domain" : new PROT_VersionParser("test2-foo-domain", 0, -1),
+      "test-white-domain" : 
+        new PROT_VersionParser("test-white-domain", 0, -1, true /* require mac*/),
+      "test-mac-domain" :
+        new PROT_VersionParser("test-mac-domain", 0, -1, true /* require mac */)
+    };
+    
+    // expose the object for unittesting
+    this.wrappedJSObject = this;
+  }

  this.tablesData = {};

@ -87,9 +133,6 @@ function PROT_ListManager() {
                                   10*60*1000 /* error time, 10min */,
                                   60*60*1000 /* backoff interval, 60min */,
                                   6*60*60*1000 /* max backoff, 6hr */);
-
-  this.dbService_ = Cc["@mozilla.org/url-classifier/dbservice;1"]
-                   .getService(Ci.nsIUrlClassifierDBService);
 }

 /**
@ -120,6 +163,7 @@ PROT_ListManager.prototype.setUpdateUrl = function(url) {
    // Remove old tables which probably aren't valid for the new provider.
    for (var name in this.tablesData) {
      delete this.tablesData[name];
+      delete this.tablesKnown_[name];
    }
  }
 }
@ -144,8 +188,11 @@ PROT_ListManager.prototype.setKeyUrl = function(url) {
 */
 PROT_ListManager.prototype.registerTable = function(tableName, 
                                                    opt_requireMac) {
-  this.tablesData[tableName] = {};
-  this.tablesData[tableName].needsUpdate = false;
+  var table = new PROT_VersionParser(tableName, 1, -1, opt_requireMac);
+  if (!table)
+    return false;
+  this.tablesKnown_[tableName] = table;
+  this.tablesData[tableName] = newUrlClassifierTable(tableName);

  return true;
 }
@ -156,7 +203,7 @@ PROT_ListManager.prototype.registerTable = function(tableName,
 */
 PROT_ListManager.prototype.enableUpdate = function(tableName) {
  var changed = false;
-  var table = this.tablesData[tableName];
+  var table = this.tablesKnown_[tableName];
  if (table) {
    G_Debug(this, "Enabling table updates for " + tableName);
    table.needsUpdate = true;
@ -173,7 +220,7 @@ PROT_ListManager.prototype.enableUpdate = function(tableName) {
 */
 PROT_ListManager.prototype.disableUpdate = function(tableName) {
  var changed = false;
-  var table = this.tablesData[tableName];
+  var table = this.tablesKnown_[tableName];
  if (table) {
    G_Debug(this, "Disabling table updates for " + tableName);
    table.needsUpdate = false;
@ -188,9 +235,14 @@ PROT_ListManager.prototype.disableUpdate = function(tableName) {
 * Determine if we have some tables that need updating.
 */
 PROT_ListManager.prototype.requireTableUpdates = function() {
-  for (var type in this.tablesData) {
+  for (var type in this.tablesKnown_) {
+    // All tables with a major of 0 are internal tables that we never
+    // update remotely.
+    if (this.tablesKnown_[type].major == 0)
+      continue;
+     
    // Tables that need updating even if other tables dont require it
-    if (this.tablesData[type].needsUpdate)
+    if (this.tablesKnown_[type].needsUpdate)
      return true;
  }

@ -211,22 +263,6 @@ PROT_ListManager.prototype.maybeStartManagingUpdates = function() {
  this.maybeToggleUpdateChecking();
 }

-PROT_ListManager.prototype.kickoffUpdate_ = function (tableData)
-{
-  this.startingUpdate_ = false;
-  // If the user has never downloaded tables, do the check now.
-  // If the user has tables, add a fuzz of a few minutes.
-  var initialUpdateDelay = 3000;
-  if (tableData != "") {
-    // Add a fuzz of 0-5 minutes.
-    initialUpdateDelay += Math.floor(Math.random() * (5 * 60 * 1000));
-  }
-
-  this.currentUpdateChecker_ =
-    new G_Alarm(BindToObject(this.checkForUpdates, this),
-                initialUpdateDelay);
-}
-
 /**
 * Determine if we have any tables that require updating.  Different
 * Wardens may call us with new tables that need to be updated.
@ -245,10 +281,26 @@ PROT_ListManager.prototype.maybeToggleUpdateChecking = function() {

    // Multiple warden can ask us to reenable updates at the same time, but we
    // really just need to schedule a single update.
-    if (!this.currentUpdateChecker && !this.startingUpdate_) {
-      this.startingUpdate_ = true;
-      // check the current state of tables in the database
-      this.dbService_.getTables(BindToObject(this.kickoffUpdate_, this));
+    if (!this.currentUpdateChecker_) {
+      // If the user has never downloaded tables, do the check now.
+      // If the user has tables, add a fuzz of a few minutes.
+      this.loadTableVersions_();
+      var hasTables = false;
+      for (var table in this.tablesKnown_) {
+        if (this.tablesKnown_[table].minor != -1) {
+          hasTables = true;
+          break;
+        }
+      }
+
+      var initialUpdateDelay = 3000;
+      if (hasTables) {
+        // Add a fuzz of 0-5 minutes.
+        initialUpdateDelay += Math.floor(Math.random() * (5 * 60 * 1000));
+      }
+      this.currentUpdateChecker_ =
+        new G_Alarm(BindToObject(this.checkForUpdates, this),
+                    initialUpdateDelay);
    }
  } else {
    G_Debug(this, "Stopping managing lists (if currently active)");
@ -311,19 +363,116 @@ PROT_ListManager.prototype.stopUpdateChecker = function() {
 *        value in the table corresponding to key.  If the table name does not
 *        exist, we return false, too.
 */
-PROT_ListManager.prototype.safeLookup = function(key, callback) {
+PROT_ListManager.prototype.safeExists = function(table, key, callback) {
  try {
-    G_Debug(this, "safeLookup: " + key);
-    var cb = new QueryAdapter(callback);
-    this.dbService_.lookup(key,
-                           BindToObject(cb.handleResponse, cb),
-                           true);
+    G_Debug(this, "safeExists: " + table + ", " + key);
+    var map = this.tablesData[table];
+    map.exists(key, callback);
  } catch(e) {
-    G_Debug(this, "safeLookup masked failure for key " + key + ": " + e);
-    callback.handleEvent("");
+    G_Debug(this, "safeExists masked failure for " + table + ", key " + key + ": " + e);
+    callback.handleEvent(false);
  }
 }

+/**
+ * We store table versions in user prefs.  This method pulls the values out of
+ * the user prefs and into the tablesKnown objects.
+ */
+PROT_ListManager.prototype.loadTableVersions_ = function() {
+  // Pull values out of prefs.
+  var prefBase = kTableVersionPrefPrefix;
+  for (var table in this.tablesKnown_) {
+    var version = this.prefs_.getPref(prefBase + table, "1.-1");
+    G_Debug(this, "loadTableVersion " + table + ": " + version);
+    var tokens = version.split(".");
+    G_Assert(this, tokens.length == 2, "invalid version number");
+    
+    this.tablesKnown_[table].major = tokens[0];
+    this.tablesKnown_[table].minor = tokens[1];
+  }
+}
+
+/**
+ * Callback from db update service.  As new tables are added to the db,
+ * this callback is fired so we can update the version number.
+ * @param versionString String containing the table update response from the
+ *        server
+ */
+PROT_ListManager.prototype.setTableVersion_ = function(versionString) {
+  G_Debug(this, "Got version string: " + versionString);
+  var versionParser = new PROT_VersionParser("");
+  if (versionParser.fromString(versionString)) {
+    var tableName = versionParser.type;
+    var versionNumber = versionParser.versionString();
+    var prefBase = kTableVersionPrefPrefix;
+
+    this.prefs_.setPref(prefBase + tableName, versionNumber);
+    
+    if (!this.tablesKnown_[tableName]) {
+      this.tablesKnown_[tableName] = versionParser;
+    } else {
+      this.tablesKnown_[tableName].ImportVersion(versionParser);
+    }
+    
+    if (!this.tablesData[tableName])
+      this.tablesData[tableName] = newUrlClassifierTable(tableName);
+  }
+
+  // Since this is called from the update server, it means there was
+  // a successful http request.  Make sure to notify the request backoff
+  // object.
+  this.requestBackoff_.noteServerResponse(200 /* ok */);
+}
+
+/**
+ * Prepares a URL to fetch upates from. Format is a squence of 
+ * type:major:minor, fields
+ * 
+ * @param url The base URL to which query parameters are appended; assumes
+ *            already has a trailing ?
+ * @returns the URL that we should request the table update from.
+ */
+PROT_ListManager.prototype.getRequestURL_ = function(url) {
+  url += "version=";
+  var firstElement = true;
+  var requestMac = false;
+
+  for (var type in this.tablesKnown_) {
+    // All tables with a major of 0 are internal tables that we never
+    // update remotely.
+    if (this.tablesKnown_[type].major == 0)
+      continue;
+
+    // Check if the table needs updating
+    if (this.tablesKnown_[type].needsUpdate == false)
+      continue;
+
+    if (!firstElement) {
+      url += ","
+    } else {
+      firstElement = false;
+    }
+    url += type + ":" + this.tablesKnown_[type].toUrl();
+
+    if (this.tablesKnown_[type].requireMac)
+      requestMac = true;
+  }
+
+  // Request a mac only if at least one of the tables to be updated requires
+  // it
+  if (requestMac) {
+    // Add the wrapped key for requesting macs
+    if (!this.urlCrypto_)
+      this.urlCrypto_ = new PROT_UrlCrypto();
+
+    url += "&wrkey=" +
+      encodeURIComponent(this.urlCrypto_.getManager().getWrappedKey());
+  }
+
+  G_Debug(this, "getRequestURL returning: " + url);
+  return url;
+}
+
 /**
 * Updates our internal tables from the update server
 *
@ -343,87 +492,56 @@ PROT_ListManager.prototype.checkForUpdates = function() {
  if (!this.requestBackoff_.canMakeRequest())
    return false;

-  // Grab the current state of the tables from the database
-  this.dbService_.getTables(BindToObject(this.makeUpdateRequest_, this));
+  // Check to make sure our tables still exist (maybe the db got corrupted or
+  // the user deleted the file).  If not, we need to reset the table version
+  // before sending the update check.
+  var tableNames = [];
+  for (var tableName in this.tablesKnown_) {
+    tableNames.push(tableName);
+  }
+  var dbService = Cc["@mozilla.org/url-classifier/dbservice;1"]
+                  .getService(Ci.nsIUrlClassifierDBService);
+  dbService.checkTables(tableNames.join(","),
+                        BindToObject(this.makeUpdateRequest_, this));
  return true;
 }

 /**
 * Method that fires the actual HTTP update request.
 * First we reset any tables that have disappeared.
- * @param tableData List of table data already in the database, in the form
- *        tablename;<chunk ranges>\n
+ * @param tableNames String comma separated list of tables that
+ *   don't exist
 */
-PROT_ListManager.prototype.makeUpdateRequest_ = function(tableData) {
-  var tableNames = {};
-  for (var tableName in this.tablesData) {
-    tableNames[tableName] = true;
+PROT_ListManager.prototype.makeUpdateRequest_ = function(tableNames) {
+  // Clear prefs that track table version if they no longer exist in the db.
+  var tables = tableNames.split(",");
+  for (var i = 0; i < tables.length; ++i) {
+    G_Debug(this, "Table |" + tables[i] + "| no longer exists, clearing pref.");
+    this.prefs_.clearPref(kTableVersionPrefPrefix + tables[i]);
  }

-  var request = "";
-
-  // For each table already in the database, include the chunk data from
-  // the database
-  var lines = tableData.split("\n");
-  for (var i = 0; i < lines.length; i++) {
-    var fields = lines[i].split(";");
-    if (tableNames[fields[0]]) {
-      request += lines[i] + "\n";
-      delete tableNames[fields[0]];
-    }
-  }
-
-  // For each requested table that didn't have chunk data in the database,
-  // request it fresh
-  for (var tableName in tableNames) {
-    request += tableName + ";\n";
-  }
+  // Ok, now reload the table version.
+  this.loadTableVersions_();

  G_Debug(this, 'checkForUpdates: scheduling request..');
+  var url = this.getRequestURL_(this.updateserverURL_);
  var streamer = Cc["@mozilla.org/url-classifier/streamupdater;1"]
                 .getService(Ci.nsIUrlClassifierStreamUpdater);
  try {
-    streamer.updateUrl = this.updateserverURL_;
+    streamer.updateUrl = url;
  } catch (e) {
    G_Debug(this, 'invalid url');
    return;
  }

-  if (!streamer.downloadUpdates(request,
-                                BindToObject(this.updateSuccess_, this),
-                                BindToObject(this.updateError_, this),
+  if (!streamer.downloadUpdates(BindToObject(this.setTableVersion_, this),
                                BindToObject(this.downloadError_, this))) {
    G_Debug(this, "pending update, wait until later");
  }
 }

 /**
- * Callback function if the update request succeeded.
- * @param waitForUpdate String The number of seconds that the client should
- *        wait before requesting again.
- */
-PROT_ListManager.prototype.updateSuccess_ = function(waitForUpdate) {
-  G_Debug(this, "update success: " + waitForUpdate);
-  if (waitForUpdate) {
-    var delay = parseInt(waitForUpdate, 10);
-    // As long as the delay is something sane (5 minutes or more), update
-    // our delay time for requesting updates
-    if (delay >= (5 * 60) && this.updateChecker_)
-      this.updateChecker_.setDelay(delay * 1000);
-  }
-}
-
-/**
- * Callback function if the update request succeeded.
- * @param result String The error code of the failure
- */
-PROT_ListManager.prototype.updateError_ = function(result) {
-  G_Debug(this, "update error: " + result);
-  // XXX: there was some trouble applying the updates.
-}
-
-/**
- * Callback function when the download failed
+ * Callback function if there's a download error.
 * @param status String http status or an empty string if connection refused.
 */
 PROT_ListManager.prototype.downloadError_ = function(status) {
@ -450,3 +568,17 @@ PROT_ListManager.prototype.QueryInterface = function(iid) {
  Components.returnCode = Components.results.NS_ERROR_NO_INTERFACE;
  return null;
 }
+
+// A simple factory function that creates nsIUrlClassifierTable instances based
+// on a name.  The name is a string of the format
+// provider_name-semantic_type-table_type.  For example, goog-white-enchash
+// or goog-black-url.
+function newUrlClassifierTable(name) {
+  G_Debug("protfactory", "Creating a new nsIUrlClassifierTable: " + name);
+  var tokens = name.split('-');
+  var type = tokens[2];
+  var table = Cc['@mozilla.org/url-classifier/table;1?type=' + type]
+                .createInstance(Ci.nsIUrlClassifierTable);
+  table.name = name;
+  return table;
+}
--- a/toolkit/components/url-classifier/content/moz/alarm.js
+++ b/toolkit/components/url-classifier/content/moz/alarm.js
@ -134,10 +134,6 @@ G_Alarm.prototype.notify = function(timer) {
  return ret;
 }

-G_Alarm.prototype.setDelay = function(delay) {
-  this.timer_.delay = delay;
-}
-
 /**
 * XPCOM cruft
 */
--- a/toolkit/components/url-classifier/content/multi-querier.js
+++ b/toolkit/components/url-classifier/content/multi-querier.js
@ -0,0 +1,152 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Google Safe Browsing.
+#
+# The Initial Developer of the Original Code is Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Tony Chang <tony@google.com> (original author)
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+/**
+ * This class helps us batch a series of async calls to the db.
+ * If any of the tokens is in the database, we fire callback with
+ * true as a param.  If all the tokens are not in the  database,
+ * we fire callback with false as a param.
+ * This is an "Abstract" base class.  Subclasses need to supply
+ * the condition_ method.
+ *
+ * @param tokens Array of strings to lookup in the db
+ * @param tableName String name of the table
+ * @param callback Function callback function that takes true if the condition
+ *        passes.
+ */
+function MultiQuerier(tokens, tableName, callback) {
+  this.tokens_ = tokens;
+  this.tableName_ = tableName;
+  this.callback_ = callback;
+  this.dbservice_ = Cc["@mozilla.org/url-classifier/dbservice;1"]
+                    .getService(Ci.nsIUrlClassifierDBService);
+  // We put the current token in this variable.
+  this.key_ = null;
+}
+
+/**
+ * Run the remaining tokens against the db.
+ */
+MultiQuerier.prototype.run = function() {
+  if (this.tokens_.length == 0) {
+    this.callback_.handleEvent(false);
+    this.dbservice_ = null;
+    this.callback_ = null;
+    return;
+  }
+  
+  this.key_ = this.tokens_.pop();
+  G_Debug(this, "Looking up " + this.key_ + " in " + this.tableName_);
+  this.dbservice_.exists(this.tableName_, this.key_,
+                         BindToObject(this.result_, this));
+}
+
+/**
+ * Callback from the db.  If the returned value passes the this.condition_
+ * test, go ahead and call the main callback.
+ */
+MultiQuerier.prototype.result_ = function(value) {
+  if (this.condition_(value)) {
+    this.callback_.handleEvent(true)
+    this.dbservice_ = null;
+    this.callback_ = null;
+  } else {
+    this.run();
+  }
+}
+
+// Subclasses must override this.
+MultiQuerier.prototype.condition_ = function(value) {
+  throw "MultiQuerier is an abstract base class";
+}
+
+
+/**
+ * Concrete MultiQuerier that stops if the key exists in the db.
+ */
+function ExistsMultiQuerier(tokens, tableName, callback) {
+  MultiQuerier.call(this, tokens, tableName, callback);
+  this.debugZone = "existsMultiQuerier";
+}
+ExistsMultiQuerier.inherits(MultiQuerier);
+
+ExistsMultiQuerier.prototype.condition_ = function(value) {
+  return value.length > 0;
+}
+
+
+/**
+ * Concrete MultiQuerier that looks up a key, decrypts it, then
+ * checks the the resulting regular expressions for a match.
+ * @param tokens Array of hosts
+ */
+function EnchashMultiQuerier(tokens, tableName, callback, url) {
+  MultiQuerier.call(this, tokens, tableName, callback);
+  this.url_ = url;
+  this.enchashDecrypter_ = new PROT_EnchashDecrypter();
+  this.debugZone = "enchashMultiQuerier";
+}
+EnchashMultiQuerier.inherits(MultiQuerier);
+
+EnchashMultiQuerier.prototype.run = function() {
+  if (this.tokens_.length == 0) {
+    this.callback_.handleEvent(false);
+    this.dbservice_ = null;
+    this.callback_ = null;
+    return;
+  }
+  var host = this.tokens_.pop();
+  this.key_ = host;
+  var lookupKey = this.enchashDecrypter_.getLookupKey(host);
+  this.dbservice_.exists(this.tableName_, lookupKey,
+                         BindToObject(this.result_, this));
+}
+
+EnchashMultiQuerier.prototype.condition_ = function(encryptedValue) {
+  if (encryptedValue.length > 0) {
+    // We have encrypted regular expressions for this host. Let's 
+    // decrypt them and see if we have a match.
+    var decrypted = this.enchashDecrypter_.decryptData(encryptedValue,
+                                                       this.key_);
+    var res = this.enchashDecrypter_.parseRegExps(decrypted);
+    for (var j = 0; j < res.length; j++) {
+      if (res[j].test(this.url_)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
--- a/toolkit/components/url-classifier/content/trtable.js
+++ b/toolkit/components/url-classifier/content/trtable.js
@ -0,0 +1,181 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Url Classifier code
+#
+# The Initial Developer of the Original Code is
+# Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Tony Chang <tony@ponderer.org>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+// XXX: This should all be moved into the dbservice class so it happens
+// in the background thread.
+
+/**
+ * Abstract base class for a lookup table.
+ * @construction
+ */
+function UrlClassifierTable() {
+  this.debugZone = "urlclassifier-table";
+  this.name = '';
+  this.needsUpdate = false;
+  this.enchashDecrypter_ = new PROT_EnchashDecrypter();
+  this.wrappedJSObject = this;
+}
+
+UrlClassifierTable.prototype.QueryInterface = function(iid) {
+  if (iid.equals(Components.interfaces.nsISupports) ||
+      iid.equals(Components.interfaces.nsIUrlClassifierTable))
+    return this;                                              
+  Components.returnCode = Components.results.NS_ERROR_NO_INTERFACE;
+  return null;
+}
+
+/**
+ * Subclasses need to implment this method.
+ */
+UrlClassifierTable.prototype.exists = function(url, callback) {
+  throw Components.results.NS_ERROR_NOT_IMPLEMENTED;
+}
+
+/////////////////////////////////////////////////////////////////////
+// Url table implementation
+function UrlClassifierTableUrl() {
+  UrlClassifierTable.call(this);
+}
+UrlClassifierTableUrl.inherits(UrlClassifierTable);
+
+/**
+ * Look up a URL in a URL table
+ */
+UrlClassifierTableUrl.prototype.exists = function(url, callback) {
+  // nsIUrlClassifierUtils.canonicalizeURL is the old way of canonicalizing a
+  // URL.  Unfortunately, it doesn't normalize numeric domains so alternate IP
+  // formats (hex, octal, etc) won't trigger a match.
+  // this.enchashDecrypter_.getCanonicalUrl does the right thing and
+  // normalizes a URL to 4 decimal numbers, but the update server may still be
+  // giving us encoded IP addresses.  So to be safe, we check both cases.
+  var urlUtils = Cc["@mozilla.org/url-classifier/utils;1"]
+                 .getService(Ci.nsIUrlClassifierUtils);
+  var oldCanonicalized = urlUtils.canonicalizeURL(url);
+  var canonicalized = this.enchashDecrypter_.getCanonicalUrl(url);
+  G_Debug(this, "Looking up: " + url + " (" + oldCanonicalized + " and " +
+                canonicalized + ")");
+  (new ExistsMultiQuerier([oldCanonicalized, canonicalized],
+                          this.name,
+                          callback)).run();
+}
+
+/////////////////////////////////////////////////////////////////////
+// Domain table implementation
+
+function UrlClassifierTableDomain() {
+  UrlClassifierTable.call(this);
+  this.debugZone = "urlclassifier-table-domain";
+  this.ioService_ = Cc["@mozilla.org/network/io-service;1"]
+                    .getService(Ci.nsIIOService);
+}
+UrlClassifierTableDomain.inherits(UrlClassifierTable);
+
+/**
+ * Look up a URL in a domain table
+ * We also try to lookup domain + first path component (e.g.,
+ * www.mozilla.org/products).
+ *
+ * @returns Boolean true if the url domain is in the table
+ */
+UrlClassifierTableDomain.prototype.exists = function(url, callback) {
+  var canonicalized = this.enchashDecrypter_.getCanonicalUrl(url);
+  var urlObj = this.ioService_.newURI(canonicalized, null, null);
+  var host = '';
+  try {
+    host = urlObj.host;
+  } catch (e) { }
+  var hostComponents = host.split(".");
+
+  // Try to get the path of the URL.  Pseudo urls (like wyciwyg:) throw
+  // errors when trying to convert to an nsIURL so we wrap in a try/catch
+  // block.
+  var path = ""
+  try {
+    urlObj.QueryInterface(Ci.nsIURL);
+    path = urlObj.filePath;
+  } catch (e) { }
+
+  var pathComponents = path.split("/");
+
+  // We don't have a good way map from hosts to domains, so we instead try
+  // each possibility. Could probably optimize to start at the second dot?
+  var possible = [];
+  for (var i = 0; i < hostComponents.length - 1; i++) {
+    host = hostComponents.slice(i).join(".");
+    possible.push(host);
+
+    // The path starts with a "/", so we are interested in the second path
+    // component if it is available
+    if (pathComponents.length >= 2 && pathComponents[1].length > 0) {
+      host = host + "/" + pathComponents[1];
+      possible.push(host);
+    }
+  }
+
+  // Run the possible domains against the db.
+  (new ExistsMultiQuerier(possible, this.name, callback)).run();
+}
+
+/////////////////////////////////////////////////////////////////////
+// Enchash table implementation
+
+function UrlClassifierTableEnchash() {
+  UrlClassifierTable.call(this);
+  this.debugZone = "urlclassifier-table-enchash";
+}
+UrlClassifierTableEnchash.inherits(UrlClassifierTable);
+
+/**
+ * Look up a URL in an enchashDB.  We try all sub domains (up to MAX_DOTS).
+ */
+UrlClassifierTableEnchash.prototype.exists = function(url, callback) {
+  url = this.enchashDecrypter_.getCanonicalUrl(url);
+  var host = this.enchashDecrypter_.getCanonicalHost(url,
+                                               PROT_EnchashDecrypter.MAX_DOTS);
+
+  var possible = [];
+  for (var i = 0; i < PROT_EnchashDecrypter.MAX_DOTS + 1; i++) {
+    possible.push(host);
+
+    var index = host.indexOf(".");
+    if (index == -1)
+      break;
+    host = host.substring(index + 1);
+  }
+  // Run the possible domains against the db.
+  (new EnchashMultiQuerier(possible, this.name, callback, url)).run();
+}
--- a/toolkit/components/url-classifier/content/wireformat.js
+++ b/toolkit/components/url-classifier/content/wireformat.js
@ -0,0 +1,262 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Google Safe Browsing.
+#
+# The Initial Developer of the Original Code is Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Niels Provos <niels@google.com> (original author)
+#   Fritz Schneider <fritz@google.com>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+
+// A class that serializes and deserializes opaque key/value string to
+// string maps to/from maps (trtables). It knows how to create
+// trtables from the serialized format, so it also understands
+// meta-information like the name of the table and the table's
+// version. See docs for the protocol description.
+// 
+// TODO: wireformatreader: if you have multiple updates for one table
+//       in a call to deserialize, the later ones will be merged 
+//       (all but the last will be ignored). To fix, merge instead
+//       of replace when you have an existing table, and only do so once.
+// TODO must have blank line between successive types -- problem?
+// TODO doesn't tolerate blank lines very well
+//
+// Maybe: These classes could use a LOT more cleanup, but it's not a
+//       priority at the moment. For example, the tablesData/Known
+//       maps should be combined into a single object, the parser
+//       for a given type should be separate from the version info,
+//       and there should be synchronous interfaces for testing.
+
+
+/**
+ * A class that knows how to serialize and deserialize meta-information.
+ * This meta information is the table name and version number, and 
+ * in its serialized form looks like the first line below:
+ * 
+ * [name-of-table X.Y update?]                
+ * ...key/value pairs to add or delete follow...
+ * <blank line ends the table>
+ *
+ * The X.Y is the version number and the optional "update" token means 
+ * that the table is a differential from the curent table the extension
+ * has. Its absence means that this is a full, new table.
+ */
+function PROT_VersionParser(type, opt_major, opt_minor, opt_requireMac) {
+  this.debugZone = "versionparser";
+  this.type = type;
+  this.major = 0;
+  this.minor = 0;
+
+  this.badHeader = false;
+
+  // Should the wireformatreader compute a mac?
+  this.mac = false;
+  this.macval = "";
+  this.macFailed = false;
+  this.requireMac = !!opt_requireMac;
+
+  this.update = false;
+  this.needsUpdate = false;  // used by ListManager to determine update policy
+  // Used by ListerManager to see if we have read data for this table from
+  // disk.  Once we read a table from disk, we are not going to do so again
+  // but instead update remotely if necessary.
+  this.didRead = false;
+  if (opt_major)
+    this.major = parseInt(opt_major);
+  if (opt_minor)
+    this.minor = parseInt(opt_minor);
+}
+
+/** Import the version information from another VersionParser
+ * @params version a version parser object
+ */
+PROT_VersionParser.prototype.ImportVersion = function(version) {
+  this.major = version.major;
+  this.minor = version.minor;
+
+  this.mac = version.mac;
+  this.macFailed = version.macFailed;
+  this.macval = version.macval;
+  // Don't set requireMac, since we create vparsers from scratch and doesn't
+  // know about it
+}
+
+/** 
+ * Creates a string like [goog-white-black 1.1] from internal information
+ * 
+ * @returns String
+ */
+PROT_VersionParser.prototype.toString = function() {
+  var s = "[" + this.type + " " + this.major + "." + this.minor + "]";
+  return s;
+}
+
+/**
+ * Creates a string like 1.123 with the version number.  This is the
+ * format we store in prefs.
+ * @return String
+ */
+PROT_VersionParser.prototype.versionString = function() {
+  return this.major + "." + this.minor;
+}
+
+/** 
+ * Creates a string like 1:1 from internal information used for
+ * fetching updates from the server. Called by the listmanager.
+ * 
+ * @returns String
+ */
+PROT_VersionParser.prototype.toUrl = function() {
+  return this.major + ":" + this.minor;
+}
+
+/**
+ * Process the old format, [type major.minor [update]]
+ *
+ * @returns true if the string could be parsed, false otherwise
+ */
+PROT_VersionParser.prototype.processOldFormat_ = function(line) {
+  if (line[0] != '[' || line.slice(-1) != ']')
+    return false;
+
+  var description = line.slice(1, -1);
+
+  // Get the type name and version number of this table
+  var tokens = description.split(" ");
+  this.type = tokens[0];
+  var majorminor = tokens[1].split(".");
+  this.major = parseInt(majorminor[0]);
+  this.minor = parseInt(majorminor[1]);
+  if (isNaN(this.major) || isNaN(this.minor))
+    return false;
+
+  if (tokens.length >= 3) {
+     this.update = tokens[2] == "update";
+  }
+
+  return true;
+}
+
+/**
+ * Takes a string like [name-of-table 1.1 [update]][mac=MAC] and figures out the
+ * type and corresponding version numbers.
+ * @returns true if the string could be parsed, false otherwise
+ */
+PROT_VersionParser.prototype.fromString = function(line) {
+  G_Debug(this, "Calling fromString with line: " + line);
+  if (line[0] != '[' || line.slice(-1) != ']')
+    return false;
+
+  // There could be two [][], so take care of it
+  var secondBracket = line.indexOf('[', 1);
+  var firstPart = null;
+  var secondPart = null;
+
+  if (secondBracket != -1) {
+    firstPart = line.substring(0, secondBracket);
+    secondPart = line.substring(secondBracket);
+    G_Debug(this, "First part: " + firstPart + " Second part: " + secondPart);
+  } else {
+    firstPart = line;
+    G_Debug(this, "Old format: " + firstPart);
+  }
+
+  if (!this.processOldFormat_(firstPart))
+    return false;
+
+  if (secondPart && !this.processOptTokens_(secondPart))
+    return false;
+
+  return true;
+}
+
+/**
+ * Process optional tokens
+ *
+ * @param line A string [token1=val1 token2=val2...]
+ * @returns true if the string could be parsed, false otherwise
+ */
+PROT_VersionParser.prototype.processOptTokens_ = function(line) {
+  if (line[0] != '[' || line.slice(-1) != ']')
+    return false;
+  var description = line.slice(1, -1);
+  // Get the type name and version number of this table
+  var tokens = description.split(" ");
+
+  for (var i = 0; i < tokens.length; i++) {
+    G_Debug(this, "Processing optional token: " + tokens[i]);
+    var tokenparts = tokens[i].split("=");
+    switch(tokenparts[0]){
+    case "mac":
+      this.mac = true;
+      if (tokenparts.length < 2) {
+        G_Debug(this, "Found mac flag but not mac value!");
+        return false;
+      }
+      // The mac value may have "=" in it, so we can't just use tokenparts[1].
+      // Instead, just take the rest of tokens[i] after the first "="
+      this.macval = tokens[i].substr(tokens[i].indexOf("=")+1);
+      break;
+    default:
+      G_Debug(this, "Found unrecognized token: " + tokenparts[0]);
+      break;
+    }
+  }
+
+  return true;
+}
+
+#ifdef DEBUG
+function TEST_PROT_WireFormat() {
+  if (G_GDEBUG) {
+    var z = "versionparser UNITTEST";
+    G_Debug(z, "Starting");
+
+    var vp = new PROT_VersionParser("dummy");
+    G_Assert(z, vp.fromString("[foo-bar-url 1.234]"),
+             "failed to parse old format");
+    G_Assert(z, "foo-bar-url" == vp.type, "failed to parse type");
+    G_Assert(z, "1" == vp.major, "failed to parse major");
+    G_Assert(z, "234" == vp.minor, "failed to parse minor");
+
+    vp = new PROT_VersionParser("dummy");
+    G_Assert(z, vp.fromString("[foo-bar-url 1.234][mac=567]"),
+             "failed to parse new format");
+    G_Assert(z, "foo-bar-url" == vp.type, "failed to parse type");
+    G_Assert(z, "1" == vp.major, "failed to parse major");
+    G_Assert(z, "234" == vp.minor, "failed to parse minor");
+    G_Assert(z, true == vp.mac, "failed to parse mac");
+    G_Assert(z, "567" == vp.macval, "failed to parse macval");
+
+    G_Debug(z, "PASSED");
+  }
+}
+#endif
--- a/toolkit/components/url-classifier/public/Makefile.in
+++ b/toolkit/components/url-classifier/public/Makefile.in
@ -10,6 +10,7 @@ XPIDL_MODULE = url-classifier

 XPIDLSRCS = nsIUrlClassifierDBService.idl \
            nsIUrlClassifierStreamUpdater.idl \
+            nsIUrlClassifierTable.idl \
            nsIUrlClassifierUtils.idl \
            nsIUrlListManager.idl \
            $(NULL)
--- a/toolkit/components/url-classifier/public/nsIUrlClassifierDBService.idl
+++ b/toolkit/components/url-classifier/public/nsIUrlClassifierDBService.idl
@ -49,34 +49,32 @@ interface nsIUrlClassifierCallback : nsISupports {
 * It provides async methods for querying and updating the database.  As the
 * methods complete, they call the callback function.
 */
-[scriptable, uuid(10928bf5-e18d-4086-854b-6c4006f2b009)]
+[scriptable, uuid(211d5360-4af6-4a1d-99c1-926d35861eaf)]
 interface nsIUrlClassifierDBService : nsISupports
 {
  /**
-   * Looks up a key in the database.
-   *
-   * @param key: The URL to search for.  This URL will be canonicalized
-   *        by the service.
-   * @param c: The callback will be called with a comma-separated list
-   *        of tables to which the key belongs.
-   * @param needsProxy: Should be true if the callback needs to be called
-   *        in the main thread, false if the callback is threadsafe.
+   * Looks up a key in the database.  After it finds a value, it calls
+   * callback with the value as the first param.  If the key is not in
+   * the db or the table does not exist, the callback is called with
+   * an empty string parameter.
   */
-  void lookup(in ACString spec,
-              in nsIUrlClassifierCallback c,
-              in boolean needsProxy);
+  void exists(in ACString tableName, in ACString key,
+              in nsIUrlClassifierCallback c);

  /**
-   * Lists the tables along with which chunks are available in each table.
-   * This list is in the format of the request body:
-   *   tablename;chunkdata\n
-   *   tablename2;chunkdata2\n
-   *
-   * For example:
-   *   goog-phish-regexp;a:10,14,30-40s:56,67
-   *   goog-white-regexp;a:1-3,5
+   * Checks to see if the tables exist.  tableNames is a comma separated list
+   * of table names to check.  The callback is called with a comma separated
+   * list of tables that no longer exist (either the db is corrupted or the
+   * user deleted the file).
   */
-  void getTables(in nsIUrlClassifierCallback c);
+  void checkTables(in ACString tableNames, in nsIUrlClassifierCallback c);
+
+  /**
+   * Updates the table in the background.  Calls callback after each table
+   * completes processing with the new table line as the parameter.  This
+   * allows us to keep track of the table version in our main thread.
+   */
+  void updateTables(in ACString updateString, in nsIUrlClassifierCallback c);

  ////////////////////////////////////////////////////////////////////////////
  // Incremental update methods.  These are named to match similar methods
@ -91,12 +89,10 @@ interface nsIUrlClassifierDBService : nsISupports
  // interface, but it's tricky because of XPCOM proxies.

  /**
-   * Finish an incremental update.  Calls successCallback with the
-   * requested delay before the next update, or failureCallback with a
-   * result code.
+   * Finish an incremental update.  This commits any pending tables and
+   * calls the callback for each completed table.
   */
-  void finish(in nsIUrlClassifierCallback successCallback,
-              in nsIUrlClassifierCallback failureCallback);
+  void finish(in nsIUrlClassifierCallback c);

  /**
   * Cancel an incremental update.  This rolls back and pending changes.
--- a/toolkit/components/url-classifier/public/nsIUrlClassifierStreamUpdater.idl
+++ b/toolkit/components/url-classifier/public/nsIUrlClassifierStreamUpdater.idl
@ -44,7 +44,7 @@
 * downloading the whole update and then updating the sqlite database, we
 * update tables as the data is streaming in.
 */
-[scriptable, uuid(adf0dfaa-ce91-4cf2-ab15-f5810408e2ec)]
+[scriptable, uuid(d9277fa4-7d51-4175-bd4e-546c080a83bf)]
 interface nsIUrlClassifierStreamUpdater : nsISupports
 {
  /**
@ -56,14 +56,11 @@ interface nsIUrlClassifierStreamUpdater : nsISupports
   * Try to download updates from updateUrl.  Only one instance of this
   * runs at a time, so we return false if another instance is already
   * running.
-   * @param aRequestBody The body for the request.
-   * @param aSuccessCallback Called after a successful update.
-   * @param aUpdateErrorCallback Called for problems applying the update
-   * @param aDownloadErrorCallback Called if we get an http error or a
-   *        connection refused error.
+   * @param aTableCallback Called once for each table that we successfully
+   *     download with the table header as the parameter.
+   * @param aErrorCallback Called if we get an http error or a connection
+   *   refused.
   */
-  boolean downloadUpdates(in ACString aRequestBody,
-                          in nsIUrlClassifierCallback aSuccessCallback,
-                          in nsIUrlClassifierCallback aUpdateErrorCallback,
-                          in nsIUrlClassifierCallback aDownloadErrorCallback);
+  boolean downloadUpdates(in nsIUrlClassifierCallback aTableCallback,
+                          in nsIUrlClassifierCallback aErrorCallback);
 };
--- a/toolkit/components/url-classifier/public/nsIUrlClassifierTable.idl
+++ b/toolkit/components/url-classifier/public/nsIUrlClassifierTable.idl
@ -1,5 +1,4 @@
-/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
 /* ***** BEGIN LICENSE BLOCK *****
 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
 *
@ -13,14 +12,14 @@
 * for the specific language governing rights and limitations under the
 * License.
 *
- * The Original Code is the Metrics extension.
+ * The Original Code is mozilla.org code.
 *
- * The Initial Developer of the Original Code is Google Inc.
- * Portions created by the Initial Developer are Copyright (C) 2006
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 1998
 * the Initial Developer. All Rights Reserved.
 *
 * Contributor(s):
- *  Brian Ryner <bryner@brianryner.com>
 *
 * Alternatively, the contents of this file may be used under the terms of
 * either the GNU General Public License Version 2 or later (the "GPL"), or
@ -36,42 +35,29 @@
 *
 * ***** END LICENSE BLOCK ***** */

-/**
- * This file contains trivial implementations of the NSS PORT_* functions
- * that sha256.c uses.
- */
+#include "nsISupports.idl"
+#include "nsIUrlListManager.idl"

-#include "prmem.h"
-#include "prerror.h"
-#include "string.h"
+// A map that contains a string keys mapped to string values.

-void*
-PORT_Alloc(size_t bytes)
+[scriptable, uuid(fd1f8334-1859-472d-b01f-4ac6b1121ce4)]
+interface nsIUrlClassifierTable : nsISupports
 {
-  /* Always allocate a non-zero amount of bytes */
-  return (void *)PR_Malloc(bytes ? bytes : 1);
-}
+    /**
+     * The name used to identify this table
+     */
+    attribute ACString name;

-void
-PORT_Free(void *ptr)
-{
-  if (ptr) {
-    PR_Free(ptr);
-  }
-}
+    /**
+     * Set to false if we don't want to update this table.
+     */
+    attribute boolean needsUpdate;

-void
-PORT_ZFree(void *ptr, size_t len)
-{
-  if (ptr) {
-    memset(ptr, 0, len);
-    PR_Free(ptr);
-  }
-}
-
-void
-PORT_SetError(int value)
-{
-  PR_SetError(value, 0);
-  return;
-}
+    /**
+     * In the simple case, exists just looks up the string in the
+     * table and call the callback after the query returns with true or
+     * false.  It's possible that something more complex happens
+     * (e.g., canonicalize the url).
+     */
+    void exists(in ACString key, in nsIUrlListManagerCallback cb);
+};
--- a/toolkit/components/url-classifier/public/nsIUrlClassifierUtils.idl
+++ b/toolkit/components/url-classifier/public/nsIUrlClassifierUtils.idl
@ -39,18 +39,27 @@
 * Some utility methods used by the url classifier.
 */

-interface nsIURI;
-
-[scriptable, uuid(e4f0e59c-b922-48b0-a7b6-1735c1f96fed)]
+[scriptable, uuid(89ea43b0-a23f-4db2-8d23-6d90dc55f67a)]
 interface nsIUrlClassifierUtils : nsISupports
 {
  /**
-   * Get the lookup string for a given URI.  This normalizes the hostname,
-   * url-decodes the string, and strips off the protocol.
+   * Canonicalize a URL.  DON'T USE THIS DIRECTLY.  Use
+   * PROT_EnchashDecrypter.prototype.getCanonicalUrl instead.  This method
+   * url-decodes a string, but it doesn't normalize the hostname.  The method
+   * in EnchashDecrypter first calls this method, then normalizes the hostname.
   *
-   * @param uri URI to get the lookup key for.
+   * @param url String to canonicalize
   *
-   * @returns String containing the canonicalized URI.
+   * @returns String containing the canonicalized url (maximally url-decoded,
+   *          then specially url-encoded)
   */
-  ACString getKeyForURI(in nsIURI uri);
+  ACString canonicalizeURL(in ACString url);
+
+  /**
+   * When canonicalizing hostnames, the final step is to url escape everything that
+   * is not alphanumeric or hyphen or dot.  The existing methods (escape,
+   * encodeURIComponent and encodeURI are close, but not exactly what we want
+   * so we write our own function to do this.
+   */
+  ACString escapeHostname(in ACString hostname);
 };
--- a/toolkit/components/url-classifier/public/nsIUrlListManager.idl
+++ b/toolkit/components/url-classifier/public/nsIUrlListManager.idl
@ -39,17 +39,16 @@
 #include "nsISupports.idl"

 /**
- * Interface for a class that manages updates of the url classifier database.
+ * Interface for a class that manages updates of multiple nsIUrlClassifierTables.
 */

 // Interface for JS function callbacks
-[scriptable, function, uuid(fa4caf12-d057-4e7e-81e9-ce066ceee90b)]
+[scriptable, function, uuid(ba913c5c-13d6-41eb-83c1-de2f4165a516)]
 interface nsIUrlListManagerCallback : nsISupports {
-  void handleEvent(in ACString value);
+  void handleEvent(in boolean value);
 };

-
-[scriptable, uuid(874d6c95-fb8b-4f89-b36d-85fe267ab356)]
+[scriptable, uuid(d39982d6-da4f-4a27-8d91-f9c7b179aa33)]
 interface nsIUrlListManager : nsISupports
 {
    /**
@ -83,12 +82,10 @@ interface nsIUrlListManager : nsISupports
    void disableUpdate(in ACString tableName);

    /**
-     * Lookup a key.  Should not raise exceptions.  Calls the callback
-     * function with a comma-separated list of tables to which the key
-     * belongs.
+     * Lookup a key in a table.  Should not raise exceptions.  Calls
+     * the callback function with a single parameter: true if the key
+     * is in the table, false if it isn't.
     */
-    void safeLookup(in ACString key,
+    void safeExists(in ACString tableName, in ACString key,
                    in nsIUrlListManagerCallback cb);
-
-    void checkForUpdates();
 };
--- a/toolkit/components/url-classifier/src/Makefile.in
+++ b/toolkit/components/url-classifier/src/Makefile.in
@ -16,7 +16,6 @@ REQUIRES = necko \
           storage \
           string \
           xpcom \
-           $(ZLIB_REQUIRES) \
           $(NULL)

 CPPSRCS = \
@ -25,29 +24,15 @@ CPPSRCS = \
          nsUrlClassifierUtils.cpp \
          $(NULL)

-CSRCS = \
-        sha512.c \
-        nssstubs.c \
-        $(NULL)
-
-# sha512.c requires NSS headers
 LOCAL_INCLUDES = \
-                 -I$(srcdir)/../../build \
-                 -I$(DIST)/public/nss \
-                 -I$(DIST)/private/nss \
-                 -I$(topsrcdir)/security/nss/lib/freebl \
-                 -I$(topsrcdir)/security/nss/lib/freebl/ecl \
-                 -I$(topsrcdir)/security/nss/lib/cryptohi \
-                 -I$(topsrcdir)/security/nss/lib/util \
+                 -I$(srcdir)/../../build
                 $(NULL)


 # Same as JS components that are run through the pre-processor.
-EXTRA_PP_COMPONENTS = nsUrlClassifierLib.js \
+EXTRA_PP_COMPONENTS = nsUrlClassifierTable.js \
+                      nsUrlClassifierLib.js \
                      nsUrlClassifierListManager.js \
                      $(NULL)

 include $(topsrcdir)/config/rules.mk
-
-export:: $(topsrcdir)/security/nss/lib/freebl/sha512.c
-	$(INSTALL) $^ .
--- a/toolkit/components/url-classifier/src/nsUrlClassifierDBService.cpp
+++ b/toolkit/components/url-classifier/src/nsUrlClassifierDBService.cpp
--- a/toolkit/components/url-classifier/src/nsUrlClassifierListManager.js
+++ b/toolkit/components/url-classifier/src/nsUrlClassifierListManager.js
@ -39,6 +39,7 @@ const Cc = Components.classes;
 const Ci = Components.interfaces;

 #include ../content/listmanager.js
+#include ../content/wireformat.js

 var modScope = this;
 function Init() {
--- a/toolkit/components/url-classifier/src/nsUrlClassifierStreamUpdater.cpp
+++ b/toolkit/components/url-classifier/src/nsUrlClassifierStreamUpdater.cpp
@ -37,14 +37,10 @@
 * ***** END LICENSE BLOCK ***** */

 #include "nsCRT.h"
-#include "nsIHttpChannel.h"
 #include "nsIObserverService.h"
-#include "nsIStringStream.h"
-#include "nsIUploadChannel.h"
 #include "nsIURI.h"
 #include "nsIUrlClassifierDBService.h"
 #include "nsStreamUtils.h"
-#include "nsStringStream.h"
 #include "nsToolkitCompsCID.h"
 #include "nsUrlClassifierStreamUpdater.h"
 #include "prlog.h"
@ -67,9 +63,8 @@ class nsUrlClassifierStreamUpdater;
 class TableUpdateListener : public nsIStreamListener
 {
 public:
-  TableUpdateListener(nsIUrlClassifierCallback *aSuccessCallback,
-                      nsIUrlClassifierCallback *aUpdateErrorCallback,
-                      nsIUrlClassifierCallback *aDownloadErrorCallback,
+  TableUpdateListener(nsIUrlClassifierCallback *aTableCallback,
+                      nsIUrlClassifierCallback *aErrorCallback,
                      nsUrlClassifierStreamUpdater* aStreamUpdater);
  nsCOMPtr<nsIUrlClassifierDBService> mDBService;

@ -81,23 +76,20 @@ private:
  ~TableUpdateListener() {}

  // Callback when table updates complete.
-  nsCOMPtr<nsIUrlClassifierCallback> mSuccessCallback;
-  nsCOMPtr<nsIUrlClassifierCallback> mUpdateErrorCallback;
-  nsCOMPtr<nsIUrlClassifierCallback> mDownloadErrorCallback;
+  nsCOMPtr<nsIUrlClassifierCallback> mTableCallback;
+  nsCOMPtr<nsIUrlClassifierCallback> mErrorCallback;

  // Reference to the stream updater that created this.
  nsUrlClassifierStreamUpdater *mStreamUpdater;
 };

 TableUpdateListener::TableUpdateListener(
-                                nsIUrlClassifierCallback *aSuccessCallback,
-                                nsIUrlClassifierCallback *aUpdateErrorCallback,
-                                nsIUrlClassifierCallback *aDownloadErrorCallback,
+                                nsIUrlClassifierCallback *aTableCallback,
+                                nsIUrlClassifierCallback *aErrorCallback,
                                nsUrlClassifierStreamUpdater* aStreamUpdater)
 {
-  mSuccessCallback = aSuccessCallback;
-  mDownloadErrorCallback = aDownloadErrorCallback;
-  mUpdateErrorCallback = aUpdateErrorCallback;
+  mTableCallback = aTableCallback;
+  mErrorCallback = aErrorCallback;
  mStreamUpdater = aStreamUpdater;
 }

@ -118,13 +110,10 @@ TableUpdateListener::OnStartRequest(nsIRequest *request, nsISupports* context)
  nsresult status;
  rv = httpChannel->GetStatus(&status);
  NS_ENSURE_SUCCESS(rv, rv);
-
-  LOG(("OnStartRequest (status %d)", status));
-
  if (NS_ERROR_CONNECTION_REFUSED == status ||
      NS_ERROR_NET_TIMEOUT == status) {
    // Assume that we're overloading the server and trigger backoff.
-    mDownloadErrorCallback->HandleEvent(nsCString());
+    mErrorCallback->HandleEvent(nsCString());
    return NS_ERROR_ABORT;
  }

@ -162,7 +151,7 @@ TableUpdateListener::OnDataAvailable(nsIRequest *request,

    nsCAutoString strStatus;
    strStatus.AppendInt(status);
-    mDownloadErrorCallback->HandleEvent(strStatus);
+    mErrorCallback->HandleEvent(strStatus);
    return NS_ERROR_ABORT;
  }

@ -191,7 +180,7 @@ TableUpdateListener::OnStopRequest(nsIRequest *request, nsISupports* context,
  // If we got the whole stream, call Finish to commit the changes.
  // Otherwise, call Cancel to rollback the changes.
  if (NS_SUCCEEDED(aStatus))
-    mDBService->Finish(mSuccessCallback, mUpdateErrorCallback);
+    mDBService->Finish(mTableCallback);
  else
    mDBService->CancelStream();

@ -246,8 +235,6 @@ nsUrlClassifierStreamUpdater::GetUpdateUrl(nsACString & aUpdateUrl)
 NS_IMETHODIMP
 nsUrlClassifierStreamUpdater::SetUpdateUrl(const nsACString & aUpdateUrl)
 {
-  LOG(("Update URL is %s\n", PromiseFlatCString(aUpdateUrl).get()));
-
  nsresult rv = NS_NewURI(getter_AddRefs(mUpdateUrl), aUpdateUrl);
  NS_ENSURE_SUCCESS(rv, rv);

@ -256,10 +243,8 @@ nsUrlClassifierStreamUpdater::SetUpdateUrl(const nsACString & aUpdateUrl)

 NS_IMETHODIMP
 nsUrlClassifierStreamUpdater::DownloadUpdates(
-                                const nsACString &aRequestBody,
-                                nsIUrlClassifierCallback *aSuccessCallback,
-                                nsIUrlClassifierCallback *aUpdateErrorCallback,
-                                nsIUrlClassifierCallback *aDownloadErrorCallback,
+                                nsIUrlClassifierCallback *aTableCallback,
+                                nsIUrlClassifierCallback *aErrorCallback,
                                PRBool *_retval)
 {
  if (mIsUpdating) {
@ -291,12 +276,8 @@ nsUrlClassifierStreamUpdater::DownloadUpdates(
  rv = NS_NewChannel(getter_AddRefs(mChannel), mUpdateUrl);
  NS_ENSURE_SUCCESS(rv, rv);

-  rv = AddRequestBody(aRequestBody);
-  NS_ENSURE_SUCCESS(rv, rv);
-
  // Bind to a different callback each time we invoke this method.
-  mListener = new TableUpdateListener(aSuccessCallback, aUpdateErrorCallback,
-                                      aDownloadErrorCallback, this);
+  mListener = new TableUpdateListener(aTableCallback, aErrorCallback, this);

  // Make the request
  rv = mChannel->AsyncOpen(mListener.get(), nsnull);
@ -308,35 +289,6 @@ nsUrlClassifierStreamUpdater::DownloadUpdates(
  return NS_OK;
 }

-nsresult
-nsUrlClassifierStreamUpdater::AddRequestBody(const nsACString &aRequestBody)
-{
-  nsresult rv;
-  nsCOMPtr<nsIStringInputStream> strStream =
-    do_CreateInstance(NS_STRINGINPUTSTREAM_CONTRACTID, &rv);
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  rv = strStream->SetData(aRequestBody.BeginReading(),
-                          aRequestBody.Length());
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  nsCOMPtr<nsIUploadChannel> uploadChannel = do_QueryInterface(mChannel, &rv);
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  rv = uploadChannel->SetUploadStream(strStream,
-                                      NS_LITERAL_CSTRING("text/plain"),
-                                      -1);
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(mChannel, &rv);
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  rv = httpChannel->SetRequestMethod(NS_LITERAL_CSTRING("POST"));
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  return NS_OK;
-}
-
 ///////////////////////////////////////////////////////////////////////////////
 // nsIObserver implementation

--- a/toolkit/components/url-classifier/src/nsUrlClassifierStreamUpdater.h
+++ b/toolkit/components/url-classifier/src/nsUrlClassifierStreamUpdater.h
@ -71,8 +71,6 @@ private:
  // Disallow copy constructor
  nsUrlClassifierStreamUpdater(nsUrlClassifierStreamUpdater&);

-  nsresult AddRequestBody(const nsACString &aRequestBody);
-
  PRBool mIsUpdating;
  PRBool mInitialized;
  nsCOMPtr<nsIURI> mUpdateUrl;
--- a/toolkit/components/url-classifier/src/nsUrlClassifierTable.js
+++ b/toolkit/components/url-classifier/src/nsUrlClassifierTable.js
@ -0,0 +1,137 @@
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is Url Classifier code
+#
+# The Initial Developer of the Original Code is
+# Google Inc.
+# Portions created by the Initial Developer are Copyright (C) 2006
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Tony Chang <tony@ponderer.org>
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+
+// moz/lang.js is needed for Function.prototype.inherts
+#include ../content/moz/lang.js
+#include ../content/enchash-decrypter.js
+#include ../content/multi-querier.js
+#include ../content/trtable.js
+
+var modScope = this;
+function Init() {
+  // Pull the library in.
+  var jslib = Cc["@mozilla.org/url-classifier/jslib;1"]
+              .getService().wrappedJSObject;
+  modScope.G_Preferences = jslib.G_Preferences;
+  modScope.G_PreferenceObserver = jslib.G_PreferenceObserver;
+  modScope.G_Debug = jslib.G_Debug;
+  modScope.G_CryptoHasher = jslib.G_CryptoHasher;
+  modScope.BindToObject = jslib.BindToObject;
+
+  // We only need to call Init once.
+  modScope.Init = function() {};
+}
+
+
+function UrlClassifierTableMod() {
+  this.components = {};
+  this.addComponent({
+      cid: "{43399ee0-da0b-46a8-9541-08721265981c}",
+      name: "UrlClassifier Table Url Module",
+      progid: "@mozilla.org/url-classifier/table;1?type=url",
+      factory: new UrlClassifierTableFactory(UrlClassifierTableUrl)
+    });
+  this.addComponent({
+      cid: "{3b5004c6-3fcd-4b12-b311-a4dfbeaf27aa}",
+      name: "UrlClassifier Table Domain Module",
+      progid: "@mozilla.org/url-classifier/table;1?type=domain",
+      factory: new UrlClassifierTableFactory(UrlClassifierTableDomain)
+    });
+  this.addComponent({
+      cid: "{04f15d1d-2db8-4b8e-91d7-82f30308b434}",
+      name: "UrlClassifier Table Enchash Module",
+      progid: "@mozilla.org/url-classifier/table;1?type=enchash",
+      factory: new UrlClassifierTableFactory(UrlClassifierTableEnchash)
+    });
+}
+
+UrlClassifierTableMod.prototype.addComponent = function(comp) {
+  this.components[comp.cid] = comp;
+};
+
+UrlClassifierTableMod.prototype.registerSelf = function(compMgr, fileSpec, loc, type) {
+  compMgr = compMgr.QueryInterface(Ci.nsIComponentRegistrar);
+  // Register all the components
+  for (var cid in this.components) {
+    var comp = this.components[cid];
+    compMgr.registerFactoryLocation(Components.ID(comp.cid),
+                                    comp.name,
+                                    comp.progid,
+                                    fileSpec,
+                                    loc,
+                                    type);
+  }
+};
+
+UrlClassifierTableMod.prototype.getClassObject = function(compMgr, cid, iid) {
+  var comp = this.components[cid.toString()];
+
+  if (!comp)
+    throw Components.results.NS_ERROR_NO_INTERFACE;
+  if (!iid.equals(Ci.nsIFactory))
+    throw Components.results.NS_ERROR_NOT_IMPLEMENTED;
+
+  return comp.factory;
+};
+
+UrlClassifierTableMod.prototype.canUnload = function(compMgr) {
+  return true;
+};
+
+/**
+ * Create a factory.
+ * @param ctor Function constructor for the object we're creating.
+ */
+function UrlClassifierTableFactory(ctor) {
+  this.ctor = ctor;
+}
+
+UrlClassifierTableFactory.prototype.createInstance = function(outer, iid) {
+  if (outer != null)
+    throw Components.results.NS_ERROR_NO_AGGREGATION;
+  Init();
+  return (new this.ctor()).QueryInterface(iid);
+};
+
+var modInst = new UrlClassifierTableMod();
+
+function NSGetModule(compMgr, fileSpec) {
+  return modInst;
+}
--- a/toolkit/components/url-classifier/src/nsUrlClassifierUtils.cpp
+++ b/toolkit/components/url-classifier/src/nsUrlClassifierUtils.cpp
@ -36,11 +36,7 @@

 #include "nsEscape.h"
 #include "nsString.h"
-#include "nsIURI.h"
-#include "nsNetUtil.h"
 #include "nsUrlClassifierUtils.h"
-#include "nsVoidArray.h"
-#include "prprf.h"

 static char int_to_hex_digit(PRInt32 i)
 {
@ -48,58 +44,6 @@ static char int_to_hex_digit(PRInt32 i)
  return static_cast<char>(((i < 10) ? (i + '0') : ((i - 10) + 'A')));
 }

-static PRBool
-IsDecimal(const nsACString & num)
-{
-  for (PRUint32 i = 0; i < num.Length(); i++) {
-    if (!isdigit(num[i])) {
-      return PR_FALSE;
-    }
-  }
-
-  return PR_TRUE;
-}
-
-static PRBool
-IsHex(const nsACString & num)
-{
-  if (num.Length() < 3) {
-    return PR_FALSE;
-  }
-
-  if (num[0] != '0' || !(num[1] == 'x' || num[1] == 'X')) {
-    return PR_FALSE;
-  }
-
-  for (PRUint32 i = 2; i < num.Length(); i++) {
-    if (!isxdigit(num[i])) {
-      return PR_FALSE;
-    }
-  }
-
-  return PR_TRUE;
-}
-
-static PRBool
-IsOctal(const nsACString & num)
-{
-  if (num.Length() < 2) {
-    return PR_FALSE;
-  }
-
-  if (num[0] != '0') {
-    return PR_FALSE;
-  }
-
-  for (PRUint32 i = 1; i < num.Length(); i++) {
-    if (!isdigit(num[i]) || num[i] == '8' || num[i] == '9') {
-      return PR_FALSE;
-    }
-  }
-
-  return PR_TRUE;
-}
-
 nsUrlClassifierUtils::nsUrlClassifierUtils() : mEscapeCharmap(nsnull)
 {
 }
@ -120,252 +64,54 @@ NS_IMPL_ISUPPORTS1(nsUrlClassifierUtils, nsIUrlClassifierUtils)
 /////////////////////////////////////////////////////////////////////////////
 // nsIUrlClassifierUtils

+/* ACString canonicalizeURL (in ACString url); */
 NS_IMETHODIMP
-nsUrlClassifierUtils::GetKeyForURI(nsIURI * uri, nsACString & _retval)
+nsUrlClassifierUtils::CanonicalizeURL(const nsACString & url, nsACString & _retval)
 {
-  nsCOMPtr<nsIURI> innerURI = NS_GetInnermostURI(uri);
-  if (!innerURI)
-    innerURI = uri;
-
-  nsCAutoString host;
-  innerURI->GetAsciiHost(host);
-
-  nsresult rv = CanonicalizeHostname(host, _retval);
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  nsCAutoString path;
-  rv = innerURI->GetPath(path);
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  // strip out anchors and query parameters
-  PRInt32 ref = path.FindChar('#');
-  if (ref != kNotFound)
-    path.SetLength(ref);
-
-  ref = path.FindChar('?');
-  if (ref != kNotFound)
-    path.SetLength(ref);
-
+  nsCAutoString decodedUrl(url);
  nsCAutoString temp;
-  rv = CanonicalizePath(path, temp);
-  NS_ENSURE_SUCCESS(rv, rv);
-
-  _retval.Append(temp);
+  while (NS_UnescapeURL(decodedUrl.get(), decodedUrl.Length(), 0, temp)) {
+    decodedUrl.Assign(temp);
+    temp.Truncate();
+  }
+  SpecialEncode(decodedUrl, _retval);
+  return NS_OK;
+}

+NS_IMETHODIMP
+nsUrlClassifierUtils::EscapeHostname(const nsACString & hostname,
+                                     nsACString & _retval)
+{
+  const char* curChar = hostname.BeginReading();
+  const char* end = hostname.EndReading();
+  while (curChar != end) {
+    unsigned char c = static_cast<unsigned char>(*curChar);
+    if (mEscapeCharmap->Contains(c)) {
+      _retval.Append('%');
+      _retval.Append(int_to_hex_digit(c / 16));
+      _retval.Append(int_to_hex_digit(c % 16));
+    } else {
+      _retval.Append(*curChar);
+    }
+    ++curChar;
+  }
+  
  return NS_OK;
 }

 /////////////////////////////////////////////////////////////////////////////
 // non-interface methods

-nsresult
-nsUrlClassifierUtils::CanonicalizeHostname(const nsACString & hostname,
-                                           nsACString & _retval)
-{
-  nsCAutoString unescaped;
-  if (!NS_UnescapeURL(PromiseFlatCString(hostname).get(),
-                      PromiseFlatCString(hostname).Length(),
-                      0, unescaped)) {
-    unescaped.Assign(hostname);
-  }
-
-  nsCAutoString cleaned;
-  CleanupHostname(unescaped, cleaned);
-
-  nsCAutoString temp;
-  ParseIPAddress(cleaned, temp);
-  if (!temp.IsEmpty()) {
-    cleaned.Assign(temp);
-  }
-
-  ToLowerCase(cleaned);
-  SpecialEncode(cleaned, PR_FALSE, _retval);
-
-  return NS_OK;
-}
-
-
-nsresult
-nsUrlClassifierUtils::CanonicalizePath(const nsACString & path,
-                                       nsACString & _retval)
-{
-  _retval.Truncate();
-
-  nsCAutoString decodedPath(path);
-  nsCAutoString temp;
-  while (NS_UnescapeURL(decodedPath.get(), decodedPath.Length(), 0, temp)) {
-    decodedPath.Assign(temp);
-    temp.Truncate();
-  }
-
-  SpecialEncode(decodedPath, PR_TRUE, _retval);
-  // XXX: lowercase the path?
-
-  return NS_OK;
-}
-
-void
-nsUrlClassifierUtils::CleanupHostname(const nsACString & hostname,
-                                      nsACString & _retval)
-{
-  _retval.Truncate();
-
-  const char* curChar = hostname.BeginReading();
-  const char* end = hostname.EndReading();
-  char lastChar = '\0';
-  while (curChar != end) {
-    unsigned char c = static_cast<unsigned char>(*curChar);
-    if (c == '.' && (lastChar == '\0' || lastChar == '.')) {
-      // skip
-    } else {
-      _retval.Append(*curChar);
-    }
-    lastChar = c;
-    ++curChar;
-  }
-
-  // cut off trailing dots
-  while (_retval[_retval.Length() - 1] == '.') {
-    _retval.SetLength(_retval.Length() - 1);
-  }
-}
-
-void
-nsUrlClassifierUtils::ParseIPAddress(const nsACString & host,
-                                     nsACString & _retval)
-{
-  _retval.Truncate();
-  nsACString::const_iterator iter, end;
-  host.BeginReading(iter);
-  host.EndReading(end);
-
-  if (host.Length() <= 15) {
-    // The Windows resolver allows a 4-part dotted decimal IP address to
-    // have a space followed by any old rubbish, so long as the total length
-    // of the string doesn't get above 15 characters. So, "10.192.95.89 xy"
-    // is resolved to 10.192.95.89.
-    // If the string length is greater than 15 characters, e.g.
-    // "10.192.95.89 xy.wildcard.example.com", it will be resolved through
-    // DNS.
-
-    if (FindCharInReadable(' ', iter, end)) {
-      end = iter;
-    }
-  }
-
-  for (host.BeginReading(iter); iter != end; iter++) {
-    if (!(isxdigit(*iter) || *iter == 'x' || *iter == 'X' || *iter == '.')) {
-      // not an IP
-      return;
-    }
-  }
-
-  host.BeginReading(iter);
-  nsCStringArray parts;
-  parts.ParseString(PromiseFlatCString(Substring(iter, end)).get(), ".");
-  if (parts.Count() > 4) {
-    return;
-  }
-
-  // If any potentially-octal numbers (start with 0 but not hex) have
-  // non-octal digits, no part of the ip can be in octal
-  // XXX: this came from the old javascript implementation, is it really
-  // supposed to be like this?
-  PRBool allowOctal = PR_TRUE;
-  for (PRInt32 i = 0; i < parts.Count(); i++) {
-    const nsCString& part = *parts[i];
-    if (part[0] == '0') {
-      for (PRUint32 j = 1; j < part.Length(); j++) {
-        if (part[j] == 'x') {
-          break;
-        }
-        if (part[j] == '8' || part[j] == '9') {
-          allowOctal = PR_FALSE;
-          break;
-        }
-      }
-    }
-  }
-
-  for (PRInt32 i = 0; i < parts.Count(); i++) {
-    nsCAutoString canonical;
-
-    if (i == parts.Count() - 1) {
-      CanonicalNum(*parts[i], 5 - parts.Count(), allowOctal, canonical);
-    } else {
-      CanonicalNum(*parts[i], 1, allowOctal, canonical);
-    }
-
-    if (canonical.IsEmpty()) {
-      _retval.Truncate();
-      return;
-    }
-
-    if (_retval.IsEmpty()) {
-      _retval.Assign(canonical);
-    } else {
-      _retval.Append('.');
-      _retval.Append(canonical);
-    }
-  }
-  return;
-}
-
-void
-nsUrlClassifierUtils::CanonicalNum(const nsACString& num,
-                                   PRUint32 bytes,
-                                   PRBool allowOctal,
-                                   nsACString& _retval)
-{
-  _retval.Truncate();
-
-  if (num.Length() < 1) {
-    return;
-  }
-
-  PRUint32 val;
-  if (allowOctal && IsOctal(num)) {
-    if (PR_sscanf(PromiseFlatCString(num).get(), "%o", &val) != 1) {
-      return;
-    }
-  } else if (IsDecimal(num)) {
-    if (PR_sscanf(PromiseFlatCString(num).get(), "%u", &val) != 1) {
-      return;
-    }
-  } else if (IsHex(num)) {
-  if (PR_sscanf(PromiseFlatCString(num).get(), num[1] == 'X' ? "0X%x" : "0x%x",
-                &val) != 1) {
-      return;
-    }
-  } else {
-    return;
-  }
-
-  while (bytes--) {
-    char buf[20];
-    PR_snprintf(buf, sizeof(buf), "%u", val & 0xff);
-    if (_retval.IsEmpty()) {
-      _retval.Assign(buf);
-    } else {
-      _retval = nsDependentCString(buf) + NS_LITERAL_CSTRING(".") + _retval;
-    }
-    val >>= 8;
-  }
-}
-
 // This function will encode all "special" characters in typical url
-// encoding, that is %hh where h is a valid hex digit.  It will also fold
-// any duplicated slashes.
+// encoding, that is %hh where h is a valid hex digit.  See the comment in
+// the header file for details.
 PRBool
-nsUrlClassifierUtils::SpecialEncode(const nsACString & url,
-                                    PRBool foldSlashes,
-                                    nsACString & _retval)
+nsUrlClassifierUtils::SpecialEncode(const nsACString & url, nsACString & _retval)
 {
  PRBool changed = PR_FALSE;
  const char* curChar = url.BeginReading();
  const char* end = url.EndReading();

-  unsigned char lastChar = '\0';
  while (curChar != end) {
    unsigned char c = static_cast<unsigned char>(*curChar);
    if (ShouldURLEscape(c)) {
@ -379,12 +125,9 @@ nsUrlClassifierUtils::SpecialEncode(const nsACString & url,
      _retval.Append(int_to_hex_digit(c % 16));

      changed = PR_TRUE;
-    } else if (foldSlashes && (c == '/' && lastChar == '/')) {
-      // skip
    } else {
      _retval.Append(*curChar);
    }
-    lastChar = c;
    curChar++;
  }
  return changed;
--- a/toolkit/components/url-classifier/src/nsUrlClassifierUtils.h
+++ b/toolkit/components/url-classifier/src/nsUrlClassifierUtils.h
@ -77,14 +77,10 @@ public:
  nsUrlClassifierUtils();
  ~nsUrlClassifierUtils() {}

-  NS_DECL_ISUPPORTS
-  NS_DECL_NSIURLCLASSIFIERUTILS
-
  nsresult Init();

-  nsresult CanonicalizeHostname(const nsACString & hostname,
-                                nsACString & _retval);
-  nsresult CanonicalizePath(const nsACString & url, nsACString & _retval);
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIURLCLASSIFIERUTILS

  // This function will encode all "special" characters in typical url encoding,
  // that is %hh where h is a valid hex digit.  The characters which are encoded
@ -92,15 +88,8 @@ public:
  // space), 37(%), and anything 127 or above (special characters).  Url is the
  // string to encode, ret is the encoded string.  Function returns true if
  // ret != url.
-  PRBool SpecialEncode(const nsACString & url,
-                       PRBool foldSlashes,
-                       nsACString & _retval);
+  PRBool SpecialEncode(const nsACString & url, nsACString & _retval);

-  void ParseIPAddress(const nsACString & host, nsACString & _retval);
-  void CanonicalNum(const nsACString & num,
-                    PRUint32 bytes,
-                    PRBool allowOctal,
-                    nsACString & _retval);
 private:
  // Disallow copy constructor
  nsUrlClassifierUtils(const nsUrlClassifierUtils&);
@ -108,8 +97,6 @@ private:
  // Function to tell if we should encode a character.
  PRBool ShouldURLEscape(const unsigned char c) const;

-  void CleanupHostname(const nsACString & host, nsACString & _retval);
-
  nsAutoPtr<Charmap> mEscapeCharmap;
 };

--- a/toolkit/components/url-classifier/tests/Makefile.in
+++ b/toolkit/components/url-classifier/tests/Makefile.in
@ -52,11 +52,15 @@ REQUIRES = \
 	string \
 	url-classifier \
 	xpcom \
-	necko \
 	$(NULL)

-# xpcshell tests
-XPCSHELL_TESTS=unit
+# mochitests
+_TEST_FILES = \
+	test_enchash-decrypter.xhtml \
+	$(NULL)
+
+libs:: $(_TEST_FILES)
+	$(INSTALL) $^ $(DEPTH)/_tests/testing/mochitest/tests/$(relativesrcdir)

 # simple c++ tests (no xpcom)
 CPPSRCS = \
@ -72,7 +76,6 @@ LOCAL_INCLUDES = \

 LIBS = \
 	../src/$(LIB_PREFIX)urlclassifier_s.$(LIB_SUFFIX) \
-	$(MOZ_COMPONENT_LIBS) \
 	$(XPCOM_LIBS) \
 	$(NSPR_LIBS) \
 	$(NULL)
--- a/toolkit/components/url-classifier/tests/TestUrlClassifierUtils.cpp
+++ b/toolkit/components/url-classifier/tests/TestUrlClassifierUtils.cpp
@ -39,8 +39,6 @@
 #include "nsEscape.h"
 #include "nsString.h"
 #include "nsUrlClassifierUtils.h"
-#include "nsNetUtil.h"
-#include "stdlib.h"

 static int gTotalTests = 0;
 static int gPassedTests = 0;
@ -119,8 +117,8 @@ void TestEncodeHelper(const char* in, const char* expected)
 {
  nsCString out, strIn(in), strExp(expected);
  nsUrlClassifierUtils utils;
-
-  utils.SpecialEncode(strIn, PR_TRUE, out);
+  
+  utils.SpecialEncode(strIn, out);
  CheckEquals(strExp, out);
 }

@ -138,7 +136,7 @@ void TestEnc()
  }
  nsUrlClassifierUtils utils;
  nsCString out;
-  utils.SpecialEncode(noenc, PR_FALSE, out);
+  utils.SpecialEncode(noenc, out);
  CheckEquals(noenc, out);

  // Test that all the chars that we should encode [0,32],37,[127,255] are
@ -153,18 +151,16 @@ void TestEnc()
  }
  
  out.Truncate();
-  utils.SpecialEncode(yesAsString, PR_FALSE, out);
+  utils.SpecialEncode(yesAsString, out);
  CheckEquals(yesExpectedString, out);
-
-  TestEncodeHelper("blah//blah", "blah/blah");
 }

 void TestCanonicalizeHelper(const char* in, const char* expected)
 {
  nsCString out, strIn(in), strExp(expected);
  nsUrlClassifierUtils utils;
-
-  utils.CanonicalizePath(strIn, out);
+  
+  utils.CanonicalizeURL(strIn, out);
  CheckEquals(strExp, out);
 }

@ -181,142 +177,19 @@ void TestCanonicalize()
      "~a!b@c#d$e%25f^00&11*22(33)44_55+");

  TestCanonicalizeHelper("", "");
-  TestCanonicalizeHelper("%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/",
-                         "168.188.99.26/.secure/www.ebay.com/");
-  TestCanonicalizeHelper("195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/",
-                         "195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/");
-}
-
-void TestParseIPAddressHelper(const char *in, const char *expected)
-{
-  nsCString out, strIn(in), strExp(expected);
-  nsUrlClassifierUtils utils;
-  utils.Init();
-
-  utils.ParseIPAddress(strIn, out);
-  CheckEquals(strExp, out);
-}
-
-void TestParseIPAddress()
-{
-  TestParseIPAddressHelper("123.123.0.0.1", "");
-  TestParseIPAddressHelper("255.0.0.1", "255.0.0.1");
-  TestParseIPAddressHelper("12.0x12.01234", "12.18.2.156");
-  TestParseIPAddressHelper("276.2.3", "20.2.0.3");
-  TestParseIPAddressHelper("012.034.01.055", "10.28.1.45");
-  TestParseIPAddressHelper("0x12.0x43.0x44.0x01", "18.67.68.1");
-  TestParseIPAddressHelper("167838211", "10.1.2.3");
-  TestParseIPAddressHelper("3279880203", "195.127.0.11");
-  TestParseIPAddressHelper("0x12434401", "18.67.68.1");
-  TestParseIPAddressHelper("413960661", "24.172.137.213");
-  TestParseIPAddressHelper("03053104725", "24.172.137.213");
-  TestParseIPAddressHelper("030.0254.0x89d5", "24.172.137.213");
-  TestParseIPAddressHelper("1.234.4.0377", "1.234.4.255");
-  TestParseIPAddressHelper("1.2.3.00x0", "");
-  TestParseIPAddressHelper("10.192.95.89 xy", "10.192.95.89");
-  TestParseIPAddressHelper("10.192.95.89 xyz", "");
-  TestParseIPAddressHelper("1.2.3.0x0", "1.2.3.0");
-  TestParseIPAddressHelper("1.2.3.4", "1.2.3.4");
-}
-
-void TestCanonicalNumHelper(const char *in, PRUint32 bytes,
-                            bool allowOctal, const char *expected)
-{
-  nsCString out, strIn(in), strExp(expected);
-  nsUrlClassifierUtils utils;
-  utils.Init();
-
-  utils.CanonicalNum(strIn, bytes, allowOctal, out);
-  CheckEquals(strExp, out);
-}
-
-void TestCanonicalNum()
-{
-  TestCanonicalNumHelper("", 1, true, "");
-  TestCanonicalNumHelper("10", 0, true, "");
-  TestCanonicalNumHelper("45", 1, true, "45");
-  TestCanonicalNumHelper("0x10", 1, true, "16");
-  TestCanonicalNumHelper("367", 2, true, "1.111");
-  TestCanonicalNumHelper("012345", 3, true, "0.20.229");
-  TestCanonicalNumHelper("0173", 1, true, "123");
-  TestCanonicalNumHelper("09", 1, false, "9");
-  TestCanonicalNumHelper("0x120x34", 2, true, "");
-  TestCanonicalNumHelper("0x12fc", 2, true, "18.252");
-  TestCanonicalNumHelper("3279880203", 4, true, "195.127.0.11");
-  TestCanonicalNumHelper("0x0000059", 1, true, "89");
-  TestCanonicalNumHelper("0x00000059", 1, true, "89");
-  TestCanonicalNumHelper("0x0000067", 1, true, "103");
-}
-
-void TestHostnameHelper(const char *in, const char *expected)
-{
-  nsCString out, strIn(in), strExp(expected);
-  nsUrlClassifierUtils utils;
-  utils.Init();
-
-  utils.CanonicalizeHostname(strIn, out);
-  CheckEquals(strExp, out);
-}
-
-void TestHostname()
-{
-  TestHostnameHelper("abcd123;[]", "abcd123;[]");
-  TestHostnameHelper("abc.123", "abc.123");
-  TestHostnameHelper("abc..123", "abc.123");
-  TestHostnameHelper("trailing.", "trailing");
-  TestHostnameHelper("i love trailing dots....", "i%20love%20trailing%20dots");
-  TestHostnameHelper(".leading", "leading");
-  TestHostnameHelper("..leading", "leading");
-  TestHostnameHelper(".dots.", "dots");
-  TestHostnameHelper(".both.", "both");
-  TestHostnameHelper(".both..", "both");
-  TestHostnameHelper("..both.", "both");
-  TestHostnameHelper("..both..", "both");
-  TestHostnameHelper("..a.b.c.d..", "a.b.c.d");
-  TestHostnameHelper("..127.0.0.1..", "127.0.0.1");
-  TestHostnameHelper("asdf!@#$a", "asdf!@#$a");
-  TestHostnameHelper("AB CD 12354", "ab%20cd%2012354");
-  TestHostnameHelper("\1\2\3\4\112\177", "%01%02%03%04j%7F");
-  TestHostnameHelper("<>.AS/-+", "<>.as/-+");
-
-}
-
-void TestLongHostname()
-{
-  static const int kTestSize = 1024 * 150;
-  char *str = static_cast<char*>(malloc(kTestSize + 1));
-  memset(str, 'x', kTestSize);
-  str[kTestSize] = '\0';
-
-  nsUrlClassifierUtils utils;
-  utils.Init();
-
-  nsCAutoString out;
-  nsDependentCString in(str);
-  PRIntervalTime clockStart = PR_IntervalNow();
-  utils.CanonicalizeHostname(in, out);
-  PRIntervalTime clockEnd = PR_IntervalNow();
-
-  CheckEquals(in, out);
-
-  printf("CanonicalizeHostname on long string (%dms)\n",
-         PR_IntervalToMilliseconds(clockEnd - clockStart));
+  TestCanonicalizeHelper("http://www.google.com", "http://www.google.com");
+  TestCanonicalizeHelper("http://%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/",
+                         "http://168.188.99.26/.secure/www.ebay.com/");
+  TestCanonicalizeHelper("http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/",
+                         "http://195.127.0.11/uploads/%20%20%20%20/.verify/.eBaysecure=updateuserdataxplimnbqmn-xplmvalidateinfoswqpcmlx=hgplmcx/");
 }

 int main(int argc, char **argv)
 {
-  NS_LogInit();
-
  TestUnescape();
  TestEnc();
  TestCanonicalize();
-  TestCanonicalNum();
-  TestParseIPAddress();
-  TestHostname();
-  TestLongHostname();
-
  printf("%d of %d tests passed\n", gPassedTests, gTotalTests);
  // Non-zero return status signals test failure to build system.
-
  return (gPassedTests != gTotalTests);
 }
--- a/toolkit/components/url-classifier/tests/jar.mn
+++ b/toolkit/components/url-classifier/tests/jar.mn
@ -0,0 +1,2 @@
+toolkit.jar:
+ content/global/url-classifier/unittests.xul         (unittests.xul)
--- a/toolkit/components/url-classifier/tests/test_enchash-decrypter.xhtml
+++ b/toolkit/components/url-classifier/tests/test_enchash-decrypter.xhtml
@ -0,0 +1,394 @@
+<html xmlns="http://www.w3.org/1999/xhtml">
+<!--
+This is a port of all the existing EnchashDecrypter unittests to the
+mochitest framework.
+-->
+<head>
+  <title>Test for enchash-decrypter.js</title>
+  <script type="text/javascript" src="/MochiKit/packed.js"></script>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+<script class="testbody" type="text/javascript">
+<![CDATA[
+
+netscape.security.PrivilegeManager.enablePrivilege('UniversalXPConnect');
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var table = Cc["@mozilla.org/url-classifier/table;1?type=url"].createInstance();
+var componentScope = table.wrappedJSObject.__parent__;
+ok(!!componentScope, "unable to get wrapped js object");
+
+////// Test PROT_EnchashDecrypter methods //////
+var PROT_EnchashDecrypter = componentScope.PROT_EnchashDecrypter;
+var l = new PROT_EnchashDecrypter();
+
+// Test our regular expressions.  Make sure they are handled the same as on
+// the server that handles remote look ups.
+// Yes this defies our naming convention, but we copy verbatim from
+// the C++ unittest, so lets just keep things clear.
+var no_dots = "abcd123;[]";
+var one_dot = "abc.123";
+var two_dots = "two..dots";
+var lots_o_dots = "I have a lovely .... bunch of dots";
+var multi_dots = "dots ... and ... more .... dots";
+var leading_dot = ".leading";
+var trailing_dot = "trailing.";
+var trailing_dots = "I love trailing dots....";
+var end_dots = ".dots.";
+
+var decimal = "1234567890";
+var hex = "0x123452FAf";
+var bad_hex = "0xFF0xGG";
+var octal = "012034056";
+var bad_octal = "012034089";
+var garbage = "lk,.:asdfa-=";
+var mixed = "1230x78034";
+var spaces = "123 0xFA 045";
+
+var r = PROT_EnchashDecrypter.REs;
+// Test regular expressions matches
+function testRE(re, inputValPairs) {
+  for (var i = 0; i < inputValPairs.length; i += 2)
+    ok(re.test(inputValPairs[i]) == inputValPairs[i + 1],
+       "RegExp broken: " + re + " (input: " + inputValPairs[i] + ")");
+};
+
+// Test regular expression replacement
+function testReplaceRE(re, replaceStr, inputValPairs) {
+  for (var i = 0; i < inputValPairs.length; i += 2) {
+    var inStr = inputValPairs[i];
+    var outStr = inputValPairs[i + 1];
+    ok(inStr.replace(re, replaceStr) == outStr,
+       uneval(inStr) + ".replace(" + uneval(re) + "," + uneval(replaceStr) + ") == " + uneval(outStr) + ")");
+  }
+};
+
+var tests = [
+  "", "",
+  "normal chars;!@#$%^&*&(", "normal chars;!@#$%^&*&(",
+  "MORE NORMAL ,./<>?;':{}", "MORE NORMAL ,./<>?;':{}",
+  "Slightly less\2 normal", "Slightly less normal",
+  "\245 stuff \4\5foo", " stuff foo",
+];
+testReplaceRE(PROT_EnchashDecrypter.REs.FIND_DODGY_CHARS_GLOBAL, "", tests);
+
+tests = [
+  "", "",
+  no_dots, no_dots,
+  one_dot, one_dot,
+  two_dots, two_dots,
+  trailing_dot, "trailing",
+  trailing_dots, "I love trailing dots",
+  leading_dot, "leading",
+  "..leading", "leading",
+  end_dots, "dots",
+  ".both.", "both",
+  ".both..", "both",
+  "..both.", "both",
+  "..both..", "both",
+  "..a.b.c.d..", "a.b.c.d",
+  "..127.0.0.1..", "127.0.0.1",
+];
+testReplaceRE(PROT_EnchashDecrypter.REs.FIND_END_DOTS_GLOBAL, "", tests);
+
+tests = [
+  "", "",
+  no_dots, no_dots,
+  one_dot, one_dot,
+  two_dots, "two.dots",
+  lots_o_dots, "I have a lovely . bunch of dots",
+  multi_dots, "dots . and . more . dots",
+  "127.0.0.1", "127.0.0.1",
+  ".127.0.0.1.", ".127.0.0.1.",
+  "127..0.0.1", "127.0.0.1",
+  "127.0..0.1", "127.0.0.1",
+  "127..0..0..1", "127.0.0.1",
+];
+testReplaceRE(PROT_EnchashDecrypter.REs.FIND_MULTIPLE_DOTS_GLOBAL, ".", tests);
+
+tests = [
+  no_dots, false,
+  one_dot, false,
+  two_dots, true,
+  lots_o_dots, true,
+  multi_dots, true
+];
+testRE(r.FIND_MULTIPLE_DOTS_GLOBAL, tests);
+
+tests = [
+  "random junk", false,
+  "123.45.6-7.89", false,
+  "012.12.123", true,
+  "0x12.0xff.123", true,
+  "225.0.0.1", true
+];
+testRE(r.POSSIBLE_IP, tests);
+
+tests = [
+  decimal, false,
+  hex, false,
+  octal, false,
+  bad_octal, true
+];
+testRE(r.FIND_BAD_OCTAL, tests);
+
+tests = [
+  decimal, false,
+  hex, false,
+  bad_octal, false,
+  garbage, false,
+  mixed, false,
+  spaces, false,
+  octal, true
+];
+testRE(r.IS_OCTAL, tests);
+
+tests = [
+  hex, false,
+  garbage, false,
+  mixed, false,
+  spaces, false,
+  octal, true,
+  bad_octal, true,
+  decimal, true
+];
+testRE(r.IS_DECIMAL, tests);
+
+tests = [
+  decimal, false,
+  octal, false,
+  bad_octal, false,
+  garbage, false,
+  mixed, false,
+  spaces, false,
+  bad_hex, false,
+  hex, true
+];
+testRE(r.IS_HEX, tests);
+
+// Test find last N
+var longstr = "";
+for(var k = 0; k < 100; k++) {
+  longstr += "a";
+}
+var shortstr = "short";
+
+var val = l.lastNChars_(longstr, 8);
+ok(val.length == 8, "find last eight broken on long str");
+val = l.lastNChars_(shortstr, 8);
+ok(val.length == 5, "find last eight broken on short str");
+
+// Test canonical num
+var tests = [
+  "", "", 1, true,
+  "", "10", 0, true,
+  "", "0x45", -1, true,
+  "45", "45", 1, true,
+  "16", "0x10", 1, true,
+  "1.111", "367", 2, true,
+  "0.20.229", "012345", 3, true,
+  "123", "0173", 1, true,
+  "9", "09", 1, false,
+  "", "0x120x34", 2, true,
+  "18.252", "0x12fc", 2, true,
+  "195.127.0.11", "3279880203", 4, true,
+  "89", "0x0000059", 1, true,
+  "89", "0x00000059", 1, true,
+  "103", "0x0000067", 1, true
+];
+for (var i = 0; i < tests.length; i+= 4) {
+  ok(tests[i] === l.canonicalNum_(tests[i + 1], tests[i + 2], tests[i + 3]),
+     "canonicalNum broken on: " + tests[i + 1]);
+}
+
+// Test parseIPAddress (these are all verifiable using ping)
+var testing = {
+  "fake ip": "",
+  "123.123.0.0.1": "",
+  "255.0.0.1": "255.0.0.1",
+  "12.0x12.01234": "12.18.2.156",
+  "276.2.3": "20.2.0.3",
+  "012.034.01.055": "10.28.1.45",
+  "0x12.0x43.0x44.0x01": "18.67.68.1",
+  "167838211": "10.1.2.3",
+  "3279880203": "195.127.0.11",
+  "0x12434401": "18.67.68.1",
+  "413960661": "24.172.137.213",
+  "03053104725": "24.172.137.213",
+  "030.0254.0x89d5": "24.172.137.213",
+  "1.234.4.0377": "1.234.4.255",
+  "1.2.3.00x0": "",
+  "10.192.95.89 xy": "10.192.95.89",
+  "10.192.95.89 xyz": "",
+  "1.2.3.0x0": "1.2.3.0",
+  "1.2.3.4": "1.2.3.4"
+};
+for (var key in testing) {
+  ok(l.parseIPAddress_(key) === testing[key],
+     "parseIPAddress broken on " + key + "(got: " + l.parseIPAddress_(key));
+}
+
+// Test escapeHostname (bug 368998)
+testing = {
+  "asdf!@#$a": "asdf%21%40%23%24a",
+  "AB CD 12354": "AB%20CD%2012354",
+  "\1\2\3\4\112\177": "%01%02%03%04J%7F",
+  "<>.AS/-+": "%3C%3E.AS%2F-%2B"
+};
+var urlUtils = Cc["@mozilla.org/url-classifier/utils;1"]
+               .getService(Ci.nsIUrlClassifierUtils);
+for (var key in testing) {
+  var out = urlUtils.escapeHostname(key);
+  ok(out === testing[key],
+     "escapeString broken on " + key + " (got: " + out + ")");
+}
+
+// Test a really long url (~130k).  getCanonicalHost takes about 55ms
+// on my 2.8ghz machine.
+var long_string = "x";
+for (var i = 0; i < 17; ++i) {
+  long_string += long_string;
+}
+var long_hostname_url = "http://" + long_string + "/foo";
+var startTime = Date.now();
+var out = l.getCanonicalHost(long_hostname_url);
+var endTime = Date.now();
+ok(out == long_string, "getCanonicalHost on long string (" +
+                       (endTime - startTime) + "ms)");
+
+// Verify that each character is escaped properly.
+for (var i = 0; i < 256; ++i) {
+  var chr = String.fromCharCode(i);
+  if ( (chr.toLowerCase() >= 'a' && chr.toLowerCase() <= 'z') ||
+       (chr >= '0' && chr <= '9') ||
+       '.' == chr || '-' == chr) {
+    ok(urlUtils.escapeHostname(chr).length == 1, 'failed on ' + i);
+  } else {
+    ok(urlUtils.escapeHostname(chr).length == 3, 'failed on ' + i);
+  }
+}
+
+// Test getCanonicalHost
+testing = {
+  "http://completely.bogus.url.with.a.whole.lot.of.dots":
+      "with.a.whole.lot.of.dots",
+  "http://poseidon.marinet.gr/~elani": "poseidon.marinet.gr",
+  "http://www.google.com..": "www.google.com",
+  "https://www.yaho%6F.com": "www.yahoo.com",
+  "http://012.034.01.0xa": "10.28.1.10",
+  "ftp://wierd..chars...%0f,%fa": "wierd.chars.%2c",
+  "http://0x18ac89d5/http.www.paypal.com/": "24.172.137.213",
+  "http://413960661/http.www.paypal.com/": "24.172.137.213",
+  "http://03053104725/http.www.paypal.com/": "24.172.137.213",
+  "http://www.barclays.co.uk.brccontrol.assruspede.org.bz/detailsconfirm":
+      "co.uk.brccontrol.assruspede.org.bz",
+  "http://www.mozilla.org/foo": "www.mozilla.org",
+  "http://,=.mozilla.org/foo": "%2c%3d.mozilla.org",
+  "http://f00.b4r.mozi=lla.org/": "f00.b4r.mozi%3dlla.org",
+  "http://a-_b.mozilla.org/": "a-%5fb.mozilla.org",
+  "http://z%38bl%61h%%2F.com/": "z8blah%25%2f.com",
+  "http://moZilla.Org/": "mozilla.org"
+}
+for (var key in testing) {
+  var out = l.getCanonicalHost(key, PROT_EnchashDecrypter.MAX_DOTS);
+  ok(out == testing[key],
+     "getCanonicalHost broken on: " + key + "(got: " + out + ")");
+}
+
+
+// Test getCanonicalUrl 
+testing = {
+  // For bug 356355.
+  "http://0x18.0xac.0x89.0xd5/http.www.paypal.com/":
+      "http://24.172.137.213/http.www.paypal.com/",
+  "http://0x18ac89d5/http.www.paypal.com/":
+      "http://24.172.137.213/http.www.paypal.com/",
+  "http://413960661/http.www.paypal.com/":
+      "http://24.172.137.213/http.www.paypal.com/",
+  "http://03053104725/http.www.paypal.com/":
+      "http://24.172.137.213/http.www.paypal.com/",
+  "http://03053104725/%68t%74p.www.paypal.c%6fm/":
+      "http://24.172.137.213/http.www.paypal.com/",
+  "http://www.barclays.co.uk.brccontrol.assruspede.org.bz/detailsconfirm":
+      "http://www.barclays.co.uk.brccontrol.assruspede.org.bz/detailsconfirm",
+
+  // For bug 366645
+  "http://030.0254.0x89d5./": "http://24.172.137.213/",
+  "http://030.0254.0x89d5.../": "http://24.172.137.213/",
+  "http://...030.0254.0x89d5.../": "http://24.172.137.213/",
+  "http://127.0.0.1./": "http://127.0.0.1/",
+  "http://127.0.0.1/": "http://127.0.0.1/",
+  "http://a.b.c.d.e.f.g/path": "http://a.b.c.d.e.f.g/path",
+  "http://a.b.c.d.e.f.g...../path": "http://a.b.c.d.e.f.g/path",
+  "http://a.b.c.d.e.f.g./path": "http://a.b.c.d.e.f.g/path"
+}
+for (var key in testing) {
+  ok(l.getCanonicalUrl(key) == testing[key],
+     "getCanonicalUrl broken on: " + key + "(got: " + l.getCanonicalUrl(key) + ")");
+}
+
+// Test for a really long url.  This 130k url takes about 80ms
+// on my 2.8ghz machine.
+startTime = Date.now();
+out = l.getCanonicalUrl(long_hostname_url);
+endTime = Date.now();
+ok(out == long_hostname_url, "getCanonicalUrl on long string (" +
+                       (endTime - startTime) + "ms)");
+
+// Test getlookupkey
+var testing = {};
+testing["www.google.com"] = "AF5638A09FDDDAFF5B7A6013B1BE69A9";
+testing["poseidon.marinet.gr"] = "01844755C8143C4579BB28DD59C23747";
+testing["80.53.164.26"] = "B775DDC22DEBF8BEBFEAC24CE40A1FBF";
+
+for (var key in testing)
+  ok(l.getLookupKey(key) === testing[key],
+           "getlookupkey broken on " + key + " (got: " +
+           l.getLookupKey(key) + ", expected: " +
+           testing[key] + ")");
+// Test decryptdata
+var tests =
+  [ "bGtEQWJuMl/z2ZxSBB2hsuWI8geMAwfSh3YBfYPejQ1O+wyRAJeJ1UW3V56zm" +
+    "EpUvnaEiECN1pndxW5rEMNzE+gppPeel7PvH+OuabL3NXlspcP0xnpK8rzNgB1" +
+    "JT1KcajQ9K3CCl24T9r8VGb0M3w==",
+    "80.53.164.26",
+    "^(?i)http\\:\\/\\/80\\.53\\.164\\.26(?:\\:80)?\\/\\.PayPal" +
+    "\\.com\\/webscr\\-id\\/secure\\-SSL\\/cmd\\-run\\=\\/login\\.htm$",
+
+    "ZTMzZjVnb3WW1Yc2ABorgQGAwYfcaCb/BG3sMFLTMDvOQxH8LkdGGWqp2tI5SK" +
+    "uNrXIHNf2cyzcVocTqUIUkt1Ud1GKieINcp4tWcU53I0VZ0ZZHCjGObDCbv9Wb" +
+    "CPSx1eS8vMREDv8Jj+UVL1yaZQ==",
+    "80.53.164.26",
+    "^(?i)http\\:\\/\\/80\\.53\\.164\\.26(?:\\:80)?\\/\\.PayPal\\.com" +
+    "\\/webscr\\-id\\/secure\\-SSL\\/cmd\\-run\\=\\/login\\.htm$",
+
+    "ZTMzZjVnb3WVb6VqoJ44hVo4V77XjDRcXTxOc2Zpn4yIHcpS0AQ0nn1TVlX4MY" +
+    "IeNL/6ggzCmcJSWOOkj06Mpo56LNLrbxNxTBuoy9GF+xcm",
+    "poseidon.marinet.gr",
+    "^(?i)http\\:\\/\\/poseidon\\.marinet\\.gr(?:\\:80)?\\/\\~eleni" +
+    "\\/eBay\\/index\\.php$",
+
+    "bGtEQWJuMl9FA3Kl5RiXMpgFU8nDJl9J0hXjUck9+mMUQwAN6llf0gJeY5DIPP" +
+    "c2f+a8MSBFJN17ANGJZl5oZVsQfSW4i12rlScsx4tweZAE",
+    "poseidon.marinet.gr",
+    "^(?i)http\\:\\/\\/poseidon\\.marinet\\.gr(?:\\:80)?\\/\\~eleni" +
+    "\\/eBay\\/index\\.php$"];
+
+for (var i = 0; i < tests.length; i += 3) {
+  var dec = l.decryptData(tests[i], tests[i + 1]);
+  ok(dec === tests[i + 2],
+     "decryptdata broken on " + tests[i] + " (got: " + dec + ", expected: "
+     + tests[i + 2] + ")");
+}
+]]>
+</script>
+</pre>
+</body>
+</html>
--- a/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js
+++ b/toolkit/components/url-classifier/tests/unit/head_urlclassifier.js
@ -1,52 +0,0 @@
-function dumpn(s) {
-  dump(s + "\n");
-}
-
-const NS_APP_USER_PROFILE_50_DIR = "ProfD";
-const NS_APP_USER_PROFILE_LOCAL_50_DIR = "ProfLD";
-const Ci = Components.interfaces;
-const Cc = Components.classes;
-const Cr = Components.results;
-
-// If there's no location registered for the profile direcotry, register one now.
-var dirSvc = Cc["@mozilla.org/file/directory_service;1"].getService(Ci.nsIProperties);
-var profileDir = null;
-try {
-  profileDir = dirSvc.get(NS_APP_USER_PROFILE_50_DIR, Ci.nsIFile);
-} catch (e) {}
-
-if (!profileDir) {
-  // Register our own provider for the profile directory.
-  // It will simply return the current directory.
-  var provider = {
-    getFile: function(prop, persistent) {
-      persistent.value = true;
-      if (prop == NS_APP_USER_PROFILE_50_DIR ||
-          prop == NS_APP_USER_PROFILE_LOCAL_50_DIR) {
-        return dirSvc.get("CurProcD", Ci.nsIFile);
-      }
-      throw Cr.NS_ERROR_FAILURE;
-    },
-    QueryInterface: function(iid) {
-      if (iid.equals(Ci.nsIDirectoryServiceProvider) ||
-          iid.equals(Ci.nsISupports)) {
-        return this;
-      }
-      throw Cr.NS_ERROR_NO_INTERFACE;
-    }
-  };
-  dirSvc.QueryInterface(Ci.nsIDirectoryService).registerProvider(provider);
-}
-
-var iosvc = Cc["@mozilla.org/network/io-service;1"].getService(Ci.nsIIOService);
-
-function cleanUp() {
-  try {
-    // Delete a previously created sqlite file
-    var file = dirSvc.get('ProfLD', Ci.nsIFile);
-    file.append("urlclassifier3.sqlite");
-    if (file.exists())
-      file.remove(false);
-  } catch (e) {}
-}
-cleanUp();
--- a/toolkit/components/url-classifier/tests/unit/tail_urlclassifier.js
+++ b/toolkit/components/url-classifier/tests/unit/tail_urlclassifier.js
@ -1 +0,0 @@
-cleanUp();
--- a/toolkit/components/url-classifier/tests/unit/test_dbservice.js
+++ b/toolkit/components/url-classifier/tests/unit/test_dbservice.js
@ -1,167 +0,0 @@
-var dbservice = Cc["@mozilla.org/url-classifier/dbservice;1"].getService(Ci.nsIUrlClassifierDBService);
-
-var checkUrls = [];
-var checkExpect;
-
-var chunk1Urls = [
-  "test.com/aba",
-  "test.com/foo/bar",
-  "foo.bar.com/a/b/c"
-];
-var chunk1 = chunk1Urls.join("\n");
-
-var chunk2Urls = [
-  "blah.com/a",
-  "baz.com/",
-  "255.255.0.1/"
-];
-var chunk2 = chunk2Urls.join("\n");
-
-var chunk3Urls = [
-  "test.com/a",
-  "foo.bar.com/a",
-  "blah.com/a",
-  ];
-var chunk3 = chunk3Urls.join("\n");
-
-// we are going to add chunks 1 and 2 to phish-simple, and chunk 2 to
-// malware-simple.  Then we'll remove the urls in chunk3 from phish-simple,
-// then expire chunk 1 from phish-simple.
-var phishExpected = {};
-var phishUnexpected = {};
-var malwareExpected = {};
-for (var i = 0; i < chunk2Urls.length; i++) {
-  phishExpected[chunk2Urls[i]] = true;
-  malwareExpected[chunk2Urls[i]] = true;
-}
-for (var i = 0; i < chunk3Urls.length; i++) {
-  delete phishExpected[chunk3Urls[i]];
-  phishUnexpected[chunk3Urls[i]] = true;
-}
-for (var i = 0; i < chunk1Urls.length; i++) {
-  // chunk1 urls are expired
-  phishUnexpected[chunk1Urls[i]] = true;
-}
-
-var numExpecting;
-
-function testFailure(arg) {
-  do_throw(arg);
-}
-
-function tablesCallback(tables)
-{
-  var parts = tables.split("\n");
-  parts.sort();
-  // there's a leading \n here because splitting left an empty string
-  // after the trailing newline, which will sort first
-  do_check_eq(parts.join("\n"),
-              "\ntesting-malware-simple;a:1\ntesting-phish-simple;a:2s:3");
-  do_test_finished();
-}
-
-function checkChunks()
-{
-  dbservice.getTables(tablesCallback);
-}
-
-function checkDone() {
-  if (--numExpecting == 0)
-    checkChunks();
-}
-
-function phishExists(result) {
-  dumpn("phishExists: " + result);
-  try {
-    do_check_true(result.indexOf("testing-phish-simple") != -1);
-  } finally {
-    checkDone();
-  }
-}
-
-function phishDoesntExist(result) {
-  dumpn("phishDoesntExist: " + result);
-  try {
-    do_check_true(result.indexOf("testing-phish-simple") == -1);
-  } finally {
-    checkDone();
-  }
-}
-
-function malwareExists(result) {
-  dumpn("malwareExists: " + result);
-
-  try {
-    do_check_true(result.indexOf("testing-malware-simple") != -1);
-  } finally {
-    checkDone();
-  }
-}
-
-function checkState()
-{
-  numExpecting = 0;
-  for (var key in phishExpected) {
-    dbservice.lookup("http://" + key, phishExists, true);
-    numExpecting++;
-  }
-
-  for (var key in phishUnexpected) {
-    dbservice.lookup("http://" + key, phishDoesntExist, true);
-    numExpecting++;
-  }
-
-  for (var key in malwareExpected) {
-    dbservice.lookup("http://" + key, malwareExists, true);
-    numExpecting++;
-  }
-}
-
-function testSubSuccess(result)
-{
-  do_check_eq(result, "1000");
-  checkState();
-}
-
-function do_subs() {
-  var data =
-    "n:1000\n" +
-    "i:testing-phish-simple\n" +
-    "s:3:" + chunk3.length + "\n" +
-    chunk3 + "\n" +
-    "ad:1\n";
-
-  dbservice.update(data);
-  dbservice.finish(testSubSuccess, testFailure);
-}
-
-function testAddSuccess(arg) {
-  do_check_eq(arg, "1000");
-
-  do_subs();
-}
-
-function do_adds() {
-  // This test relies on the fact that only -regexp tables are ungzipped,
-  // and only -hash tables are assumed to be pre-md5'd.  So we use
-  // a 'simple' table type to get simple hostname-per-line semantics.
-
-  var data =
-    "n:1000\n" +
-    "i:testing-phish-simple\n" +
-    "a:1:" + chunk1.length + "\n" +
-    chunk1 + "\n" +
-    "a:2:" + chunk2.length + "\n" +
-    chunk2 + "\n" +
-    "i:testing-malware-simple\n" +
-    "a:1:" + chunk2.length + "\n" +
-    chunk2 + "\n";
-
-  dbservice.update(data);
-  dbservice.finish(testAddSuccess, testFailure);
-}
-
-function run_test() {
-  do_adds();
-  do_test_pending();
-}
--- a/toolkit/components/url-classifier/tests/unit/test_urikeys.js
+++ b/toolkit/components/url-classifier/tests/unit/test_urikeys.js
@ -1,35 +0,0 @@
-
-var utils = Cc["@mozilla.org/url-classifier/utils;1"].createInstance(Ci.nsIUrlClassifierUtils);
-
-function testKey(spec, expect)
-{
-  var uri = iosvc.newURI(spec, null, null);
-  do_check_eq('"' + expect + '"', '"' + utils.getKeyForURI(uri) + '"');
-}
-
-function run_test() {
-//  testKey("http://poseidon.marinet.gr/~elani", "poseidon.marinet.gr/%7Eelani"),
-  testKey("http://www.google.com..", "www.google.com/");
-  testKey("https://www.yaho%6F.com", "www.yahoo.com/");
-  testKey("http://012.034.01.0xa", "10.28.1.10/");
-  testKey("ftp://wierd..chars...%0f,%fa", "wierd.chars.%2c/");
-  testKey("http://0x18ac89d5/http.www.paypal.com/", "24.172.137.213/http.www.paypal.com/");
-  testKey("http://413960661/http.www.paypal.com/", "24.172.137.213/http.www.paypal.com/");
-  testKey("http://03053104725/http.www.paypal.com/", "24.172.137.213/http.www.paypal.com/");
-  testKey("http://www.barclays.co.uk.brccontrol.assruspede.org.bz/detailsconfirm", "www.barclays.co.uk.brccontrol.assruspede.org.bz/detailsconfirm");
-  testKey("http://www.mozilla.org/foo", "www.mozilla.org/foo");
-  testKey("http://,=.mozilla.org/foo", "%2c%3d.mozilla.org/foo");
-  testKey("http://f00.b4r.mozi=lla.org/", "f00.b4r.mozi%3dlla.org/");
-  testKey("http://a-_b.mozilla.org/", "a-%5fb.mozilla.org/");
-  testKey("http://z%38bl%61h%%2F.com/", "z8blah%25%2f.com/");
-  testKey("http://moZilla.Org/", "mozilla.org/");
-  testKey("http://030.0254.0x89d5./", "24.172.137.213/");
-  testKey("http://030.0254.0x89d5.../", "24.172.137.213/");
-  testKey("http://...030.0254.0x89d5.../", "24.172.137.213/");
-  testKey("http://127.0.0.1./", "127.0.0.1/");
-  testKey("http://127.0.0.1/", "127.0.0.1/");
-  testKey("http://a.b.c.d.e.f.g/path", "a.b.c.d.e.f.g/path");
-  testKey("http://a.b.c.d.e.f.g...../path", "a.b.c.d.e.f.g/path");
-  testKey("http://a.b.c.d.e.f.g./path", "a.b.c.d.e.f.g/path");
-  testKey("http://a.b.c.d.e.f.g./path/to/../", "a.b.c.d.e.f.g/path/");
-}
--- a/toolkit/components/url-classifier/tests/unittests.xul
+++ b/toolkit/components/url-classifier/tests/unittests.xul
@ -0,0 +1,188 @@
+<?xml version="1.0"?>
+<window id="PROT_unittest"
+        xmlns="http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" 
+        onload="onProtUnittestLoad();"
+        title="prot unittests">
+
+<script><![CDATA[
+  const Cc = Components.classes;
+  const Ci = Components.interfaces;
+
+  function G_Debug(zone, s) {
+    var label = document.createElement('label');
+    var txt = "[" + zone + "] " + s;
+    label.appendChild(document.createTextNode(txt));
+
+    document.documentElement.appendChild(label);
+  }
+  
+  function G_Assert(zone, cond, msg) {
+    if (!cond) {
+      G_Debug(zone, msg);
+      throw msg;
+    }
+  }
+
+  function ProtectionTableTests() {
+    var z = "trtable UNITTEST";
+
+    G_Debug(z, "Starting");  
+
+    var url = "http://www.yahoo.com?foo=bar";
+    var url2 = "http://168.188.99.26/.secure/www.ebay.com/";
+    var urlTable = Cc['@mozilla.org/url-classifier/table;1?type=url']
+                     .createInstance(Ci.nsIUrlClassifierTable);
+    urlTable.insert(url, "1");
+    urlTable.insert(url2, "1");
+    G_Assert(z, urlTable.exists(url), "URL lookups broken");
+    G_Assert(z, !urlTable.exists("about:config"), "about:config breaks domlook");
+    G_Assert(z, urlTable.exists(url2), "URL lookups broken");
+    G_Assert(z, urlTable.exists("http://%31%36%38%2e%31%38%38%2e%39%39%2e%32%36/%2E%73%65%63%75%72%65/%77%77%77%2E%65%62%61%79%2E%63%6F%6D/") == true,
+             "URL Canonicalization broken");
+    G_Assert(z, urlTable.count == 2, 'urlTable: wrong size');
+
+    var dom1 = "bar.com";
+    var dom2 = "amazon.co.uk";
+    var dom3 = "127.0.0.1";
+    var domainTable = Cc['@mozilla.org/url-classifier/table;1?type=domain']
+                        .createInstance(Ci.nsIUrlClassifierTable);
+    domainTable.insert(dom1, "1");
+    domainTable.insert(dom2, "1");
+    domainTable.insert(dom3, "1");
+    G_Assert(z, domainTable.exists("http://www.bar.com/?zaz=asdf#url"),
+             "Domain lookups broken (single dot)");
+    G_Assert(z, domainTable.exists("http://www.amazon.co.uk/?z=af#url"),
+             "Domain lookups broken (two dots)");
+    G_Assert(z, domainTable.exists("http://127.0.0.1/?z=af#url"),
+             "Domain lookups broken (IP)");
+    G_Assert(z, domainTable.count == 3, 'domainTable: wrong size');
+
+    var site1 = "google.com/safebrowsing/";
+    var site2 = "www.foo.bar/";
+    var site3 = "127.0.0.1/";
+    var siteTable = Cc['@mozilla.org/url-classifier/table;1?type=site']
+                      .createInstance(Ci.nsIUrlClassifierTable);
+    siteTable.insert(site1, "1");
+    siteTable.insert(site2, "1");
+    siteTable.insert(site3, "1");
+    G_Assert(z, siteTable.exists("http://www.google.com/safebrowsing/1.php"),
+             "Site lookups broken - reducing");
+    G_Assert(z, siteTable.exists("http://www.foo.bar/some/random/path"),
+             "Site lookups broken - fqdn");
+    G_Assert(z, siteTable.exists("http://127.0.0.1/something?hello=1"),
+             "Site lookups broken - IP");
+    G_Assert(z, !siteTable.exists("http://www.google.com/search/"),
+             "Site lookups broken - overreaching");
+    G_Assert(z, siteTable.count == 3, 'siteTable: wrong size');
+
+    var url1 = "http://poseidon.marinet.gr/~eleni/eBay/index.php";
+    var domainHash = "01844755C8143C4579BB28DD59C23747";
+    var enchashTable = Cc['@mozilla.org/url-classifier/table;1?type=enchash']
+                         .createInstance(Ci.nsIUrlClassifierTable);
+    enchashTable.insert(domainHash, "bGtEQWJuMl9FA3Kl5RiXMpgFU8nDJl9J0hXjUck9+"
+                                    + "mMUQwAN6llf0gJeY5DIPPc2f+a8MSBFJN17ANGJ"
+                                    + "Zl5oZVsQfSW4i12rlScsx4tweZAE");
+    G_Assert(z, enchashTable.exists(url1), 'enchash lookup failed');
+    G_Assert(z, !enchashTable.exists(url1 + '/foo'),
+             "enchash lookup broken - overreaching");
+    G_Assert(z, enchashTable.count == 1, 'enchashTable: wrong size');
+
+    // TODO: test replace
+    G_Debug(z, "PASSED");
+  }
+  
+  function ProtectionListManagerTests() {
+    var z = "listmanager UNITTEST";
+    G_Debug(z, "Starting");
+
+    // test lookup and register
+    var listManagerInst = Cc["@mozilla.org/url-classifier/listmanager;1"]
+                            .createInstance(Ci.nsIUrlListManager);
+    var listName = 'foo-bar-url';
+    listManagerInst.registerTable(listName, false);
+    listManagerInst.safeInsert(listName, 'test', '1');
+    G_Assert(z, listManagerInst.safeExists(listName, 'test'),
+                'insert/exist failed');
+
+    // test serialization
+    var baseName = (new Date().getTime()) + ".tmp";
+    var tempDir = Cc["@mozilla.org/file/directory_service;1"]
+                  .getService(Ci.nsIProperties)
+                  .get("TmpD", Ci.nsILocalFile);
+    tempDir.append(baseName);
+    tempDir.createUnique(tempDir.DIRECTORY_TYPE, 0744);
+
+    var listManager = Cc["@mozilla.org/url-classifier/listmanager;1"]
+                        .getService(Ci.nsIUrlListManager);
+    listManager.setAppDir(tempDir);
+    
+    var data = "";
+
+    var set1Name = "test1-foo-domain";
+    data += "[" + set1Name + " 1.2]\n";
+    var set1 = {};
+    for (var i = 0; i < 10; i++) {
+      set1["http://" + i + ".com"] = 1;
+      data += "+" + i + ".com\t1\n";
+    }
+
+    data += "\n";
+    var set2Name = "test2-foo-domain";
+    // TODO must have blank line
+    data += "\n[" + set2Name + " 1.7]\n";
+    var set2 = {};
+    for (var i = 0; i < 5; i++) {
+      set2["http://" + i + ".com"] = 1;
+      data += "+" + i + ".com\t1\n";
+    }
+
+    function deserialized(tablesKnown, tablesData) {
+      listManager.wrappedJSObject.dataReady(tablesKnown, tablesData);
+
+      var file = tempDir.clone();
+      file.append(set1Name + ".sst");
+      G_Assert(z, file.exists() && file.isFile() && file.isReadable(), 
+               "Failed to write out: " + file.path);
+      
+      file = tempDir.clone();
+      file.append(set2Name + ".sst");
+      G_Assert(z, file.exists() && file.isFile() && file.isReadable(), 
+               "Failed to write out: " + file.path);
+      
+      // now try to read them back from disk
+      listManager = Cc["@mozilla.org/url-classifier/listmanager;1"]
+                       .createInstance(Ci.nsIUrlListManager);
+      listManager.setAppDir(tempDir);
+      var tables = [ set1Name, set2Name ];
+      listManager.enableUpdate(set1Name);
+      listManager.enableUpdate(set2Name);
+      listManager.wrappedJSObject.readDataFiles();
+      
+      // assert that the values match
+      for (var prop in set1) {
+        G_Assert(z, 
+                 listManager.wrappedJSObject.tablesData[set1Name].exists(prop), 
+                 "Couldn't find member " + prop + "of set1 from disk.");
+      }
+      
+      for (var prop in set2) {
+        G_Assert(z,
+                 listManager.wrappedJSObject.tablesData[set2Name].exists(prop), 
+                 "Couldn't find member " + prop + "of set2 from disk.");
+      }
+
+      tempDir.remove(true);
+      
+      G_Debug(z, "PASSED");
+    };
+
+    // Use the unwrapped object for the unittest
+    listManager.wrappedJSObject.deserialize_(data, deserialized);
+  }
+
+  function onProtUnittestLoad() {
+    ProtectionTableTests();
+    ProtectionListManagerTests();
+  }
+]]></script>
+</window>