Bug 709388 - Restrict set of enabled engines when running TPS tests; r=rnewmana a=testonly

services/sync/tests/tps/test_bookmarks_in_same_named_folder.js

@@ -8,6 +8,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["bookmarks"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_bug501528.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["passwords"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_bug530717.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["prefs"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_bug531489.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["bookmarks"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_bug535326.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["tabs"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2"};

services/sync/tests/tps/test_bug538298.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["bookmarks"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_bug546807.js

@@ -7,6 +7,8 @@
  * testrunner (no single quotes, extra comma's, etc).
  */
 
+EnableEngines(["tabs"]);
+
 var phases = { "phase1": "profile1",
                "phase2": "profile2"};
 

services/sync/tests/tps/test_bug556509.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["bookmarks"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2"};

services/sync/tests/tps/test_bug562515.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["bookmarks"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_bug563989.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["bookmarks"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_bug575423.js

@@ -7,6 +7,8 @@
  * testrunner (no single quotes, extra comma's, etc).
  */
 
+EnableEngines(["history"]);
+
 var phases = { "phase1": "profile1",
                "phase2": "profile2"};
 

services/sync/tests/tps/test_formdata.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["forms"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_history.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["history"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2" };

services/sync/tests/tps/test_history_collision.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["history"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_passwords.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["passwords"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_prefs.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["prefs"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_privbrw_formdata.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["forms"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_privbrw_passwords.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["passwords"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_privbrw_tabs.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["tabs"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tests/tps/test_special_tabs.js

@@ -8,6 +8,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["tabs"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2" };

services/sync/tests/tps/test_tabs.js

@@ -6,6 +6,7 @@
  * here must be in strict JSON format, as it will get parsed by the Python
  * testrunner (no single quotes, extra comma's, etc).
  */
+EnableEngines(["tabs"]);
 
 var phases = { "phase1": "profile1",
                "phase2": "profile2",

services/sync/tps/extensions/tps/components/tps-cmdline.js

@@ -77,16 +77,22 @@ TPSCmdLineHandler.prototype =
 
   /* nsICommandLineHandler */
   handle : function handler_handle(cmdLine) {
-    var uristr = cmdLine.handleFlagWithParam("tps", false);
+    let options = {};
+
+    let uristr = cmdLine.handleFlagWithParam("tps", false);
     if (uristr == null)
         return;
-    var phase = cmdLine.handleFlagWithParam("tpsphase", false);
+    let phase = cmdLine.handleFlagWithParam("tpsphase", false);
     if (phase == null)
         throw("must specify --tpsphase with --tps");
-    var logfile = cmdLine.handleFlagWithParam("tpslogfile", false);
+    let logfile = cmdLine.handleFlagWithParam("tpslogfile", false);
     if (logfile == null)
         logfile = "";
 
+    options.ignoreUnusedEngines = cmdLine.handleFlag("ignore-unused-engines",
+                                                     false);
+
+
     /* Ignore the platform's online/offline status while running tests. */
     var ios = Components.classes["@mozilla.org/network/io-service;1"]
               .getService(Components.interfaces.nsIIOService2);
@@ -96,14 +102,15 @@ TPSCmdLineHandler.prototype =
     Components.utils.import("resource://tps/tps.jsm");
     Components.utils.import("resource://tps/quit.js", TPS);
     let uri = cmdLine.resolveURI(uristr).spec;
-    TPS.RunTestPhase(uri, phase, logfile);
+    TPS.RunTestPhase(uri, phase, logfile, options);
 
     //cmdLine.preventDefault = true;
   },
 
   helpInfo : "  -tps <file>               Run TPS tests with the given test file.\n" +
              "  -tpsphase <phase>         Run the specified phase in the TPS test.\n" +
-             "  -tpslogfile <file>        Logfile for TPS output.\n",
+             "  -tpslogfile <file>        Logfile for TPS output.\n" +
+             "  --ignore-unused-engines   Don't load engines not used in tests.\n",
 };
 
 

services/sync/tps/extensions/tps/modules/tps.jsm

@@ -46,6 +46,7 @@ const {classes: CC, interfaces: CI, utils: CU} = Components;
 
 CU.import("resource://services-sync/service.js");
 CU.import("resource://services-sync/constants.js");
+CU.import("resource://services-sync/engines.js");
 CU.import("resource://services-sync/async.js");
 CU.import("resource://services-sync/util.js");
 CU.import("resource://gre/modules/XPCOMUtils.jsm");
@@ -103,6 +104,7 @@ let TPS =
   _phaselist: {},
   _operations_pending: 0,
   _loggedIn: false,
+  _enabledEngines: null,
 
   DumpError: function (msg) {
     this._errors++;
@@ -489,8 +491,35 @@ let TPS =
     this.RunNextTestAction();
   },
 
-  RunTestPhase: function (file, phase, logpath) {
+  /**
+   * Runs a single test phase.
+   *
+   * This is the main entry point for each phase of a test. The TPS command
+   * line driver loads this module and calls into the function with the
+   * arguments from the command line.
+   *
+   * When a phase is executed, the file is loaded as JavaScript into the
+   * current object.
+   *
+   * The following keys in the options argument have meaning:
+   *
+   *   - ignoreUnusedEngines  If true, unused engines will be unloaded from
+   *                          Sync. This makes output easier to parse and is
+   *                          useful for debugging test failures.
+   *
+   * @param  file
+   *         String URI of the file to open.
+   * @param  phase
+   *         String name of the phase to run.
+   * @param  logpath
+   *         String path of the log file to write to.
+   * @param  options
+   *         Object defining addition run-time options.
+   */
+  RunTestPhase: function (file, phase, logpath, options) {
     try {
+      let settings = options || {};
+
       Logger.init(logpath);
       Logger.logInfo("Sync version: " + WEAVE_VERSION);
       Logger.logInfo("Firefox builddate: " + Services.appinfo.appBuildID);
@@ -522,6 +551,23 @@ let TPS =
         this.DumpError("no profile defined for phase " + this._currentPhase);
         return;
       }
+
+      // If we have restricted the active engines, unregister engines we don't
+      // care about.
+      if (settings.ignoreUnusedEngines && Array.isArray(this._enabledEngines)) {
+        let names = {};
+        for each (let name in this._enabledEngines) {
+          names[name] = true;
+        }
+
+        for each (let engine in Engines.getEnabled()) {
+          if (!(engine.name in names)) {
+            Logger.logInfo("Unregistering unused engine: " + engine.name);
+            Engines.unregister(engine);
+          }
+        }
+      }
+
       Logger.logInfo("Starting phase " + parseInt(phase, 10) + "/" +
                      Object.keys(this._phaselist).length);
 
@@ -559,10 +605,42 @@ let TPS =
     }
   },
 
+  /**
+   * Register a single phase with the test harness.
+   *
+   * This is called when loading individual test files.
+   *
+   * @param  phasename
+   *         String name of the phase being loaded.
+   * @param  fnlist
+   *         Array of functions/actions to perform.
+   */
   Phase: function Test__Phase(phasename, fnlist) {
     this._phaselist[phasename] = fnlist;
   },
 
+  /**
+   * Restrict enabled Sync engines to a specified set.
+   *
+   * This can be called by a test to limit what engines are enabled. It is
+   * recommended to call it to reduce the overhead and log clutter for the
+   * test.
+   *
+   * The "clients" engine is special and is always enabled, so there is no
+   * need to specify it.
+   *
+   * @param  names
+   *         Array of Strings for engines to make active during the test.
+   */
+  EnableEngines: function EnableEngines(names) {
+    if (!Array.isArray(names)) {
+      throw new Error("Argument to RestrictEngines() is not an array: "
+                      + typeof(names));
+    }
+
+    this._enabledEngines = names;
+  },
+
   RunMozmillTest: function TPS__RunMozmillTest(testfile) {
     var mozmillfile = CC["@mozilla.org/file/local;1"]
                       .createInstance(CI.nsILocalFile);

testing/tps/tps/cli.py

@@ -87,6 +87,12 @@ def main():
                     default = None,
                     help = "path to file containing a pulse message in "
                            "json format that you want to inject into the monitor")
+  parser.add_option("--ignore-unused-engines",
+                     default=False,
+                     action="store_true",
+                     dest="ignore_unused_engines",
+                     help="If defined, don't load unused engines in individual tests."
+                           " Has no effect for pulse monitor.")
   (options, args) = parser.parse_args()
 
   configfile = options.configfile
@@ -153,7 +159,8 @@ def main():
                       config=config,
                       rlock=rlock,
                       mobile=options.mobile,
-                      autolog=options.autolog)
+                      autolog=options.autolog,
+                      ignore_unused_engines=options.ignore_unused_engines)
   TPS.run_tests()
 
 if __name__ == "__main__":

testing/tps/tps/phase.py

@@ -44,7 +44,7 @@ class TPSTestPhase(object):
       r"^(.*?)test phase (?P<matchphase>\d+): (?P<matchstatus>.*)$")
 
   def __init__(self, phase, profile, testname, testpath, logfile, env,
-               firefoxRunner, logfn):
+               firefoxRunner, logfn, ignore_unused_engines=False):
     self.phase = phase
     self.profile = profile
     self.testname = str(testname) # this might be passed in as unicode
@@ -53,6 +53,7 @@ class TPSTestPhase(object):
     self.env = env
     self.firefoxRunner = firefoxRunner
     self.log = logfn
+    self.ignore_unused_engines = ignore_unused_engines
     self._status = None
     self.errline = ''
 
@@ -68,14 +69,17 @@ class TPSTestPhase(object):
 
   def run(self):
     # launch Firefox
-    args = [ '-tps', self.testpath, 
-             '-tpsphase', self.phasenum, 
+    args = [ '-tps', self.testpath,
+             '-tpsphase', self.phasenum,
              '-tpslogfile', self.logfile ]
 
-    self.log("\nlaunching firefox for phase %s with args %s\n" % 
+    if self.ignore_unused_engines:
+        args.append('--ignore-unused-engines')
+
+    self.log("\nlaunching Firefox for phase %s with args %s\n" %
              (self.phase, str(args)))
     returncode = self.firefoxRunner.run(env=self.env,
-                                        args=args, 
+                                        args=args,
                                         profile=self.profile)
 
     # parse the logfile and look for results from the current test phase

testing/tps/tps/testrunner.py

@@ -118,12 +118,14 @@ class TPSTestRunner(object):
 
   def __init__(self, extensionDir, emailresults=False, testfile="sync.test",
                binary=None, config=None, rlock=None, mobile=False,
-               autolog=False, logfile="tps.log"):
+               autolog=False, logfile="tps.log",
+               ignore_unused_engines=False):
     self.extensions = []
     self.emailresults = emailresults
     self.testfile = testfile
     self.logfile = os.path.abspath(logfile)
     self.binary = binary
+    self.ignore_unused_engines = ignore_unused_engines
     self.config = config if config else {}
     self.repo = None
     self.changeset = None
@@ -212,14 +214,16 @@ class TPSTestRunner(object):
                                         addons = self.extensions)
 
       # create the test phase
-      phaselist.append(TPSTestPhase(phase,
-                                    profiles[profilename],
-                                    testname,
-                                    tmpfile.filename,
-                                    self.logfile,
-                                    self.env,
-                                    self.firefoxRunner,
-                                    self.log))
+      phaselist.append(TPSTestPhase(
+          phase,
+          profiles[profilename],
+          testname,
+          tmpfile.filename,
+          self.logfile,
+          self.env,
+          self.firefoxRunner,
+          self.log,
+          ignore_unused_engines=self.ignore_unused_engines))
 
     # sort the phase list by name
     phaselist = sorted(phaselist, key=lambda phase: phase.phase)